AI observability

For instructions on how to authenticate to use this endpoint, see API overview.

Endpoints

`GET`
`POST`
`POST`
`GET`
`POST`
`GET`
`PATCH`
`POST`
`POST`
`GET`
`GET`
`GET`
`GET`
`GET`
`GET`
`GET`
`POST`
`GET`
`GET`
`GET`

Retrieve ai observability instrumentation checklist

Grade every instrumentation check for this project.

Required API key scopes

llm_analytics:read

Query parameters

refresh
boolean
Default: false

Response

Example request

GET /api/projects/:project_id/ai_observability/instrumentation_checklist
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/instrumentation_checklist/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/instrumentation_checklist/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "window_days": 0,
  "checks": [
    {
      "key": "sessions",
      "status": "ok",
      "title": "string",
      "detail": "string",
      "docs_url": "http://example.com",
      "stats": {
        "property1": 0,
        "property2": 0
      }
    }
  ]
}

Create ai observability instrumentation checklist dismiss

Mark a check as not applicable to this project.

Required API key scopes

llm_analytics:write

Request parameters

check

Response

Example request

POST /api/projects/:project_id/ai_observability/instrumentation_checklist/dismiss
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/instrumentation_checklist/dismiss/\
	-d check=undefined
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/instrumentation_checklist/dismiss/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={}
).json()

Example response

Status 200

RESPONSE
{
  "window_days": 0,
  "checks": [
    {
      "key": "sessions",
      "status": "ok",
      "title": "string",
      "detail": "string",
      "docs_url": "http://example.com",
      "stats": {
        "property1": 0,
        "property2": 0
      }
    }
  ]
}

Create ai observability instrumentation checklist restore

Bring a dismissed check back into grading.

Required API key scopes

llm_analytics:write

Request parameters

check

Response

Example request

POST /api/projects/:project_id/ai_observability/instrumentation_checklist/restore
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/instrumentation_checklist/restore/\
	-d check=undefined
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/instrumentation_checklist/restore/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={}
).json()

Example response

Status 200

RESPONSE
{
  "window_days": 0,
  "checks": [
    {
      "key": "sessions",
      "status": "ok",
      "title": "string",
      "detail": "string",
      "docs_url": "http://example.com",
      "stats": {
        "property1": 0,
        "property2": 0
      }
    }
  ]
}

List all ai observability offline experiments

Required API key scopes

evaluation:read

Query parameters

application_version
string
created_by
integer
created_by_is_null
boolean
cursor
string
dataset_identifier
string
dataset_revision_identifier
string
dataset_source
string
date_from
string
date_to
string
limit
integer
Default: 50
model_version
string
prompt_version
string
run_source
string
scorer_definition_id
string
scorer_version_ids
string
search
string
statuses
string
suite_key
string
tag_match
string
Default: any
One of: "any""all"
tags
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 0,
  "next_cursor": "string",
  "results": [
    {
      "created_by": {
        "id": 0,
        "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
        "distinct_id": "string",
        "first_name": "string",
        "last_name": "string",
        "email": "user@example.com",
        "is_email_verified": true,
        "hedgehog_config": {},
        "role_at_organization": "engineering"
      },
      "tags": [
        "string"
      ],
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "name": "string",
      "run_source": "ci",
      "status": "uploading",
      "started_at": "2019-08-24T14:15:22Z",
      "created_at": "2019-08-24T14:15:22Z",
      "finished_at": "2019-08-24T14:15:22Z",
      "expected_item_count": 0,
      "expected_result_count": 0,
      "accepted_item_count": 0,
      "visible_result_count": 0,
      "visible_scorer_definition_count": 0,
      "visible_scorer_version_count": 0,
      "result_count_scope": "string",
      "suite_key": "string",
      "dataset_source": "string",
      "dataset_identifier": "string",
      "dataset_revision_identifier": "string",
      "dataset_revision_id": "e7047d72-9b1c-4feb-9c11-cc3d637eeab5",
      "application_version": "string",
      "model_version": "string",
      "prompt_version": "string"
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Create ai observability offline experiments

Required API key scopes

offline_evaluation_ingestion:write

Request parameters

id
string
name
string
started_at
string
run_source
expected_item_count
integer | null
expected_result_count
integer | null
suite_key
string | null
dataset_source
string | null
dataset_identifier
string | null
dataset_revision_identifier
string | null
dataset_revision_id
string | null
application_version
string | null
model_version
string | null
prompt_version
string | null

Response

Example request

POST /api/projects/:project_id/ai_observability/offline_experiments
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/\
	-d id="string",\
	-d name="string",\
	-d started_at="string"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/".format(
        project_id=project_id
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"id": "string",
		"name": "string",
		"started_at": "string"
	}
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "uploading",
  "created": true,
  "started_at": "2019-08-24T14:15:22Z",
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0
}

Status 201

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "uploading",
  "created": true,
  "started_at": "2019-08-24T14:15:22Z",
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 409

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 413

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Retrieve ai observability offline experiments

Required API key scopes

evaluation:read

Path parameters

id
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "tags": [
    "string"
  ],
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "name": "string",
  "run_source": "ci",
  "status": "uploading",
  "started_at": "2019-08-24T14:15:22Z",
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "visible_result_count": 0,
  "visible_scorer_definition_count": 0,
  "visible_scorer_version_count": 0,
  "result_count_scope": "string",
  "suite_key": "string",
  "dataset_source": "string",
  "dataset_identifier": "string",
  "dataset_revision_identifier": "string",
  "dataset_revision_id": "e7047d72-9b1c-4feb-9c11-cc3d637eeab5",
  "application_version": "string",
  "model_version": "string",
  "prompt_version": "string"
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Update ai observability offline experiments

Required API key scopes

evaluation:write

Path parameters

id
string

Request parameters

tags
array

Response

Example request

PATCH /api/projects/:project_id/ai_observability/offline_experiments/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl -X PATCH \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/\
	-d tags="array"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.patch(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"tags": "array"
	}
).json()

Example response

Status 200

RESPONSE
{
  "created_by": {
    "id": 0,
    "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
    "distinct_id": "string",
    "first_name": "string",
    "last_name": "string",
    "email": "user@example.com",
    "is_email_verified": true,
    "hedgehog_config": {},
    "role_at_organization": "engineering"
  },
  "tags": [
    "string"
  ],
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "name": "string",
  "run_source": "ci",
  "status": "uploading",
  "started_at": "2019-08-24T14:15:22Z",
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "visible_result_count": 0,
  "visible_scorer_definition_count": 0,
  "visible_scorer_version_count": 0,
  "result_count_scope": "string",
  "suite_key": "string",
  "dataset_source": "string",
  "dataset_identifier": "string",
  "dataset_revision_identifier": "string",
  "dataset_revision_id": "e7047d72-9b1c-4feb-9c11-cc3d637eeab5",
  "application_version": "string",
  "model_version": "string",
  "prompt_version": "string"
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 409

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 413

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Create ai observability offline experiments complete

Required API key scopes

offline_evaluation_ingestion:write

Path parameters

id
string

Response

Example request

POST /api/projects/:project_id/ai_observability/offline_experiments/:id/complete
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/complete/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/complete/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "uploading",
  "created": true,
  "started_at": "2019-08-24T14:15:22Z",
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 409

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 413

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Create ai observability offline experiments fail

Required API key scopes

offline_evaluation_ingestion:write

Path parameters

id
string

Response

Example request

POST /api/projects/:project_id/ai_observability/offline_experiments/:id/fail
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/fail/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/fail/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "status": "uploading",
  "created": true,
  "started_at": "2019-08-24T14:15:22Z",
  "created_at": "2019-08-24T14:15:22Z",
  "finished_at": "2019-08-24T14:15:22Z",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 409

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 413

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

List all ai observability offline experiments items

Required API key scopes

evaluation:read

Path parameters

id
string

Query parameters

cursor
string
limit
integer
Default: 50
scorer_version_ids
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/items
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/items/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/items/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 0,
  "next_cursor": "string",
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "experiment_id": "916afd89-cac5-4339-9c59-dd068abdfa69",
      "case_key": "string",
      "trial": "string",
      "dataset_item_identifier": "string",
      "dataset_item_version_identifier": "string",
      "dataset_item_version_id": "987e10b5-2243-43e5-b94b-a21eb028fb88",
      "application_trace_id": "string",
      "accepted_at": "2019-08-24T14:15:22Z",
      "payload_state": "not_provided",
      "payload_expires_at": "2019-08-24T14:15:22Z",
      "results": [
        {
          "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
          "item_id": "4d8cd62e-a579-4dae-af8c-3172f96f8f7c",
          "status": "ok",
          "value": 0.1,
          "error_code": "string",
          "evaluator_trace_id": "string",
          "evaluated_at": "2019-08-24T14:15:22Z",
          "accepted_at": "2019-08-24T14:15:22Z",
          "payload_state": "not_provided",
          "payload_expires_at": "2019-08-24T14:15:22Z",
          "scorer_version_id": "d07df8d1-3b1c-4e6f-8d6d-dd2c83c1a72e"
        }
      ]
    }
  ],
  "scorer_versions": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "definition_id": "91c1994f-b1db-4fef-840d-7d3ab2984871",
      "version": 0,
      "kind": "categorical",
      "name": "string",
      "short_name": "string",
      "description": "string",
      "archived": true,
      "config": {
        "options": [
          {
            "key": "string",
            "label": "string"
          }
        ],
        "selection_mode": "single",
        "min_selections": 1,
        "max_selections": 1,
        "passing_rule": {
          "categories": [
            "string"
          ]
        }
      }
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Retrieve ai observability offline experiments items

Required API key scopes

evaluation:read

Path parameters

id
string
item_id
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/items/:item_id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/items/:item_id/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/items/{item_id}/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>",
		item_id="<the item id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "experiment_id": "916afd89-cac5-4339-9c59-dd068abdfa69",
  "case_key": "string",
  "trial": "string",
  "dataset_item_identifier": "string",
  "dataset_item_version_identifier": "string",
  "dataset_item_version_id": "987e10b5-2243-43e5-b94b-a21eb028fb88",
  "application_trace_id": "string",
  "accepted_at": "2019-08-24T14:15:22Z",
  "payload_state": "not_provided",
  "payload_expires_at": "2019-08-24T14:15:22Z",
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "item_id": "4d8cd62e-a579-4dae-af8c-3172f96f8f7c",
      "status": "ok",
      "value": 0.1,
      "error_code": "string",
      "evaluator_trace_id": "string",
      "evaluated_at": "2019-08-24T14:15:22Z",
      "accepted_at": "2019-08-24T14:15:22Z",
      "payload_state": "not_provided",
      "payload_expires_at": "2019-08-24T14:15:22Z",
      "scorer_version_id": "d07df8d1-3b1c-4e6f-8d6d-dd2c83c1a72e"
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Retrieve ai observability offline experiments items payload

Required API key scopes

evaluation:read

Path parameters

id
string
item_id
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/items/:item_id/payload
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/items/:item_id/payload/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/items/{item_id}/payload/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>",
		item_id="<the item id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "payload_state": "not_provided",
  "payload_expires_at": "2019-08-24T14:15:22Z",
  "available": true,
  "data": {
    "input": {},
    "output": {},
    "expected_output": {},
    "metadata": {}
  }
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

List all ai observability offline experiments items results

Required API key scopes

evaluation:readllm_analytics:read

Path parameters

id
string
item_id
string

Query parameters

cursor
string
limit
integer
Default: 50
scorer_definition_id
string
scorer_version_ids
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/items/:item_id/results
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/items/:item_id/results/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/items/{item_id}/results/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>",
		item_id="<the item id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 0,
  "next_cursor": "string",
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "item_id": "4d8cd62e-a579-4dae-af8c-3172f96f8f7c",
      "status": "ok",
      "value": 0.1,
      "error_code": "string",
      "evaluator_trace_id": "string",
      "evaluated_at": "2019-08-24T14:15:22Z",
      "accepted_at": "2019-08-24T14:15:22Z",
      "payload_state": "not_provided",
      "payload_expires_at": "2019-08-24T14:15:22Z",
      "scorer": {
        "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
        "definition_id": "91c1994f-b1db-4fef-840d-7d3ab2984871",
        "version": 0,
        "kind": "categorical",
        "name": "string",
        "short_name": "string",
        "description": "string",
        "archived": true,
        "config": {
          "options": [
            {
              "key": "string",
              "label": "string"
            }
          ],
          "selection_mode": "single",
          "min_selections": 1,
          "max_selections": 1,
          "passing_rule": {
            "categories": [
              "string"
            ]
          }
        }
      }
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Retrieve ai observability offline experiments result cells

Required API key scopes

evaluation:readllm_analytics:read

Path parameters

id
string

Query parameters

item_ids
string
scorer_version_ids
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/result_cells
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/result_cells/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/result_cells/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "scorer_versions": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "definition_id": "91c1994f-b1db-4fef-840d-7d3ab2984871",
      "version": 0,
      "kind": "categorical",
      "name": "string",
      "short_name": "string",
      "description": "string",
      "archived": true,
      "config": {
        "options": [
          {
            "key": "string",
            "label": "string"
          }
        ],
        "selection_mode": "single",
        "min_selections": 1,
        "max_selections": 1,
        "passing_rule": {
          "categories": [
            "string"
          ]
        }
      }
    }
  ],
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "item_id": "4d8cd62e-a579-4dae-af8c-3172f96f8f7c",
      "status": "ok",
      "value": 0.1,
      "error_code": "string",
      "evaluator_trace_id": "string",
      "evaluated_at": "2019-08-24T14:15:22Z",
      "accepted_at": "2019-08-24T14:15:22Z",
      "payload_state": "not_provided",
      "payload_expires_at": "2019-08-24T14:15:22Z",
      "scorer_version_id": "d07df8d1-3b1c-4e6f-8d6d-dd2c83c1a72e"
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Retrieve ai observability offline experiments results payload

Required API key scopes

evaluation:readllm_analytics:read

Path parameters

id
string
result_id
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/results/:result_id/payload
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/results/:result_id/payload/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/results/{result_id}/payload/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>",
		result_id="<the result id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
  "payload_state": "not_provided",
  "payload_expires_at": "2019-08-24T14:15:22Z",
  "available": true,
  "data": {
    "reasoning": "string",
    "error_message": "string",
    "metadata": {}
  }
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

List all ai observability offline experiments scorer summaries

Required API key scopes

evaluation:readllm_analytics:read

Path parameters

id
string

Query parameters

cursor
string
limit
integer
Default: 50
scorer_definition_id
string
scorer_version_ids
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_experiments/:id/scorer_summaries
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/scorer_summaries/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/scorer_summaries/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 0,
  "next_cursor": "string",
  "results": [
    {
      "scorer": {
        "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
        "definition_id": "91c1994f-b1db-4fef-840d-7d3ab2984871",
        "version": 0,
        "kind": "categorical",
        "name": "string",
        "short_name": "string",
        "description": "string",
        "archived": true,
        "config": {
          "options": [
            {
              "key": "string",
              "label": "string"
            }
          ],
          "selection_mode": "single",
          "min_selections": 1,
          "max_selections": 1,
          "passing_rule": {
            "categories": [
              "string"
            ]
          }
        }
      },
      "observed_item_count": 0,
      "result_count": 0,
      "status_counts": {
        "ok": 0,
        "error": 0,
        "skipped": 0,
        "not_applicable": 0
      },
      "missing_result_count": 0,
      "distinct_case_count": 0,
      "items_with_case_key_count": 0,
      "items_without_case_key_count": 0,
      "trial_item_count": 0,
      "distinct_trial_count": 0,
      "mean": 0,
      "true_count": 0,
      "false_count": 0,
      "true_rate": 0,
      "pass_count": 0,
      "fail_count": 0,
      "pass_rate": 0,
      "categories": [
        {
          "key": "string",
          "count": 0,
          "rate": 0,
          "label": "string"
        }
      ]
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Create ai observability offline experiments upload

Required API key scopes

offline_evaluation_ingestion:write

Path parameters

id
string

Request parameters

items
array
results
array

Response

Example request

POST /api/projects/:project_id/ai_observability/offline_experiments/:id/upload
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl 
    -H 'Content-Type: application/json'\
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_experiments/:id/upload/\
	-d results="array"
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.post(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_experiments/{ai_observability_id}/upload/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
	data={
		"results": "array"
	}
).json()

Example response

Status 200

RESPONSE
{
  "items": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "created": true,
      "accepted_at": "2019-08-24T14:15:22Z"
    }
  ],
  "results": [
    {
      "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
      "created": true,
      "accepted_at": "2019-08-24T14:15:22Z",
      "item_id": "4d8cd62e-a579-4dae-af8c-3172f96f8f7c",
      "scorer_version_id": "d07df8d1-3b1c-4e6f-8d6d-dd2c83c1a72e"
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 409

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 413

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

List all ai observability offline scorers history

Required API key scopes

evaluation:readllm_analytics:read

Path parameters

id
string

Query parameters

application_version
string
created_by
integer
created_by_is_null
boolean
cursor
string
dataset_identifier
string
dataset_revision_identifier
string
dataset_source
string
date_from
string
date_to
string
limit
integer
Default: 50
model_version
string
prompt_version
string
run_source
string
scorer_version_ids
string
search
string
statuses
string
suite_key
string
tag_match
string
Default: any
One of: "any""all"
tags
string

Response

Example request

GET /api/projects/:project_id/ai_observability/offline_scorers/:id/history
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/offline_scorers/:id/history/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/offline_scorers/{ai_observability_id}/history/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "count": 0,
  "next_cursor": "string",
  "results": [
    {
      "experiment": {
        "created_by": {
          "id": 0,
          "uuid": "095be615-a8ad-4c33-8e9c-c7612fbf6c9f",
          "distinct_id": "string",
          "first_name": "string",
          "last_name": "string",
          "email": "user@example.com",
          "is_email_verified": true,
          "hedgehog_config": {},
          "role_at_organization": "engineering"
        },
        "tags": [
          "string"
        ],
        "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
        "name": "string",
        "run_source": "ci",
        "status": "uploading",
        "started_at": "2019-08-24T14:15:22Z",
        "created_at": "2019-08-24T14:15:22Z",
        "finished_at": "2019-08-24T14:15:22Z",
        "expected_item_count": 0,
        "expected_result_count": 0,
        "accepted_item_count": 0,
        "visible_result_count": 0,
        "visible_scorer_definition_count": 0,
        "visible_scorer_version_count": 0,
        "result_count_scope": "string",
        "suite_key": "string",
        "dataset_source": "string",
        "dataset_identifier": "string",
        "dataset_revision_identifier": "string",
        "dataset_revision_id": "e7047d72-9b1c-4feb-9c11-cc3d637eeab5",
        "application_version": "string",
        "model_version": "string",
        "prompt_version": "string"
      },
      "summary": {
        "scorer": {
          "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
          "definition_id": "91c1994f-b1db-4fef-840d-7d3ab2984871",
          "version": 0,
          "kind": "categorical",
          "name": "string",
          "short_name": "string",
          "description": "string",
          "archived": true,
          "config": {
            "options": [
              {
                "key": "string",
                "label": "string"
              }
            ],
            "selection_mode": "single",
            "min_selections": 1,
            "max_selections": 1,
            "passing_rule": {
              "categories": [
                "string"
              ]
            }
          }
        },
        "observed_item_count": 0,
        "result_count": 0,
        "status_counts": {
          "ok": 0,
          "error": 0,
          "skipped": 0,
          "not_applicable": 0
        },
        "missing_result_count": 0,
        "distinct_case_count": 0,
        "items_with_case_key_count": 0,
        "items_without_case_key_count": 0,
        "trial_item_count": 0,
        "distinct_trial_count": 0,
        "mean": 0,
        "true_count": 0,
        "false_count": 0,
        "true_rate": 0,
        "pass_count": 0,
        "fail_count": 0,
        "pass_rate": 0,
        "categories": [
          {
            "key": "string",
            "count": 0,
            "rate": 0,
            "label": "string"
          }
        ]
      }
    }
  ]
}

Status 400

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 401

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 403

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 404

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 429

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Status 503

RESPONSE
{
  "type": "string",
  "code": "string",
  "detail": "string",
  "attr": "string",
  "expected_item_count": 0,
  "expected_result_count": 0,
  "accepted_item_count": 0,
  "accepted_result_count": 0,
  "errors": [
    {
      "code": "string",
      "detail": "string",
      "attr": "string"
    }
  ]
}

Retrieve ai observability traces

A trace ready to render: its tree with roll-ups, timeline, totals and person, without inputs or outputs.

Required API key scopes

llm_analytics:read

Path parameters

id
string

Query parameters

timestamp_hint
string

Response

Example request

GET /api/projects/:project_id/ai_observability/traces/:id
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/traces/:id/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/traces/{ai_observability_id}/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "id": "string",
  "name": "string",
  "createdAt": "2019-08-24T14:15:22Z",
  "sessionId": "string",
  "person": {
    "distinctId": "string",
    "label": "string"
  },
  "totals": {
    "costUsd": 0,
    "inputTokens": 0,
    "outputTokens": 0,
    "cacheReadTokens": 0,
    "cacheWriteTokens": 0,
    "latencyMs": 0
  },
  "hasError": true,
  "errorCount": 0,
  "tree": [
    {
      "id": "string",
      "kind": "trace",
      "name": "string",
      "model": "string",
      "stats": {
        "costUsd": 0,
        "inputTokens": 0,
        "outputTokens": 0,
        "cacheReadTokens": 0,
        "cacheWriteTokens": 0,
        "latencyMs": 0
      },
      "hasError": true,
      "children": [
        {}
      ]
    }
  ],
  "timeline": [
    {
      "id": "string",
      "kind": "trace",
      "name": "string",
      "depth": 0,
      "startMs": 0,
      "durationMs": 0,
      "hasError": true
    }
  ],
  "totalMs": 0,
  "threadNodeIds": [
    "string"
  ],
  "defaultNodeId": "string",
  "nodeIdsByAlias": {
    "property1": "string",
    "property2": "string"
  }
}

Status 400 The timestamp hint or the encoded trace id is malformed.

Status 404 No trace with this id in the project.

Status 429 Too many queries are running for the project or organization.

Retrieve ai observability traces evaluations

A trace's evaluation results, newest first and at most 250, each with its outcome and label ready to render.

Required API key scopes

llm_analytics:read

Path parameters

id
string

Response

Example request

GET /api/projects/:project_id/ai_observability/traces/:id/evaluations
export POSTHOG_PERSONAL_API_KEY=[your personal api key]
curl \
    -H "Authorization: Bearer $POSTHOG_PERSONAL_API_KEY" \
    <ph_app_host>/api/projects/:project_id/ai_observability/traces/:id/evaluations/
api_key = "[your personal api key]"
project_id = "[your project id]"
response = requests.get(
    "<ph_app_host>/api/projects/{project_id}/ai_observability/traces/{ai_observability_id}/evaluations/".format(
        project_id=project_id,
		ai_observability_id="<the ai observability id>"
    ),
    headers={"Authorization": "Bearer {}".format(api_key)},
).json()

Example response

Status 200

RESPONSE
{
  "results": [
    {
      "id": "string",
      "evaluationId": "string",
      "evaluationName": "string",
      "targetNodeId": "string",
      "outcome": "pass",
      "label": "string",
      "reasoning": "string",
      "timestamp": "2019-08-24T14:15:22Z",
      "isBackfill": true
    }
  ]
}