# AUTO-GENERATED tier-A mirror spec — do not edit.
# Source: openapi/runtime/evaluations.yaml (runtime 0.124.0)
#  + policy: openapi/mirror-policies/evaluations.json
# Regenerate: node scripts/generate-mirror-specs.mjs
openapi: 3.0.3
info:
  title: naturali.ai — Evaluations API
  version: 1.0.0
  description: >-
    Evaluations: datasets of test cases, the evals that score an agent against them, and the runs
    that record each attempt — the repeatable suites a change is verified against before it rolls
    out. A fully runtime-backed module — this spec is generated verbatim from the runtime's own,
    re-rooted under /v1/projects/{project_id}. The project in the path is authorized by naturali and
    enforced upstream by the project's scoped credential.


    This module mirrors the upstream runtime verbatim (tier A): paths are the runtime's own
    re-rooted under /v1/projects/{project_id}, and every field, method, status code and error shape
    passes through unchanged. Errors raised by the runtime arrive in its envelope; errors raised by
    naturali itself (authentication, project resolution, an unreachable runtime) use naturali's.
  contact:
    name: naturali.ai
    url: https://naturali.ai
servers:
  - url: "{baseUrl}"
    description: Host of your naturali.ai deployment; every path carries the /v1 prefix.
    variables:
      baseUrl:
        description: Base host URL.
        default: https://api.naturali.ai
tags:
  - name: Evaluations
    description: Datasets, evals, and eval runs
security:
  - bearerAuth: []
  - oauth2:
      - mcp:access
paths:
  /v1/projects/{project_id}/datasets:
    get:
      tags:
        - Evaluations
      summary: List datasets
      description: Returns the datasets defined in a project
      operationId: listDatasets
      parameters:
        - $ref: "#/components/parameters/Limit"
        - $ref: "#/components/parameters/Cursor"
      responses:
        "200":
          description: List of datasets
          content:
            application/json:
              schema:
                allOf:
                  - $ref: "#/components/schemas/Page"
                  - properties:
                      data:
                        type: array
                        items:
                          $ref: "#/components/schemas/Dataset"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          $ref: "#/components/responses/ProjectNotFound"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
    post:
      tags:
        - Evaluations
      summary: Create a dataset
      description: >-
        Creates a project-scoped dataset — a named collection of test cases an eval runs an agent
        against. Names are unique per project.


        Datasets are operator-owned **fixtures**. The platform's content purge never deletes or
        mutates a dataset item, so erasing a generation cannot silently stop a test suite from being
        runnable.
      operationId: createDataset
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required:
                - name
              additionalProperties: false
              properties:
                name:
                  type: string
                  description: Unique name within the project
                  example: billing-regressions
                description:
                  type: string
                  nullable: true
                  description: What this suite covers
                  example: Questions the billing agent regressed on in Q2
      responses:
        "201":
          description: Dataset created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Dataset"
        "400":
          description: Bad request (missing or invalid name)
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          $ref: "#/components/responses/ProjectNotFound"
        "409":
          description: A dataset with that name already exists in the project
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/datasets/{dataset_id}:
    get:
      tags:
        - Evaluations
      summary: Get a dataset
      description: Returns a specific dataset
      operationId: getDataset
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
      responses:
        "200":
          description: Dataset details
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Dataset"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
    put:
      tags:
        - Evaluations
      summary: Update a dataset
      description: Updates a dataset's name and/or description
      operationId: updateDataset
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              additionalProperties: false
              properties:
                name:
                  type: string
                  example: billing-regressions
                description:
                  type: string
                  nullable: true
                  example: Questions the billing agent regressed on in Q2
      responses:
        "200":
          description: Dataset updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Dataset"
        "400":
          description: Bad request
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "409":
          description: A dataset with that name already exists in the project
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    delete:
      tags:
        - Evaluations
      summary: Delete a dataset
      description: Deletes a dataset, its items, and every eval bound to it. Results of runs that already
        scored those items keep their frozen copies of the input and expected output.
      operationId: deleteDataset
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
      responses:
        "204":
          description: Dataset deleted successfully
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/datasets/{dataset_id}/items:
    get:
      tags:
        - Evaluations
      summary: List dataset items
      description: Returns the test cases in a dataset, oldest first
      operationId: listDatasetItems
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
        - $ref: "#/components/parameters/Limit"
        - $ref: "#/components/parameters/Cursor"
      responses:
        "200":
          description: List of dataset items
          content:
            application/json:
              schema:
                allOf:
                  - $ref: "#/components/schemas/Page"
                  - properties:
                      data:
                        type: array
                        items:
                          $ref: "#/components/schemas/DatasetItem"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
    post:
      tags:
        - Evaluations
      summary: Add a dataset item
      description: Adds one test case. `input` is replayed verbatim as the generation's messages, so it
        must be a non-empty array of `{ role, content }`.
      operationId: createDatasetItem
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required:
                - input
              additionalProperties: false
              properties:
                input:
                  $ref: "#/components/schemas/DatasetItemInput"
                expected_output:
                  type: string
                  nullable: true
                  description: Reference answer for exact_match / embedding_similarity / llm_judge scorers
                  example: Your invoice is issued on the first of each month.
                metadata:
                  description: Free-form tags, opaque to the platform
                  example:
                    topic: billing
                  allOf:
                    - $ref: "#/components/schemas/NullableMetadataBag"
      responses:
        "201":
          description: Dataset item created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/DatasetItem"
        "400":
          description: Bad request (input is not message-shaped)
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "409":
          description: The project's `storage_bytes` quota is exceeded (`QUOTA_STORAGE_EXCEEDED`). A fixture
            is a corpus write like a document, so it is bounded by the same cap; delete stored
            content or raise the quota — no `Retry-After` is sent, since no window reset clears a
            stored total.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/datasets/{dataset_id}/items/from-generation:
    post:
      tags:
        - Evaluations
      summary: Curate a dataset item from a generation
      description: >-
        Promotes a real, completed generation into a test case: its input messages become the item's
        `input`, and its own answer becomes `expected_output` unless you supply one. Use it to build
        an evaluation set out of production traffic rather than hand-authoring fixtures.


        The item is a **copy**, not a view. It keeps working after the source generation's content
        is purged, and `source_generation_id` goes null if that generation is deleted — a purge can
        never quietly stop a suite from being runnable.


        Requires both `evaluations:CreateDataset` and `generations:GetGeneration`: the call copies
        content out of a generation, so a principal that may not read that generation may not curate
        it either.


        Only a **completed** generation can be promoted (`409 GENERATION_NOT_COMPLETED`), and only
        while its content is still available: an agent or project running with `trace_content_mode:
        none` never stored the input, and a purged or expired generation no longer has it (`409
        GENERATION_CONTENT_UNAVAILABLE`). Generations that predate input recording answer the same
        way.
      operationId: createDatasetItemFromGeneration
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required:
                - generation_id
              additionalProperties: false
              properties:
                generation_id:
                  type: string
                  description: The completed generation to promote. Must belong to the same project as the dataset.
                  example: gen_V1StGXR8Z5jdHi6B
                expected_output:
                  type: string
                  nullable: true
                  description: Reference answer. Omit to use the generation's own answer; pass `null` to store the
                    item with no reference answer.
                  example: Your invoice is issued on the first of each month.
                metadata:
                  description: Free-form tags, opaque to the platform
                  example:
                    topic: billing
                  allOf:
                    - $ref: "#/components/schemas/NullableMetadataBag"
      responses:
        "201":
          description: Dataset item created from the generation
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/DatasetItem"
        "400":
          description: Bad request (generation_id missing, or the generation belongs to a different project
            than the dataset)
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset or generation not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "409":
          description: The generation has not completed, its content was never stored or has been purged, or
            the project's `storage_bytes` quota is exceeded (`QUOTA_STORAGE_EXCEEDED`; delete stored
            content or raise the quota — no `Retry-After` is sent, since no window reset clears a
            stored total)
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/datasets/{dataset_id}/items/{item_id}:
    put:
      tags:
        - Evaluations
      summary: Update a dataset item
      description: Updates a test case. Runs that already scored it are unaffected — each result carries
        its own frozen copy of the input and expected output.
      operationId: updateDatasetItem
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
        - name: item_id
          in: path
          required: true
          description: Dataset item ID
          schema:
            type: string
            example: dsit_V1StGXR8Z5jdHi6B
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              additionalProperties: false
              properties:
                input:
                  $ref: "#/components/schemas/DatasetItemInput"
                expected_output:
                  type: string
                  nullable: true
                  example: Your invoice is issued on the first of each month.
                metadata:
                  example:
                    topic: billing
                  allOf:
                    - $ref: "#/components/schemas/NullableMetadataBag"
      responses:
        "200":
          description: Dataset item updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/DatasetItem"
        "400":
          description: Bad request
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset or item not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    delete:
      tags:
        - Evaluations
      summary: Delete a dataset item
      description: Deletes a test case. Results of runs that already scored it stay readable; their
        `dataset_item_id` becomes null.
      operationId: deleteDatasetItem
      parameters:
        - name: dataset_id
          in: path
          required: true
          description: Dataset ID
          schema:
            type: string
            example: dset_V1StGXR8Z5jdHi6B
        - name: item_id
          in: path
          required: true
          description: Dataset item ID
          schema:
            type: string
            example: dsit_V1StGXR8Z5jdHi6B
      responses:
        "204":
          description: Dataset item deleted successfully
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Dataset or item not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/evals:
    get:
      tags:
        - Evaluations
      summary: List evals
      description: Returns the evals defined in a project
      operationId: listEvals
      parameters:
        - $ref: "#/components/parameters/Limit"
        - $ref: "#/components/parameters/Cursor"
      responses:
        "200":
          description: List of evals
          content:
            application/json:
              schema:
                allOf:
                  - $ref: "#/components/schemas/Page"
                  - properties:
                      data:
                        type: array
                        items:
                          $ref: "#/components/schemas/Eval"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          $ref: "#/components/responses/ProjectNotFound"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
    post:
      tags:
        - Evaluations
      summary: Create an eval
      description: >-
        Binds an agent under test to a dataset and a list of scorers. The agent and the dataset must
        belong to the same project as the eval; a cross-project reference is rejected with 400.


        Scorer config is frozen here rather than read from the agent at run time, so two runs of the
        same eval are always judged by the same criteria and their comparison measures the agent
        instead of the config drifting underneath it. Each scorer `type` may appear at most once.
      operationId: createEval
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required:
                - name
                - agent_id
                - dataset_id
                - scorers
                - policy_ids
              additionalProperties: false
              properties:
                policy_ids:
                  description: The policies every run of the eval acts under.
                  allOf:
                    - $ref: "#/components/schemas/AttachedPolicyIds"
                name:
                  type: string
                  description: Unique name within the project
                  example: billing-regression-suite
                agent_id:
                  x-naturali-ref: agents
                  type: string
                  description: The agent under test
                  example: agent_V1StGXR8Z5jdHi6B
                dataset_id:
                  type: string
                  description: The dataset to run it against
                  example: dset_V1StGXR8Z5jdHi6B
                scorers:
                  $ref: "#/components/schemas/Scorers"
                pass_threshold:
                  type: number
                  nullable: true
                  description: 0–1. A run's verdict reads the 95% interval of its pass rate (passed items over
                    non-errored items) against this. Null reports scores without a verdict.
                  example: 0.8
                group_by:
                  type: string
                  nullable: true
                  minLength: 1
                  maxLength: 255
                  description: A key of the items' `metadata`. A run rolls its scores up per string value of that key
                    in `aggregate_scores.grouping`. Null reports no grouping.
                  example: kind
                evidence:
                  $ref: "#/components/schemas/Evidence"
                claim:
                  $ref: "#/components/schemas/Claim"
      responses:
        "201":
          description: Eval created successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Eval"
        "400":
          description: Bad request (unknown scorer type, cross-project reference, invalid threshold)
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          $ref: "#/components/responses/ProjectNotFound"
        "409":
          description: An eval with that name already exists in the project
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/evals/{eval_id}:
    get:
      tags:
        - Evaluations
      summary: Get an eval
      description: Returns a specific eval
      operationId: getEval
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
      responses:
        "200":
          description: Eval details
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Eval"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
    put:
      tags:
        - Evaluations
      summary: Update an eval
      description: Updates an eval. Changing `agent_id` re-validates the scorers against the new agent,
        since an `output_schema` scorer that was legal against the old one may not be.
      operationId: updateEval
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              additionalProperties: false
              properties:
                policy_ids:
                  description: Replaces the policies every run of the eval acts under.
                  allOf:
                    - $ref: "#/components/schemas/AttachedPolicyIds"
                name:
                  type: string
                  example: billing-regression-suite
                agent_id:
                  x-naturali-ref: agents
                  type: string
                  example: agent_V1StGXR8Z5jdHi6B
                dataset_id:
                  type: string
                  example: dset_V1StGXR8Z5jdHi6B
                scorers:
                  $ref: "#/components/schemas/Scorers"
                pass_threshold:
                  type: number
                  nullable: true
                  example: 0.8
                group_by:
                  type: string
                  nullable: true
                  minLength: 1
                  maxLength: 255
                  description: A key of the items' `metadata` to group run scores by; null clears it, omitting it
                    leaves it unchanged. Refused while `evidence.required_groups` names a group.
                  example: kind
                evidence:
                  description: Replaces the stored evidence; null clears it, omitting it leaves it unchanged.
                  allOf:
                    - $ref: "#/components/schemas/Evidence"
                claim:
                  description: Replaces the stored claim as a whole; null clears it, omitting it leaves it unchanged.
                  allOf:
                    - $ref: "#/components/schemas/Claim"
      responses:
        "200":
          description: Eval updated successfully
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/Eval"
        "400":
          description: Bad request
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "409":
          description: An eval with that name already exists in the project
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    delete:
      tags:
        - Evaluations
      summary: Delete an eval
      description: Deletes an eval, its runs, and their results
      operationId: deleteEval
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
      responses:
        "204":
          description: Eval deleted successfully
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/evals/{eval_id}/runs:
    get:
      tags:
        - Evaluations
      summary: List eval runs
      description: Returns an eval's runs, newest first
      operationId: listEvalRuns
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
        - $ref: "#/components/parameters/Limit"
        - $ref: "#/components/parameters/Cursor"
      responses:
        "200":
          description: List of eval runs
          content:
            application/json:
              schema:
                allOf:
                  - $ref: "#/components/schemas/Page"
                  - properties:
                      data:
                        type: array
                        items:
                          $ref: "#/components/schemas/EvalRun"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
    post:
      tags:
        - Evaluations
      summary: Start an eval run
      description: >-
        Runs the eval against its dataset, creating one real agent generation per item and scoring
        the outputs.


        `wait: true` executes the run synchronously and returns it terminal, with its scores. The
        dataset is capped at 25 items for a synchronous run; a larger one is rejected with 400
        rather than partially scored.


        `wait: false` (the default) enqueues one task per item and returns immediately with `status:
        "queued"`. A worker executes the items and the run settles itself; poll `GET
        /evals/{eval_id}/runs/{eval_run_id}` for the terminal status. There is no item cap on a
        queued run.


        The whole run is pinned to **one** agent version, stamped on `agent_version`: pass one
        explicitly to evaluate a canary before promoting it, or omit it to use the active release's
        stable version (or the live draft when no release is in effect). Without the pin, release
        assignment would bucket each item independently and blend two configs into a single score.


        With `baseline_run_id`, the finished run's `aggregate_scores.baseline` carries per-scorer
        deltas against that run, computed over the items present and scorable in **both** runs, with
        the divergence counted. A delta over a shifted dataset is therefore never presented as a
        clean comparison.
      operationId: startEvalRun
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              additionalProperties: false
              properties:
                wait:
                  type: boolean
                  default: false
                  description: True runs the eval synchronously (25-item cap) and returns a terminal run with its
                    scores. False — the default — enqueues the items and returns a `queued` run
                    immediately.
                  example: true
                agent_version:
                  type: integer
                  nullable: true
                  description: An archived agent version to evaluate. Defaults to the active release's stable version,
                    or the live draft version when no release is in effect.
                  example: 3
                baseline_run_id:
                  type: string
                  nullable: true
                  description: A terminal run of the same eval to compare against. The finished run's
                    `aggregate_scores.baseline` reports per-scorer deltas over the item
                    intersection. A run of a different eval is rejected with 400.
                  example: evrun_V1StGXR8Z5jdHi6B
                metadata:
                  description: >-
                    Caller-supplied key/value metadata attached to the run record for attribution —
                    what this measurement was of (the commit or release candidate being scored, the
                    CI job that asked for it). Round-trips verbatim on every read of the run, the
                    list included.


                    The bag is caller-owned and no key is reserved: everything the platform decides
                    about a run (`status`, `agent_version`, `baseline_run_id`, `aggregate_scores`,
                    `verdict`, the counts) is a field of its own and cannot be written from here.
                    Nothing in the scoring path reads it. A non-object is rejected with `400
                    VALIDATION_FAILED` and no run is created.
                  example:
                    commit_sha: 9f2c1ab
                    ci_job: nightly-evals
                  allOf:
                    - $ref: "#/components/schemas/MetadataBag"
                tool_context:
                  type: object
                  additionalProperties:
                    type: string
                  description: >-
                    Key/value context forwarded to every item's generation, so an agent whose tools
                    authorize through `tool_context` is scored against the configuration it runs in
                    production rather than with an empty bag. Each key is forwarded as one
                    `X-Naturali-Context-<key>` header and resolves any `{{context:<key>}}` token in
                    a bound tool's headers or `preset_parameters`.


                    Stored on the run and re-read per item, since a queued run (the default) is
                    driven by a worker with no request behind it. **Write-only**: no read of the run
                    returns it, unlike `metadata` — a run is a report other people read, and a
                    credential in it is not theirs to see. Cleared once the run reaches a terminal
                    state.


                    An eval generation has no session, so the reserved keys `session_id`, `actor_id`
                    and `actor_external_id` are dropped (in any casing) rather than forwarded. Every
                    other key becomes an HTTP header name and must match that grammar, or the
                    request is rejected with `400 INVALID_TOOL_CONTEXT_KEY` and no run is created.
                  example:
                    ocaToken: eyJhbGciOiJIUzI1NiJ9.abc
                    tenant: acme
      responses:
        "201":
          description: "Eval run finished (`wait: true`) or queued (`wait: false`)"
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/EvalRun"
        "400":
          description: Bad request (non-boolean wait, dataset empty or over the synchronous cap, unknown
            agent_version, invalid baseline, scorers no longer valid against the agent, a
            `tool_context` key that cannot become a header)
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "409":
          description: PROJECT_PAUSED — the project is paused, so nothing new starts in it; `error.meta`
            carries `paused_at` and `pause_reason`.
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "413":
          $ref: "#/components/responses/PayloadTooLarge"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}:
    get:
      tags:
        - Evaluations
      summary: Get an eval run
      description: Returns a run's status, counts, and aggregate scores
      operationId: getEvalRun
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
        - name: eval_run_id
          in: path
          required: true
          description: Eval run ID
          schema:
            type: string
            example: evrun_V1StGXR8Z5jdHi6B
      responses:
        "200":
          description: Eval run details
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/EvalRun"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval or run not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}/results:
    get:
      tags:
        - Evaluations
      summary: List eval run results
      description: Returns the per-item results of a run, oldest first
      operationId: listEvalResults
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
        - name: eval_run_id
          in: path
          required: true
          description: Eval run ID
          schema:
            type: string
            example: evrun_V1StGXR8Z5jdHi6B
        - $ref: "#/components/parameters/Limit"
        - $ref: "#/components/parameters/Cursor"
      responses:
        "200":
          description: List of eval results
          content:
            application/json:
              schema:
                allOf:
                  - $ref: "#/components/schemas/Page"
                  - properties:
                      data:
                        type: array
                        items:
                          $ref: "#/components/schemas/EvalResult"
        "400":
          $ref: "#/components/responses/BadRequest"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval or run not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          $ref: "#/components/responses/InternalError"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
  /v1/projects/{project_id}/evals/{eval_id}/runs/{eval_run_id}/cancel:
    post:
      tags:
        - Evaluations
      summary: Cancel an eval run
      description: >-
        Cancels a queued or running run: its outstanding item tasks are dropped so it stops
        consuming provider budget, and the run settles as `cancelled`.


        Results already written are kept — they are real measurements of generations that were
        really paid for — and `completed_count` / `errored_count` report what ran.
        `aggregate_scores` is deliberately left null: a partial roll-up in the same field a
        completed run uses would read as a whole-dataset verdict.


        A run that has already finished is rejected with 400.
      operationId: cancelEvalRun
      parameters:
        - name: eval_id
          in: path
          required: true
          description: Eval ID
          schema:
            type: string
            example: eval_V1StGXR8Z5jdHi6B
        - name: eval_run_id
          in: path
          required: true
          description: Eval run ID
          schema:
            type: string
            example: evrun_V1StGXR8Z5jdHi6B
      responses:
        "200":
          description: Eval run cancelled
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/EvalRun"
        "400":
          description: The run has already finished
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "401":
          description: Unauthorized
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "403":
          description: Forbidden
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "404":
          description: Eval or run not found
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "429":
          $ref: "#/components/responses/TooManyRequests"
        "500":
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: "#/components/schemas/ErrorResponse"
        "502":
          $ref: "#/components/responses/AdmissionUnavailable"
        4XX:
          $ref: "#/components/responses/AdmissionRefused"
    parameters:
      - $ref: "#/components/parameters/ProjectId"
components:
  schemas:
    DatasetItemInput:
      type: array
      description: Messages replayed verbatim as the generation's input
      items:
        type: object
        required:
          - role
          - content
        additionalProperties: false
        properties:
          role:
            type: string
            example: user
          content:
            description: Message content — a string, or AI SDK content parts
            example: When is my invoice issued?
    Scorers:
      type: array
      description: >-
        Scorer configs, a discriminated union on `type`. Each type may appear at most once. Every
        scorer produces `{ score: 0–1, passed: boolean }`; binary scorers emit 0 or 1.


        `exact_match` compares the trimmed output text to `expected_output`. `contains` looks for
        `value` in the output text. `json_logic` evaluates `expression` over `{ input, output,
        object, expected, item.metadata }`, where `object` is the structured output (absent when the
        agent has no `output_schema`). `output_schema` validates the structured output against the
        scorer's own `schema`, falling back to the agent's; it requires the agent to carry an
        `output_schema`, because without one the platform emits no structured output and every item
        would score 0.


        `llm_judge` grades the output with a model completion, returning a continuous score plus its
        `reasoning`. Its `pass_threshold` is required: a continuous score says nothing about where
        "good enough" is, and a defaulted cutoff would silently decide the gate.


        `embedding_similarity` embeds the output text and `expected_output` with the platform's
        configured embedding model (`EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` — the same stack
        document ingestion uses) and scores their cosine similarity, clamped to 0-1. Its
        `pass_threshold` is required for the same reason as the judge's. An item without an
        `expected_output` scores 0; an embedding backend failure marks the **item** errored, never a
        score of 0.


        `tool` runs a custom scoring algorithm: a server-callable project tool the engine invokes
        once per item with the item's context. Unlike the built-in types it may appear several
        times, each under a distinct `name` — outcomes and aggregates key on the name.


        `decider` grades each item with a decision of a project decider: its `score` expression
        reads the decision's `answers`. Like `tool`, it keys on its `name` and may appear several
        times.
      items:
        oneOf:
          - $ref: "#/components/schemas/ExactMatchScorer"
          - $ref: "#/components/schemas/ContainsScorer"
          - $ref: "#/components/schemas/JsonLogicScorer"
          - $ref: "#/components/schemas/OutputSchemaScorer"
          - $ref: "#/components/schemas/EmbeddingSimilarityScorer"
          - $ref: "#/components/schemas/LlmJudgeScorer"
          - $ref: "#/components/schemas/ToolScorer"
          - $ref: "#/components/schemas/DeciderScorer"
    ExactMatchScorer:
      type: object
      required:
        - type
      properties:
        type:
          type: string
          enum:
            - exact_match
    ContainsScorer:
      type: object
      required:
        - type
        - value
      properties:
        type:
          type: string
          enum:
            - contains
        value:
          type: string
          example: invoice
        case_sensitive:
          type: boolean
          default: false
    JsonLogicScorer:
      type: object
      required:
        - type
        - expression
      properties:
        type:
          type: string
          enum:
            - json_logic
        expression:
          type: object
          additionalProperties: true
          description: A JSON Logic expression; a truthy result scores 1
    OutputSchemaScorer:
      type: object
      required:
        - type
      properties:
        type:
          type: string
          enum:
            - output_schema
        schema:
          type: object
          additionalProperties: true
          description: JSON Schema the structured output is validated against. Frozen here so two runs stay
            comparable; falls back to the agent's `output_schema` when omitted.
    EmbeddingSimilarityScorer:
      type: object
      required:
        - type
        - pass_threshold
      properties:
        type:
          type: string
          enum:
            - embedding_similarity
        pass_threshold:
          type: number
          minimum: 0
          maximum: 1
          description: The item passes this scorer when the cosine similarity between the embeddings of the
            output text and `expected_output` is greater than or equal to this value. Required.
          example: 0.85
    LlmJudgeScorer:
      type: object
      required:
        - type
        - prompt
        - pass_threshold
      properties:
        type:
          type: string
          enum:
            - llm_judge
        prompt:
          type: string
          description: The judge prompt. `{{input}}`, `{{output}}` and `{{expected}}` are replaced with the
            item's input messages, the agent's output text, and the item's `expected_output`. Slots
            are filled in one pass, so a slot value that itself contains `{{output}}` is not
            re-expanded. The judge must answer with a JSON object carrying a numeric `score` between
            0 and 1 and an optional `reasoning` string; a reply that does not marks the **item**
            errored, never the run failed and never a score of 0.
          example: 'Rate 0-1 how well the answer matches the reference. Answer with {"score": <0-1>,
            "reasoning": "<why>"}. Question: {{input}} Answer: {{output}} Reference: {{expected}}'
        pass_threshold:
          type: number
          minimum: 0
          maximum: 1
          description: The item passes this scorer when the judge's score is greater than or equal to this
            value. Required.
          example: 0.7
        ai_provider_id:
          type: string
          nullable: true
          description: The AI provider that runs the judge; it must belong to the eval's project. Omit to use
            the project's default model route.
          example: aip_V1StGXR8Z5jdHi6B
        model:
          type: string
          nullable: true
          description: Overrides the provider's default model. Pinned per scorer, because deltas between runs
            judged by different models are not comparable.
          example: gpt-4o-mini
    ToolScorer:
      type: object
      description: A custom scoring algorithm — a server-callable project tool the engine invokes once per
        item. The tool receives the same variables a `json_logic` expression reads — `input`,
        `output`, `object` (when the agent has an `output_schema`), `expected`, and `item.metadata`
        — with `preset_parameters` merged in at the top level, and must answer with a JSON object
        carrying a numeric `score` between 0 and 1, an optional boolean `passed`, and an optional
        `reasoning` string. A malformed answer or a failed call marks the **item** errored, never
        the run failed and never a score of 0.
      required:
        - type
        - name
        - tool_id
      properties:
        type:
          type: string
          enum:
            - tool
        name:
          type: string
          description: Keys this scorer's outcomes and aggregate scores, so it must be unique within the eval
            and must not shadow a built-in scorer type. Unlike the built-in types, several `tool`
            scorers may coexist under distinct names.
          example: toxicity
        tool_id:
          x-naturali-ref: tools
          type: string
          description: The tool that scores each item. It must belong to the eval's project and be
            server-callable (`http`, `mcp`, `builtin`, or `pipeline` — a `client` tool pauses for a
            calling client an eval run does not have).
          example: tool_V1StGXR8Z5jdHi6B
        action:
          type: string
          nullable: true
          description: The operation to invoke; required when the tool type is `builtin` or `mcp`.
          example: score-toxicity
        preset_parameters:
          type: object
          nullable: true
          additionalProperties: true
          description: Fixed values merged into every call's input at the top level. The engine-injected keys
            (`input`, `output`, `object`, `expected`, `item`) are reserved and rejected.
        pass_threshold:
          type: number
          nullable: true
          minimum: 0
          maximum: 1
          description: "Fallback verdict cutoff when the tool answers without a `passed` flag: the item passes
            this scorer when `score` is greater than or equal to this value. A tool-returned
            `passed` always wins. When the tool omits `passed` and no threshold is set, the item is
            recorded as errored — the scorer produced no verdict."
          example: 0.5
    DeciderScorer:
      type: object
      description: Grades each item with a decision of a project decider, under the decider version pinned
        when the run started (`decider_versions` on the run). The decision is requested as the run,
        with `metadata` naming the eval, run and dataset item. A failed decision, or a `score` that
        is not a number between 0 and 1, marks the **item** errored, never a score of 0.
      required:
        - type
        - name
        - decider_id
        - score
        - pass_threshold
      properties:
        type:
          type: string
          enum:
            - decider
        name:
          type: string
          description: Keys this scorer's outcomes and aggregate scores, so it must be unique within the eval
            and must not shadow a built-in scorer type.
          example: review
        decider_id:
          x-naturali-ref: deciders
          type: string
          description: The decider that judges each item; it must belong to the eval's project.
          example: dcd_V1StGXR8Z5jdHi6B
        input:
          description: JSON Logic mapping over `{ input, output, object, expected, item }` building the input
            the decider judges. Omitted, the input is that item context itself.
          example:
            customer:
              var: input.0.content
            reply:
              var: output
        score:
          description: JSON Logic expression over `{ answers, answers_by_name }`, the decision's answers in
            question order and keyed by question name, that evaluates to the item's score in 0–1.
          example:
            var: answers_by_name.resolves_issue.probability
        pass_threshold:
          type: number
          minimum: 0
          maximum: 1
          description: 'The item passes this scorer when `score` is greater than or equal to this value.
            Required: a score read from a distribution says nothing about where "good enough" is.'
          example: 0.7
    ScorerResult:
      type: object
      properties:
        scorer:
          type: string
          description: The scorer that produced this entry — the scorer type, or for a `tool` scorer its `name`
          example: contains
        score:
          type: number
          example: 1
        passed:
          type: boolean
        reasoning:
          type: string
          description: The stated rationale; present for `llm_judge` and for `tool` scorers whose tool
            returned one
        decision_id:
          type: string
          description: The decision that graded this entry; present for `decider` scorers
          example: dec_V1StGXR8Z5jdHi6B
    Dataset:
      type: object
      properties:
        id:
          type: string
          example: dset_V1StGXR8Z5jdHi6B
        project_id:
          x-naturali-ref: projects
          type: string
        name:
          type: string
        description:
          type: string
          nullable: true
        created_at:
          type: string
          format: date-time
        updated_at:
          type: string
          format: date-time
    DatasetItem:
      type: object
      properties:
        id:
          type: string
          example: dsit_V1StGXR8Z5jdHi6B
        dataset_id:
          type: string
          example: dset_V1StGXR8Z5jdHi6B
        input:
          $ref: "#/components/schemas/DatasetItemInput"
        expected_output:
          type: string
          nullable: true
        metadata:
          $ref: "#/components/schemas/MetadataBag"
        source_generation_id:
          type: string
          nullable: true
          description: "The generation this item was curated from. A curated item is a deliberate fixture:
            erasing the source generation neither deletes nor mutates it."
          example: gen_V1StGXR8Z5jdHi6B
        created_at:
          type: string
          format: date-time
        updated_at:
          type: string
          format: date-time
    Eval:
      type: object
      properties:
        policy_ids:
          description: The policies every run of the eval acts under. Empty for one written without any, which
            does not run until a write gives it one.
          type: array
          items:
            x-naturali-ref: policies
            type: string
          example:
            - pol_V1StGXR8Z5jdHi6B
        id:
          type: string
          example: eval_V1StGXR8Z5jdHi6B
        project_id:
          x-naturali-ref: projects
          type: string
        name:
          type: string
        agent_id:
          x-naturali-ref: agents
          type: string
          example: agent_V1StGXR8Z5jdHi6B
        dataset_id:
          type: string
          example: dset_V1StGXR8Z5jdHi6B
        scorers:
          $ref: "#/components/schemas/Scorers"
        pass_threshold:
          type: number
          nullable: true
        group_by:
          type: string
          nullable: true
          description: The item `metadata` key a run rolls its scores up by; null when the eval groups nothing
          example: kind
        evidence:
          $ref: "#/components/schemas/Evidence"
        claim:
          $ref: "#/components/schemas/Claim"
        created_at:
          type: string
          format: date-time
        updated_at:
          type: string
          format: date-time
    Claim:
      type: object
      nullable: true
      additionalProperties: false
      required:
        - statement
      description: "What a run of the eval tests: the statement a `supported` verdict backs, the
        population its dataset stands for, and the limits outside which the verdict says nothing.
        Each run copies it when created. Null states none."
      properties:
        statement:
          type: string
          minLength: 1
          description: The proposition the eval's verdict supports or refutes
          example: The agent answers refund questions with the current policy.
        population:
          type: string
          nullable: true
          minLength: 1
          description: Who or what the dataset's items are drawn from; the verdict generalizes to this and no
            further. Null when unstated.
          example: Refund requests from retail customers, 2026
        limits:
          type: array
          nullable: true
          description: Conditions the verdict does not cover. Null when unstated.
          items:
            type: string
            minLength: 1
          example:
            - English only
            - No escalations to a human
    Evidence:
      type: object
      nullable: true
      additionalProperties: false
      description: The rules a run's verdict reads beyond the threshold. Null declares none.
      properties:
        required_groups:
          type: array
          description: Values of `group_by` each judged on its own interval against `pass_threshold`. A
            required group with no scored items is `insufficient`; the run's verdict is the worst
            across the run and every required group. Needs `group_by`. Names are non-empty and
            unique.
          items:
            type: string
            minLength: 1
          example:
            - pix
            - card
            - boleto
    VerdictReason:
      type: object
      additionalProperties: false
      required:
        - scope
        - verdict
        - reason
      properties:
        scope:
          type: object
          additionalProperties: false
          required:
            - kind
          description: What this reason judges
          properties:
            kind:
              type: string
              enum:
                - run
                - group
            name:
              type: string
              description: The group, on a `group` scope
              example: boleto
        verdict:
          type: string
          enum:
            - supported
            - refuted
            - insufficient
        reason:
          type: string
          example: interval 0.91–0.96 at or above 0.9
    AggregateScores:
      type: object
      nullable: true
      description: Per-scorer rollup plus the run-level pass rate; null until the run is terminal
      properties:
        scorers:
          type: object
          additionalProperties:
            type: object
            properties:
              mean:
                type: number
              pass_rate:
                type: number
        pass_rate:
          type: number
          nullable: true
          description: Passed items over non-errored items; null when nothing was scorable
        pass_rate_interval:
          $ref: "#/components/schemas/PassRateInterval"
        scored_item_count:
          type: integer
          description: Items that produced a score — errored items are excluded
        baseline:
          $ref: "#/components/schemas/BaselineComparison"
        grouping:
          $ref: "#/components/schemas/RunGrouping"
    ScoreRollup:
      type: object
      description: One group's figures, computed as the run-level ones are
      properties:
        scorers:
          type: object
          additionalProperties:
            type: object
            properties:
              mean:
                type: number
              pass_rate:
                type: number
        pass_rate:
          type: number
          nullable: true
          description: Passed items over non-errored items in the group
        pass_rate_interval:
          $ref: "#/components/schemas/PassRateInterval"
        scored_item_count:
          type: integer
          description: Non-errored items in the group
    PassRateInterval:
      type: object
      nullable: true
      description: "95% Wilson score interval around `pass_rate`: the range the pass rate of this agent
        over items like these plausibly spans, given how many items were scored. Null exactly when
        `pass_rate` is."
      properties:
        low:
          type: number
          example: 0.63
        high:
          type: number
          example: 0.84
        level:
          type: number
          description: The confidence level, always 0.95
          example: 0.95
    FlippedItems:
      type: object
      description: Compared items that changed verdict. `pass_rate_delta` is `(improved - regressed) /
        compared_item_count`.
      properties:
        improved:
          type: integer
          description: Passed in this run, failed in the baseline
        regressed:
          type: integer
          description: Failed in this run, passed in the baseline
    RunGrouping:
      type: object
      description: Scores rolled up per value of the eval's `group_by` key; absent when the eval declares
        none. An item belongs to the group its `metadata` names under that key, read when the run
        settles; only a string names a group. Errored items count in no group.
      properties:
        group_by:
          type: string
          description: The metadata key the groups were read from
          example: kind
        groups:
          type: object
          description: Group value → that group's figures
          additionalProperties:
            $ref: "#/components/schemas/ScoreRollup"
        ungrouped_item_count:
          type: integer
          description: Scored items whose `metadata` holds no string under `group_by`, or whose dataset item
            was deleted
    GroupComparison:
      type: object
      description: One group's baseline comparison, computed as the run-level one is
      properties:
        compared_item_count:
          type: integer
        added_item_count:
          type: integer
        removed_item_count:
          type: integer
        pass_rate_delta:
          type: number
          nullable: true
        flipped:
          $ref: "#/components/schemas/FlippedItems"
        p_value:
          type: number
          nullable: true
        scorers:
          type: object
          additionalProperties:
            type: object
            properties:
              mean_delta:
                type: number
              pass_rate_delta:
                type: number
    BaselineGrouping:
      type: object
      description: The baseline comparison per group; absent when the eval declares no `group_by`. Both
        runs' items are grouped by the same labels, so an item relabelled between the runs is
        compared within one group.
      properties:
        group_by:
          type: string
          example: kind
        groups:
          type: object
          additionalProperties:
            $ref: "#/components/schemas/GroupComparison"
        ungrouped_item_count:
          type: integer
          description: Items scorable in both runs that name no group
    BaselineComparison:
      type: object
      nullable: true
      description: >-
        Comparison against the run named by `baseline_run_id`; absent when the run named none.


        Every number here is computed over the **item intersection** — the dataset items present and
        scorable in both runs — because a delta only means something when both sides answered the
        same question. The compared/added/removed counts make any dataset drift visible instead of
        letting it read as agent regression. Positive deltas mean this run scored higher than the
        baseline.
      properties:
        run_id:
          type: string
          example: evrun_V1StGXR8Z5jdHi6B
        compared_item_count:
          type: integer
          description: Items scorable in both runs — the basis of every delta
        added_item_count:
          type: integer
          description: Scorable here but not in the baseline (added, or errored there)
        removed_item_count:
          type: integer
          description: Scorable in the baseline but not here (removed, or errored here)
        pass_rate_delta:
          type: number
          nullable: true
          description: Run-level pass-rate delta over the intersection; null when the two runs share no
            comparable item
        flipped:
          $ref: "#/components/schemas/FlippedItems"
        p_value:
          type: number
          nullable: true
          description: "Exact McNemar p-value of the `flipped` split: the probability of a split at least this
            uneven were an item equally likely to flip either way, which is what an unchanged agent
            produces. `1` when no item flipped; null when the runs share no comparable item."
          example: 0.03
        scorers:
          type: object
          description: Per-scorer deltas, keyed by scorer type. A scorer only one of the two runs ran is
            omitted rather than compared against nothing.
          additionalProperties:
            type: object
            properties:
              mean_delta:
                type: number
              pass_rate_delta:
                type: number
        grouping:
          $ref: "#/components/schemas/BaselineGrouping"
    EvalRun:
      type: object
      properties:
        id:
          type: string
          example: evrun_V1StGXR8Z5jdHi6B
        eval_id:
          type: string
          example: eval_V1StGXR8Z5jdHi6B
        agent_id:
          x-naturali-ref: agents
          type: string
          nullable: true
          description: The agent every item of this run executed against, resolved when the run was created
            and kept after that agent is deleted or the eval is pointed at another. Null for a run
            that recorded none, which satisfies no promotion gate.
          example: agent_V1StGXR8Z5jdHi6B
        agent_version:
          type: integer
          description: The one agent version every item in this run executed against
          example: 3
        definition:
          $ref: "#/components/schemas/EvalRunDefinition"
        claim:
          description: "The eval's claim as it was when the run was created: what this run tested. A later
            edit to the eval reaches only runs created after it. Null when the eval stated none."
          allOf:
            - $ref: "#/components/schemas/Claim"
        decider_versions:
          type: object
          nullable: true
          additionalProperties:
            type: integer
          description: The decider version each `decider` scorer grades under, keyed by scorer name and
            resolved when the run started. Null when the eval has no decider scorer.
          example:
            review: 2
        status:
          type: string
          enum:
            - queued
            - running
            - completed
            - failed
            - cancelled
        baseline_run_id:
          type: string
          nullable: true
          example: evrun_V1StGXR8Z5jdHi6B
        trigger_id:
          type: string
          nullable: true
          description: The trigger that started this run — set when a schedule (or a manual trigger fire)
            started it, null for a run started through this API. Kept even if the trigger is later
            deleted.
          example: trg_V1StGXR8Z5jdHi6B
        aggregate_scores:
          $ref: "#/components/schemas/AggregateScores"
        verdict:
          type: string
          nullable: true
          enum:
            - supported
            - refuted
            - insufficient
            - null
          description: "`supported` when the pass rate's 95% interval sits at or above
            `definition.pass_threshold`, `refuted` when it sits below, and `insufficient` when it
            holds the threshold or nothing was scored; the worst of that and every required group.
            Null when the run has no threshold, and until it completes."
        verdict_reasons:
          type: array
          nullable: true
          description: One entry per judged scope, the run first. Null when `verdict` is.
          items:
            $ref: "#/components/schemas/VerdictReason"
        item_count:
          type: integer
        completed_count:
          type: integer
        errored_count:
          type: integer
        metadata:
          description: The caller-owned key/value metadata supplied when the run was started, returned
            verbatim. Absent when the run was started without any (a trigger-started run included —
            see `trigger_id` for that provenance). The server writes nothing here.
          example:
            commit_sha: 9f2c1ab
            ci_job: nightly-evals
          allOf:
            - $ref: "#/components/schemas/MetadataBag"
        started_at:
          type: string
          format: date-time
          nullable: true
        finished_at:
          type: string
          format: date-time
          nullable: true
        created_at:
          type: string
          format: date-time
    EvalRunDefinition:
      type: object
      nullable: true
      description: The eval's scorers, pass threshold, grouping key and evidence as they were when the run
        was created. Every item is scored and the run settled by these; a later edit to the eval
        reaches only runs created after it. Null when the run recorded none.
      properties:
        scorers:
          $ref: "#/components/schemas/Scorers"
        pass_threshold:
          type: number
          nullable: true
          description: Null when the run reports without gating
          example: 0.8
        group_by:
          type: string
          nullable: true
          example: kind
        evidence:
          $ref: "#/components/schemas/Evidence"
    EvalResult:
      type: object
      properties:
        id:
          type: string
          example: evres_V1StGXR8Z5jdHi6B
        eval_run_id:
          type: string
          example: evrun_V1StGXR8Z5jdHi6B
        dataset_item_id:
          type: string
          nullable: true
          description: Null once the dataset item has been deleted
          example: dsit_V1StGXR8Z5jdHi6B
        input:
          $ref: "#/components/schemas/DatasetItemInput"
        expected_output:
          type: string
          nullable: true
        generation_id:
          type: string
          nullable: true
          example: gen_V1StGXR8Z5jdHi6B
        output:
          type: string
          nullable: true
          description: The agent's final output text. Cleared when the linked generation's content is purged;
            the scores and the frozen input survive.
        scores:
          type: array
          items:
            $ref: "#/components/schemas/ScorerResult"
        passed:
          type: boolean
          description: AND over the per-scorer passed flags
        error:
          allOf:
            - $ref: "#/components/schemas/StoredError"
          nullable: true
          description: Item-level failure reason. A generation that did not complete — a `requires_action`
            pause (`GENERATION_NOT_COMPLETED`), a provider failure — or a scorer that could not
            grade the output is recorded here and excluded from the aggregates rather than scored 0.
        created_at:
          type: string
          format: date-time
    Page:
      type: object
      description: One page of a list. Rows inserted or removed between two calls neither repeat nor skip
        a row on the next page.
      required:
        - data
        - next_cursor
      properties:
        data:
          type: array
          description: This page's items, in the list's order.
          items: {}
        next_cursor:
          type: string
          nullable: true
          description: Pass as `cursor` to fetch the next page; `null` on the last page.
    ErrorResponse:
      type: object
      required:
        - error
      properties:
        error:
          type: object
          description: Structured error. Every error response uses this shape, so `code` can be read without
            first checking the type of `error`.
          required:
            - code
            - message
            - hint
          properties:
            code:
              type: string
              pattern: ^[A-Z][A-Z0-9_]*$
              description: Machine-readable error code, UPPER_SNAKE whether naturali or the runtime raises it.
              example: FORBIDDEN
            message:
              type: string
              description: Human-readable explanation.
              example: Your role in this project does not carry this action.
            hint:
              type: string
              description: What to do about it, naming the routes of this API to call. For people and agents; it
                may change, so branch on `code`.
              example: Use a credential whose scope includes this action, or ask a project `admin` or `owner` to
                act; the caller's role or key excludes it.
            meta:
              type: object
              additionalProperties: true
              description: Structured context, such as the `resource` and `limit` of a `PLAN_LIMIT_REACHED`.
    NullableMetadataBag:
      type: object
      nullable: true
      description: "A `MetadataBag` on a request field where `null` is meaningful: a full-replacement
        update that clears the bag. A record whose bag is unset leaves the field out."
      example:
        author: John
        revision: 2
    AttachedPolicyIds:
      type: array
      minItems: 1
      items:
        x-naturali-ref: policies
        type: string
      description: The policies the resource acts under; an event subscriber receives an event only when
        they allow reading its resource. The caller must hold every grant of each one it attaches
        inside the project, else `403 POLICY_GRANT_NOT_HELD`.
      example:
        - pol_V1StGXR8Z5jdHi6B
    MetadataBag:
      type: object
      description: >-
        Caller-owned annotations on a resource, stored as the object they were written as: the types
        a value was written with are the types a read returns, so a filter can ask an ordering
        question about a number. Unlike other body fields, keys are stored and returned verbatim in
        the casing supplied — they are not converted between snake_case and camelCase.


        No key is reserved, and that is the point: every piece of state the platform owns lives in
        its own typed column, so nothing written here reaches platform state. The platform never
        reads the bag — it is not an IAM context, not a policy input and not part of a prompt —
        which is what separates it from a tag bag.
      example:
        author: John
        revision: 2
    StoredError:
      type: object
      description: Why a resource's work failed, in the shape of an error response's `error` without
        `hint`. Every `error` field a resource records carries it, so `code` can be read without
        first checking the type of `error`.
      required:
        - code
        - message
      properties:
        code:
          type: string
          description: A key from the server's ERROR_CODES registry, documented on the Error Codes page.
            `INTERNAL_ERROR`, with the message `Internal Server Error`, records an unexpected
            failure whose detail reaches the server log only.
          example: AI_PROVIDER_ERROR
        message:
          type: string
          example: "Provider returned 402: insufficient credits"
        meta:
          type: object
          additionalProperties: true
          description: The failure's own structured context, when it names any.
  parameters:
    ProjectId:
      name: project_id
      in: path
      required: true
      description: Project public ID (proj_ prefix).
      schema:
        type: string
        example: proj_V1StGXR8Z5jdHi6B
    Limit:
      name: limit
      in: query
      required: false
      description: Results per page, 1 to 100 (default 50). Above 100 it is clamped; below 1 or not an
        integer it is `400`.
      schema:
        type: integer
        minimum: 1
        maximum: 100
        default: 50
    Cursor:
      name: cursor
      in: query
      required: false
      description: The previous page's `next_cursor`; omit it for the first page. A cursor this list did
        not issue is `400`.
      schema:
        type: string
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      description: A naturali API key (nat_sk_…) or a session JWT.
    oauth2:
      type: oauth2
      description: "A connected app's OAuth access token, issued by this API's authorization server
        (discovery: /.well-known/oauth-authorization-server). Its one scope carries every operation,
        confined to the projects the user chose when approving the app."
      flows:
        authorizationCode:
          authorizationUrl: https://api.naturali.ai/authorize
          tokenUrl: https://api.naturali.ai/token
          refreshUrl: https://api.naturali.ai/token
          scopes:
            mcp:access: Every operation this API serves, on the projects the grant covers.
  responses:
    BadRequest:
      description: The request was malformed or failed validation.
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
    TooManyRequests:
      description: "`QUOTA_EXCEEDED` — a `requests` quota on the calling API key or its project is spent
        for the window; `Retry-After` says when it reopens"
      headers:
        Retry-After:
          description: Seconds until the quota window admits a request again
          schema:
            type: integer
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
    PayloadTooLarge:
      description: "`REQUEST_REJECTED` — the request body is over the size the body parser accepts"
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
    AdmissionUnavailable:
      description: The deployment's admission service is asked about this write and did not answer —
        `ADMISSION_UNAVAILABLE`; the write did not happen
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
    AdmissionRefused:
      description: The deployment's admission service refused this write — answered under the service's
        own `4xx` status, with its `code` (matching `^[A-Za-z_][A-Za-z0-9_]{0,63}$`, not necessarily
        an `ERROR_CODES` key), `message` and `meta`; the write did not happen. A status this
        operation also declares on its own is documented there
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
    InternalError:
      description: Unexpected server error — `INTERNAL_ERROR`, with the raw message never forwarded
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
    ProjectNotFound:
      description: The project does not exist or the caller is not a member of it (`RESOURCE_NOT_FOUND`).
      content:
        application/json:
          schema:
            $ref: "#/components/schemas/ErrorResponse"
