> ## Documentation Index
> Fetch the complete documentation index at: https://docs.orq.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Evals SDK Reference

> Manage evaluators with the Node.js and Python SDKs: create, list, update, invoke, and delete evals and list evaluator versions for LLM scoring.

## Evals

### List Evals

List all evaluators in the workspace.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.all(limit=10)

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.all({});

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "limit": Optional[int],
        "starting_after": Optional[str],
        "ending_before": Optional[str],
        "search": Optional[str],
        "sort": Optional[Literal["asc", "desc"]],
        "project_id": Optional[str],
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      limit?: number;
      startingAfter?: string;
      endingBefore?: string;
      search?: string;
      sort?: "asc" | "desc";
      projectId?: string;
    }
    ```
  </CodeGroup>
</Expandable>

<Expandable title="Response">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "object": Literal["list"],
        "data": List[Union[EvaluatorResponseLlm, EvaluatorResponseJSONSchema, EvaluatorResponseHTTP, EvaluatorResponsePython, EvaluatorResponseFunction, EvaluatorResponseRagas, EvaluatorResponseTypescript]],
        "has_more": bool,
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      object: "list";
      data: (EvaluatorResponseLlm | EvaluatorResponseJsonSchema | EvaluatorResponseHttp | EvaluatorResponsePython | EvaluatorResponseFunction | EvaluatorResponseRagas | EvaluatorResponseTypescript)[];
      hasMore: boolean;
    }
    ```
  </CodeGroup>
</Expandable>

### Create an Eval

Create a new evaluator in the workspace.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.create(request={
          "code": "<value>",
          "type": "python_eval",
          "path": "Default",
          "description": "",
          "key": "<key>",
      })

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.create({
      code: "<value>",
      type: "python_eval",
      path: "Default",
      description: "",
      key: "<key>",
    });

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "guardrail_config": Optional[Any],
        "output_type": Optional[Literal["boolean", "categorical", "number", "string"]],
        "type": Literal["llm_eval"],  # required
        "repetitions": Optional[int],
        "prompt": str,  # required
        "categories": List[str],  # optional
        "categorical_labels": [{  # optional
            "value": str,  # required
            "description": Optional[str],
        }],
        "dataset_id": Optional[str],
        "path": Optional[str],
        "project_id": Optional[str],
        "description": Optional[str],
        "key": str,  # required
        "mode": Literal["single"],  # required
        "model": str,  # required
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      guardrailConfig?: any;
      outputType?: "boolean" | "categorical" | "number" | "string";
      type: "llm_eval";  // required
      repetitions?: number;
      prompt: string;  // required
      categories?: string[];
      categoricalLabels?: {
        value: string;  // required
        description?: string;
      }[];
      datasetId?: string;
      path?: string;
      projectId?: string;
      description?: string;
      key: string;  // required
      mode: "single";  // required
      model: string;  // required
    }
    ```
  </CodeGroup>
</Expandable>

<Expandable title="Response">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,
        "description": str,
        "created": Optional[str],
        "updated": Optional[str],
        "updated_by_id": Optional[str],
        "project_id": Optional[str],
        "guardrail_config": Optional[Any],
        "type": Literal["llm_eval"],
        "repetitions": Optional[int],
        "prompt": str,
        "categories": List[str],  # optional
        "categorical_labels": [{  # optional
            "value": str,
            "description": Optional[str],
        }],
        "dataset_id": Optional[str],
        "key": str,
        "mode": Literal["single", "jury"],
        "model": Optional[str],
        "jury": {  # optional
            "judges": [{
                "model": str,
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": str,
                }],
            }],
            "replacement_judges": [{  # optional
                "model": str,
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": str,
                }],
            }],
            "min_successful_judges": Optional[int],
            "tie_value": Optional[Literal["Tie"]],
        },
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;
      description: string;
      created?: string;
      updated?: string;
      updatedById?: string;
      projectId?: string;
      guardrailConfig?: any;
      type: "llm_eval";
      repetitions?: number;
      prompt: string;
      categories?: string[];
      categoricalLabels?: {
        value: string;
        description?: string;
      }[];
      datasetId?: string;
      key: string;
      mode: "single" | "jury";
      model?: string;
      jury?: {
        judges: {
          model: string;
          retry?: {
            count?: number;
            onCodes?: number[];
          };
          fallbacks?: {
            model: string;
          }[];
        }[];
        replacementJudges?: {
          model: string;
          retry?: {
            count?: number;
            onCodes?: number[];
          };
          fallbacks?: {
            model: string;
          }[];
        }[];
        minSuccessfulJudges?: number;
        tieValue?: "Tie";
      };
    }
    ```
  </CodeGroup>
</Expandable>

### Retrieve an Eval

Retrieve a single evaluator by ID with more detail than the list endpoint: full type-specific config, owner, domain\_id, metadata and enabled.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.get(id="01JMDPA3QW5C1V0NJ1PW34T4E5")

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.get({
      id: "01JMDPA3QW5C1V0NJ1PW34T4E5",
    });

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,  # required
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;  // required
    }
    ```
  </CodeGroup>
</Expandable>

<Expandable title="Response">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "enabled": Optional[bool],
        "metadata": {
            "required_model_with_tools_support": Optional[bool],
            "required_retrieval_context": Optional[bool],
            "required_expected_output": Optional[bool],
            "supported_on_input_type": Optional[bool],
            "supported_on_output_type": Optional[bool],
            "support_use_as_guardrail": Optional[bool],
        },
        "id": str,
        "display_name": str,
        "description": str,
        "owner": str,
        "created": Optional[str],
        "updated": Optional[str],
        "created_by_id": Optional[str],
        "updated_by_id": Optional[str],
        "domain_id": str,
        "project_id": Optional[str],
        "guardrail_config": Optional[Any],
        "output_type": Optional[Literal["boolean", "categorical", "number", "string"]],
        "type": Literal["llm_eval"],
        "mode": Optional[Literal["single", "jury"]],
        "repetitions": Optional[int],
        "model": {  # optional
            "id": str,
            "integration_id": Optional[str],
            "model_parameters": {  # optional
                "temperature": Optional[float],
                "max_tokens": Optional[float],
                "top_k": Optional[float],
                "top_p": Optional[float],
                "frequency_penalty": Optional[float],
                "presence_penalty": Optional[float],
                "reasoning_effort": Optional[str],
                "budget_tokens": Optional[float],
            },
        },
        "jury": {  # optional
            "judges": [{
                "model": {
                    "id": str,
                    "integration_id": Optional[str],
                    "model_parameters": {  # optional
                        "temperature": Optional[float],
                        "max_tokens": Optional[float],
                        "top_k": Optional[float],
                        "top_p": Optional[float],
                        "frequency_penalty": Optional[float],
                        "presence_penalty": Optional[float],
                        "reasoning_effort": Optional[str],
                        "budget_tokens": Optional[float],
                    },
                },
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": {
                        "id": str,
                        "integration_id": Optional[str],
                        "model_parameters": {  # optional
                            "temperature": Optional[float],
                            "max_tokens": Optional[float],
                            "top_k": Optional[float],
                            "top_p": Optional[float],
                            "frequency_penalty": Optional[float],
                            "presence_penalty": Optional[float],
                            "reasoning_effort": Optional[str],
                            "budget_tokens": Optional[float],
                        },
                    },
                }],
            }],
            "replacement_judges": [{  # optional
                "model": {
                    "id": str,
                    "integration_id": Optional[str],
                    "model_parameters": {  # optional
                        "temperature": Optional[float],
                        "max_tokens": Optional[float],
                        "top_k": Optional[float],
                        "top_p": Optional[float],
                        "frequency_penalty": Optional[float],
                        "presence_penalty": Optional[float],
                        "reasoning_effort": Optional[str],
                        "budget_tokens": Optional[float],
                    },
                },
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": {
                        "id": str,
                        "integration_id": Optional[str],
                        "model_parameters": {  # optional
                            "temperature": Optional[float],
                            "max_tokens": Optional[float],
                            "top_k": Optional[float],
                            "top_p": Optional[float],
                            "frequency_penalty": Optional[float],
                            "presence_penalty": Optional[float],
                            "reasoning_effort": Optional[str],
                            "budget_tokens": Optional[float],
                        },
                    },
                }],
            }],
            "min_successful_judges": Optional[int],
            "tie_value": Optional[Literal["Tie"]],
        },
        "prompt": str,
        "categories": List[str],  # optional
        "categorical_labels": [{  # optional
            "value": str,
            "description": Optional[str],
        }],
        "dataset_id": Optional[str],
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      enabled?: boolean;
      metadata: {
        requiredModelWithToolsSupport?: boolean;
        requiredRetrievalContext?: boolean;
        requiredExpectedOutput?: boolean;
        supportedOnInputType?: boolean;
        supportedOnOutputType?: boolean;
        supportUseAsGuardrail?: boolean;
      };
      id: string;
      displayName: string;
      description: string;
      owner: string;
      created?: string;
      updated?: string;
      createdById?: string;
      updatedById?: string;
      domainId: string;
      projectId?: string;
      guardrailConfig?: any;
      outputType?: "boolean" | "categorical" | "number" | "string";
      type: "llm_eval";
      mode?: "single" | "jury";
      repetitions?: number;
      model?: {
        id: string;
        integrationId?: string;
        modelParameters?: {
          temperature?: number;
          maxTokens?: number;
          topK?: number;
          topP?: number;
          frequencyPenalty?: number;
          presencePenalty?: number;
          reasoningEffort?: string;
          budgetTokens?: number;
        };
      };
      jury?: {
        judges: {
          model: {
            id: string;
            integrationId?: string;
            modelParameters?: {
              temperature?: number;
              maxTokens?: number;
              topK?: number;
              topP?: number;
              frequencyPenalty?: number;
              presencePenalty?: number;
              reasoningEffort?: string;
              budgetTokens?: number;
            };
          };
          retry?: {
            count?: number;
            onCodes?: number[];
          };
          fallbacks?: {
            model: {
              id: string;
              integrationId?: string;
              modelParameters?: {
                temperature?: number;
                maxTokens?: number;
                topK?: number;
                topP?: number;
                frequencyPenalty?: number;
                presencePenalty?: number;
                reasoningEffort?: string;
                budgetTokens?: number;
              };
            };
          }[];
        }[];
        replacementJudges?: {
          model: {
            id: string;
            integrationId?: string;
            modelParameters?: {
              temperature?: number;
              maxTokens?: number;
              topK?: number;
              topP?: number;
              frequencyPenalty?: number;
              presencePenalty?: number;
              reasoningEffort?: string;
              budgetTokens?: number;
            };
          };
          retry?: {
            count?: number;
            onCodes?: number[];
          };
          fallbacks?: {
            model: {
              id: string;
              integrationId?: string;
              modelParameters?: {
                temperature?: number;
                maxTokens?: number;
                topK?: number;
                topP?: number;
                frequencyPenalty?: number;
                presencePenalty?: number;
                reasoningEffort?: string;
                budgetTokens?: number;
              };
            };
          }[];
        }[];
        minSuccessfulJudges?: number;
        tieValue?: "Tie";
      };
      prompt: string;
      categories?: string[];
      categoricalLabels?: {
        value: string;
        description?: string;
      }[];
      datasetId?: string;
    }
    ```
  </CodeGroup>
</Expandable>

### Delete an Eval

Delete an evaluator by its unique identifier.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      orq.evals.delete(id="<id>")

      # Use the SDK ...

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    await orq.evals.delete({
      id: "<id>",
    });

  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,  # required
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;  // required
    }
    ```
  </CodeGroup>
</Expandable>

### Update an Eval

Update an evaluator by ID with the provided fields.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.update(id="<id>", path="Default", project_id="01JMDPA3QW5C1V0NJ1PW34T4E5")

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.update({
      id: "<id>",
      requestBody: {
        path: "Default",
        projectId: "01JMDPA3QW5C1V0NJ1PW34T4E5",
      },
    });

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,  # required
        "type": Optional[str],
        "path": Optional[str],
        "project_id": Optional[str],
        "key": Optional[str],
        "description": Optional[str],
        "prompt": Optional[str],
        "output_type": Optional[str],
        "categories": List[str],  # optional
        "categorical_labels": [{  # optional
            "value": str,  # required
            "description": Optional[str],
        }],
        "dataset_id": Optional[str],
        "repetitions": Optional[float],
        "mode": Optional[Literal["single", "jury"]],
        "model": Optional[str],
        "jury": {  # optional
            "judges": [{  # required
                "model": str,  # required
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": str,  # required
                }],
            }],
            "replacement_judges": [{  # optional
                "model": str,  # required
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": str,  # required
                }],
            }],
            "min_successful_judges": Optional[int],
            "tie_value": Optional[Literal["Tie"]],
        },
        "schema_": Optional[str],
        "url": Optional[str],
        "method": Optional[str],
        "headers": Dict[str, str],  # optional
        "payload": Dict[str, Any],  # optional
        "code": Optional[str],
        "guardrail_config": Optional[Any],
        "version_increment": Optional[Literal["major", "minor", "patch"]],
        "version_description": Optional[str],
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;  // required
      requestBody?: {
        type?: string;
        path?: string;
        projectId?: string;
        key?: string;
        description?: string;
        prompt?: string;
        outputType?: string;
        categories?: string[];
        categoricalLabels?: {
          value: string;  // required
          description?: string;
        }[];
        datasetId?: string;
        repetitions?: number;
        mode?: "single" | "jury";
        model?: string;
        jury?: {
          judges: {  // required
            model: string;  // required
            retry?: {
              count?: number;
              onCodes?: number[];
            };
            fallbacks?: {
              model: string;  // required
            }[];
          }[];
          replacementJudges?: {
            model: string;  // required
            retry?: {
              count?: number;
              onCodes?: number[];
            };
            fallbacks?: {
              model: string;  // required
            }[];
          }[];
          minSuccessfulJudges?: number;
          tieValue?: "Tie";
        };
        schema?: string;
        url?: string;
        method?: string;
        headers?: Record<string, string>;
        payload?: Record<string, any>;
        code?: string;
        guardrailConfig?: any;
        versionIncrement?: "major" | "minor" | "patch";
        versionDescription?: string;
      };
    }
    ```
  </CodeGroup>
</Expandable>

<Expandable title="Response">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,
        "description": str,
        "created": Optional[str],
        "updated": Optional[str],
        "updated_by_id": Optional[str],
        "project_id": Optional[str],
        "guardrail_config": Optional[Any],
        "type": Literal["llm_eval"],
        "repetitions": Optional[int],
        "prompt": str,
        "categories": List[str],  # optional
        "categorical_labels": [{  # optional
            "value": str,
            "description": Optional[str],
        }],
        "dataset_id": Optional[str],
        "key": str,
        "mode": Literal["single", "jury"],
        "model": Optional[str],
        "jury": {  # optional
            "judges": [{
                "model": str,
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": str,
                }],
            }],
            "replacement_judges": [{  # optional
                "model": str,
                "retry": {  # optional
                    "count": Optional[int],
                    "on_codes": List[int],  # optional
                },
                "fallbacks": [{  # optional
                    "model": str,
                }],
            }],
            "min_successful_judges": Optional[int],
            "tie_value": Optional[Literal["Tie"]],
        },
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;
      description: string;
      created?: string;
      updated?: string;
      updatedById?: string;
      projectId?: string;
      guardrailConfig?: any;
      type: "llm_eval";
      repetitions?: number;
      prompt: string;
      categories?: string[];
      categoricalLabels?: {
        value: string;
        description?: string;
      }[];
      datasetId?: string;
      key: string;
      mode: "single" | "jury";
      model?: string;
      jury?: {
        judges: {
          model: string;
          retry?: {
            count?: number;
            onCodes?: number[];
          };
          fallbacks?: {
            model: string;
          }[];
        }[];
        replacementJudges?: {
          model: string;
          retry?: {
            count?: number;
            onCodes?: number[];
          };
          fallbacks?: {
            model: string;
          }[];
        }[];
        minSuccessfulJudges?: number;
        tieValue?: "Tie";
      };
    }
    ```
  </CodeGroup>
</Expandable>

### List Versions

Returns version history for a specific evaluator.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.list_versions(id="<id>")

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.listVersions({
      id: "<id>",
    });

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,  # required
        "limit": Optional[int],
        "starting_after": Optional[str],
        "ending_before": Optional[str],
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;  // required
      limit?: number;
      startingAfter?: string;
      endingBefore?: string;
    }
    ```
  </CodeGroup>
</Expandable>

<Expandable title="Response">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "object": str,
        "data": List[Dict[str, Any]],
        "has_more": bool,
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      object: string;
      data: Record<string, unknown>[];
      hasMore: boolean;
    }
    ```
  </CodeGroup>
</Expandable>

### Get Version

Returns a specific version of an evaluator.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.get_version(id="<id>", version_id="<id>")

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.getVersion({
      id: "<id>",
      versionId: "<id>",
    });

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,  # required
        "version_id": str,  # required
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;  // required
      versionId: string;  // required
    }
    ```
  </CodeGroup>
</Expandable>

### Invoke an Eval

Runs an evaluator that already exists in the workspace. Accepts either a conversation or the structured input and output fields; when both are present the conversation wins.

<CodeGroup>
  ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
  from orq_ai_sdk import Orq
  import os

  with Orq(
      api_key=os.getenv("ORQ_API_KEY", ""),
  ) as orq:

      res = orq.evals.invoke(id="<id>")

      # Handle response
      print(res)

  ```

  ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
  import { Orq } from "@orq-ai/node";

  const orq = new Orq({
    apiKey: process.env["ORQ_API_KEY"] ?? "",
  });

  async function run() {
    const result = await orq.evals.invoke({
      id: "<id>",
      invokeEvaluatorRequest: {},
    });

    console.log(result);
  }

  run();
  ```
</CodeGroup>

<Expandable title="Parameters">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "id": str,  # required
        "context": {  # optional
            "messages": List[Dict[str, Any]],  # optional
            "input": {  # optional
                "system_instructions": Optional[str],
                "user_query": Optional[str],
                "retrievals": List[str],  # optional
                "expected_output": Optional[str],
            },
            "output": {  # optional
                "response": Optional[str],
                "tools_called": [{  # optional
                    "name": Optional[str],
                    "arguments": Optional[str],
                    "output": Optional[str],
                }],
            },
            "variables": Dict[str, Any],  # optional
        },
        "model": Optional[str],
        "query": Optional[str],
        "output": Optional[str],
        "reference": Optional[str],
        "retrievals": List[str],  # optional
        "messages": List[Dict[str, Any]],  # optional
        "variables": Dict[str, Any],  # optional
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      id: string;  // required
      invokeEvaluatorRequest: {  // required
        context?: {
          messages?: Record<string, any>[];
          input?: {
            systemInstructions?: string;
            userQuery?: string;
            retrievals?: string[];
            expectedOutput?: string;
          };
          output?: {
            response?: string;
            toolsCalled?: {
              name?: string;
              arguments?: string;
              output?: string;
            }[];
          };
          variables?: Record<string, any>;
        };
        model?: string;
        query?: string;
        output?: string;
        reference?: string;
        retrievals?: string[];
        messages?: Record<string, any>[];
        variables?: Record<string, any>;
      };
    }
    ```
  </CodeGroup>
</Expandable>

<Expandable title="Response">
  <CodeGroup>
    ```python Python theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
        "type": Optional[str],
        "value": Optional[Any],
        "trace_id": Optional[str],
        "span_id": Optional[str],
        "evaluator_id": Optional[str],
        "status": Optional[str],
        "passed": Optional[bool],
        "explanation": Optional[str],
        "categories": List[str],  # optional
        "confidence": Optional[float],
    }
    ```

    ```typescript Node.js theme={"theme":{"light":"github-light","dark":"github-dark"}}
    {
      type?: string;
      value?: any;
      traceId?: string;
      spanId?: string;
      evaluatorId?: string;
      status?: string;
      passed?: boolean;
      explanation?: string;
      categories?: string[];
      confidence?: number;
    }
    ```
  </CodeGroup>
</Expandable>


This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.