Start an evaluation run

⚠️

Preview — this feature is being built; the design below is the spec.

POST https://app.syntic.ai/v1/evals/runs

Run one or more prompt and model variants over every item in a dataset and grade the outputs. Generation goes through the batch path, so runs cost the batch rate and usually finish within the hour; set priority: interactive to pay the standard rate for faster results.

For how this fits into a full workflow, see Evaluations.

Headers

Header	Required	Value
`Authorization`	Yes	`Bearer <key>`: a private inference key (`sk_live_...`). See Authentication.
`syntic-beta`	Yes, while in preview	`evals-2026-12-01`. Without it the call fails with `400 beta_required`. See Versioning.
`Content-Type`	Yes	`application/json`
`Idempotency-Key`	No	Up to 255 characters. Retrying with the same key within 24 hours returns the first result instead of acting twice. See Idempotency.

Body parameters

Parameter	Type	Required	Notes
`dataset_id`	string	Yes
`variants`	array	Yes	1 to 4 items, each with `label`, `model`, and either `prompt_version` (from prompt templates) or an inline `messages` template using `{{variables}}`.
`grader_ids`	array of strings	Yes	1 to 5 graders; an item passes when every grader passes.
`name`	string	No
`priority`	string	No	`batch` (default) or `interactive`.
`item_tags`	array of strings	No	Run only items with these tags.

Response

202 Accepted, Content-Type: application/json

{
  "id": "evrun_01JD35D7F9H1K3M5P7R9",
  "type": "eval_run",
  "name": "shorter system prompt vs baseline",
  "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
  "status": "queued",
  "variants": [
    {
      "label": "baseline",
      "model": "Amara S1.2",
      "prompt_version": "support-v12"
    },
    {
      "label": "short",
      "model": "Amara S1.2",
      "prompt_version": "support-v13"
    }
  ],
  "grader_ids": [
    "evgr_01JD46E8G0J2L4N6Q8S0"
  ],
  "progress": {
    "total": 400,
    "done": 0,
    "errored": 0
  },
  "summary": null,
  "created_at": "2026-11-05T10:00:00Z",
  "finished_at": null
}

Field	Type	Description
`variants`	array	Up to 4 prompt and model combinations, each run on every item.
`progress`	object	Items done per variant summed across variants.
`summary`	array	Per variant: pass rate, mean score, mean output length and cost, once finished.
`status`	string	`queued`, `running`, `succeeded`, `failed` or `canceled`.

Examples

curl -sS "https://app.syntic.ai/v1/evals/runs" \
  -H "Authorization: Bearer $SYNTIC_API_KEY" \
  -H "syntic-beta: evals-2026-12-01" \
  -H "Idempotency-Key: 8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69" \
  -H "Content-Type: application/json" \
  -d '{
      "name": "shorter system prompt vs baseline",
      "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
      "variants": [
        {
          "label": "baseline",
          "model": "Amara S1.2",
          "prompt_version": "support-v12"
        },
        {
          "label": "short",
          "model": "Amara S1.2",
          "prompt_version": "support-v13"
        }
      ],
      "grader_ids": [
        "evgr_01JD46E8G0J2L4N6Q8S0"
      ]
    }'

import os
 
import requests
 
resp = requests.post(
    "https://app.syntic.ai/v1/evals/runs",
    headers={
        "Authorization": f"Bearer {os.environ['SYNTIC_API_KEY']}",
        "syntic-beta": "evals-2026-12-01",
        "Idempotency-Key": "8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69",
    },
    json={
        "name": "shorter system prompt vs baseline",
        "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
        "variants": [
            {
                "label": "baseline",
                "model": "Amara S1.2",
                "prompt_version": "support-v12"
            },
            {
                "label": "short",
                "model": "Amara S1.2",
                "prompt_version": "support-v13"
            }
        ],
        "grader_ids": [
            "evgr_01JD46E8G0J2L4N6Q8S0"
        ]
    },
    timeout=60,
)
data = resp.json()
if not resp.ok:
    raise RuntimeError(f"{resp.status_code} {data['error']['code']}: {data['error']['message']}")
print(data)

const res = await fetch('https://app.syntic.ai/v1/evals/runs', {
  method: 'POST',
  headers: {
    Authorization: `Bearer ${process.env.SYNTIC_API_KEY}`,
    'syntic-beta': 'evals-2026-12-01',
    'Idempotency-Key': '8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69',
    'Content-Type': 'application/json',
  },
  body: JSON.stringify({
    "name": "shorter system prompt vs baseline",
    "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
    "variants": [
      {
        "label": "baseline",
        "model": "Amara S1.2",
        "prompt_version": "support-v12"
      },
      {
        "label": "short",
        "model": "Amara S1.2",
        "prompt_version": "support-v13"
      }
    ],
    "grader_ids": [
      "evgr_01JD46E8G0J2L4N6Q8S0"
    ]
  }),
})
const text = await res.text()
const data = text ? JSON.parse(text) : null
if (!res.ok) throw new Error(`${res.status} ${data.error.code}: ${data.error.message}`)
console.log(data ?? res.status)

package main
 
import (
	"fmt"
	"io"
	"net/http"
	"os"
	"strings"
)
 
func main() {
	body := strings.NewReader(`{
  "name": "shorter system prompt vs baseline",
  "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
  "variants": [
    {
      "label": "baseline",
      "model": "Amara S1.2",
      "prompt_version": "support-v12"
    },
    {
      "label": "short",
      "model": "Amara S1.2",
      "prompt_version": "support-v13"
    }
  ],
  "grader_ids": [
    "evgr_01JD46E8G0J2L4N6Q8S0"
  ]
}`)
	req, err := http.NewRequest("POST", "https://app.syntic.ai/v1/evals/runs", body)
	if err != nil {
		panic(err)
	}
	req.Header.Set("Authorization", "Bearer "+os.Getenv("SYNTIC_API_KEY"))
	req.Header.Set("syntic-beta", "evals-2026-12-01")
	req.Header.Set("Idempotency-Key", "8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69")
	req.Header.Set("Content-Type", "application/json")
 
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
 
	out, _ := io.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		panic(fmt.Sprintf("%d: %s", resp.StatusCode, out))
	}
	fmt.Println(string(out))
}

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
 
public class Example {
    public static void main(String[] args) throws Exception {
        String body = """
            {
              "name": "shorter system prompt vs baseline",
              "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
              "variants": [
                {
                  "label": "baseline",
                  "model": "Amara S1.2",
                  "prompt_version": "support-v12"
                },
                {
                  "label": "short",
                  "model": "Amara S1.2",
                  "prompt_version": "support-v13"
                }
              ],
              "grader_ids": [
                "evgr_01JD46E8G0J2L4N6Q8S0"
              ]
            }
            """;
        HttpRequest request = HttpRequest.newBuilder(URI.create("https://app.syntic.ai/v1/evals/runs"))
            .header("Authorization", "Bearer " + System.getenv("SYNTIC_API_KEY"))
            .header("syntic-beta", "evals-2026-12-01")
            .header("Idempotency-Key", "8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69")
            .header("Content-Type", "application/json")
            .method("POST", HttpRequest.BodyPublishers.ofString(body))
            .build();
        HttpResponse<String> response = HttpClient.newHttpClient()
            .send(request, HttpResponse.BodyHandlers.ofString());
        if (response.statusCode() >= 300) {
            throw new RuntimeException(response.statusCode() + ": " + response.body());
        }
        System.out.println(response.body());
    }
}

using System.Net.Http.Headers;
using System.Text;
 
using var http = new HttpClient();
using var request = new HttpRequestMessage(new HttpMethod("POST"), "https://app.syntic.ai/v1/evals/runs");
request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", Environment.GetEnvironmentVariable("SYNTIC_API_KEY"));
request.Headers.Add("syntic-beta", "evals-2026-12-01");
request.Headers.Add("Idempotency-Key", "8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69");
request.Content = new StringContent("""
    {
      "name": "shorter system prompt vs baseline",
      "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
      "variants": [
        {
          "label": "baseline",
          "model": "Amara S1.2",
          "prompt_version": "support-v12"
        },
        {
          "label": "short",
          "model": "Amara S1.2",
          "prompt_version": "support-v13"
        }
      ],
      "grader_ids": [
        "evgr_01JD46E8G0J2L4N6Q8S0"
      ]
    }
    """, Encoding.UTF8, "application/json");
 
using var response = await http.SendAsync(request);
var text = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode) throw new Exception($"{(int)response.StatusCode}: {text}");
Console.WriteLine(text);

<?php
$body = <<<'JSON'
{
  "name": "shorter system prompt vs baseline",
  "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
  "variants": [
    {
      "label": "baseline",
      "model": "Amara S1.2",
      "prompt_version": "support-v12"
    },
    {
      "label": "short",
      "model": "Amara S1.2",
      "prompt_version": "support-v13"
    }
  ],
  "grader_ids": [
    "evgr_01JD46E8G0J2L4N6Q8S0"
  ]
}
JSON;
 
$ch = curl_init('https://app.syntic.ai/v1/evals/runs');
curl_setopt_array($ch, [
    CURLOPT_CUSTOMREQUEST => 'POST',
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_HTTPHEADER => [
        'Authorization: Bearer ' . getenv('SYNTIC_API_KEY'),
        'syntic-beta: evals-2026-12-01',
        'Idempotency-Key: 8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69',
        'Content-Type: application/json',
    ],
    CURLOPT_POSTFIELDS => $body,
]);
$response = curl_exec($ch);
$status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
curl_close($ch);
 
if ($status >= 300) {
    throw new RuntimeException("$status: $response");
}
print_r(json_decode($response, true) ?? $status);

require "json"
require "net/http"
require "uri"
 
uri = URI("https://app.syntic.ai/v1/evals/runs")
req = Net::HTTP::Post.new(uri)
req["Authorization"] = "Bearer #{ENV.fetch('SYNTIC_API_KEY')}"
req["syntic-beta"] = "evals-2026-12-01"
req["Idempotency-Key"] = "8d6f0c1e-4b2a-4a57-9f0e-2f1d3c4b5a69"
req["Content-Type"] = "application/json"
req.body = <<~'JSON'
  {
    "name": "shorter system prompt vs baseline",
    "dataset_id": "evds_01JD13B5D7F9H1K3M5P7",
    "variants": [
      {
        "label": "baseline",
        "model": "Amara S1.2",
        "prompt_version": "support-v12"
      },
      {
        "label": "short",
        "model": "Amara S1.2",
        "prompt_version": "support-v13"
      }
    ],
    "grader_ids": [
      "evgr_01JD46E8G0J2L4N6Q8S0"
    ]
  }
JSON
 
res = Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |http| http.request(req) }
raise "#{res.code}: #{res.body}" unless res.is_a?(Net::HTTPSuccess)
puts res.body.empty? ? res.code : JSON.pretty_generate(JSON.parse(res.body))

Errors

Errors use the error envelope and carry a request_id. Codes specific to this operation:

Status	`type`	`code`	When
400	`invalid_request_error`	`beta_required`	The `syntic-beta` header is missing.
400	`invalid_request_error`	`unknown_variable`	A template uses a `{{variable}}` not present in the dataset’s inputs.
404	`not_found_error`	`dataset_not_found`	No dataset with this id.
401	`authentication_error`	`invalid_api_key`	The key is missing, unknown, revoked or expired.
429	`rate_limit_error`	`rate_limited`	Too many requests; wait for `Retry-After`. See Rate-limit headers.

Test a grader List runs

Was this page helpful?