Testing A8

import json
import os
from collections import defaultdict

from openai import OpenAI

client = OpenAI(
    base_url="https://api.u22a8.ai/eval/v1", api_key=os.environ["U22A8_API_KEY"]
)

criterion = "Does the reply promise a refund?"
verdict = {
    "type": "json_schema",
    "json_schema": {
        "name": "verdict",
        "schema": {"properties": {"refund": {"type": "boolean"}}},
    },
}
tests = [  # new questions, with the answers you expect
    ("We will refund you in full by Friday.", True),
    ("We will sort it out on Monday.", False),
]

answered, right = defaultdict(int), defaultdict(int)
for reply, expected in tests:
    resp = client.chat.completions.create(
        model="a8-1-1:95@20260715t140322z",  # pinned, minimum accuracy 95
        messages=[
            {"role": "system", "content": criterion},
            {"role": "user", "content": reply},
        ],
        response_format=verdict,
    )
    message = resp.choices[0].message
    if message.refusal:  # an abstention: it lowers coverage
        continue
    level = resp.model_extra["min_accuracy"]  # the level it answered at
    answered[level] += 1
    right[level] += json.loads(message.content)["refund"] == expected

for level in sorted(answered):
    print(f"at {level}: {right[level]}/{answered[level]} right")
print(f"answered {sum(answered.values())} of {len(tests)}")

import OpenAI from "openai";

const client = new OpenAI({
  baseURL: "https://api.u22a8.ai/eval/v1",
  apiKey: process.env.U22A8_API_KEY,
});

const criterion = "Does the reply promise a refund?";
const tests: [string, boolean][] = [
  // new questions, with the answers you expect
  ["We will refund you in full by Friday.", true],
  ["We will sort it out on Monday.", false],
];

const answered = new Map<number, number>();
const right = new Map<number, number>();
for (const [reply, expected] of tests) {
  const resp = await client.chat.completions.create({
    model: "a8-1-1:95@20260715t140322z", // pinned, at a minimum accuracy of 95
    messages: [
      { role: "system", content: criterion },
      { role: "user", content: reply },
    ],
    response_format: {
      type: "json_schema",
      json_schema: {
        name: "verdict",
        schema: { properties: { refund: { type: "boolean" } } },
      },
    },
  });
  const message = resp.choices[0].message;
  if (message.refusal) continue; // an abstention: it lowers coverage
  // the level it answered at
  const level = (resp as unknown as { min_accuracy: number }).min_accuracy;
  const verdict = JSON.parse(message.content ?? "{}");
  answered.set(level, (answered.get(level) ?? 0) + 1);
  const correct = Number(verdict.refund === expected);
  right.set(level, (right.get(level) ?? 0) + correct);
}

for (const [level, count] of [...answered].sort((a, b) => a[0] - b[0])) {
  console.log(`at ${level}: ${right.get(level)}/${count} right`);
}
const total = [...answered.values()].reduce((a, b) => a + b, 0);
console.log(`answered ${total} of ${tests.length}`);