Autoevals

npm install autoevals

yarn add autoevals

import { Factuality } from "autoevals";
 
(async () => {
  const input = "Which country has the highest population?";
  const output = "People's Republic of China";
  const expected = "China";
 
  const result = await Factuality({ output, expected, input });
  console.log(`Factuality score: ${result.score}`);
  console.log(`Factuality metadata: ${result.metadata?.rationale}`);
})();

import { Eval } from "braintrust";
import { Factuality } from "autoevals";
 
Eval("Autoevals", {
  data: () => [
    {
      input: "Which country has the highest population?",
      expected: "China",
    },
  ],
  task: () => "People's Republic of China",
  scores: [Factuality],
});

npx braintrust run example.eval.js

import { LLMClassifierFromTemplate } from "autoevals";
 
(async () => {
  const promptTemplate = `You are a technical project manager who helps software engineers generate better titles for their GitHub issues.
You will look at the issue description, and pick which of two titles better describes it.
 
I'm going to provide you with the issue description, and two possible titles.
 
Issue Description: {{input}}
 
1: {{output}}
2: {{expected}}`;
 
  const choiceScores = { 1: 1, 2: 0 };
 
  const evaluator = LLMClassifierFromTemplate<{ input: string }>({
    name: "TitleQuality",
    promptTemplate,
    choiceScores,
    useCoT: true,
  });
 
  const input = `As suggested by Nicolo, we should standardize the error responses coming from GoTrue, postgres, and realtime (and any other/future APIs) so that it's better DX when writing a client,
We can make this change on the servers themselves, but since postgrest and gotrue are fully/partially external may be harder to change, it might be an option to transform the errors within the client libraries/supabase-js, could be messy?
Nicolo also dropped this as a reference: http://spec.openapis.org/oas/v3.0.3#openapi-specification`;
  const output = `Standardize error responses from GoTrue, Postgres, and Realtime APIs for better DX`;
  const expected = `Standardize Error Responses across APIs`;
 
  const response = await evaluator({ input, output, expected });
 
  console.log("Score", response.score);
  console.log("Metadata", response.metadata);
})();

import { Score } from "autoevals";
 
const bananaScorer = ({
  output,
  expected,
  input,
}: {
  output: string;
  expected: string;
  input: string;
}): Score => {
  return { name: "banana_scorer", score: output.includes("banana") ? 1 : 0 };
};
 
(async () => {
  const input = "What is 1 banana + 2 bananas?";
  const output = "3";
  const expected = "3 bananas";
 
  const result = bananaScorer({ output, expected, input });
  console.log(`Banana score: ${result.score}`);
})();