Hallucination Scorer

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { createHallucinationScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createHallucinationScorer({ model: openai("gpt-4o-mini"), options: {
  context: [
    "The iPhone was first released in 2007.",
    "Steve Jobs unveiled it at Macworld.",
    "The original model had a 3.5-inch screen."
  ]
});
 
const query = "When was the first iPhone released?";
const response = "The iPhone was first released in 2007, when Steve Jobs unveiled it at Macworld. The original iPhone featured a 3.5-inch screen.";
 
const result = await scorer.run({
  input: [{ role: 'user', content: query }],
  output: { text: response },
});
 
console.log(result);

{
  score: 0,
  reason: 'The score is 0 because none of the statements from the context were contradicted by the output.'
}

import { openai } from "@ai-sdk/openai";
import { createHallucinationScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createHallucinationScorer({ model: openai("gpt-4o-mini"), options: {
  context: [
    "The first Star Wars movie was released in 1977.",
    "It was directed by George Lucas.",
    "The film earned $775 million worldwide.",
    "The movie was filmed in Tunisia and England."
  ]
});
 
const query = "Tell me about the first Star Wars movie.";
const response = "The first Star Wars movie came out in 1977 and was directed by George Lucas. It made over $1 billion at the box office and was filmed entirely in California.";
 
const result = await scorer.run({
  input: [{ role: 'user', content: query }],
  output: { text: response },
});
 
console.log(result);

{
  score: 0.5,
  reason: 'The score is 0.5 because two out of four statements from the output were contradicted by claims in the context, indicating a balance of accurate and inaccurate information.'
}

import { openai } from "@ai-sdk/openai";
import { createHallucinationScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createHallucinationScorer({ model: openai("gpt-4o-mini"), options: {
  context: [
    "The Wright brothers made their first flight in 1903.",
    "The flight lasted 12 seconds.",
    "It covered a distance of 120 feet."
  ]
});
 
const query = "When did the Wright brothers first fly?";
const response = "The Wright brothers achieved their historic first flight in 1908. The flight lasted about 2 minutes and covered nearly a mile.";
 
const result = await scorer.run({
  input: [{ role: 'user', content: query }],
  output: { text: response },
});
 
console.log(result);
 

{
  score: 1,
  reason: 'The score is 1.0 because all three statements from the output directly contradict the context: the first flight was in 1903, not 1908; it lasted 12 seconds, not about 2 minutes; and it covered 120 feet, not nearly a mile.'
}

const scorer = createHallucinationScorer({ model: openai("gpt-4o-mini"), options: {
  context: [""],
  scale: 1
});

{
  runId: string,
  extractStepResult: { claims: string[] },
  extractPrompt: string,
  analyzeStepResult: {
    verdicts: Array<{ statement: string, verdict: 'yes' | 'no', reason: string }>
  },
  analyzePrompt: string,
  score: number,
  reason: string,
  reasonPrompt: string
}