Faithfulness Scorer

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { createFaithfulnessScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createFaithfulnessScorer({ model: openai("gpt-4o-mini"), options: {
  context: [
    "The Tesla Model 3 was launched in 2017.",
    "It has a range of up to 358 miles.",
    "The base model accelerates 0-60 mph in 5.8 seconds."
  ]
});
 
const query = "Tell me about the Tesla Model 3.";
const response = "The Tesla Model 3 was introduced in 2017. It can travel up to 358 miles on a single charge and the base version goes from 0 to 60 mph in 5.8 seconds.";
 
const result = await scorer.run({
  input: [{ role: 'user', content: query }],
  output: { text: response },
});
 
console.log(result);

{
  score: 1,
  reason: 'The score is 1 because all claims made in the output are supported by the provided context.'
}

import { openai } from "@ai-sdk/openai";
import { createFaithfulnessScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createFaithfulnessScorer({ model: openai("gpt-4o-mini"), options: {
  context: [
    "Python was created by Guido van Rossum.",
    "The first version was released in 1991.",
    "Python emphasizes code readability."
  ]
});
 
const query = "What can you tell me about Python?";
const response = "Python was created by Guido van Rossum and released in 1991. It is the most popular programming language today and is used by millions of developers worldwide.";
 
const result = await scorer.run({
  input: [{ role: 'user', content: query }],
  output: { text: response },
});
 
console.log(result);

{
  score: 0.5,
  reason: "The score is 0.5 because while two claims are supported by the context (Python was created by Guido van Rossum and Python was released in 1991), the other two claims regarding Python's popularity and usage cannot be verified as they are not mentioned in the context."
}

import { openai } from "@ai-sdk/openai";
import { createFaithfulnessScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createFaithfulnessScorer({ model: openai("gpt-4o-mini"), options: {
  context: [
    "Mars is the fourth planet from the Sun.",
    "It has a thin atmosphere of mostly carbon dioxide.",
    "Two small moons orbit Mars: Phobos and Deimos."
  ]
});
 
const query = "What do we know about Mars?";
const response = "Mars is the third planet from the Sun. It has a thick atmosphere rich in oxygen and nitrogen, and is orbited by three large moons.";
 
const result = await scorer.run({
  input: [{ role: 'user', content: query }],
  output: { text: response },
});
 
console.log(result);

{
  score: 0,
  reason: "The score is 0 because all claims made in the output contradict the provided context. The output states that Mars is the third planet from the Sun, while the context clearly states it is the fourth. Additionally, it claims that Mars has a thick atmosphere rich in oxygen and nitrogen, contradicting the context's description of a thin atmosphere mostly composed of carbon dioxide. Finally, the output mentions that Mars is orbited by three large moons, while the context specifies that it has only two small moons, Phobos and Deimos. Therefore, there are no supported claims, leading to a score of 0."
}

const scorer = createFaithfulnessScorer({ model: openai("gpt-4o-mini"), options: {
  context: [""],
  scale: 1
});

{
  runId: string,
  extractStepResult: string[],
  extractPrompt: string,
  analyzeStepResult: {
    verdicts: Array<{ verdict: 'yes' | 'no' | 'unsure', reason: string }>
  },
  analyzePrompt: string,
  score: number,
  reason: string,
  reasonPrompt: string
}