Answer Relevancy Scorer

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createAnswerRelevancyScorer({ model: openai("gpt-4o-mini") });
 
const inputMessages = [{ role: 'user', content: "What are the health benefits of regular exercise?" }];
const outputMessage = { text: "Regular exercise improves cardiovascular health, strengthens muscles, boosts metabolism, and enhances mental well-being through the release of endorphins." };
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 1,
  reason: 'The score is 1 because the output directly addresses the question by providing multiple explicit health benefits of regular exercise, including improvements in cardiovascular health, muscle strength, metabolism, and mental well-being. Each point is relevant and contributes to a comprehensive understanding of the health benefits.'
}

import { openai } from "@ai-sdk/openai";
import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createAnswerRelevancyScorer({ model: openai("gpt-4o-mini") });
 
const inputMessages = [{ role: 'user', content: "What should a healthy breakfast include?" }];
const outputMessage = { text: "A nutritious breakfast should include whole grains and protein. However, the timing of your breakfast is just as important - studies show eating within 2 hours of waking optimizes metabolism and energy levels throughout the day." };
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.25,
  reason: 'The score is 0.25 because the output provides a direct answer by mentioning whole grains and protein as components of a healthy breakfast, which is relevant. However, the additional information about the timing of breakfast and its effects on metabolism and energy levels is not directly related to the question, leading to a lower overall relevance score.'
}

import { openai } from "@ai-sdk/openai";
import { createAnswerRelevancyScorer } from "@mastra/evals/scorers/llm";
 
const scorer = createAnswerRelevancyScorer({ model: openai("gpt-4o-mini") });
 
const inputMessages = [{ role: 'user', content: "What are the benefits of meditation?" }];
const outputMessage = { text: "The Great Wall of China is over 13,000 miles long and was built during the Ming Dynasty to protect against invasions." };
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0,
  reason: 'The score is 0 because the output about the Great Wall of China is completely unrelated to the benefits of meditation, providing no relevant information or context that addresses the input question.'
}

const scorer = createAnswerRelevancyScorer({ model: openai("gpt-4o-mini"), options: { uncertaintyWeight: 0.3, scale: 1 } });

{
  runId: string,
  score: number,
  extractPrompt: string,
  extractStepResult: { statements: string[] },
  analyzePrompt: string,
  analyzeStepResult: { results: Array<{ result: 'yes' | 'unsure' | 'no', reason: string }> },
  reasonPrompt: string,
  reason: string
}