Answer Relevancy Evaluation

OPENAI_API_KEY=your_api_key_here

import { openai } from "@ai-sdk/openai";
import { AnswerRelevancyMetric } from "@mastra/evals/llm";

const metric = new AnswerRelevancyMetric(openai("gpt-4o-mini"), {
  uncertaintyWeight: 0.3, // Weight for 'unsure' verdicts
  scale: 1, // Scale for the final score
});

const query1 = "What are the health benefits of regular exercise?";
const response1 =
  "Regular exercise improves cardiovascular health, strengthens muscles, boosts metabolism, and enhances mental well-being through the release of endorphins.";
 
console.log("Example 1 - High Relevancy:");
console.log("Query:", query1);
console.log("Response:", response1);
 
const result1 = await metric.measure(query1, response1);
console.log("Metric Result:", {
  score: result1.score,
  reason: result1.info.reason,
});
// Example Output:
// Metric Result: { score: 1, reason: 'The response is highly relevant to the query. It provides a comprehensive overview of the health benefits of regular exercise.' }

const query2 = "What should a healthy breakfast include?";
const response2 =
  "A nutritious breakfast should include whole grains and protein. However, the timing of your breakfast is just as important - studies show eating within 2 hours of waking optimizes metabolism and energy levels throughout the day.";
 
console.log("Example 2 - Partial Relevancy:");
console.log("Query:", query2);
console.log("Response:", response2);
 
const result2 = await metric.measure(query2, response2);
console.log("Metric Result:", {
  score: result2.score,
  reason: result2.info.reason,
});
// Example Output:
// Metric Result: { score: 0.7, reason: 'The response is partially relevant to the query. It provides some information about healthy breakfast choices but misses the timing aspect.' }

const query3 = "What are the benefits of meditation?";
const response3 =
  "The Great Wall of China is over 13,000 miles long and was built during the Ming Dynasty to protect against invasions.";
 
console.log("Example 3 - Low Relevancy:");
console.log("Query:", query3);
console.log("Response:", response3);
 
const result3 = await metric.measure(query3, response3);
console.log("Metric Result:", {
  score: result3.score,
  reason: result3.info.reason,
});
// Example Output:
// Metric Result: { score: 0.1, reason: 'The response is not relevant to the query. It provides information about the Great Wall of China but does not mention meditation.' }