回答の関連性評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { AnswerRelevancyMetric } from "@mastra/evals/llm";
 
const metric = new AnswerRelevancyMetric(openai("gpt-4o-mini"));
 
const query = "What are the health benefits of regular exercise?";
const response = "Regular exercise improves cardiovascular health, strengthens muscles, boosts metabolism, and enhances mental well-being through the release of endorphins.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 1,
  info: {
    reason: 'The score is 1 because the output directly addresses the question by providing multiple explicit health benefits of regular exercise, including improvements in cardiovascular health, muscle strength, metabolism, and mental well-being. Each point is relevant and contributes to a comprehensive understanding of the health benefits.'
  }
}

import { openai } from "@ai-sdk/openai";
import { AnswerRelevancyMetric } from "@mastra/evals/llm";
 
const metric = new AnswerRelevancyMetric(openai("gpt-4o-mini"));
 
const query = "What should a healthy breakfast include?";
const response =
  "A nutritious breakfast should include whole grains and protein. However, the timing of your breakfast is just as important - studies show eating within 2 hours of waking optimizes metabolism and energy levels throughout the day.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.25,
  info: {
    reason: 'スコアが 0.25 なのは、健全な朝食の構成要素として全粒穀物とタンパク質に言及しており、直接的な回答になっているためです。しかし、朝食のタイミングやそれが代謝や一日のエネルギーレベルに及ぼす影響に関する追加情報は、質問に直接関係していないため、全体の関連性スコアが低くなっています。'
  }
}

import { openai } from "@ai-sdk/openai";
import { AnswerRelevancyMetric } from "@mastra/evals/llm";
 
const metric = new AnswerRelevancyMetric(openai("gpt-4o-mini"));
 
const query = "What are the benefits of meditation?";
const response = "The Great Wall of China is over 13,000 miles long and was built during the Ming Dynasty to protect against invasions.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0,
  info: {
    reason: 'The score is 0 because the output about the Great Wall of China is completely unrelated to the benefits of meditation, providing no relevant information or context that addresses the input question.'
  }
}

const metric = new AnswerRelevancyMetric(openai("gpt-4o-mini"), {
  uncertaintyWeight: 0.3,
  scale: 1,
});

{
  score: number,
  info: {
    reason: string
  }
}