コンテキスト適合性の評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { ContextRelevancyMetric } from "@mastra/evals/llm";
 
const metric = new ContextRelevancyMetric(openai("gpt-4o-mini"), {
  context: [
    "Einstein won the Nobel Prize for his discovery of the photoelectric effect.",
    "He published his theory of relativity in 1905.",
    "His work revolutionized modern physics."
  ]
});
 
const query = "What were some of Einstein's achievements?";
const response = "Einstein won the Nobel Prize for discovering the photoelectric effect and published his groundbreaking theory of relativity.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 1,
  info: {
    reason: "The score is 1 because the retrieval context directly addresses the input by highlighting Einstein's significant achievements, making it entirely relevant."
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextRelevancyMetric } from "@mastra/evals/llm";
 
const metric = new ContextRelevancyMetric(openai("gpt-4o-mini"), {
  context: [
    "Solar eclipses occur when the Moon blocks the Sun.",
    "The Moon moves between the Earth and Sun during eclipses.",
    "The Moon is visible at night.",
    "The Moon has no atmosphere."
  ]
});
 
const query = "What causes solar eclipses?";
const response =
  "Solar eclipses happen when the Moon moves between Earth and the Sun, blocking sunlight.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.5,
  info: {
    reason: "スコアが0.5であるのは、取得されたコンテキストに入力と無関係な記述（例：「The Moon is visible at night」「The Moon has no atmosphere」）が含まれており、これらは日食の原因を説明していないためです。関連情報の不足が、文脈的な関連性スコアを大きく下げています。"
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextRelevancyMetric } from "@mastra/evals/llm";
 
const metric = new ContextRelevancyMetric(openai("gpt-4o-mini"), {
  context: [
    "The Great Barrier Reef is in Australia.",
    "Coral reefs need warm water to survive.",
    "Marine life depends on coral reefs.",
    "The capital of Australia is Canberra."
  ]
});
 
const query = "What is the capital of Australia?";
const response = "The capital of Australia is Canberra.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.25,
  info: {
    reason: "The score is 0.25 because the retrieval context contains statements that are completely irrelevant to the input question about the capital of Australia. For instance, 'The Great Barrier Reef is in Australia' and 'Coral reefs need warm water to survive' do not provide any geographical or political information related to the capital, thus failing to address the inquiry."
  }
}

const metric = new ContextRelevancyMetric(openai("gpt-4o-mini"), {
  context: [""],
  scale: 1
});

{
  score: number,
  info: {
    reason: string
  }
}