コンテキスト位置の評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { ContextPositionMetric } from "@mastra/evals/llm";
 
const metric = new ContextPositionMetric(openai("gpt-4o-mini"), {
  context: [
    "The capital of France is Paris.",
    "Paris has been the capital since 508 CE.",
    "Paris serves as France's political center.",
    "The capital city hosts the French government."
  ]
});
 
const query = "What is the capital of France?";
const response = "The capital of France is Paris.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 1,
  info: {
    reason: 'The score is 1 because all provided context directly supports the output by confirming that Paris is the capital of France, with each statement reinforcing the answer through historical, political, and functional relevance.'
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextPositionMetric } from "@mastra/evals/llm";
 
const metric = new ContextPositionMetric(openai("gpt-4o-mini"), {
  context: [
    "Elephants are herbivores.",
    "Adult elephants can weigh up to 13,000 pounds.",
    "Elephants are the largest land animals.",
    "Elephants eat plants and grass."
  ]
});
 
const query = "How much do elephants weigh?";
const response = "Adult elephants can weigh up to 13,000 pounds, making them the largest land animals.";
 
const result = await metric.measure(query, response);
 
console.log(result);
 

{
  score: 0.4,
  info: {
    reason: 'スコアが0.4であるのは、応答に成体のゾウの体重に関する直接的な答えが含まれており、これは高い関連性がある一方で、ゾウが草食動物であることや食性に関する、体重の理解に寄与しない不要な情報も含まれているためです。関連性の高い文脈と低い文脈が併存していることが、全体として低めのスコアにつながっています。'
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextPositionMetric } from "@mastra/evals/llm";
 
const metric = new ContextPositionMetric(openai("gpt-4o-mini"), {
  context: [
    "Rainbows appear in the sky.",
    "Rainbows have different colors.",
    "Rainbows are curved in shape.",
    "Rainbows form when sunlight hits water droplets."
  ]
});
 
const query = "How do rainbows form?";
const response = "Rainbows are created when sunlight interacts with water droplets in the air.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.12,
  info: {
    reason: 'The score is 0.12 because the relevant context directly explains how rainbows form, while the other statements provide information that is either unrelated or only tangentially related to the formation process.'
  }
}

const metric = new ContextPositionMetric(openai("gpt-4o-mini"), {
  context: [""],
  scale: 1
});

{
  score: number,
  info: {
    reason: string
  }
}