コンテキスト精度の評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { ContextPrecisionMetric } from "@mastra/evals/llm";
 
const metric = new ContextPrecisionMetric(openai("gpt-4o-mini"), {
  context: [
    "Photosynthesis converts sunlight into energy.",
    "Plants use chlorophyll for photosynthesis.",
    "Photosynthesis produces oxygen as a byproduct.",
    "The process requires sunlight and chlorophyll."
  ]
});
 
const query = "What is photosynthesis and how does it work?";
const response = "Photosynthesis is a process where plants convert sunlight into energy using chlorophyll, producing oxygen as a byproduct.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 1,
  info: {
    reason: 'The score is 1 because all context nodes are relevant and provide direct definitions, explanations, and essential components of photosynthesis, with the first node defining the process, followed by supporting details about chlorophyll and oxygen production, all in optimal ranking order.'
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextPrecisionMetric } from "@mastra/evals/llm";
 
const metric = new ContextPrecisionMetric(openai("gpt-4o-mini"), {
  context: [
    "Volcanoes are openings in the Earth's crust.",
    "Volcanoes can be active, dormant, or extinct.",
    "Hawaii has many active volcanoes.",
    "The Pacific Ring of Fire has many volcanoes."
  ]
});
 
const query = "What are the different types of volcanoes?";
const response = "Volcanoes can be classified as active, dormant, or extinct based on their activity status.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.58,
  info: {
    reason: 'The score is 0.58 because while the second and third nodes provided direct definitions and examples of volcano types, the first and fourth nodes were irrelevant, leading to a lower precision score. The relevant nodes were not optimally ordered, as the most useful context was not the first, which affected the overall effectiveness.'
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextPrecisionMetric } from "@mastra/evals/llm";
 
const metric = new ContextPrecisionMetric(openai("gpt-4o-mini"), {
  context: [
    "The Nile River is in Africa.",
    "The Nile is the longest river.",
    "Ancient Egyptians used the Nile.",
    "The Nile flows north."
  ]
});
 
const query = "Which direction does the Nile River flow?";
const response = "The Nile River flows northward.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.25,
  info: {
    reason: "スコアが0.25なのは、ナイル川の流れる方向に関する質問に直接答えているのが4つ目のコンテキストノードだけであり、最初の3つのノードは無関係で有用な情報を提供していないためです。これは、取得されたコンテキスト全体の関連性に大きな制約があることを示しており、その大半が期待される出力に寄与しなかったことを浮き彫りにしています。"
  }
}

const metric = new ContextPrecisionMetric(openai("gpt-4o-mini"), {
  context: [""],
  scale: 1
});

{
  score: number,
  info: {
    reason: string
  }
}