コンテキスト想起の評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { ContextualRecallMetric } from "@mastra/evals/llm";
 
const metric = new ContextualRecallMetric(openai("gpt-4o-mini"), {
  context: [
    "Product features include cloud sync.",
    "Offline mode is available.",
    "Supports multiple devices."
  ]
});
 
const query = "What are the key features of the product?";
const response = "The product features cloud synchronization, offline mode support, and the ability to work across multiple devices.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 1,
  info: {
    reason: 'The score is 1 because all elements of the expected output are fully supported by the corresponding nodes in retrieval context, specifically node(s) that detail cloud synchronization, offline mode support, and multi-device functionality.'
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextualRecallMetric } from "@mastra/evals/llm";
 
const metric = new ContextualRecallMetric(openai("gpt-4o-mini"), {
  context: [
    "Python is a high-level programming language.",
    "Python emphasizes code readability.",
    "Python supports multiple programming paradigms.",
    "Python is widely used in data science."
  ]
});
 
const query = "What are Python's key characteristics?";
const response = "Python is a high-level programming language. It is also a type of snake.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.25,
  info: {
    reason: "The score is 0.25 because while the sentence 'Python is a high-level programming language' aligns with node 1 in the retrieval context, the lack of mention of other relevant information from nodes 2, 3, and 4 indicates significant gaps in the overall context."
  }
}

import { openai } from "@ai-sdk/openai";
import { ContextualRecallMetric } from "@mastra/evals/llm";
 
const metric = new ContextualRecallMetric(openai("gpt-4o-mini"), {
  context: [
    "The solar system has eight planets.",
    "Mercury is closest to the Sun.",
    "Venus is the hottest planet.",
    "Mars is called the Red Planet."
  ]
});
 
const query = "Tell me about the solar system.";
const response = "Jupiter is the largest planet in the solar system.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0,
  info: {
    reason: "The score is 0 because the output lacks any relevant information from the node(s) in retrieval context, failing to address key aspects such as the number of planets, Mercury's position, Venus's temperature, and Mars's nickname."
  }
}

const metric = new ContextualRecallMetric(openai("gpt-4o-mini"), {
  context: [""],
  scale: 1
});

{
  score: number,
  info: {
    reason: string
  }
}