要約の評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { SummarizationMetric } from "@mastra/evals/llm";
 
const metric = new SummarizationMetric(openai("gpt-4o-mini"));
 
const query =
  "The electric car company Tesla was founded in 2003 by Martin Eberhard and Marc Tarpenning. Elon Musk joined in 2004 as the largest investor and became CEO in 2008. The company's first car, the Roadster, was launched in 2008.";
const response =
  "Tesla, founded by Martin Eberhard and Marc Tarpenning in 2003, launched its first car, the Roadster, in 2008. Elon Musk joined as the largest investor in 2004 and became CEO in 2008.";
 
const result = await metric.measure(query, response);

{
  score: 1,
  info: {
    reason: 'The score is 1 because the summary is completely factual and covers all key information from the original text.',
    alignmentScore: 1,
    coverageScore: 1
  }
}

import { openai } from "@ai-sdk/openai";
import { SummarizationMetric } from "@mastra/evals/llm";
 
const metric = new SummarizationMetric(openai("gpt-4o-mini"));
 
const query =
  "The Python programming language was created by Guido van Rossum and was first released in 1991. It emphasizes code readability with its notable use of significant whitespace. Python is dynamically typed and garbage-collected. It supports multiple programming paradigms, including structured, object-oriented, and functional programming.";
const response = "Python, created by Guido van Rossum, is a programming language known for its readable code and use of whitespace. It was released in 1991.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.7,
  info: {
    reason: "The score is 0.7 because the summary accurately captures key facts about Python's creation, release date, and emphasis on readability, achieving a perfect alignment score. However, it fails to mention that Python is dynamically typed, garbage-collected, and supports multiple programming paradigms, which affects the coverage score.",
    alignmentScore: 1,
    coverageScore: 0.7
  }
}

import { openai } from "@ai-sdk/openai";
import { SummarizationMetric } from "@mastra/evals/llm";
 
const metric = new SummarizationMetric(openai("gpt-4o-mini"));
 
const query =
  "The World Wide Web was invented by Tim Berners-Lee in 1989 while working at CERN. He published the first website in 1991. Berners-Lee made the Web freely available, with no patent and no royalties due.";
const response = "The Internet was created by Tim Berners-Lee at MIT in the early 1990s, and he went on to commercialize the technology through patents.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0,
  info: {
    reason: 'The score is 0 because the summary contains factual inaccuracies and fails to cover essential details from the original text. The claim that the Internet was created at MIT in the early 1990s contradicts the original text, which states that the World Wide Web was invented at CERN in 1989. Additionally, the summary incorrectly states that Berners-Lee commercialized the technology through patents, while the original text clearly mentions that he made the Web freely available with no patents or royalties.',
    alignmentScore: 0,
    coverageScore: 0.17
  }
}

const metric = new SummarizationMetric(openai("gpt-4o-mini"));

{
  score: number,
  info: {
    reason: string,
    alignmentScore: number,
    coverageScore: number
  }
}