SummarizationMetric

import { openai } from "@ai-sdk/openai";
import { SummarizationMetric } from "@mastra/evals/llm";
 
// Configure the model for evaluation
const model = openai("gpt-4o-mini");
 
const metric = new SummarizationMetric(model);
 
const result = await metric.measure(
  "The company was founded in 1995 by John Smith. It started with 10 employees and grew to 500 by 2020. The company is based in Seattle.",
  "Founded in 1995 by John Smith, the company grew from 10 to 500 employees by 2020.",
);
 
console.log(result.score); // Score from 0-1
console.log(result.info); // Object containing detailed metrics about the summary

import { openai } from "@ai-sdk/openai";
import { SummarizationMetric } from "@mastra/evals/llm";
 
// Configure the model for evaluation
const model = openai("gpt-4o-mini");
 
const metric = new SummarizationMetric(model);
 
const result = await metric.measure(
  "The electric car company Tesla was founded in 2003 by Martin Eberhard and Marc Tarpenning. Elon Musk joined in 2004 as the largest investor and became CEO in 2008. The company's first car, the Roadster, was launched in 2008.",
  "Tesla, founded by Elon Musk in 2003, revolutionized the electric car industry starting with the Roadster in 2008.",
);
 
// Example output:
// {
//   score: 0.5,
//   info: {
//     reason: "The score is 0.5 because while the coverage is good (0.75) - mentioning the founding year,
//           first car model, and launch date - the alignment score is lower (0.5) due to incorrectly
//           attributing the company's founding to Elon Musk instead of Martin Eberhard and Marc Tarpenning.
//           The final score takes the minimum of these two scores to ensure both factual accuracy and
//           coverage are necessary for a good summary."
//     alignmentScore: 0.5,
//     coverageScore: 0.75,
//   }
// }