npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { HallucinationMetric } from "@mastra/evals/llm";
 
const metric = new HallucinationMetric(openai("gpt-4o-mini"), {
  context: [
    "The iPhone was first released in 2007.",
    "Steve Jobs unveiled it at Macworld.",
    "The original model had a 3.5-inch screen."
  ]
});
 
const query = "When was the first iPhone released?";
const response = "The iPhone was first released in 2007, when Steve Jobs unveiled it at Macworld. The original iPhone featured a 3.5-inch screen.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0,
  info: {
    reason: 'The score is 0 because none of the statements from the context were contradicted by the output.'
  }
}

import { openai } from "@ai-sdk/openai";
import { HallucinationMetric } from "@mastra/evals/llm";
 
const metric = new HallucinationMetric(openai("gpt-4o-mini"), {
  context: [
    "The first Star Wars movie was released in 1977.",
    "It was directed by George Lucas.",
    "The film earned $775 million worldwide.",
    "The movie was filmed in Tunisia and England."
  ]
});
 
const query = "Tell me about the first Star Wars movie.";
const response = "The first Star Wars movie came out in 1977 and was directed by George Lucas. It made over $1 billion at the box office and was filmed entirely in California.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.5,
  info: {
    reason: 'スコアが0.5であるのは、出力の4つの記述のうち2つがコンテキスト内の主張と矛盾しており、正確な情報と不正確な情報のバランスが取れていることを示しているためです。'
  }
}

import { openai } from "@ai-sdk/openai";
import { HallucinationMetric } from "@mastra/evals/llm";
 
const metric = new HallucinationMetric(openai("gpt-4o-mini"), {
  context: [
    "The Wright brothers made their first flight in 1903.",
    "The flight lasted 12 seconds.",
    "It covered a distance of 120 feet."
  ]
});
 
const query = "When did the Wright brothers first fly?";
const response = "The Wright brothers achieved their historic first flight in 1908. The flight lasted about 2 minutes and covered nearly a mile.";
 
const result = await metric.measure(query, response);
 
console.log(result);
 

{
  score: 1,
  info: {
    reason: 'スコアが1.0なのは、出力の3つの記述すべてがコンテキストと直接矛盾しているためです。初飛行は1908年ではなく1903年、飛行時間は約2分ではなく12秒、距離はほぼ1マイルではなく120フィートです。'
  }
}

const metric = new HallucinationMetric(openai("gpt-4o-mini"), {
  context: [""],
  scale: 1
});

{
  score: number,
  info: {
    reason: string
  }
}