回答類似度スコアラー

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { runExperiment } from "@mastra/core/scores";
import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/llm";
import { myAgent } from "./agent";
 
const scorer = createAnswerSimilarityScorer({ model: openai("gpt-4o-mini") });
 
const result = await runExperiment({
  data: [
    { 
      input: "What is 2+2?",
      groundTruth: "4"
    }
  ],
  scorers: [scorer],
  target: myAgent,
});
 
console.log(result.scores);

{
  "Answer Similarity Scorer": {
    score: 1.0,
    reason: "スコアは1.0/1です。出力が正解と完全に一致しており、エージェントは数値解を正しく提示しました。応答は完全に正確なため、改善の必要はありません。"
  }
}

import { openai } from "@ai-sdk/openai";
import { runExperiment } from "@mastra/core/scores";
import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/llm";
import { myAgent } from "./agent";
 
const scorer = createAnswerSimilarityScorer({ model: openai("gpt-4o-mini") });
 
const result = await runExperiment({
  data: [
    { 
      input: "What is the capital of France?",
      groundTruth: "The capital of France is Paris",
    }
  ],
  scorers: [scorer],
  target: myAgent,
});
 
console.log(result.scores);

{
  "Answer Similarity Scorer": {
    score: 0.9,
    reason: "スコアは0.9/1です。両方の回答が「ParisはFranceの首都である」という同じ情報を伝えているためです。エージェントは主要な事実を、表現はやや異なるものの正しく特定しました。構造にわずかな違いはありますが、意味的には同等です。"
  }
}

import { openai } from "@ai-sdk/openai";
import { runExperiment } from "@mastra/core/scores";
import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/llm";
import { myAgent } from "./agent";
 
const scorer = createAnswerSimilarityScorer({ model: openai("gpt-4o-mini") });
 
const result = await runExperiment({
  data: [
    { 
      input: "What are the primary colors?",
      groundTruth: "The primary colors are red, blue, and yellow",
    }
  ],
  scorers: [scorer],
  target: myAgent,
});
 
console.log(result.scores);

{
  "Answer Similarity Scorer": {
    score: 0.6,
    reason: "The score is 0.6/1 because the answer captures some key elements but is incomplete. The agent correctly identified red and blue as primary colors. However, it missed the critical color yellow, which is essential for a complete answer."
  }
}

import { openai } from "@ai-sdk/openai";
import { runExperiment } from "@mastra/core/scores";
import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/llm";
import { myAgent } from "./agent";
 
const scorer = createAnswerSimilarityScorer({ model: openai("gpt-4o-mini") });
 
const result = await runExperiment({
  data: [
    { 
      input: "Who wrote Romeo and Juliet?",
      groundTruth: "William Shakespeare wrote Romeo and Juliet",
    }
  ],
  scorers: [scorer],
  target: myAgent,
});
 
console.log(result.scores);

{
  "Answer Similarity Scorer": {
    score: 0.0,
    reason: "スコアが 0.0/1 であるのは、著者に関する重大な誤りが含まれているためです。エージェントは戯曲の題名は正しく特定したものの、著者をウィリアム・シェイクスピアではなくクリストファー・マーロウと誤って帰属しており、これは本質的な矛盾です。"
  }
}

import { describe, it, expect } from 'vitest';
import { openai } from "@ai-sdk/openai";
import { runExperiment } from "@mastra/core/scores";
import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/llm";
import { myAgent } from "./agent";
 
describe('Agent Consistency Tests', () => {
  const scorer = createAnswerSimilarityScorer({ model: openai("gpt-4o-mini") });
 
  it('should provide accurate factual answers', async () => {
    const result = await runExperiment({
      data: [
        { 
          input: "What is the speed of light?",
          groundTruth: "The speed of light in vacuum is 299,792,458 meters per second"
        },
        { 
          input: "What is the capital of Japan?",
          groundTruth: "Tokyo is the capital of Japan"
        }
      ],
      scorers: [scorer],
      target: myAgent,
    });
 
    // すべての回答が類似度のしきい値を満たしていることを検証
    expect(result.scores['Answer Similarity Scorer'].score).toBeGreaterThan(0.8);
  });
 
  it('should maintain consistency across runs', async () => {
    const testData = { 
      input: "Define machine learning",
      groundTruth: "Machine learning is a subset of AI that enables systems to learn and improve from experience"
    };
 
    // 複数回実行して一貫性を確認
    const results = await Promise.all([
      runExperiment({ data: [testData], scorers: [scorer], target: myAgent }),
      runExperiment({ data: [testData], scorers: [scorer], target: myAgent }),
      runExperiment({ data: [testData], scorers: [scorer], target: myAgent })
    ]);
 
    // すべての実行で類似したスコアになることを確認（許容差 0.1）
    const scores = results.map(r => r.scores['Answer Similarity Scorer'].score);
    const maxDiff = Math.max(...scores) - Math.min(...scores);
    expect(maxDiff).toBeLessThan(0.1);
  });
});

import { openai } from "@ai-sdk/openai";
import { runExperiment } from "@mastra/core/scores";
import { createAnswerSimilarityScorer } from "@mastra/evals/scorers/llm";
import { myAgent } from "./agent";
 
// 厳密な完全一致と高いスケールで設定
const strictScorer = createAnswerSimilarityScorer({ 
  model: openai("gpt-4o-mini"),
  options: {
    exactMatchBonus: 0.5,        // 完全一致のボーナスを高める
    contradictionPenalty: 2.0,   // 矛盾に非常に厳しくする
    missingPenalty: 0.3,         // 欠落情報へのペナルティを高める
    scale: 10                    // スコアを1ではなく10点満点にする
  }
});
 
// 寛容な意味一致で設定
const lenientScorer = createAnswerSimilarityScorer({ 
  model: openai("gpt-4o-mini"),
  options: {
    semanticThreshold: 0.6,      // 意味的一致のしきい値を下げる
    contradictionPenalty: 0.5,   // 軽微な矛盾には寛容にする
    extraInfoPenalty: 0,         // 追加情報にはペナルティを科さない
    requireGroundTruth: false    // 正解データがなくても許容する
  }
});
 
const result = await runExperiment({
  data: [
    { 
      input: "Explain photosynthesis",
      groundTruth: "Photosynthesis is the process by which plants convert light energy into chemical energy"
    }
  ],
  scorers: [strictScorer, lenientScorer],
  target: myAgent,
});
 
console.log('Strict scorer:', result.scores['Answer Similarity Scorer'].score); // 10点満点
console.log('Lenient scorer:', result.scores['Answer Similarity Scorer'].score); // 1点満点