コンテキスト精度スコアラー

npm install @mastra/evals

import { openai } from '@ai-sdk/openai';
import { createContextPrecisionScorer } from '@mastra/evals';
 
const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen.',
      'The process occurs in the chloroplasts of plant cells, specifically in the thylakoids.',
      'Light-dependent reactions happen in the thylakoid membranes, while the Calvin cycle occurs in the stroma.',
    ],
    scale: 1,
  },
});
 
const result = await scorer.run({
  input: {
    inputMessages: [
      {
        id: '1',
        role: 'user',
        content: 'How does photosynthesis work in plants?',
      },
    ],
  },
  output: [
    {
      id: '2', 
      role: 'assistant',
      content: 'Photosynthesis is the process where plants convert sunlight, CO2, and water into glucose and oxygen using chloroplasts.',
    },
  ],
});
 
console.log(result);
// 出力:
// {
//   score: 1.0,
//   reason: "スコアが 1.0 なのは、すべてのコンテキストが光合成の説明に高度に関連しており、期待される出力を支える最適な順序で並んでいるためです。"
// }

import { openai } from '@ai-sdk/openai';
import { createContextPrecisionScorer } from '@mastra/evals';
 
const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'Regular exercise improves cardiovascular health by strengthening the heart muscle.',
      'A balanced diet should include fruits, vegetables, and whole grains.',
      'Physical activity releases endorphins which improve mood and reduce stress.',
      'The average person should drink 8 glasses of water per day.',
      'Exercise also helps maintain healthy body weight and muscle mass.',
    ],
    scale: 1,
  },
});
 
const result = await scorer.run({
  input: {
    inputMessages: [
      {
        id: '1',
        role: 'user', 
        content: 'What are the mental and physical benefits of exercise?',
      },
    ],
  },
  output: [
    {
      id: '2',
      role: 'assistant', 
      content: 'Exercise provides cardiovascular benefits, improves mood through endorphin release, and helps maintain healthy body composition.',
    },
  ],
});
 
console.log(result);
// 出力:
// {
//   score: 0.72,
//   reason: "スコアが0.72なのは、コンテキスト1、3、5は運動の利点に関連している一方で、食事や水分補給に関する無関係なコンテキストが精度スコアを下げているためです。"
// }

import { openai } from '@ai-sdk/openai';
import { createContextPrecisionScorer } from '@mastra/evals';
 
const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'The weather forecast shows sunny skies this weekend.',
      'Coffee is one of the world\'s most popular beverages.',
      'Machine learning requires large amounts of training data.',
      'Cats typically sleep 12-16 hours per day.',
      'The capital of France is Paris.',
    ],
    scale: 1,
  },
});
 
const result = await scorer.run({
  input: {
    inputMessages: [
      {
        id: '1',
        role: 'user',
        content: 'How does photosynthesis work?',
      },
    ],
  },
  output: [
    {
      id: '2',
      role: 'assistant',
      content: 'Photosynthesis is the process by which plants convert sunlight into energy using chlorophyll.',
    },
  ],
});
 
console.log(result);
// 出力:
// {
//   score: 0.0,
//   reason: "取得されたコンテキストのいずれも光合成の説明に関連していないため、スコアは 0.0 です。"
// }

const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'Machine learning models require training data.',
      'Deep learning uses neural networks with multiple layers.',
    ],
    scale: 10, // スコアを0–1ではなく0–10の範囲にスケーリング
  },
});
 
// 結果はスケーリングされます: score は 0.85 ではなく 8.5

const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    contextExtractor: (input, output) => {
      // クエリに基づいて動的にコンテキストを抽出
      const query = input?.inputMessages?.[0]?.content || '';
      
      // 例: ベクターデータベースから取得
      const searchResults = vectorDB.search(query, { limit: 10 });
      return searchResults.map(result => result.content);
    },
    scale: 1,
  },
});

const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'), 
  options: {
    context: [
      // ベクターデータベースから取得したドキュメントを想定
      'Document 1: Highly relevant content...',
      'Document 2: Somewhat related content...',
      'Document 3: Tangentially related...',
      'Document 4: Not relevant...',
      'Document 5: Highly relevant content...',
      // ... コンテキストは数十件まで拡張可能
    ],
  },
});