Context Precision Scorer

npm install @mastra/evals

import { openai } from '@ai-sdk/openai';
import { createContextPrecisionScorer } from '@mastra/evals';
 
const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'Photosynthesis is the process by which plants convert sunlight, carbon dioxide, and water into glucose and oxygen.',
      'The process occurs in the chloroplasts of plant cells, specifically in the thylakoids.',
      'Light-dependent reactions happen in the thylakoid membranes, while the Calvin cycle occurs in the stroma.',
    ],
    scale: 1,
  },
});
 
const result = await scorer.run({
  input: {
    inputMessages: [
      {
        id: '1',
        role: 'user',
        content: 'How does photosynthesis work in plants?',
      },
    ],
  },
  output: [
    {
      id: '2', 
      role: 'assistant',
      content: 'Photosynthesis is the process where plants convert sunlight, CO2, and water into glucose and oxygen using chloroplasts.',
    },
  ],
});
 
console.log(result);
// Output:
// {
//   score: 1.0,
//   reason: "The score is 1.0 because all context pieces are highly relevant to explaining photosynthesis and are optimally ordered to support the expected output."
// }

import { openai } from '@ai-sdk/openai';
import { createContextPrecisionScorer } from '@mastra/evals';
 
const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'Regular exercise improves cardiovascular health by strengthening the heart muscle.',
      'A balanced diet should include fruits, vegetables, and whole grains.',
      'Physical activity releases endorphins which improve mood and reduce stress.',
      'The average person should drink 8 glasses of water per day.',
      'Exercise also helps maintain healthy body weight and muscle mass.',
    ],
    scale: 1,
  },
});
 
const result = await scorer.run({
  input: {
    inputMessages: [
      {
        id: '1',
        role: 'user', 
        content: 'What are the mental and physical benefits of exercise?',
      },
    ],
  },
  output: [
    {
      id: '2',
      role: 'assistant', 
      content: 'Exercise provides cardiovascular benefits, improves mood through endorphin release, and helps maintain healthy body composition.',
    },
  ],
});
 
console.log(result);
// Output:
// {
//   score: 0.72,
//   reason: "The score is 0.72 because contexts 1, 3, and 5 are relevant to exercise benefits, but irrelevant contexts about diet and hydration reduce the precision score."
// }

import { openai } from '@ai-sdk/openai';
import { createContextPrecisionScorer } from '@mastra/evals';
 
const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'The weather forecast shows sunny skies this weekend.',
      'Coffee is one of the world\'s most popular beverages.',
      'Machine learning requires large amounts of training data.',
      'Cats typically sleep 12-16 hours per day.',
      'The capital of France is Paris.',
    ],
    scale: 1,
  },
});
 
const result = await scorer.run({
  input: {
    inputMessages: [
      {
        id: '1',
        role: 'user',
        content: 'How does photosynthesis work?',
      },
    ],
  },
  output: [
    {
      id: '2',
      role: 'assistant',
      content: 'Photosynthesis is the process by which plants convert sunlight into energy using chlorophyll.',
    },
  ],
});
 
console.log(result);
// Output:
// {
//   score: 0.0,
//   reason: "The score is 0.0 because none of the retrieved context pieces are relevant to explaining photosynthesis."
// }

const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    context: [
      'Machine learning models require training data.',
      'Deep learning uses neural networks with multiple layers.',
    ],
    scale: 10, // Scale scores from 0-10 instead of 0-1
  },
});
 
// Result will be scaled: score: 8.5 instead of 0.85

const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'),
  options: {
    contextExtractor: (input, output) => {
      // Extract context dynamically based on the query
      const query = input?.inputMessages?.[0]?.content || '';
      
      // Example: Retrieve from a vector database
      const searchResults = vectorDB.search(query, { limit: 10 });
      return searchResults.map(result => result.content);
    },
    scale: 1,
  },
});

const scorer = createContextPrecisionScorer({
  model: openai('gpt-4o-mini'), 
  options: {
    context: [
      // Simulate retrieved documents from vector database
      'Document 1: Highly relevant content...',
      'Document 2: Somewhat related content...',
      'Document 3: Tangentially related...',
      'Document 4: Not relevant...',
      'Document 5: Highly relevant content...',
      // ... up to dozens of context pieces
    ],
  },
});