プロンプト整合性スコアラー

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { createPromptAlignmentScorerLLM } from "@mastra/evals/scorers/llm";
 
const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini") 
});
 
const inputMessages = [{ 
  role: 'user', 
  content: "Write a Python function to calculate factorial with error handling for negative numbers" 
}];
 
const outputMessage = { 
  text: `def factorial(n):
    """Calculate factorial of a number."""
    if n < 0:
        raise ValueError("Factorial not defined for negative numbers")
    if n == 0 or n == 1:
        return 1
    return n * factorial(n - 1)` 
};
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.95,
  reason: 'The score is 0.95 because the response perfectly addresses the primary intent of creating a factorial function and fulfills all requirements including Python implementation, error handling for negative numbers, and proper documentation. The code format is appropriate and the implementation is complete.'
}

import { openai } from "@ai-sdk/openai";
import { createPromptAlignmentScorerLLM } from "@mastra/evals/scorers/llm";
 
const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini") 
});
 
const inputMessages = [{ 
  role: 'user', 
  content: "List the benefits of TypeScript in bullet points" 
}];
 
const outputMessage = { 
  text: "TypeScript provides static typing, better IDE support, and enhanced code reliability through compile-time error checking." 
};
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.75,
  reason: 'The score is 0.75 because the response addresses the intent of explaining TypeScript benefits and provides accurate information, but fails to use the requested bullet point format, resulting in lower appropriateness scoring.'
}

import { openai } from "@ai-sdk/openai";
import { createPromptAlignmentScorerLLM } from "@mastra/evals/scorers/llm";
 
const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini") 
});
 
const inputMessages = [{ 
  role: 'user', 
  content: "Write a Python class with initialization, validation, error handling, and documentation" 
}];
 
const outputMessage = { 
  text: `class Example:
    def __init__(self, value):
        self.value = value` 
};
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.35,
  reason: 'スコアが 0.35 なのは、初期化を備えた Python クラスを作成するという基本的な意図には応えているものの、明示的に求められていた検証、エラー処理、ドキュメンテーションが含まれておらず、要件の充足が不完全だからです。'
}

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"), 
  options: { 
    scale: 10, // 0-1 ではなく 0-10 のスコア
    evaluationMode: 'both' // 'user'、'system'、または 'both'（既定）
  }
});

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"),
  options: { evaluationMode: 'user' }
});
 
const result = await scorer.run({
  input: {
    inputMessages: [{ 
      role: 'user', 
      content: "Explain recursion with an example" 
    }],
    systemMessages: [{ 
      role: 'system', 
      content: "Always provide code examples in Python" 
    }]
  },
  output: { 
    text: "Recursion is when a function calls itself. For example: factorial(5) = 5 * factorial(4)" 
  }
});
// Python のコード例がなくても、ユーザーの要望に応えているため高評価

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"),
  options: { evaluationMode: 'system' }
});
 
const result = await scorer.run({
  input: {
    systemMessages: [{ 
      role: 'system', 
      content: "You are a helpful assistant. Always be polite, concise, and provide examples." 
    }],
    inputMessages: [{ 
      role: 'user', 
      content: "What is machine learning?" 
    }]
  },
  output: { 
    text: "Machine learning is a subset of AI where computers learn from data. For example, spam filters learn to identify unwanted emails by analyzing patterns in previously marked spam." 
  }
});
// 丁寧さ、簡潔さ、例示の有無を評価

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"),
  options: { evaluationMode: 'both' } // 既定設定
});
 
const result = await scorer.run({
  input: {
    systemMessages: [{ 
      role: 'system', 
      content: "Always provide code examples when explaining programming concepts" 
    }],
    inputMessages: [{ 
      role: 'user', 
      content: "Explain how to reverse a string" 
    }]
  },
  output: { 
    text: `To reverse a string, you can iterate through it backwards. Here's an example in Python:
    
    def reverse_string(s):
        return s[::-1]
    
    # Usage: reverse_string("hello") returns "olleh"` 
  }
});
// ユーザーの要望への対応とシステムガイドラインの遵守の両面で高評価

{
  runId: string,
  score: number,
  reason: string,
  analyzeStepResult: {
    intentAlignment: {
      score: number,
      primaryIntent: string,
      isAddressed: boolean,
      reasoning: string
    },
    requirementsFulfillment: {
      requirements: Array<{
        requirement: string,
        isFulfilled: boolean,
        reasoning: string
      }>,
      overallScore: number
    },
    completeness: {
      score: number,
      missingElements: string[],
      reasoning: string
    },
    responseAppropriateness: {
      score: number,
      formatAlignment: boolean,
      toneAlignment: boolean,
      reasoning: string
    },
    overallAssessment: string
  }
}