Prompt Alignment Scorer

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { createPromptAlignmentScorerLLM } from "@mastra/evals/scorers/llm";
 
const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini") 
});
 
const inputMessages = [{ 
  role: 'user', 
  content: "Write a Python function to calculate factorial with error handling for negative numbers" 
}];
 
const outputMessage = { 
  text: `def factorial(n):
    """Calculate factorial of a number."""
    if n < 0:
        raise ValueError("Factorial not defined for negative numbers")
    if n == 0 or n == 1:
        return 1
    return n * factorial(n - 1)` 
};
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.95,
  reason: 'The score is 0.95 because the response perfectly addresses the primary intent of creating a factorial function and fulfills all requirements including Python implementation, error handling for negative numbers, and proper documentation. The code format is appropriate and the implementation is complete.'
}

import { openai } from "@ai-sdk/openai";
import { createPromptAlignmentScorerLLM } from "@mastra/evals/scorers/llm";
 
const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini") 
});
 
const inputMessages = [{ 
  role: 'user', 
  content: "List the benefits of TypeScript in bullet points" 
}];
 
const outputMessage = { 
  text: "TypeScript provides static typing, better IDE support, and enhanced code reliability through compile-time error checking." 
};
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.75,
  reason: 'The score is 0.75 because the response addresses the intent of explaining TypeScript benefits and provides accurate information, but fails to use the requested bullet point format, resulting in lower appropriateness scoring.'
}

import { openai } from "@ai-sdk/openai";
import { createPromptAlignmentScorerLLM } from "@mastra/evals/scorers/llm";
 
const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini") 
});
 
const inputMessages = [{ 
  role: 'user', 
  content: "Write a Python class with initialization, validation, error handling, and documentation" 
}];
 
const outputMessage = { 
  text: `class Example:
    def __init__(self, value):
        self.value = value` 
};
 
const result = await scorer.run({
  input: inputMessages,
  output: outputMessage,
});
 
console.log(result);

{
  score: 0.35,
  reason: 'The score is 0.35 because while the response addresses the basic intent of creating a Python class with initialization, it fails to include validation, error handling, and documentation as specifically requested, resulting in incomplete requirement fulfillment.'
}

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"), 
  options: { 
    scale: 10, // Score from 0-10 instead of 0-1
    evaluationMode: 'both' // 'user', 'system', or 'both' (default)
  }
});

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"),
  options: { evaluationMode: 'user' }
});
 
const result = await scorer.run({
  input: {
    inputMessages: [{ 
      role: 'user', 
      content: "Explain recursion with an example" 
    }],
    systemMessages: [{ 
      role: 'system', 
      content: "Always provide code examples in Python" 
    }]
  },
  output: { 
    text: "Recursion is when a function calls itself. For example: factorial(5) = 5 * factorial(4)" 
  }
});
// Scores high for addressing user request, even without Python code

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"),
  options: { evaluationMode: 'system' }
});
 
const result = await scorer.run({
  input: {
    systemMessages: [{ 
      role: 'system', 
      content: "You are a helpful assistant. Always be polite, concise, and provide examples." 
    }],
    inputMessages: [{ 
      role: 'user', 
      content: "What is machine learning?" 
    }]
  },
  output: { 
    text: "Machine learning is a subset of AI where computers learn from data. For example, spam filters learn to identify unwanted emails by analyzing patterns in previously marked spam." 
  }
});
// Evaluates politeness, conciseness, and example provision

const scorer = createPromptAlignmentScorerLLM({ 
  model: openai("gpt-4o-mini"),
  options: { evaluationMode: 'both' } // This is the default
});
 
const result = await scorer.run({
  input: {
    systemMessages: [{ 
      role: 'system', 
      content: "Always provide code examples when explaining programming concepts" 
    }],
    inputMessages: [{ 
      role: 'user', 
      content: "Explain how to reverse a string" 
    }]
  },
  output: { 
    text: `To reverse a string, you can iterate through it backwards. Here's an example in Python:
    
    def reverse_string(s):
        return s[::-1]
    
    # Usage: reverse_string("hello") returns "olleh"` 
  }
});
// High score for both addressing the user's request AND following system guidelines

{
  runId: string,
  score: number,
  reason: string,
  analyzeStepResult: {
    intentAlignment: {
      score: number,
      primaryIntent: string,
      isAddressed: boolean,
      reasoning: string
    },
    requirementsFulfillment: {
      requirements: Array<{
        requirement: string,
        isFulfilled: boolean,
        reasoning: string
      }>,
      overallScore: number
    },
    completeness: {
      score: number,
      missingElements: string[],
      reasoning: string
    },
    responseAppropriateness: {
      score: number,
      formatAlignment: boolean,
      toneAlignment: boolean,
      reasoning: string
    },
    overallAssessment: string
  }
}