Prompt Alignment Scorer

Weighted Score = (intent_score × 0.4) + (requirements_score × 0.3) + 
                 (completeness_score × 0.2) + (appropriateness_score × 0.1)
Final Score = Weighted Score × scale

Weighted Score = (intent_score × 0.35) + (requirements_score × 0.35) + 
                 (completeness_score × 0.15) + (appropriateness_score × 0.15)
Final Score = Weighted Score × scale

User Score = (user dimensions with user weights)
System Score = (system dimensions with system weights)  
Weighted Score = (User Score × 0.7) + (System Score × 0.3)
Final Score = Weighted Score × scale

import { openai } from '@ai-sdk/openai';
import { createPromptAlignmentScorerLLM } from '@mastra/evals';
 
const scorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o'),
});
 
// Evaluate a code generation task
const result = await scorer.run({
  input: [{
    role: 'user',
    content: 'Write a Python function to calculate factorial with error handling'
  }],
  output: {
    role: 'assistant', 
    text: `def factorial(n):
    if n < 0:
        raise ValueError("Factorial not defined for negative numbers")
    if n == 0:
        return 1
    return n * factorial(n-1)`
  }
});
// Result: { score: 0.95, reason: "Excellent alignment - function addresses intent, includes error handling..." }

// Configure scale and evaluation mode
const scorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o'),
  options: {
    scale: 10, // Score from 0-10 instead of 0-1
    evaluationMode: 'both' // 'user', 'system', or 'both' (default)
  },
});
 
// User-only evaluation - focus on user satisfaction
const userScorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o'),
  options: { evaluationMode: 'user' }
});
 
// System-only evaluation - focus on compliance
const systemScorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o'),
  options: { evaluationMode: 'system' }
});
 
const result = await scorer.run(testRun);
// Result: { score: 8.5, reason: "Score: 8.5 out of 10 - Good alignment with both user intent and system guidelines..." }

// Evaluate bullet point formatting
const result = await scorer.run({
  input: [{
    role: 'user',
    content: 'List the benefits of TypeScript in bullet points'
  }],
  output: {
    role: 'assistant',
    text: 'TypeScript provides static typing, better IDE support, and enhanced code reliability.'
  }
});
// Result: Lower appropriateness score due to format mismatch (paragraph vs bullet points)

// Example: API endpoint creation
const codePrompt = "Create a REST API endpoint with authentication and rate limiting";
// Scorer evaluates: intent (API creation), requirements (auth + rate limiting), 
// completeness (full implementation), format (code structure)

// Example: Multi-requirement task
const taskPrompt = "Write a Python class with initialization, validation, error handling, and documentation";
// Scorer tracks each requirement individually and provides detailed breakdown

// Example: Structured output
const formatPrompt = "Explain the differences between let and const in JavaScript using bullet points";
// Scorer evaluates content accuracy AND format compliance

const agent = new Agent({
  name: 'CodingAssistant',
  instructions: 'You are a helpful coding assistant. Always provide working code examples.',
  model: openai('gpt-4o'),
});
 
// Evaluate comprehensive alignment (default)
const scorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o-mini'),
  options: { evaluationMode: 'both' } // Evaluates both user intent and system guidelines
});
 
// Evaluate just user satisfaction
const userScorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o-mini'),
  options: { evaluationMode: 'user' } // Focus only on user request fulfillment
});
 
// Evaluate system compliance
const systemScorer = createPromptAlignmentScorerLLM({
  model: openai('gpt-4o-mini'),
  options: { evaluationMode: 'system' } // Check adherence to system instructions
});
 
const result = await scorer.run(agentRun);

const prompts = [
  'Write a function to calculate factorial',
  'Create a Python function that calculates factorial with error handling for negative inputs',
  'Implement a factorial calculator in Python with: input validation, error handling, and docstring'
];
 
// Compare alignment scores to find the best prompt
for (const prompt of prompts) {
  const result = await scorer.run(createTestRun(prompt, response));
  console.log(`Prompt alignment: ${result.score}`);
}

const agents = [agent1, agent2, agent3];
const testPrompts = [...]; // Array of test prompts
 
for (const agent of agents) {
  let totalScore = 0;
  for (const prompt of testPrompts) {
    const response = await agent.run(prompt);
    const evaluation = await scorer.run({ input: prompt, output: response });
    totalScore += evaluation.score;
  }
  console.log(`${agent.name} average alignment: ${totalScore / testPrompts.length}`);
}

// Missing user prompt
try {
  await scorer.run({ input: [], output: response });
} catch (error) {
  // Error: "Both user prompt and agent response are required for prompt alignment scoring"
}
 
// Empty response
const result = await scorer.run({ 
  input: [userMessage], 
  output: { role: 'assistant', text: '' } 
});
// Returns low scores with detailed reasoning about incompleteness