プロンプト整合性評価

npm install @mastra/evals

import { openai } from "@ai-sdk/openai";
import { PromptAlignmentMetric } from "@mastra/evals/llm";
 
const metric = new PromptAlignmentMetric(openai("gpt-4o-mini"), {
  instructions: [
    "Use complete sentences",
    "Include temperature in Celsius",
    "Mention wind conditions", "State precipitation chance"
  ]
});
 
const query = "What is the weather like?";
const response = "The temperature is 22 degrees Celsius with moderate winds from the northwest. There is a 30% chance of rain.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 1,
  info: {
    reason: 'The score is 1 because the output fully aligns with all applicable instructions, providing a comprehensive weather report that includes temperature, wind conditions, and chance of precipitation, all presented in complete sentences.',
    scoreDetails: {
      totalInstructions: 4,
      applicableInstructions: 4,
      followedInstructions: 4,
      naInstructions: 0
    }
  }
}

import { openai } from "@ai-sdk/openai";
import { PromptAlignmentMetric } from "@mastra/evals/llm";
 
const metric = new PromptAlignmentMetric(openai("gpt-4o-mini"), {
  instructions: [
    "Use bullet points",
    "Include prices in USD",
    "Show stock status", "Add product descriptions"
  ]
});
 
const query = "List the available products";
const response = "• Coffee - $4.99 (In Stock)\n• Tea - $3.99\n• Water - $1.99 (Out of Stock)";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0.75,
  info: {
    reason: 'スコアが0.75なのは、箇条書きの使用、USDでの価格表示、在庫状況の表示により、ほとんどの指示を満たしているためです。ただし、商品説明を提供するという指示には完全には沿っておらず、全体のスコアに影響しています。',
    scoreDetails: {
      totalInstructions: 4,
      applicableInstructions: 4,
      followedInstructions: 3,
      naInstructions: 0
    }
  }
}

import { openai } from "@ai-sdk/openai";
import { PromptAlignmentMetric } from "@mastra/evals/llm";
 
const metric = new PromptAlignmentMetric(openai("gpt-4o-mini"), {
  instructions: [
    "Show account balance",
    "List recent transactions",
    "Display payment history"
  ]
});
 
const query = "What is the weather like?";
const response = "It is sunny and warm outside.";
 
const result = await metric.measure(query, response);
 
console.log(result);

{
  score: 0,
  info: {
    reason: 'The score is 0 because the output does not follow any applicable instructions related to the context of a weather query, as the instructions provided are irrelevant to the input.',
    scoreDetails: {
      totalInstructions: 3,
      applicableInstructions: 0,
      followedInstructions: 0,
      naInstructions: 3
    }
  }
}

const metric = new PromptAlignmentMetric(openai("gpt-4o-mini"), {
  instructions: [""],
  scale: 1
});

{
  score: number,
  info: {
    reason: string,
    scoreDetails: {
      followed: string[],
      missed: string[],
      notApplicable: string[]
    }
  }
}