npm install @reaatech/agent-eval-harness-judge

import { JudgeEngine } from '@reaatech/agent-eval-harness-judge';
 
const judge = new JudgeEngine({
  model: 'claude-opus',
  provider: 'claude',
  temperature: 0.1,
});
 
const result = await judge.judge({
  type: 'faithfulness',
  context: 'The account balance is $42.50',
  response: 'Your balance is $42.50. Would you like to make a payment?',
});
 
console.log(`Score: ${result.score}, Confidence: ${result.confidence}`);
console.log(result.explanation);

import { JudgeCalibrator, JudgeEngine } from '@reaatech/agent-eval-harness-judge';
 
const calibrator = new JudgeCalibrator('temperature_scaling');
 
// Collect human-labeled samples
const humanLabels = [
  { sampleId: 's1', score: 0.80, type: 'faithfulness' },
  { sampleId: 's2', score: 0.95, type: 'faithfulness' },
  { sampleId: 's3', score: 0.60, type: 'faithfulness' },
];
 
// Get raw judge scores for the same samples
const judge = new JudgeEngine({ model: 'claude-sonnet-4-20250514', provider: 'claude' });
const judgeScores = await Promise.all([
  judge.judge({ type: 'faithfulness', context: '...', response: '...' }),
  judge.judge({ type: 'faithfulness', context: '...', response: '...' }),
  judge.judge({ type: 'faithfulness', context: '...', response: '...' }),
]);
 
calibrator.addCalibrationData(humanLabels, judgeScores);
const result = calibrator.calibrate();
 
console.log(`MAE: ${result.beforeMAE} → ${result.afterMAE} (${result.improvement}% improvement)`);
 
// Apply calibration to future scores
const futureScore = await judge.judge({ type: 'faithfulness', context: '...', response: '...' });
const calibrated = calibrator.apply(futureScore.score);
console.log(`Raw: ${futureScore.score}, Calibrated: ${calibrated}`);

import { ConsensusEngine } from '@reaatech/agent-eval-harness-judge';
 
const consensusEngine = new ConsensusEngine({
  enabled: true,
  models: [
    { id: 'claude-opus', weight: 0.5 },
    { id: 'gpt-4-turbo', weight: 0.3 },
    { id: 'gemini-pro', weight: 0.2 },
  ],
  votingStrategy: 'weighted',
  minAgreement: 0.7,
  tieBreaker: 'highest_confidence',
});
 
// Assume scores collected from three separate JudgeEngine instances
const consensusResult = consensusEngine.consensus([
  { model: 'claude-opus', score: { score: 0.85, confidence: 0.9, ... } },
  { model: 'gpt-4-turbo', score: { score: 0.78, confidence: 0.85, ... } },
  { model: 'gemini-pro', score: { score: 0.82, confidence: 0.8, ... } },
]);
 
console.log(`Consensus score: ${consensusResult.score}`);
console.log(`Agreement: ${consensusResult.agreement}`);
console.log(`Consensus reached: ${consensusResult.consensusReached}`);