npm install @reaatech/agent-eval-harness-cli
# or
npm install -g @reaatech/agent-eval-harness-cli

# Install globally
npm install -g @reaatech/agent-eval-harness-cli
 
# Run evaluation on a directory of JSONL trajectories
agent-eval-harness eval trajectories/ --config eval-config.yaml --output results/
 
# Judge a single response on faithfulness
agent-eval-harness judge faithfulness \
  --context "The user's account is associated with email john@example.com" \
  --response "I've sent the password reset to john@example.com"
 
# Compare two evaluation runs
agent-eval-harness compare results/baseline.json results/candidate.json --format markdown
 
# Check CI regression gates
agent-eval-harness gate results/results.json --preset standard --exit-code
 
# List golden trajectories
agent-eval-harness golden --list
 
# Generate HTML report
agent-eval-harness report results/results.json --format html --output report.html
 
# Start MCP server
agent-eval-harness serve

import {
  evalCommand,
  judgeCommand,
  compareCommand,
  gateCommand,
  goldenCommand,
  reportCommand,
  cliOut,
  cliError,
  cliWarn,
} from "@reaatech/agent-eval-harness-cli";

# Build the image
docker build -t agent-eval-harness .
 
# Run evaluation with mounted volumes
docker run -v ./trajectories:/app/trajectories \
  -v ./results:/app/results \
  -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
  agent-eval-harness eval trajectories/ --output results/
 
# Start MCP server in stdio mode
docker run -i agent-eval-harness serve

# .github/workflows/eval.yml
name: Agent Evaluation
 
on:
  pull_request:
    branches: [main]
 
jobs:
  evaluate:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
 
      - name: Run evaluation suite
        run: |
          npx @reaatech/agent-eval-harness-cli eval trajectories/ \
            --config eval-config.yaml \
            --output results/
 
      - name: Run regression gates
        run: |
          npx @reaatech/agent-eval-harness-cli gate results/results.json \
            --preset standard \
            --exit-code
 
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-results
          path: results/