name: Compare agent candidates
on:
  workflow_dispatch:
    inputs:
      experiment_kind:
        description: Change one experiment axis
        type: choice
        options: [model, embedding_cache, adaptive_routing, simulation]
        default: model
      candidate_model:
        description: Candidate assessment model
        type: choice
        options:
          [
            anthropic/claude-haiku-4.5,
            openai/gpt-5.4-mini,
            google/gemini-3.8-flash,
            qwen/qwen3.7-plus,
          ]
        default: anthropic/claude-haiku-4.5
      repeats:
        description: Paired repetitions (3 or more for the automated gate)
        type: string
        default: "3"
      case_ids:
        description: Scenario IDs (blank selects the default suite for this experiment)
        type: string
        default: ""
      cost_limit:
        description: Reported cost guard; stops between requests, provider limit is the hard cap
        type: string
        default: "0.10"
      semantic_judge:
        description: Optional uncalibrated semantic grader (extra cost)
        type: boolean
        default: false
      enforce_gate:
        description: Fail the job unless automated workflow criteria are met
        type: boolean
        default: true
permissions:
  contents: read
jobs:
  compare:
    runs-on: ubuntu-latest
    timeout-minutes: 20
    concurrency:
      group: evaluate-portfolio-agent
      cancel-in-progress: false
    steps:
      - uses: actions/checkout@v4
      - run: test "$GITHUB_REF" = refs/heads/main
      - uses: actions/setup-node@v4
        with:
          node-version: "22"
      - name: Run paired experiment
        env:
          RAG_EVAL_SECRET: ${{ secrets.RAG_EVAL_SECRET }}
          QUALITY_EXPERIMENT_KIND: ${{ inputs.experiment_kind }}
          QUALITY_CANDIDATE_MODEL: ${{ inputs.candidate_model }}
          QUALITY_REPEATS: ${{ inputs.repeats }}
          QUALITY_CASE_IDS: ${{ inputs.case_ids }}
          QUALITY_COST_LIMIT: ${{ inputs.cost_limit }}
          QUALITY_JUDGE: ${{ inputs.semantic_judge }}
          QUALITY_ENFORCE_GATE: ${{ inputs.enforce_gate }}
          QUALITY_PUBLISH: "true"
        run: node worker/scripts/quality.mjs
      - uses: actions/upload-artifact@v4
        if: always()
        with:
          name: agent-quality-${{ github.run_id }}
          path: worker/evals/results/quality/
          retention-days: 30