{
  "schemaVersion": 1,
  "evaluatedAt": "2026-10-11T21:34:35.839Z",
  "codeVersion": "a83ff2f699df860d9be0f758771d1b08dad41024",
  "corpusVersion": "3900faa87e296ea256d10f3c6526e50238b92f2e66301b7e73e6282cf4804602",
  "datasetVersion": "01013594e067e1581c7c31fdd81c547cbe05521aa8c1c63d466eb58149aa7621",
  "rubricVersion": "portfolio-quality-v1",
  "promptVersion": "portfolio-evidence-v2",
  "reportedCostUsd": 0.083109826,
  "reportedCostLimitUsd": 0.15,
  "calibratedByHuman": false,
  "comparisonSummary": [
    {
      "model": "anthropic/claude-haiku-4.5",
      "cases": 2,
      "passed": 1,
      "validated": 2,
      "semanticPassed": 1,
      "correctionAttempts": 0,
      "costUsd": 0.020624,
      "graderCostUsd": 0.0070019999999999995,
      "meanLatencyMs": 13072
    },
    {
      "model": "openai/gpt-5.4-mini",
      "cases": 2,
      "passed": 2,
      "validated": 2,
      "semanticPassed": 2,
      "correctionAttempts": 0,
      "costUsd": 0.0130485,
      "graderCostUsd": 0.006776,
      "meanLatencyMs": 10512
    },
    {
      "model": "google/gemini-3.8-flash",
      "cases": 2,
      "passed": 2,
      "validated": 2,
      "semanticPassed": 2,
      "correctionAttempts": 0,
      "costUsd": 0.01418175,
      "graderCostUsd": 0.006671,
      "meanLatencyMs": 9832
    },
    {
      "model": "qwen/qwen3.7-plus",
      "cases": 2,
      "passed": 2,
      "validated": 2,
      "semanticPassed": 2,
      "correctionAttempts": 1,
      "costUsd": 0.007296576000000001,
      "graderCostUsd": 0.00751,
      "meanLatencyMs": 31725
    }
  ],
  "scenarios": [
    {
      "id": "fit-applied-ai-anthropic-claude-haiku-4.5",
      "model": "anthropic/claude-haiku-4.5",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 14248,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.010935,
      "graderCostUsd": 0.003725
    },
    {
      "id": "fit-applied-ai-openai-gpt-5.4-mini",
      "model": "openai/gpt-5.4-mini",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 10683,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.00777225,
      "graderCostUsd": 0.002983
    },
    {
      "id": "fit-applied-ai-google-gemini-3.8-flash",
      "model": "google/gemini-3.8-flash",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 9085,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.00690375,
      "graderCostUsd": 0.002955
    },
    {
      "id": "fit-applied-ai-qwen-qwen3.7-plus",
      "model": "qwen/qwen3.7-plus",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 24028,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.00313472,
      "graderCostUsd": 0.004244
    },
    {
      "id": "fit-mlops-anthropic-claude-haiku-4.5",
      "model": "anthropic/claude-haiku-4.5",
      "status": "failed",
      "semantic": "failed",
      "latencyMs": 11896,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.009689,
      "graderCostUsd": 0.003277
    },
    {
      "id": "fit-mlops-openai-gpt-5.4-mini",
      "model": "openai/gpt-5.4-mini",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 10341,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.00527625,
      "graderCostUsd": 0.003793
    },
    {
      "id": "fit-mlops-google-gemini-3.8-flash",
      "model": "google/gemini-3.8-flash",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 10579,
      "validated": true,
      "modelCalls": 1,
      "costUsd": 0.007278,
      "graderCostUsd": 0.003716
    },
    {
      "id": "fit-mlops-qwen-qwen3.7-plus",
      "model": "qwen/qwen3.7-plus",
      "status": "passed",
      "semantic": "passed",
      "latencyMs": 39422,
      "validated": true,
      "modelCalls": 2,
      "costUsd": 0.004161856,
      "graderCostUsd": 0.003266
    }
  ],
  "reviewNotes": [
    "All eight assessments passed structure and exact-source quotation validation.",
    "The Haiku MLOps semantic failure was a grader false positive: the listed unsupported claims were not affirmative claims actually made by that assessment. Do not interpret its 1/2 automated grade as an established model reliability difference.",
    "Two cases per model are a screening sample. Grading is not human calibrated. End-to-end latency includes optional grading; actor costs below exclude grading.",
    "Gemini selected for its observed speed, no correction attempts and appropriate treatment of explicitly listed skills; GPT Mini is a close lower-cost alternative. Qwen was cheapest but slower and required one correction."
  ]
}