import { compareRuns } from "../src/quality.mjs";

// Transport and scorer are injected: the same harness runs workerd E2Es and
// real-provider experiments. Only curated evaluation conversations enter it.
export async function readAgentResponse(response) {
  if (!response.ok) throw new Error(`Evaluation HTTP ${response.status}`);
  const text = await response.text();
  const result = {
    answer: "",
    sources: [],
    trace: null,
    assessment: null,
    grade: null,
    error: false,
    done: false,
  };
  for (const frame of text.split(/\r?\n\r?\n/)) {
    const event = frame.match(/^event: (.+)$/m)?.[1];
    const data = frame
      .split(/\r?\n/)
      .filter(line => line.startsWith("data:"))
      .map(line => line.slice(5).trimStart())
      .join("\n");
    if (!data) continue;
    if (data === "[DONE]") {
      result.done = true;
      continue;
    }
    const value = JSON.parse(data);
    if (!event) result.answer += value.choices?.[0]?.delta?.content ?? "";
    if (["sources", "trace", "assessment", "grade"].includes(event))
      result[event] = value;
    if (event === "error") result.error = true;
  }
  return result;
}
export function scoreTurn(result, scenario) {
  const checks = [];
  if (
    (scenario.answerPatterns ?? []).some(
      pattern => !new RegExp(pattern, "i").test(result.answer)
    )
  )
    checks.push("answer_coverage");
  if (result.error || !result.done || !result.answer.trim() || !result.trace)
    checks.push("workflow");
  if (
    scenario.sourceIncludes &&
    !result.trace?.sources.some(source =>
      source.id.includes(scenario.sourceIncludes)
    )
  )
    checks.push("retrieval");
  if (
    scenario.codeProject &&
    !result.sources.some(
      source =>
        source.kind === "code" &&
        source.id.startsWith(`code-${scenario.codeProject}-`)
    )
  )
    checks.push("tool_selection");
  const citations = [...result.answer.matchAll(/\[(\d+)\]/g)].map(match =>
    Number(match[1])
  );
  if (
    scenario.mode !== "assessment" &&
    !scenario.allowNoCitations &&
    (!citations.length ||
      !result.sources.length ||
      citations.some(
        citation => !result.sources.some(source => source.citation === citation)
      ))
  )
    checks.push("validation");
  if (scenario.mode === "assessment") {
    if (!result.assessment?.requirements?.length) checks.push("validation");
    for (const skill of scenario.unknownSkills ?? []) {
      const matched =
        result.assessment?.requirements.filter(item =>
          item.requirement.toLowerCase().includes(skill.toLowerCase())
        ) ?? [];
      if (
        !matched.length ||
        matched.some(item => item.status !== "not_established")
      )
        checks.push("validation");
    }
  }
  if (result.trace?.tools.length > 3 || result.trace?.modelCalls > 5)
    checks.push("workflow");
  if (
    result.grade &&
    (!result.grade.valid ||
      !result.grade.grounded ||
      !result.grade.answersQuestion ||
      !result.grade.honestGaps ||
      result.grade.unsupportedClaims?.length)
  )
    checks.push("judge");
  return [...new Set(checks)];
}
export async function runConversation(
  scenario,
  send,
  scorer = scoreTurn,
  onTurn = () => {}
) {
  let history = [],
    previous;
  const turns = [];
  const steps = scenario.steps ?? [{ user: scenario.messages.at(-1).content }];
  if (!scenario.steps) history = scenario.messages.slice(0, -1);
  for (const step of steps) {
    const selected =
      !step.when ||
      previous?.sources.some(source => source.kind === step.when.sourcesKind);
    const user = selected ? step.user : step.otherwise;
    if (typeof user !== "string" || !user.trim())
      throw new Error("Invalid simulated conversation step");
    history.push({ role: "user", content: user });
    let result;
    try {
      result = await readAgentResponse(
        await send({ messages: history, mode: scenario.mode ?? "chat" })
      );
    } catch {
      result = {
        answer: "",
        sources: [],
        trace: null,
        assessment: null,
        grade: null,
        error: true,
        done: false,
      };
    }
    onTurn(result);
    const checks = scorer(result, { ...scenario, ...(step.checks ?? {}) });
    turns.push({ ...result, checks, messages: structuredClone(history) });
    previous = result;
    if (result.error || !result.answer.trim()) break;
    history.push({ role: "assistant", content: result.answer });
  }
  return turns;
}
function knownTurnCost(turn) {
  return (
    Number.isFinite(turn.trace?.costUsd) &&
    (turn.trace.stages ?? [])
      .filter(stage => stage.kind === "model")
      .every(stage => Number.isFinite(stage.costUsd)) &&
    (!turn.grade || Number.isFinite(turn.grade.costUsd))
  );
}
export async function runExperiment({
  cases,
  repeats,
  variants,
  send,
  costLimit,
  onResult = () => {},
}) {
  const rows = [],
    results = [];
  let spent = 0,
    unknownCost = false;
  for (const scenario of cases)
    for (let repeat = 0; repeat < repeats; repeat++) {
      // Alternate ordering to reduce systematic provider/time-order effects.
      const order = repeat % 2 ? [...variants].reverse() : variants;
      for (const variant of order) {
        if (spent >= costLimit || unknownCost) break;
        const started = Date.now();
        let turns = [],
          transportError = false;
        try {
          turns = await runConversation(
            scenario,
            body => {
              if (spent >= costLimit || unknownCost)
                throw new Error("Experiment budget reached");
              return send(body, variant, scenario);
            },
            scoreTurn,
            turn => {
              if (!knownTurnCost(turn)) unknownCost = true;
              if (Number.isFinite(turn.trace?.costUsd))
                spent += turn.trace.costUsd;
              if (Number.isFinite(turn.grade?.costUsd))
                spent += turn.grade.costUsd;
            }
          );
        } catch {
          transportError = true;
        }
        const traces = turns.map(turn => turn.trace).filter(Boolean);
        const checks = [...new Set(turns.flatMap(turn => turn.checks))];
        if (transportError || turns.length !== (scenario.steps?.length ?? 1))
          checks.push("workflow");
        const failure = !checks.length
          ? "none"
          : traces
              .flatMap(trace => trace.stages ?? [])
              .find(stage => stage.failure)?.failure ??
            checks[0] ??
            "none";
        const knownCost = turns.length > 0 && turns.every(knownTurnCost);
        const costUsd = knownCost
          ? turns.reduce(
              (sum, turn) =>
                sum + (turn.trace?.costUsd ?? 0) + (turn.grade?.costUsd ?? 0),
              0
            )
          : null;
        if (turns.some(turn => turn.error)) transportError = true;
        const row = {
          caseId: scenario.id,
          variant: variant.name,
          repeat,
          status: transportError
            ? "error"
            : checks.length
            ? "failed"
            : "passed",
          latencyMs: Date.now() - started,
          costUsd,
          corpusVersion: traces[0]?.corpusVersion ?? "0".repeat(64),
          codeVersion:
            traces[0]?.codeVersion ?? variant.expectedCommit ?? "0".repeat(40),
          model: traces[0]?.model ?? variant.model ?? "unknown",
          promptVersion: traces[0]?.promptVersion ?? "unknown",
          failure,
          embeddingCache: Boolean(variant.embeddingCache),
          adaptiveRouting: Boolean(variant.adaptiveRouting),
        };
        if (
          variant.expectedCommit &&
          traces.some(trace => trace.codeVersion !== variant.expectedCommit)
        ) {
          row.status = "failed";
          row.failure = "workflow";
        }
        if (
          new Set(traces.map(trace => trace.corpusVersion)).size > 1 ||
          new Set(traces.map(trace => trace.promptVersion)).size > 1 ||
          new Set(traces.map(trace => trace.codeVersion)).size > 1
        ) {
          row.status = "failed";
          row.failure = "workflow";
        }
        rows.push(row);
        results.push({ ...row, turns });
        onResult(row);
      }
    }
  return {
    rows,
    caseIds: cases.map(item => item.id),
    results,
    reportedCostUsd: spent,
    ...compareRuns(
      rows,
      repeats,
      cases.map(item => item.id)
    ),
  };
}