{
  "artifact_version": "2",
  "report_id": "support-routing-v2-support-routing-1791531672788",
  "generated_at": "2026-10-09T09:52:38.881Z",
  "partial": false,
  "dataset": {
    "id": "support-routing",
    "version": "v1",
    "hash": "sha256:2a2683c23a9cc1aa0d8996762c1b0d53c8bc55d513f8e22a89354098943a0b5b",
    "label_policy_hash": "sha256:476220aef7fd81e74c78b1453e7558c15feda33bdd965ea4d64e2bf39c18fd42",
    "license": {
      "id": "CC-BY-4.0",
      "name": "Creative Commons Attribution 4.0 International",
      "url": "https://creativecommons.org/licenses/by/4.0/"
    },
    "sample_count": 100
  },
  "headline_metric": {
    "id": "macro_f1",
    "weighting": "equal_label",
    "rationale": "Each support label has equal weight in the primary comparison; traffic-weighted results may be secondary only when their source distribution is disclosed."
  },
  "run": {
    "safety": {
      "environment_id": "runner-v1",
      "region": "unspecified"
    },
    "max_calls": 500,
    "call_totals": {
      "planned": 500,
      "issued": 500,
      "skipped": 0,
      "completed": 500,
      "succeeded": 500,
      "failed": 0
    },
    "id": "support-routing-1791531672788",
    "started_at": "2026-10-09T07:41:12.803Z",
    "finished_at": "2026-10-09T09:52:38.881Z",
    "generated_at": "2026-10-09T09:52:38.881Z",
    "runner_commit": "912b2c4ca4eabf6adf283912c5f5889fd52664dd",
    "instruction_hash": "sha256:4a6d1fa83986f1d4c8616df4fe5a86c8a321bee6760da1837df367178904a6ed",
    "harness_version": "support-routing-benchmark-v1",
    "environment_id": "runner-v1",
    "approved_spend_ceiling_usd": "1.10",
    "spend_ceiling_enforced_by_runner": true
  },
  "methodology": {
    "same_frozen_cases": true,
    "same_instructions": true,
    "concurrency": 1,
    "retry_policy": "none",
    "cache": "disabled",
    "failed_attempts_remain_in_denominators": true
  },
  "models": [
    {
      "provider": "typesafe",
      "model_id": "jev",
      "run_id": "support-routing-1791531672788",
      "runner_commit": "912b2c4ca4eabf6adf283912c5f5889fd52664dd",
      "requested_revision": "jev-latest",
      "resolved_revision": "jev-1.13.0",
      "run_started_at": "2026-10-09T07:41:12.806Z",
      "run_finished_at": "2026-10-09T07:44:42.880Z",
      "attempted": 100,
      "valid": 100,
      "correct": 92,
      "error_count": 0,
      "status_counts": {
        "succeeded": 100,
        "invalid": 0,
        "timeout": 0,
        "refused": 0,
        "unsupported": 0,
        "error": 0,
        "cancelled": 0
      },
      "pricing_snapshot_id": null,
      "pricing_snapshot_date": null,
      "evidence": {
        "decisions": "independently_measured",
        "latency": "independently_measured",
        "errors": "independently_measured",
        "usage": "provider_reported",
        "cost": "not_available"
      }
    },
    {
      "provider": "cloudflare",
      "model_id": "clef",
      "run_id": "support-routing-1791531672788",
      "runner_commit": "912b2c4ca4eabf6adf283912c5f5889fd52664dd",
      "requested_revision": "clef",
      "resolved_revision": "clef",
      "run_started_at": "2026-10-09T07:41:13.004Z",
      "run_finished_at": "2026-10-09T07:44:43.127Z",
      "attempted": 100,
      "valid": 100,
      "correct": 92,
      "error_count": 0,
      "status_counts": {
        "succeeded": 100,
        "invalid": 0,
        "timeout": 0,
        "refused": 0,
        "unsupported": 0,
        "error": 0,
        "cancelled": 0
      },
      "pricing_snapshot_id": null,
      "pricing_snapshot_date": null,
      "evidence": {
        "decisions": "independently_measured",
        "latency": "independently_measured",
        "errors": "independently_measured",
        "usage": "provider_reported",
        "cost": "not_available"
      }
    },
    {
      "provider": "cloudflare",
      "model_id": "clef-flash",
      "run_id": "support-routing-1791531672788",
      "runner_commit": "912b2c4ca4eabf6adf283912c5f5889fd52664dd",
      "requested_revision": "clef-flash",
      "resolved_revision": "clef-flash",
      "run_started_at": "2026-10-09T07:41:13.504Z",
      "run_finished_at": "2026-10-09T07:44:43.373Z",
      "attempted": 100,
      "valid": 100,
      "correct": 87,
      "error_count": 0,
      "status_counts": {
        "succeeded": 100,
        "invalid": 0,
        "timeout": 0,
        "refused": 0,
        "unsupported": 0,
        "error": 0,
        "cancelled": 0
      },
      "pricing_snapshot_id": null,
      "pricing_snapshot_date": null,
      "evidence": {
        "decisions": "independently_measured",
        "latency": "independently_measured",
        "errors": "independently_measured",
        "usage": "provider_reported",
        "cost": "not_available"
      }
    },
    {
      "provider": "openai",
      "model_id": "gpt-6-luna",
      "run_id": "support-routing-1791531672788",
      "runner_commit": "912b2c4ca4eabf6adf283912c5f5889fd52664dd",
      "requested_revision": "gpt-6-luna",
      "resolved_revision": "gpt-6-luna",
      "run_started_at": "2026-10-09T07:41:13.669Z",
      "run_finished_at": "2026-10-09T07:44:44.578Z",
      "attempted": 100,
      "valid": 100,
      "correct": 98,
      "error_count": 0,
      "status_counts": {
        "succeeded": 100,
        "invalid": 0,
        "timeout": 0,
        "refused": 0,
        "unsupported": 0,
        "error": 0,
        "cancelled": 0
      },
      "pricing_snapshot_id": null,
      "pricing_snapshot_date": null,
      "evidence": {
        "decisions": "independently_measured",
        "latency": "independently_measured",
        "errors": "independently_measured",
        "usage": "provider_reported",
        "cost": "not_available"
      }
    },
    {
      "provider": "openai",
      "model_id": "openai-decisions",
      "run_id": "support-routing-1791539546056",
      "runner_commit": "01cba49be9cbba0cc85e64a7c4bd31f43b0394cc",
      "requested_revision": "gpt-6-luna",
      "resolved_revision": "gpt-6-luna",
      "run_started_at": "2026-10-09T09:52:26.071Z",
      "run_finished_at": "2026-10-09T09:52:38.880Z",
      "attempted": 100,
      "valid": 100,
      "correct": 94,
      "error_count": 0,
      "status_counts": {
        "succeeded": 100,
        "invalid": 0,
        "timeout": 0,
        "refused": 0,
        "unsupported": 0,
        "error": 0,
        "cancelled": 0
      },
      "pricing_snapshot_id": null,
      "pricing_snapshot_date": null,
      "evidence": {
        "decisions": "independently_measured",
        "latency": "independently_measured",
        "errors": "independently_measured",
        "usage": "provider_reported",
        "cost": "not_available"
      }
    }
  ],
  "limitations": [
    "The cases are fictional and do not estimate real-world support-intent prevalence.",
    "AI review is disclosed but was not counted toward the two required human reviews.",
    "Calibration uses only available normalized distributions; missing and failed rows remain in probability coverage. Provider-reported distributions are not independent correctness guarantees.",
    "This task-specific synthetic benchmark does not establish a universally best model.",
    "Latency reflects the disclosed run environment and date, not a service-level guarantee.",
    "Cost is shown only when an immutable pricing snapshot or provider-reported amount is available."
  ],
  "files": {
    "manifest": {
      "path": "/benchmarks/support-routing-v2/manifest.json"
    },
    "results_json": {
      "path": "/benchmarks/support-routing-v2/results.json",
      "row_count": 500,
      "sha256": "sha256:385324be823dcbe67c476139f1531c398d564da715ccc3f0255fb99f16c3fbac"
    },
    "results_csv": {
      "path": "/benchmarks/support-routing-v2/results.csv",
      "row_count": 500,
      "sha256": "sha256:589094640a1fcb6726f476c20bbb33306b00e9a87646fe7ea4f8dc6f27f47f8e"
    },
    "report": {
      "path": "data/benchmarks/support-routing/v2/report.json",
      "sha256": "sha256:68309a01b21d343a0988ad3111fa49fc65911d55c83d67cfecdd60cd75a0f260"
    }
  }
}