{
  "artifact_type": "resyst_labs_benchmark_rankings",
  "source_artifact": "docs/results/updated-model-comparison-table-20260612-kimi-k27-code.json",
  "generated_at": "2026-06-12T21:57:03.236011+00:00",
  "synced_at": "2026-06-12T21:57:03.236011+00:00",
  "summary": {
    "title": "Resyst Labs LLM Benchmark Rankings",
    "formula": "Overall = 50% Full/Agentic Final + 50% SWE MVP for models with both lanes measured",
    "public_note": "Local and API-backed models share one tournament table. Runtime/provider/cost/resource metadata remain visible so the comparison stays honest.",
    "row_count": 11
  },
  "notes": [
    "Full/Agentic and SWE are separate benchmark lanes; the overall score uses an explicit 50/50 formula when both lanes are available.",
    "Provider, runtime, suite version, cost, latency, reliability, and artifact references are exposed as metadata for auditability.",
    "Mixed-basis rows are labeled when their Full/Agentic and SWE measurements come from independently versioned benchmark runs.",
    "SWE rows use the software-engineering MVP suite and are distinct from SWE-bench Verified.",
    "Arena replays publish board states, legal actions, events, and tactical telemetry while excluding raw model completions."
  ],
  "generated_from": [
    "docs/results/updated-model-comparison-table-20260601.json",
    "results/scored/20260602-160147-727708120Z-gemini35-flash-openrouter-full-suite-scorer-v28-benchmark_core_v1.json",
    "results/scored/20260601-220351-835410174Z-benchmark_core_v1-rubric-calibrated-e55eb55.json",
    "results/scored/20260601-214959-974793792Z-benchmark_core_v1-rubric-calibrated-e55eb55.json",
    "results/scored/20260602-155938-808333481Z-qwen37max-swe-n3-scorer-v28-software_engineering_mvp.json",
    "results/scored/20260602-155939-709454110Z-claude48-swe-n3-scorer-v28-software_engineering_mvp.json",
    "results/scored/20260602-155937-035075937Z-deepseek-direct-swe-n3-scorer-v28-software_engineering_mvp.json",
    "results/scored/20260602-155940-644318161Z-minimax-m3-openrouter-xhigh-swe-n3-scorer-v28-software_engineering_mvp.json",
    "results/scored/20260602-155941-549430977Z-gpt55-swe-n3-scorer-v28-software_engineering_mvp.json",
    "results/scored/20260602-155936-146466996Z-gemini35-flash-openrouter-swe-n3-scorer-v28-software_engineering_mvp.json",
    "results/scored/20260602-190208-287273652Z-software_engineering_mvp.json",
    "docs/results/updated-model-comparison-table-20260602.json",
    "results/scored/20260605-155227-552272755Z-benchmark_core_v1.json",
    "results/scored/20260605-164152-171931398Z-software_engineering_mvp.json",
    "results/raw/20260605-155227-552272755Z-benchmark_core_v1.jsonl",
    "results/raw/20260605-164152-171931398Z-software_engineering_mvp.jsonl",
    "results/scored/20260609-205416-330274915Z-benchmark_core_v1.json",
    "results/scored/20260609-210324-246641587Z-software_engineering_mvp.json",
    "results/raw/20260609-205416-330274915Z-benchmark_core_v1.jsonl",
    "results/raw/20260609-210324-246641587Z-software_engineering_mvp.jsonl",
    "results/scored/20260612-213040-480906794Z-benchmark_core_v1.json",
    "results/raw/20260612-213040-480906794Z-benchmark_core_v1.jsonl",
    "results/reports/20260612-213040-480906794Z-benchmark_core_v1.md",
    "results/scored/20260612-213947-405764714Z-software_engineering_mvp.json",
    "results/raw/20260612-213947-405764714Z-software_engineering_mvp.jsonl",
    "results/reports/20260612-213947-405764714Z-software_engineering_mvp.md"
  ],
  "rows": [
    {
      "id": "deepseek-v4-flash-direct",
      "label": "DeepSeek V4 Flash",
      "basis": "DeepSeek direct API",
      "full": {
        "suite": "benchmark_core_v1 · 43 prompts · rubric-calibrated-e55eb55",
        "artifact": "results/scored/20260601-220351-835410174Z-benchmark_core_v1-rubric-calibrated-e55eb55.json",
        "final": 95.8,
        "capability": 96.8,
        "agentic": 96.32,
        "daily": 95.35,
        "efficiency": 94.55,
        "performance": 93.5,
        "reliability": 100.0,
        "pass_rate": 97.67,
        "avg_s": 4.602,
        "decode": 77.095,
        "cost": 0.079311,
        "cost_basis": "provider_reported_or_project_pricing",
        "prompt_count": 43,
        "total_time_s": 197.88
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155937-035075937Z-deepseek-direct-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 86.95,
        "focused_final": 86.59,
        "capability": 84.53,
        "daily": 87.04,
        "performance": 85.59,
        "efficiency": 91.35,
        "reliability": 100.0,
        "cost": 0.007089,
        "time_s": 198.43,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 1,
      "swe_rank": 3,
      "overall_score": 91.375,
      "overall_rank": 1
    },
    {
      "id": "qwen3.7-max-openrouter-xhigh",
      "label": "Qwen3.7 Max",
      "basis": "OpenRouter · extra-high reasoning",
      "full": {
        "suite": "Prior models use full_suite_v2 with v3 16K/32K native-tool patch; Qwen3.7 Max uses full_suite_v3.",
        "artifact": "docs/results/updated-model-comparison-table-20260601.json",
        "final": 83.24,
        "capability": 92.37,
        "agentic": 94.0,
        "daily": 79.71,
        "efficiency": 63.72,
        "performance": 71.49,
        "reliability": 100.0,
        "pass_rate": 93.02,
        "avg_s": 16.212,
        "decode": 44.996,
        "cost": 0.66922,
        "cost_basis": "provider_reported",
        "prompt_count": 43
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155938-808333481Z-qwen37max-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 88.99,
        "focused_final": 78.27,
        "capability": 88.63,
        "daily": 74.14,
        "performance": 59.78,
        "efficiency": 63.83,
        "reliability": 100.0,
        "cost": 0.181609,
        "time_s": 740.56,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 9,
      "swe_rank": 1,
      "overall_score": 86.115,
      "overall_rank": 2
    },
    {
      "id": "minimax-m3-openrouter-xhigh",
      "label": "MiniMax M3",
      "basis": "OpenRouter · extra-high reasoning",
      "full": {
        "suite": "Prior models use full_suite_v2 with v3 16K/32K native-tool patch; Qwen3.7 Max uses full_suite_v3.",
        "artifact": "docs/results/updated-model-comparison-table-20260601.json",
        "final": 85.33,
        "capability": 89.6,
        "agentic": 81.72,
        "daily": 83.48,
        "efficiency": 82.14,
        "performance": 74.91,
        "reliability": 100.0,
        "pass_rate": 86.05,
        "avg_s": 13.764,
        "decode": 24.334,
        "cost": 0.108666,
        "cost_basis": "provider_reported",
        "prompt_count": 43
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155940-644318161Z-minimax-m3-openrouter-xhigh-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 86.88,
        "focused_final": 77.11,
        "capability": 86.15,
        "daily": 73.36,
        "performance": 55.88,
        "efficiency": 71.7,
        "reliability": 100.0,
        "cost": 0.047882,
        "time_s": 1116.23,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 6,
      "swe_rank": 4,
      "overall_score": 86.105,
      "overall_rank": 3
    },
    {
      "id": "claude-opus-4.8-openrouter-xhigh",
      "label": "Claude Opus 4.8",
      "basis": "OpenRouter · extra-high reasoning",
      "full": {
        "suite": "Prior models use full_suite_v2 with v3 16K/32K native-tool patch; Qwen3.7 Max uses full_suite_v3.",
        "artifact": "docs/results/updated-model-comparison-table-20260601.json",
        "final": 83.32,
        "capability": 94.37,
        "agentic": 94.68,
        "daily": 79.29,
        "efficiency": 44.44,
        "performance": 79.0,
        "reliability": 100.0,
        "pass_rate": 93.02,
        "avg_s": 11.653,
        "decode": 29.802,
        "cost": 5.53418,
        "cost_basis": "provider_reported",
        "prompt_count": 43
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155939-709454110Z-claude48-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 88.67,
        "focused_final": 84.92,
        "capability": 87.69,
        "daily": 83.74,
        "performance": 84.67,
        "efficiency": 70.15,
        "reliability": 100.0,
        "cost": 0.44712,
        "time_s": 209.77,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 8,
      "swe_rank": 2,
      "overall_score": 85.995,
      "overall_rank": 4
    },
    {
      "id": "deepseek-v4-pro-direct",
      "label": "DeepSeek V4 Pro",
      "basis": "DeepSeek direct API",
      "full": {
        "suite": "benchmark_core_v1 · 43 prompts · rubric-calibrated-e55eb55",
        "artifact": "results/scored/20260601-214959-974793792Z-benchmark_core_v1-rubric-calibrated-e55eb55.json",
        "final": 92.04,
        "capability": 99.37,
        "agentic": 97.18,
        "daily": 89.28,
        "efficiency": 80.14,
        "performance": 81.44,
        "reliability": 100.0,
        "pass_rate": 100.0,
        "avg_s": 10.403,
        "decode": 37.726,
        "cost": 0.247676,
        "cost_basis": "provider_reported_or_project_pricing",
        "prompt_count": 43,
        "total_time_s": 447.32
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155937-035075937Z-deepseek-direct-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 79.68,
        "focused_final": 65.14,
        "capability": 75.75,
        "daily": 60.42,
        "performance": 28.45,
        "efficiency": 75.81,
        "reliability": 100.0,
        "cost": 0.032801,
        "time_s": 696.7,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 2,
      "swe_rank": 7,
      "overall_score": 85.86,
      "overall_rank": 5
    },
    {
      "id": "gpt-5.5-openrouter-xhigh",
      "label": "GPT‑5.5",
      "basis": "OpenRouter · extra-high reasoning",
      "full": {
        "suite": "Prior models use full_suite_v2 with v3 16K/32K native-tool patch; Qwen3.7 Max uses full_suite_v3.",
        "artifact": "docs/results/updated-model-comparison-table-20260601.json",
        "final": 85.76,
        "capability": 91.71,
        "agentic": 87.89,
        "daily": 83.58,
        "efficiency": 56.1,
        "performance": 87.64,
        "reliability": 100.0,
        "pass_rate": 93.02,
        "avg_s": 7.214,
        "decode": 24.473,
        "cost": 2.437015,
        "cost_basis": "provider_reported",
        "prompt_count": 43
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155941-549430977Z-gpt55-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 85.51,
        "focused_final": 70.46,
        "capability": 83.35,
        "daily": 65.44,
        "performance": 56.56,
        "efficiency": 36.53,
        "reliability": 100.0,
        "cost": 1.63434,
        "time_s": 1050.23,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 5,
      "swe_rank": 5,
      "overall_score": 85.635,
      "overall_rank": 6
    },
    {
      "id": "gemini-3.5-flash-openrouter",
      "label": "Gemini 3.5 Flash",
      "basis": "OpenRouter · google/gemini-3.5-flash",
      "full": {
        "suite": "full_suite_v3 · scorer_v2.8_gemini35_swe_aliases",
        "artifact": "results/scored/20260602-160147-727708120Z-gemini35-flash-openrouter-full-suite-scorer-v28-benchmark_core_v1.json",
        "final": 88.79,
        "capability": 93.55,
        "agentic": 91.87,
        "daily": 87.0,
        "efficiency": 69.56,
        "performance": 87.61,
        "reliability": 100.0,
        "pass_rate": 93.02,
        "avg_s": 7.228,
        "decode": 99.619,
        "cost": 1.010705,
        "cost_basis": "provider_reported_or_openrouter_pricing",
        "prompt_count": 43,
        "total_time_s": 310.8
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260602-155936-146466996Z-gemini35-flash-openrouter-swe-n3-scorer-v28-software_engineering_mvp.json",
        "swe_score": 73.59,
        "focused_final": 72.96,
        "capability": 70.23,
        "daily": 73.5,
        "performance": 78.11,
        "efficiency": 65.33,
        "reliability": 100.0,
        "cost": 0.491144,
        "time_s": 290.68,
        "prompt_count": 24
      },
      "notes": [],
      "full_rank": 3,
      "swe_rank": 8,
      "overall_score": 81.19,
      "overall_rank": 7
    },
    {
      "id": "claude-fable-5-openrouter-xhigh",
      "label": "Claude Fable 5",
      "basis": "OpenRouter · extra-high reasoning",
      "full": {
        "suite": "benchmark_core_v1 · 43 prompts · full_suite_v3",
        "artifact": "results/scored/20260609-205416-330274915Z-benchmark_core_v1.json",
        "final": 77.02,
        "capability": 86.0,
        "agentic": 82.26,
        "daily": 73.64,
        "efficiency": 36.64,
        "performance": 76.91,
        "reliability": 100.0,
        "pass_rate": 86.05,
        "avg_s": 12.735,
        "decode": 18.708,
        "cost": 10.79056,
        "cost_basis": "provider_reported",
        "prompt_count": 43,
        "total_time_s": 547.61,
        "prompt_tokens": 1028491,
        "output_tokens": 10113,
        "reasoning_tokens": 675
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260609-210324-246641587Z-software_engineering_mvp.json",
        "swe_score": 81.42,
        "focused_final": 76.57,
        "capability": 79.17,
        "daily": 75.36,
        "performance": 79.02,
        "efficiency": 55.18,
        "reliability": 100.0,
        "cost": 1.02434,
        "time_s": 279.59,
        "prompt_count": 24,
        "prompt_tokens": 12039,
        "output_tokens": 18079,
        "reasoning_tokens": 1804,
        "task_type_scores": {
          "multi_file_patch": 83.33,
          "component_creation": 50.0,
          "software_audit_review": 92.92,
          "edge_case_discovery": 85.86,
          "security_supply_chain": 95.0
        }
      },
      "notes": [
        "Exact OpenRouter model: anthropic/claude-fable-5."
      ],
      "full_rank": 11,
      "swe_rank": 6,
      "overall_score": 79.22,
      "overall_rank": 8
    },
    {
      "id": "minimax-m3-direct-anthropic",
      "label": "MiniMax M3 Direct Plus",
      "basis": "MiniMax Plus · direct API · SWE xhigh",
      "full": {
        "suite": "Prior models use full_suite_v2 with v3 16K/32K native-tool patch; Qwen3.7 Max uses full_suite_v3.",
        "artifact": "docs/results/updated-model-comparison-table-20260601.json",
        "final": 84.32,
        "capability": 87.74,
        "agentic": 79.86,
        "daily": 82.74,
        "efficiency": 84.23,
        "performance": 73.71,
        "reliability": 100.0,
        "pass_rate": 88.37,
        "avg_s": 14.377,
        "decode": 23.392,
        "cost": 0.003093,
        "cost_basis": "minimax_plus_quota_equivalent_20usd_1.7b_tokens",
        "prompt_count": 43
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8 · direct xhigh",
        "artifact": "results/scored/20260602-190208-287273652Z-software_engineering_mvp.json",
        "model_id": "minimax-m3-direct-anthropic-xhigh",
        "swe_score": 67.77,
        "focused_final": 61.86,
        "capability": 62.62,
        "daily": 60.93,
        "performance": 59.67,
        "efficiency": 50.0,
        "reliability": 100.0,
        "cost": 0.000601,
        "cost_basis": "minimax_plus_quota_equivalent_20usd_1.7b_tokens_includes_cache_fields",
        "quota_fraction_pct": 0.003003529411764706,
        "provider_token_usage": {
          "input_tokens": 2762,
          "output_tokens": 39450,
          "cache_creation_input_tokens": 0,
          "cache_read_input_tokens": 8848
        },
        "provider_tokens_total_for_quota": 51060,
        "time_s": 751.18,
        "prompt_count": 24
      },
      "notes": [
        "MiniMax Direct Plus combines independently measured Full and SWE lanes; suite metadata remains visible per lane."
      ],
      "full_rank": 7,
      "swe_rank": 9,
      "overall_score": 76.045,
      "overall_rank": 9
    },
    {
      "id": "kimi-k2.7-code-openrouter-xhigh",
      "label": "Kimi K2.7 Code",
      "basis": "OpenRouter · extra-high reasoning",
      "full": {
        "suite": "benchmark_core_v1 · full_suite_v3 · 43 prompts",
        "artifact": "results/scored/20260612-213040-480906794Z-benchmark_core_v1.json",
        "final": 87.95,
        "capability": 96.87,
        "agentic": 92.22,
        "daily": 84.59,
        "efficiency": 69.56,
        "performance": 76.94,
        "reliability": 100.0,
        "pass_rate": 97.67,
        "avg_s": 12.713,
        "decode": 25.227,
        "cost": 0.439739,
        "cost_basis": "provider_reported",
        "prompt_count": 43,
        "total_time_s": 546.65
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260612-213947-405764714Z-software_engineering_mvp.json",
        "swe_score": 58.61,
        "focused_final": 57.66,
        "capability": 50.84,
        "daily": 59.24,
        "performance": 58.53,
        "efficiency": 67.09,
        "reliability": 100.0,
        "cost": 0.129164,
        "time_s": 860.76,
        "prompt_count": 24
      },
      "notes": [
        "Exact OpenRouter model: moonshotai/kimi-k2.7-code.",
        "Full/Agentic lane is strong; SWE lane exposed patch-format and implementation-delivery failures in the MVP fixtures."
      ],
      "overall_score": 73.28,
      "full_rank": 4,
      "swe_rank": 11,
      "overall_rank": 10
    },
    {
      "id": "nemotron-3-ultra-openrouter-xhigh",
      "label": "NVIDIA Nemotron 3 Ultra",
      "basis": "OpenRouter · nvidia/nemotron-3-ultra-550b-a55b · xhigh reasoning",
      "full": {
        "suite": "full_suite_v3 · benchmark_core_v1 · scorer_v2.8_gemini35_swe_aliases",
        "artifact": "results/scored/20260605-155227-552272755Z-benchmark_core_v1.json",
        "raw_artifact": "results/raw/20260605-155227-552272755Z-benchmark_core_v1.jsonl",
        "report": "results/reports/20260605-155227-552272755Z-benchmark_core_v1.md",
        "final": 79.54,
        "capability": 95.29,
        "agentic": 92.57,
        "daily": 73.49,
        "efficiency": 60.02,
        "performance": 52.5,
        "reliability": 100.0,
        "pass_rate": 93.02,
        "avg_s": 68.938,
        "decode": 5.735,
        "cost": 0.312325,
        "cost_basis": "provider_reported_openrouter_upstream_cost",
        "prompt_count": 43,
        "total_time_s": 2964.34,
        "provider_token_usage": {
          "prompt_tokens": 547025,
          "output_tokens": 15928,
          "reasoning_tokens": 10899
        }
      },
      "swe": {
        "suite": "software_engineering_mvp_v1 · n=3 · scorer_v2.8",
        "artifact": "results/scored/20260605-164152-171931398Z-software_engineering_mvp.json",
        "raw_artifact": "results/raw/20260605-164152-171931398Z-software_engineering_mvp.jsonl",
        "report": "results/reports/20260605-164152-171931398Z-software_engineering_mvp.md",
        "swe_score": 58.63,
        "focused_final": 60.73,
        "capability": 58.63,
        "daily": 60.63,
        "performance": 52.5,
        "efficiency": 69.03,
        "reliability": 100.0,
        "cost": 0.056355,
        "cost_basis": "provider_reported_openrouter_upstream_cost",
        "time_s": 2234.05,
        "prompt_count": 24,
        "pass_rate": 50.0,
        "provider_token_usage": {
          "prompt_tokens": 8067,
          "output_tokens": 21027,
          "reasoning_tokens": 14712
        }
      },
      "notes": [
        "Exact OpenRouter model: nvidia/nemotron-3-ultra-550b-a55b.",
        "Full lane uses the current full_suite_v3 run; suite metadata remains visible per row.",
        "SWE lane is software_engineering_mvp_v1 n=3, not the external SWE-bench Verified benchmark.",
        "Low SWE rank is driven by patch formatting/application failures: no_patch_found=1, patch_rejected=7, tests_failed=3 across 24 SWE records."
      ],
      "full_rank": 10,
      "swe_rank": 10,
      "overall_score": 69.085,
      "overall_rank": 11
    }
  ]
}