{
  "title": "FAST54 · Model evidence",
  "updated": "2026-10-31",
  "description": "Retained selected-case observations, not a controlled leaderboard.",
  "suite": {"contested": 50, "frontier": 4, "components": {"ARC": 18, "IFEval": 27, "HumanEval+": 5, "Tool-Eval": 4}},
  "groups": [
    {
      "id": "publisher",
      "title": "Publisher-setting observations",
      "description": "Each model used its own documented scenario settings. Context, case eligibility, graders and runtime still differ.",
      "runs": [
        {"name": "SenseNova Loop 9B · Q4", "short": "SenseNova Loop 9B · Q4", "date": "2026-10-31", "environment": "Dunamis · RTX 5090 · llama.cpp fork / Vulkan · shared 256K · up to C16", "settings": "Thinking on; temperature 1.0, top-p 0.95, top-k 50; no primary request cap or fixed request seed.", "scores": [[15,17],[17,24],[5,5],[3,4]], "notes": ["All 17 contested cases completed.","All 24 contested cases completed; one frontier case was stopped in a loop.","Chat-adapted HumanEval+ with native base and plus checks.","TC-80 passed. TC-82 passed natively but supplied a path, not a file attachment."], "source": "quality-store-shared-c16-fast54-20261031", "frontier": "0/4; one case manually stopped in a confirmed loop", "color": "#39d0ff"},
        {"name": "Nanbeige 4.3 33B · AutoRound · pass 1", "short": "Nanbeige 33B · AutoRound 1", "date": "2026-10-26", "environment": "Dunamis · RTX 5090 · vLLM · 32K", "settings": "Thinking on; publisher task-specific sampling; no fixed request seed.", "scores": [[15,17],[22,24],[4,5],[1,3]], "notes": ["All 17 contested cases eligible.","All 24 contested cases eligible.","Chat-adapted HumanEval+.","TC-80 excluded due to the old negation-scoring defect. The preserved TC-75 failure has a separate clarification-scoring caveat."], "source": "20261026T223144Z-nanbeige43-autoround-recommended54-pass1-think-32k", "frontier": "3/4", "color": "#ff6671"},
        {"name": "Nanbeige 4.3 33B · AutoRound · pass 2", "short": "Nanbeige 33B · AutoRound 2", "date": "2026-10-26", "environment": "Dunamis · RTX 5090 · vLLM · 32K", "settings": "Independent second sampled pass; same publisher task-specific settings.", "scores": [[15,17],[22,24],[4,5],[1,3]], "notes": ["Same family count as pass 1; individual outcomes differ.","Same family count as pass 1; individual outcomes differ.","Chat-adapted HumanEval+.","TC-80 excluded; TC-75 retains the native caveat."], "source": "20261026T223144Z-nanbeige43-autoround-recommended54-pass2-think-32k", "frontier": "3/4", "color": "#ff8d82"},
        {"name": "Ciru Flash Next 5.5 · IU4", "short": "Ciru Flash Next 5.5 · IU4", "date": "2026-10-26", "environment": "Ciru · AMD Strix Halo · IU4 / MTP3 · 32K · C1", "settings": "Publisher nonthinking recipe: temperature 0.7, top-p 0.8, top-k 20, presence penalty 1.5.", "scores": [[9,15],[16,24],[4,5],[3,4]], "notes": ["Two contested ARC responses hit the 256-token cap; this is an audited 15-case subset, not the same denominator as 17/17 coverage.","All 24 contested cases completed.","Chat-adapted HumanEval+.","TC-80 passed under the corrected scorer."], "source": "ciru-iu4-v55-fast54-recommended-20261026", "frontier": "IFEval 2/3; ARC frontier withheld from native collection", "color": "#f6c046"}
      ]
    },
    {
      "id": "historical",
      "title": "Historical nonthinking runs",
      "description": "These older regression runs used different or non-publisher-default protocols. They are retained for context and should not be ranked against the publisher-setting group.",
      "runs": [
        {"name":"Nanbeige 4.3 33B · Q6_K", "short":"Nanbeige 33B · Q6_K", "date":"2026-10-26", "environment":"Dunamis · RTX 5090 · llama.cpp / CUDA · 32K", "settings":"Historical nonthinking/default, with pinned seeds in some components.", "scores":[[12,17],[17,23],[3,5],[2,3]], "notes":["17 eligible contested cases.","One contested response hit an output cutoff; 23 eligible.","Five selected coding cases.","TC-80 excluded due to old grader defect."], "source":"20261026T162354Z-nanbeige43-q6k-contested-fast54-nothink", "frontier":"ARC 0/1; IFEval 1/3", "color":"#e5785d"},
        {"name":"Nanbeige 4.3 33B · EXL3 5 bpw", "short":"Nanbeige 33B · EXL3 5bpw", "date":"2026-10-26", "environment":"Dunamis · RTX 5090 · ExLlamaV3 / TabbyAPI · 32K", "settings":"Historical nonthinking/default, with pinned seeds in some components.", "scores":[[12,17],[19,24],[1,5],[2,3]], "notes":["17 eligible contested cases.","24 eligible contested cases.","Five selected coding cases.","TC-80 excluded due to old grader defect."], "source":"20261026T162400Z-nanbeige43-exl3-5bpw-contested-fast54-nothink", "frontier":"ARC 0/1; IFEval 1/3", "color":"#e5865f"},
        {"name":"Nanbeige 4.3 33B · AutoRound historical", "short":"Nanbeige 33B · AutoRound old", "date":"2026-10-26", "environment":"Dunamis · RTX 5090 · vLLM · 32K", "settings":"Historical nonthinking/default; separate from the two publisher-setting passes.", "scores":[[12,17],[17,22],[4,5],[1,3]], "notes":["17 eligible contested cases.","Two contested responses hit output cutoffs; 22 eligible.","Five selected coding cases.","TC-80 excluded due to old grader defect."], "source":"20261026T202803Z-nanbeige43-autoround-attempt3-contested-fast54-nothink", "frontier":"ARC 0/1; IFEval 0/2 eligible", "color":"#dc9162"},
        {"name":"Nanbeige 4.3 33B · NVFP4", "short":"Nanbeige 33B · NVFP4", "date":"2026-10-26", "environment":"Dunamis · RTX 5090 · vLLM · 32K", "settings":"Historical nonthinking/default, with pinned seeds in some components.", "scores":[[11,17],[19,23],[4,5],[1,3]], "notes":["17 eligible contested cases.","One contested response hit a cutoff; 23 eligible.","Five selected coding cases.","TC-80 excluded due to old grader defect."], "source":"20261026T213510Z-nanbeige43-nvfp4-attempt3-contested-fast54-nothink", "frontier":"ARC 0/1; IFEval 0/2 eligible", "color":"#d69e68"},
        {"name":"Ornith 1.5 · Ciru Ultra", "short":"Ornith 1.5 · Ciru Ultra", "date":"2026-10-26", "environment":"Sozo · AMD Strix Halo · 256K · C8", "settings":"Historical nonthinking controlled regression; capped generation.", "scores":[null,[15,24],[5,5],[1,3]], "notes":["ARC family was withheld; no interpretable component score.","24 contested strict IFEval cases.","Five selected coding cases.","TC-80 excluded due to old grader defect."], "source":"20261026T211008Z-sozo-ornith-ultra-fast54", "frontier":"IFEval 0/3; ARC family withheld", "color":"#9d9a76"},
        {"name":"Raon · Ciru Ultra Q8", "short":"Raon · Ciru Ultra Q8", "date":"2026-10-26", "environment":"Sozo · AMD Strix Halo · 256K · C8", "settings":"Historical nonthinking controlled regression; capped generation.", "scores":[[10,17],[15,24],[2,5],[1,3]], "notes":["17 contested ARC cases completed.","Audited one-to-one against the raw strict grades; the collector withheld the family after a frontier cutoff.","One of the three non-passes was a 1-second execution timeout; a 10-second diagnostic returned the expected result.","TC-80 excluded; TC-75 has a clarification-matching defect."], "source":"20261026T212525Z-sozo-raon-ultra-fast54", "frontier":"ARC 0/1; IFEval 0/2 eligible, one cutoff", "color":"#9f967e"}
      ]
    }
  ]
}