{
  "release": "robojev-27b-da",
  "model_revision": "e089a9b69a365744b80765057d86583647af95a9",
  "date": "2026-10-23",
  "metrics": {
    "RoboJev-27B": {
      "so101_auroc": 0.9932,
      "so101_bacc": 0.952,
      "unseen_auroc": 0.8904,
      "unseen_bacc": 0.7816,
      "latency_s": 0.257,
      "usd_per_1000": 0.1919,
      "check_auroc": 0.9844,
      "check_acc": 0.9333,
      "check_n": 135,
      "protocol": "dense-v1"
    },
    "GPT-6 Astra": {
      "so101_auroc": 0.9869,
      "so101_bacc": 0.9329,
      "unseen_auroc": 0.8622,
      "unseen_bacc": 0.7785,
      "latency_s": 6.928,
      "usd_per_1000": 9.2862,
      "check_auroc": 0.9907,
      "check_acc": 0.9259,
      "check_n": 135,
      "protocol": "clip-v1"
    },
    "GPT-6.1 Sol": {
      "so101_auroc": 0.9891,
      "so101_bacc": 0.9284,
      "unseen_auroc": 0.8654,
      "unseen_bacc": 0.7908,
      "latency_s": 5.629,
      "usd_per_1000": 2.083,
      "check_auroc": 0.9898,
      "check_acc": 0.9259,
      "check_n": 135,
      "protocol": "clip-v1"
    },
    "Claude Opus 5.5": {
      "so101_auroc": 0.9777,
      "so101_bacc": 0.9186,
      "unseen_auroc": 0.8615,
      "unseen_bacc": 0.7393,
      "latency_s": 7.118,
      "usd_per_1000": 5.8212,
      "check_auroc": 0.9878,
      "check_acc": 0.9407,
      "check_n": 135,
      "protocol": "clip-v1"
    },
    "Claude Sonnet 5.5": {
      "so101_auroc": 0.9802,
      "so101_bacc": 0.925,
      "unseen_auroc": 0.8687,
      "unseen_bacc": 0.7436,
      "latency_s": 5.165,
      "usd_per_1000": 3.565,
      "check_auroc": 0.9716,
      "check_acc": 0.9259,
      "check_n": 135,
      "protocol": "clip-v1"
    },
    "JEV-27B-VL": {
      "so101_auroc": 0.9827279508808171,
      "so101_bacc": 0.8765134561312905,
      "unseen_auroc": 0.8797431520136572,
      "unseen_bacc": 0.7740746488709552,
      "latency_s": null,
      "usd_per_1000": null,
      "check_auroc": null,
      "check_acc": null,
      "check_n": null,
      "protocol": "clip-v1"
    },
    "TypeSafe Jev 1.14": {
      "so101_auroc": 0.9750100418890227,
      "so101_bacc": 0.8288288288288288,
      "unseen_auroc": 0.8407697679832389,
      "unseen_bacc": 0.7208427097074571,
      "latency_s": 0.721,
      "usd_per_1000": 0.094,
      "check_auroc": null,
      "check_acc": null,
      "check_n": null,
      "protocol": "clip-v1"
    },
    "Perplexity Decider": {
      "so101_auroc": 0.9536351638262466,
      "so101_bacc": 0.8108108108108107,
      "unseen_auroc": 0.8845930006983782,
      "unseen_bacc": 0.7608151625669279,
      "latency_s": 0.899,
      "usd_per_1000": 0.0159,
      "check_auroc": null,
      "check_acc": null,
      "check_n": null,
      "protocol": "clip-v1"
    },
    "Liquid D1": {
      "protocol": "clip-v1",
      "so101_bacc": 0.8279680954840191,
      "latency_s": 0.774,
      "usd_per_1000": 0.0206,
      "so101_auroc": 0.8967980719573077,
      "unseen_bacc": 0.7474295801970978,
      "unseen_auroc": 0.8280340653371615
    },
    "Mercury Decide": {
      "protocol": "clip-v1",
      "so101_bacc": 0.8160899753256441,
      "latency_s": 0.871,
      "usd_per_1000": 0.0152,
      "so101_auroc": 0.9382854191771389,
      "unseen_bacc": 0.7563145029875068,
      "unseen_auroc": 0.8594610848141538
    },
    "Cloudflare Clef": {
      "protocol": "clip-v1",
      "so101_bacc": 0.7747747747747749,
      "latency_s": 1.001,
      "usd_per_1000": 0.2018,
      "so101_auroc": 0.9072416365409997,
      "unseen_bacc": 0.719542950259952,
      "unseen_auroc": 0.802921548847676
    },
    "cloudflare/clef-flash": {
      "protocol": "clip-v1",
      "so101_bacc": 0.7355540253629426,
      "so101_auroc": 0.810036150800482,
      "latency_s": 1.1836887545068748,
      "usd_per_1000": 0.03200110447761194,
      "unseen_bacc": 0.7189415690230465,
      "unseen_auroc": 0.8149588732831535
    },
    "cloudflare/clef-omni": {
      "protocol": "clip-v1",
      "so101_bacc": 0.5630630630630631,
      "so101_auroc": 0.6995466804384002,
      "latency_s": 0.9200090550002642,
      "usd_per_1000": 0.12587014925373136,
      "unseen_bacc": 0.5810021727322108,
      "unseen_auroc": 0.7287576627609219
    },
    "liquid/d1-ultra": {
      "protocol": "clip-v1",
      "so101_bacc": 0.8136799219601767,
      "so101_auroc": 0.918488552246514,
      "latency_s": 0.8931571754510514,
      "usd_per_1000": 0.0432780223880597,
      "unseen_bacc": 0.7353049584852953,
      "unseen_auroc": 0.8586851090245984
    },
    "open/intern-decision-4b": {
      "protocol": "clip-v1",
      "so101_bacc": 0.7286394674929707,
      "so101_auroc": 0.7717908991794342,
      "latency_s": 0.08262790899999928,
      "usd_per_1000": null,
      "unseen_bacc": 0.6876794444013347,
      "unseen_auroc": 0.8080235896640025
    },
    "open/robojev-27b-v1@0.5": {
      "protocol": "clip-v1",
      "so101_bacc": 0.9241980834337522,
      "so101_auroc": 0.9869168531588913,
      "latency_s": 0.11895434550000061,
      "usd_per_1000": null,
      "unseen_bacc": 0.7607181655932336,
      "unseen_auroc": 0.8720997904865369
    },
    "open/robojev-27b": {
      "protocol": "clip-v1",
      "so101_bacc": 0.9228782923050439,
      "so101_auroc": 0.9871463820508406,
      "latency_s": 0.20359430499956943,
      "usd_per_1000": null,
      "unseen_bacc": 0.764568945448902,
      "unseen_auroc": 0.8770660355396912
    },
    "open/robojev-4b-v2": {
      "protocol": "clip-v1",
      "so101_bacc": 0.6621621621621622,
      "so101_auroc": 0.8042692373902565,
      "latency_s": 0.25079799950000137,
      "usd_per_1000": null,
      "unseen_bacc": 0.5731066190734849,
      "unseen_auroc": 0.6349518895010475
    },
    "open/robojev-4b-v3@0.25": {
      "protocol": "clip-v1",
      "so101_bacc": 0.839932288976875,
      "so101_auroc": 0.9496184082171343,
      "latency_s": 0.24857095599999468,
      "usd_per_1000": null,
      "unseen_bacc": 0.7263521378133002,
      "unseen_auroc": 0.820080313494219
    },
    "open/robojev-4b-v3@0.5": {
      "protocol": "clip-v1",
      "so101_bacc": 0.8682791071326104,
      "so101_auroc": 0.9784242841567682,
      "latency_s": 0.2487721580000013,
      "usd_per_1000": null,
      "unseen_bacc": 0.6897260805462869,
      "unseen_auroc": 0.792125785675487
    },
    "open/robojev-4b-v3@0.75": {
      "protocol": "clip-v1",
      "so101_bacc": 0.8926665519022208,
      "so101_auroc": 0.9791415619441097,
      "latency_s": 0.25151374949999905,
      "usd_per_1000": null,
      "unseen_bacc": 0.6409463024753628,
      "unseen_auroc": 0.7731337782261194
    },
    "open/robojev-4b-v3": {
      "protocol": "clip-v1",
      "so101_bacc": 0.8913467607735124,
      "so101_auroc": 0.9784529752682619,
      "latency_s": 0.2298006779999966,
      "usd_per_1000": null,
      "unseen_bacc": 0.6137289516567084,
      "unseen_auroc": 0.7496799099868084
    },
    "openai/gpt-6-luna-decisions": {
      "protocol": "clip-v1",
      "so101_bacc": 0.7477477477477478,
      "so101_auroc": 0.8413381534400642,
      "latency_s": 0.8074028005357832,
      "usd_per_1000": 0.08680186567164179,
      "unseen_bacc": 0.6510533871343214,
      "unseen_auroc": 0.7363428261038255
    },
    "perplexity/pplx-decider-flash-v1.2": {
      "protocol": "clip-v1",
      "so101_bacc": 0.7485224077580765,
      "so101_auroc": 0.9148447810868193,
      "latency_s": 0.8324902235763147,
      "usd_per_1000": 0.007981343283582088,
      "unseen_bacc": 0.7334038178008846,
      "unseen_auroc": 0.8478893458524094
    },
    "respan/span-02": {
      "protocol": "clip-v1",
      "so101_bacc": 0.6541573420554312,
      "so101_auroc": 0.7545762322832387,
      "latency_s": 0.8558713545207866,
      "usd_per_1000": 0.012961343283582089,
      "unseen_bacc": 0.6735760844261659,
      "unseen_auroc": 0.7741522464499108
    },
    "typesafe/jev-1.13": {
      "protocol": "clip-v1",
      "so101_bacc": 0.7941125839215011,
      "so101_auroc": 0.9358466747001779,
      "latency_s": 0.7168990290374495,
      "usd_per_1000": 0.09397484328358209,
      "unseen_bacc": 0.7214440909443625,
      "unseen_auroc": 0.8433401877861411
    },
    "JEV-27B-VL (10 frames)": {
      "protocol": "dense-v1",
      "so101_bacc": 0.8812474895277442,
      "so101_auroc": 0.9790554886096288,
      "latency_s": 0.2496537695005827,
      "usd_per_1000": null,
      "unseen_bacc": 0.7943858151625669,
      "unseen_auroc": 0.8996275316210134
    },
    "open/robojev-27b (10 frames)": {
      "protocol": "dense-v1",
      "so101_bacc": 0.9519997704711081,
      "so101_auroc": 0.9931715154645091,
      "latency_s": 0.25680013499732013,
      "usd_per_1000": null,
      "unseen_bacc": 0.7815822146349034,
      "unseen_auroc": 0.8903934197253046
    },
    "open/robometer-4b": {
      "protocol": "video",
      "so101_bacc": 0.676995466804384,
      "so101_auroc": 0.7768405348023183,
      "latency_s": 0.6086026594999936,
      "usd_per_1000": null,
      "unseen_bacc": 0.5975595561418484,
      "unseen_auroc": 0.8178978815860946
    }
  },
  "paired": {
    "GPT-6 Astra": {
      "so101_bacc": {
        "n": 268,
        "robojev": 0.952,
        "other": 0.9329,
        "diff": 0.0191,
        "ci95": [
          -0.0156,
          0.0521
        ],
        "clear": false
      },
      "unseen_auroc": {
        "n": 459,
        "robojev": 0.8904,
        "other": 0.8622,
        "diff": 0.0282,
        "ci95": [
          0.0049,
          0.0514
        ],
        "clear": true
      },
      "unseen_bacc": {
        "n": 459,
        "robojev": 0.7816,
        "other": 0.7785,
        "diff": 0.0031,
        "ci95": [
          -0.0321,
          0.0396
        ],
        "clear": false
      },
      "check_acc": {
        "n": 135,
        "robojev": 0.9333,
        "other": 0.9259,
        "diff": 0.0074,
        "ci95": [
          -0.0444,
          0.0593
        ],
        "clear": false
      },
      "check_auroc": {
        "n": 135,
        "robojev": 0.9844,
        "other": 0.9907,
        "diff": -0.0062,
        "ci95": [
          -0.0234,
          0.0071
        ],
        "clear": false
      }
    },
    "GPT-6.1 Sol": {
      "so101_bacc": {
        "n": 268,
        "robojev": 0.952,
        "other": 0.9284,
        "diff": 0.0236,
        "ci95": [
          -0.0093,
          0.0578
        ],
        "clear": false
      },
      "unseen_auroc": {
        "n": 458,
        "robojev": 0.8903,
        "other": 0.8654,
        "diff": 0.0249,
        "ci95": [
          0.0014,
          0.0498
        ],
        "clear": true
      },
      "unseen_bacc": {
        "n": 458,
        "robojev": 0.7812,
        "other": 0.7908,
        "diff": -0.0096,
        "ci95": [
          -0.0471,
          0.0282
        ],
        "clear": false
      },
      "check_acc": {
        "n": 135,
        "robojev": 0.9333,
        "other": 0.9259,
        "diff": 0.0074,
        "ci95": [
          -0.0444,
          0.0593
        ],
        "clear": false
      },
      "check_auroc": {
        "n": 135,
        "robojev": 0.9844,
        "other": 0.9898,
        "diff": -0.0053,
        "ci95": [
          -0.0217,
          0.0087
        ],
        "clear": false
      }
    },
    "Claude Opus 5.5": {
      "so101_bacc": {
        "n": 268,
        "robojev": 0.952,
        "other": 0.9186,
        "diff": 0.0334,
        "ci95": [
          -0.0015,
          0.0684
        ],
        "clear": false
      },
      "unseen_auroc": {
        "n": 459,
        "robojev": 0.8904,
        "other": 0.8615,
        "diff": 0.0289,
        "ci95": [
          0.0071,
          0.0512
        ],
        "clear": true
      },
      "unseen_bacc": {
        "n": 459,
        "robojev": 0.7816,
        "other": 0.7393,
        "diff": 0.0423,
        "ci95": [
          0.0086,
          0.0756
        ],
        "clear": true
      },
      "check_acc": {
        "n": 135,
        "robojev": 0.9333,
        "other": 0.9407,
        "diff": -0.0074,
        "ci95": [
          -0.0593,
          0.037
        ],
        "clear": false
      },
      "check_auroc": {
        "n": 135,
        "robojev": 0.9844,
        "other": 0.9878,
        "diff": -0.0033,
        "ci95": [
          -0.0186,
          0.0102
        ],
        "clear": false
      }
    },
    "Claude Sonnet 5.5": {
      "so101_bacc": {
        "n": 268,
        "robojev": 0.952,
        "other": 0.925,
        "diff": 0.027,
        "ci95": [
          -0.0045,
          0.0589
        ],
        "clear": false
      },
      "unseen_auroc": {
        "n": 459,
        "robojev": 0.8904,
        "other": 0.8687,
        "diff": 0.0217,
        "ci95": [
          -0.0001,
          0.0434
        ],
        "clear": false
      },
      "unseen_bacc": {
        "n": 459,
        "robojev": 0.7816,
        "other": 0.7436,
        "diff": 0.038,
        "ci95": [
          0.0049,
          0.0723
        ],
        "clear": true
      },
      "check_acc": {
        "n": 135,
        "robojev": 0.9333,
        "other": 0.9259,
        "diff": 0.0074,
        "ci95": [
          -0.037,
          0.0593
        ],
        "clear": false
      },
      "check_auroc": {
        "n": 135,
        "robojev": 0.9844,
        "other": 0.9716,
        "diff": 0.0129,
        "ci95": [
          -0.0109,
          0.0437
        ],
        "clear": false
      }
    },
    "TypeSafe Jev 1.14": {
      "so101_bacc": {
        "n": 268,
        "robojev": 0.952,
        "other": 0.8288,
        "diff": 0.1232,
        "ci95": [
          0.0744,
          0.1732
        ],
        "clear": true
      },
      "unseen_auroc": {
        "n": 459,
        "robojev": 0.8904,
        "other": 0.8408,
        "diff": 0.0496,
        "ci95": [
          0.026,
          0.0736
        ],
        "clear": true
      },
      "unseen_bacc": {
        "n": 459,
        "robojev": 0.7816,
        "other": 0.7208,
        "diff": 0.0607,
        "ci95": [
          0.0266,
          0.0947
        ],
        "clear": true
      },
      "check_acc": null,
      "check_auroc": null
    },
    "Perplexity Decider": {
      "so101_bacc": {
        "n": 268,
        "robojev": 0.952,
        "other": 0.8108,
        "diff": 0.1412,
        "ci95": [
          0.091,
          0.191
        ],
        "clear": true
      },
      "unseen_auroc": {
        "n": 459,
        "robojev": 0.8904,
        "other": 0.8846,
        "diff": 0.0058,
        "ci95": [
          -0.0105,
          0.0214
        ],
        "clear": false
      },
      "unseen_bacc": {
        "n": 459,
        "robojev": 0.7816,
        "other": 0.7608,
        "diff": 0.0208,
        "ci95": [
          -0.0137,
          0.0544
        ],
        "clear": false
      },
      "check_acc": null,
      "check_auroc": null
    }
  },
  "sample": {
    "episodes": 68555,
    "datasets": 7641,
    "hours": 501.1743
  },
  "protocol": "Each model row lists its protocol: dense-v1 = ten front frames + optional other views; clip-v1 = four front frames; video = whole video. Frontier LLMs answered zero-shot. RoboJev trained on other ArmnetBench sessions.",
  "catalogue_note": "Metadata only. Old four-frame predictions are omitted; the current release has not regraded this catalogue.",
  "featured_models": [
    "RoboJev-27B",
    "GPT-6 Astra",
    "GPT-6.1 Sol",
    "Claude Opus 5.5",
    "Claude Sonnet 5.5",
    "JEV-27B-VL",
    "TypeSafe Jev 1.14",
    "Perplexity Decider",
    "Liquid D1",
    "Mercury Decide",
    "Cloudflare Clef",
    "JEV-27B-VL (10 frames)",
    "open/robometer-4b"
  ]
}