{
  "version": "1.0.1",
  "notebook_version": "V2",
  "mode": "finetune",
  "parent_version": "1.0.0",
  "lineage": [
    "jhu-clsp/ettin-encoder-150m",
    "FalconDec-v1.0.0"
  ],
  "config": {
    "MODE": "finetune",
    "FINETUNE_FROM": "falcondec_runs/FalconDec-v1.0.0",
    "MODEL_NAME": "LightDec",
    "BACKBONE": "jhu-clsp/ettin-encoder-150m",
    "BACKBONE_INIT": "pretrained",
    "PRESET": "full",
    "CUSTOM_DATA_JSONL": "",
    "CUSTOM_WEIGHT": 3.0,
    "INCLUDE_BUILDERS": [],
    "EXCLUDE_BUILDERS": [
      "mind2web"
    ],
    "MAX_LEN": 512,
    "LONG_MAX_LEN": 2048,
    "LONG_OPTS_THRESHOLD": 24,
    "HEAD_MAX_LEN": 192,
    "MAX_TOK_PER_OPT": 24,
    "MAX_OPTS_SINGLE_PASS": 96,
    "INTERACT_LAYERS": 2,
    "INTERACT_HEADS": 8,
    "BATCH_SIZE": 32,
    "TOKENS_PER_BATCH": 16384,
    "GRAD_ACCUM": 1,
    "LR_ENCODER": 4e-05,
    "LR_HEAD": 0.0003,
    "LLRD": 0.9,
    "WEIGHT_DECAY": 0.01,
    "WARMUP_FRAC": 0.06,
    "EPOCHS": 0,
    "EMA_DECAY": 0.999,
    "GRAD_CLIP": 1.0,
    "FINETUNE_LR_SCALE": 0.5,
    "SPHERICAL_W": 0.5,
    "BRIER_W": 0.0,
    "RPS_W": 1.0,
    "TASK_SAMPLING_ALPHA": 0.5,
    "NOTA_PROB": 0.08,
    "RLCD_EPOCHS": 0.0,
    "RLCD_SAMPLES": 4,
    "RLCD_SIGMA": 0.3,
    "RLCD_CE_ANCHOR": 0.5,
    "DEVICE": "cuda",
    "SEED": 42,
    "NUM_WORKERS": 0,
    "USE_COMPILE": false,
    "ATTN_IMPL": "auto",
    "OUTPUT_ROOT": "./falcondec_runs",
    "EXPORT_INT8": true,
    "SIZE_TARGET_MB": 250,
    "DEFER_THRESHOLD": 0.7,
    "EVAL_BASELINE_PROOF_V2": true,
    "BASELINE_MAX_PER_TASK": 120,
    "RUN_CPU_BENCH": true,
    "PUSH_TO_HUB": false,
    "HUB_REPO": ""
  },
  "epochs": 2,
  "best_epoch": 1,
  "rlcd_kept": false,
  "history": [
    {
      "epoch": 1,
      "train_loss": 0.47463753718974483,
      "train_acc": 0.8307763076996187,
      "val_macro": 0.8253118682436162,
      "val_micro": 0.834065625876414,
      "val_nll": 0.4055366800702441
    },
    {
      "epoch": 2,
      "train_loss": 0.36265113689414885,
      "train_acc": 0.8738680252304237,
      "val_macro": 0.8252539304231531,
      "val_micro": 0.8348602411891184,
      "val_nll": 0.4680047751477799
    }
  ],
  "train_minutes": 172.17712485790253,
  "temperatures": [
    [
      1.3723914623260498,
      1.3805755376815796,
      1.5033340454101562,
      1.320592999458313
    ],
    [
      1.3903899192810059,
      1.3903899192810059,
      1.3903899192810059,
      1.3903899192810059
    ],
    [
      1.4738539457321167,
      1.4738539457321167,
      1.4738539457321167,
      1.4738539457321167
    ]
  ],
  "data_counts": {
    "train": 946028,
    "validation": 21394,
    "test": 26597
  },
  "skipped_builders": [],
  "test_overall": {
    "n": 26597.0,
    "micro_acc": 0.7810655337068091,
    "macro_acc": 0.7694407230754166,
    "nll": 0.5288964978022598,
    "brier": 0.2897912847111225,
    "ece": 0.01969622960708085,
    "aurc": 0.06403486126686972,
    "score_mae": 0.4928175845338412,
    "coverage@0.7": 0.6595104711057638,
    "acc_on_covered@0.7": 0.9167094236360527
  },
  "test_per_domain": {
    "agentic": 0.7781070271610528,
    "classification": 0.615,
    "code": 0.8717717677968562,
    "guardrails": 0.7951432854293641,
    "intents": 0.8278333333333333,
    "policy": 0.7873333333333333,
    "reasoning": 0.7006267469170804,
    "support": 1.0,
    "workflows": 0.755
  },
  "test_per_task": [
    {
      "task": "ag_news/topic",
      "n": 500,
      "acc": 0.888,
      "chance": 0.25,
      "nll": 0.28043824258210953,
      "brier": 0.154232526432949,
      "heldout": false,
      "domain": "classification",
      "ece": 0.03968544936180117,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "agentharm/refuse",
      "n": 416,
      "acc": 0.7427884615384616,
      "chance": 0.5,
      "nll": 0.5331332034928402,
      "brier": 0.35988097897984717,
      "heldout": true,
      "domain": "guardrails",
      "ece": 0.06706968007179408,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "anli/nli",
      "n": 498,
      "acc": 0.41566265060240964,
      "chance": 0.33333333333333326,
      "nll": 1.1498472186171984,
      "brier": 0.7016487136039561,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.18407419437624845,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "aqua_rat/math",
      "n": 247,
      "acc": 0.27530364372469635,
      "chance": 0.2,
      "nll": 1.6108178135746645,
      "brier": 0.7991190068538744,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.04157990880823329,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "arc_challenge/mcq",
      "n": 500,
      "acc": 0.46,
      "chance": 0.25006666666666666,
      "nll": 1.2087727874126286,
      "brier": 0.653150732152712,
      "heldout": true,
      "domain": "reasoning",
      "ece": 0.08824656188488007,
      "proof_v2 (card)": 0.308,
      "\u0394 vs proof_v2": 0.15200000000000002
    },
    {
      "task": "arc_easy/mcq",
      "n": 500,
      "acc": 0.594,
      "chance": 0.24980000000000002,
      "nll": 0.9366558539052494,
      "brier": 0.5142523023860749,
      "heldout": true,
      "domain": "reasoning",
      "ece": 0.0377397992014885,
      "proof_v2 (card)": 0.425,
      "\u0394 vs proof_v2": 0.16899999999999998
    },
    {
      "task": "banking77/intent",
      "n": 500,
      "acc": 0.93,
      "chance": 0.3214666666666666,
      "nll": 0.21486007268911725,
      "brier": 0.11047492184269238,
      "heldout": true,
      "domain": "intents",
      "ece": 0.04031616294384006,
      "proof_v2 (card)": 0.883,
      "\u0394 vs proof_v2": 0.04700000000000004
    },
    {
      "task": "banking77/intent_77",
      "n": 300,
      "acc": 0.5333333333333333,
      "chance": 0.012987012987012986,
      "nll": 2.063840282613334,
      "brier": 0.6459762259581862,
      "heldout": true,
      "domain": "intents",
      "ece": 0.14717036068439482,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "bigclonebench/clone",
      "n": 500,
      "acc": 0.952,
      "chance": 0.5,
      "nll": 0.14842138955372502,
      "brier": 0.07646761075289837,
      "heldout": false,
      "domain": "code",
      "ece": 0.015586107730865495,
      "proof_v2 (card)": 0.383,
      "\u0394 vs proof_v2": 0.569
    },
    {
      "task": "bitext/category",
      "n": 500,
      "acc": 1.0,
      "chance": 0.16128124098124094,
      "nll": 0.0018036427325441764,
      "brier": 0.000553129204988908,
      "heldout": false,
      "domain": "support",
      "ece": 0.0015759592056274496,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "bitext/route",
      "n": 500,
      "acc": 1.0,
      "chance": 0.19016313131313134,
      "nll": 0.0011070070102664432,
      "brier": 7.369536871875006e-05,
      "heldout": false,
      "domain": "support",
      "ece": 0.0010853933095931502,
      "proof_v2 (card)": 0.958,
      "\u0394 vs proof_v2": 0.04200000000000004
    },
    {
      "task": "boolq/yes_no",
      "n": 500,
      "acc": 0.816,
      "chance": 0.5,
      "nll": 0.4159143784940243,
      "brier": 0.26228454396614875,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.02717309403419496,
      "proof_v2 (card)": 0.717,
      "\u0394 vs proof_v2": 0.09899999999999998
    },
    {
      "task": "civil_comments/toxic",
      "n": 500,
      "acc": 0.906,
      "chance": 0.5,
      "nll": 0.2400463573904126,
      "brier": 0.13785060332637392,
      "heldout": false,
      "domain": "guardrails",
      "ece": 0.035369446873664855,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "clinc150/intent",
      "n": 500,
      "acc": 0.942,
      "chance": 0.12404474984829318,
      "nll": 0.16339450041492637,
      "brier": 0.08445263661020003,
      "heldout": false,
      "domain": "intents",
      "ece": 0.036179060459136984,
      "proof_v2 (card)": 0.85,
      "\u0394 vs proof_v2": 0.09199999999999997
    },
    {
      "task": "codexglue/code_to_doc",
      "n": 504,
      "acc": 0.9841269841269841,
      "chance": 0.26233465608465617,
      "nll": 0.04367021227094591,
      "brier": 0.02423435580505018,
      "heldout": false,
      "domain": "code",
      "ece": 0.011909238107147682,
      "proof_v2 (card)": 0.969,
      "\u0394 vs proof_v2": 0.0151269841269841
    },
    {
      "task": "codexglue/doc_to_code",
      "n": 504,
      "acc": 0.9761904761904762,
      "chance": 0.2786044973544974,
      "nll": 0.05729491385996503,
      "brier": 0.030591094765289834,
      "heldout": false,
      "domain": "code",
      "ece": 0.01021853999959103,
      "proof_v2 (card)": 0.961,
      "\u0394 vs proof_v2": 0.015190476190476199
    },
    {
      "task": "codexglue/func_name",
      "n": 467,
      "acc": 0.9528907922912205,
      "chance": 0.25499643112062814,
      "nll": 0.17043053883545994,
      "brier": 0.08725536960997705,
      "heldout": false,
      "domain": "code",
      "ece": 0.023257157468183163,
      "proof_v2 (card)": 0.901,
      "\u0394 vs proof_v2": 0.05189079229122051
    },
    {
      "task": "codexglue/lang_id",
      "n": 504,
      "acc": 1.0,
      "chance": 0.2354166666666667,
      "nll": 0.0005584346231148719,
      "brier": 3.0963766792822474e-06,
      "heldout": false,
      "domain": "code",
      "ece": 0.0005574606004214999,
      "proof_v2 (card)": 0.997,
      "\u0394 vs proof_v2": 0.0030000000000000027
    },
    {
      "task": "commonsense_qa/mcq",
      "n": 493,
      "acc": 0.6389452332657201,
      "chance": 0.19999999999999998,
      "nll": 0.904400206106629,
      "brier": 0.47107552904904443,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.04405802657589711,
      "proof_v2 (card)": 0.442,
      "\u0394 vs proof_v2": 0.19694523326572005
    },
    {
      "task": "counsel/critique_quality",
      "n": 201,
      "acc": 0.5920398009950248,
      "chance": 0.3333333333333333,
      "nll": 1.3054839486733487,
      "brier": 0.6194979652829024,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.22704330650135063,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "counsel/step_has_error",
      "n": 201,
      "acc": 0.8009950248756219,
      "chance": 0.5,
      "nll": 0.8227317674037697,
      "brier": 0.3473574496250388,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.16924872979595879,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "devign/vulnerability",
      "n": 500,
      "acc": 0.596,
      "chance": 0.5,
      "nll": 0.6676419580578804,
      "brier": 0.4753553111644763,
      "heldout": false,
      "domain": "code",
      "ece": 0.05009425449371341,
      "proof_v2 (card)": 0.542,
      "\u0394 vs proof_v2": 0.05399999999999994
    },
    {
      "task": "emotion/6way",
      "n": 500,
      "acc": 0.502,
      "chance": 0.16666666666666663,
      "nll": 1.3070907976925372,
      "brier": 0.6468766489597455,
      "heldout": true,
      "domain": "classification",
      "ece": 0.08215402710437775,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "gsm8k/math",
      "n": 500,
      "acc": 0.662,
      "chance": 0.25,
      "nll": 0.7718028125888668,
      "brier": 0.4427847443219045,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.06086966800689696,
      "proof_v2 (card)": 0.275,
      "\u0394 vs proof_v2": 0.387
    },
    {
      "task": "hellaswag/continuation",
      "n": 500,
      "acc": 0.58,
      "chance": 0.25,
      "nll": 0.9953804229423404,
      "brier": 0.5371978334623486,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.07333891946077346,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "hotpotqa/comparison_yes_no",
      "n": 26,
      "acc": 0.8461538461538461,
      "chance": 0.5,
      "nll": 0.43042818936877525,
      "brier": 0.2727966735853732,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.15112700829139125,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "hotpotqa/retrieve",
      "n": 497,
      "acc": 0.8732394366197183,
      "chance": 0.16741720162243304,
      "nll": 0.4343729186731075,
      "brier": 0.20390003436957638,
      "heldout": false,
      "domain": "agentic",
      "ece": 0.08316291382974779,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "humaneval/completion",
      "n": 119,
      "acc": 0.6722689075630253,
      "chance": 0.39355742296918783,
      "nll": 0.7120104984726616,
      "brier": 0.44585727429903876,
      "heldout": true,
      "domain": "code",
      "ece": 0.10659772783768277,
      "proof_v2 (card)": 0.575,
      "\u0394 vs proof_v2": 0.09726890756302531
    },
    {
      "task": "jailbreak/detect",
      "n": 262,
      "acc": 0.9541984732824428,
      "chance": 0.5,
      "nll": 0.10692274935157883,
      "brier": 0.06256811361157136,
      "heldout": false,
      "domain": "guardrails",
      "ece": 0.0232666851455019,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "massive_en/intent",
      "n": 500,
      "acc": 0.906,
      "chance": 0.12958727789085603,
      "nll": 0.26250843075065133,
      "brier": 0.12679194471370903,
      "heldout": false,
      "domain": "intents",
      "ece": 0.022001883208751682,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "mbpp/bugspot",
      "n": 256,
      "acc": 0.73046875,
      "chance": 0.40559895833333326,
      "nll": 0.5201096873639273,
      "brier": 0.3294220878204956,
      "heldout": false,
      "domain": "code",
      "ece": 0.1015084749087691,
      "proof_v2 (card)": 0.475,
      "\u0394 vs proof_v2": 0.25546875
    },
    {
      "task": "mbpp/solution",
      "n": 500,
      "acc": 0.982,
      "chance": 0.25,
      "nll": 0.06299449819558049,
      "brier": 0.03283427225700136,
      "heldout": false,
      "domain": "code",
      "ece": 0.014863664209842696,
      "proof_v2 (card)": 0.992,
      "\u0394 vs proof_v2": -0.010000000000000009
    },
    {
      "task": "mmlu/mcq",
      "n": 500,
      "acc": 0.356,
      "chance": 0.25,
      "nll": 1.370702707901597,
      "brier": 0.738678903692766,
      "heldout": true,
      "domain": "reasoning",
      "ece": 0.10921649205684662,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "mnli/claim",
      "n": 500,
      "acc": 0.838,
      "chance": 0.33333333333333326,
      "nll": 0.41442326370440424,
      "brier": 0.23035331540149503,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.07134601444005968,
      "proof_v2 (card)": 0.492,
      "\u0394 vs proof_v2": 0.346
    },
    {
      "task": "openbookqa/mcq",
      "n": 500,
      "acc": 0.61,
      "chance": 0.25,
      "nll": 1.024268753977958,
      "brier": 0.5355113333378367,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.09224175179004669,
      "proof_v2 (card)": 0.292,
      "\u0394 vs proof_v2": 0.318
    },
    {
      "task": "policy/access_control_transfer",
      "n": 500,
      "acc": 1.0,
      "chance": 0.33333333333333326,
      "nll": 0.0002645651292309594,
      "brier": 1.1635768837216181e-06,
      "heldout": false,
      "domain": "policy",
      "ece": 0.00026422095298772597,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/count_threshold_transfer",
      "n": 500,
      "acc": 0.846,
      "chance": 0.17848809523809525,
      "nll": 0.4453474920627887,
      "brier": 0.239936219050859,
      "heldout": false,
      "domain": "policy",
      "ece": 0.10926523756980895,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/free_shipping_transfer",
      "n": 500,
      "acc": 0.766,
      "chance": 0.5,
      "nll": 0.39174148760084065,
      "brier": 0.2747179012451008,
      "heldout": false,
      "domain": "policy",
      "ece": 0.10137953925132755,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/invoice_overdue_transfer",
      "n": 500,
      "acc": 0.6,
      "chance": 0.5,
      "nll": 0.8301094107693061,
      "brier": 0.5458620193712137,
      "heldout": false,
      "domain": "policy",
      "ece": 0.20100946557521818,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/invoice_total_transfer",
      "n": 500,
      "acc": 0.486,
      "chance": 0.5,
      "nll": 0.69399339812994,
      "brier": 0.5008197939882997,
      "heldout": false,
      "domain": "policy",
      "ece": 0.03210310280323031,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/refund_approval_transfer",
      "n": 500,
      "acc": 0.78,
      "chance": 0.33333333333333326,
      "nll": 0.8478962361293161,
      "brier": 0.38071118861218445,
      "heldout": false,
      "domain": "policy",
      "ece": 0.18587953794002532,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/return_window_transfer",
      "n": 500,
      "acc": 0.998,
      "chance": 0.33333333333333326,
      "nll": 0.02459742584443666,
      "brier": 0.005749234483560448,
      "heldout": false,
      "domain": "policy",
      "ece": 0.019777906179428068,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/sla_urgency_transfer",
      "n": 500,
      "acc": 0.698,
      "chance": 0.25,
      "nll": 0.9843494065143168,
      "brier": 0.49182705060356263,
      "heldout": false,
      "domain": "policy",
      "ece": 0.21234694278240196,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "policy/table_extreme_transfer",
      "n": 500,
      "acc": 0.912,
      "chance": 0.23716666666666666,
      "nll": 0.22187845180515342,
      "brier": 0.12271458379265106,
      "heldout": false,
      "domain": "policy",
      "ece": 0.013898970842361484,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "prompt_injections/detect",
      "n": 116,
      "acc": 0.5775862068965517,
      "chance": 0.5,
      "nll": 1.0671347150446622,
      "brier": 0.628204618498277,
      "heldout": true,
      "domain": "guardrails",
      "ece": 0.2859799378904803,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "qasc/mcq",
      "n": 500,
      "acc": 0.99,
      "chance": 0.125,
      "nll": 0.0315895584258742,
      "brier": 0.01700795004425532,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.006339998722076429,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "sciq/mcq",
      "n": 498,
      "acc": 0.9538152610441767,
      "chance": 0.25,
      "nll": 0.11897318925779031,
      "brier": 0.06399753046801272,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.03059945360244999,
      "proof_v2 (card)": 0.692,
      "\u0394 vs proof_v2": 0.26181526104417674
    },
    {
      "task": "scitail/support",
      "n": 500,
      "acc": 0.964,
      "chance": 0.5,
      "nll": 0.10391226119734347,
      "brier": 0.05616228693429717,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.020327572941780014,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "snli/contradicts",
      "n": 500,
      "acc": 0.986,
      "chance": 0.33333333333333326,
      "nll": 0.048629490838502536,
      "brier": 0.024212671137030826,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.01413622707128527,
      "proof_v2 (card)": 0.892,
      "\u0394 vs proof_v2": 0.09399999999999997
    },
    {
      "task": "snli/must_be_true",
      "n": 500,
      "acc": 0.98,
      "chance": 0.33333333333333326,
      "nll": 0.060363937412155795,
      "brier": 0.02947848703942969,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.02905823123455048,
      "proof_v2 (card)": 0.908,
      "\u0394 vs proof_v2": 0.07199999999999995
    },
    {
      "task": "snli/nli",
      "n": 988,
      "acc": 0.8795546558704453,
      "chance": 0.33333333333333326,
      "nll": 0.3359528171680472,
      "brier": 0.18385021007121183,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.04961462647688053,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "sst5/score",
      "n": 500,
      "acc": 0.416,
      "chance": 0.2,
      "nll": 1.246531030535698,
      "brier": 0.6734807272262489,
      "heldout": true,
      "domain": "classification",
      "ece": 0.04567561271786689,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/agent_trace_observability",
      "n": 500,
      "acc": 0.712,
      "chance": 0.3,
      "nll": 0.7305186161920428,
      "brier": 0.42510022971515116,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.11957757359743115,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/customer_service",
      "n": 500,
      "acc": 0.74,
      "chance": 0.28,
      "nll": 0.6714302012249828,
      "brier": 0.3723593006177565,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.08548467075824738,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/invoice_processing",
      "n": 500,
      "acc": 0.838,
      "chance": 0.35,
      "nll": 0.5010932807754725,
      "brier": 0.26905854083557984,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.12220623338222505,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "typed_decisions/security_incidents",
      "n": 500,
      "acc": 0.73,
      "chance": 0.34,
      "nll": 0.6859722163826227,
      "brier": 0.3929527039537006,
      "heldout": false,
      "domain": "workflows",
      "ece": 0.143579844892025,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "winogrande/blank",
      "n": 500,
      "acc": 0.612,
      "chance": 0.5,
      "nll": 0.7296091562472283,
      "brier": 0.508893263835874,
      "heldout": false,
      "domain": "reasoning",
      "ece": 0.11118709647655486,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    },
    {
      "task": "yelp/score",
      "n": 500,
      "acc": 0.654,
      "chance": 0.2,
      "nll": 0.7885563414096832,
      "brier": 0.4562026964016218,
      "heldout": false,
      "domain": "classification",
      "ece": 0.06915212672948838,
      "proof_v2 (card)": NaN,
      "\u0394 vs proof_v2": NaN
    }
  ],
  "head_to_head": null,
  "env": {
    "torch": "2.11.0+cu128",
    "transformers": "5.17.0",
    "python": "3.14.4",
    "gpu": "NVIDIA GeForce RTX 5090 Laptop GPU"
  }
}