{ "schema_version": "bci-report-questions-in-language-update-v1", "release_id": "questions-in-language-update-20261008", "generated_at": "2026-10-08", "status": "aggregate_preview", "question": "Can an EEG model be asked its questions in words, including ones it was never trained on?", "metric_units": "Balanced accuracy, AUROC, flip rates and chance levels are proportions in [0,1], and so are levels and descriptive contrasts. An entry with unit \"difference of proportions\" is the language arm minus the reference (or the reworded minus the training wording), stored as a proportion: multiply by 100 for percentage points, the unit the margin of 2 pp is in. Unit \"difference of AUROC\" is a difference of AUROC. Unit \"log R\" is the natural log of R, the head's remaining error (1 - AUROC) over the read-off's, and r gives R itself. Log loss is mean binary cross-entropy in nats. Counts of people, seeds, fits, intervals and cells are whole numbers.", "scope": "Route 3 of the decision-research roadmap: one question-conditioned head asked each question by a question number, a label template or a description. Seen questions in their training wording, rewordings and questions the EEG head was never trained on are separate results, never pooled. Every figure compares set-ups; none is a deployment error rate. Every BOAS figure travels with the three stated gaps and the attribution in boas_conditions. The wording lists, the held-out partition and the derangements are in questions-in-language-wordings.json.", "results": { "questions-in-language": { "id": "questions-in-language", "title": "Route 3, questions in language: task identifiers, label templates and natural-language descriptions", "run_id": "decision-route3-v1/20261007", "route": { "site_text": "Compare task identifiers, label templates and natural-language descriptions. Paraphrases and unseen concepts are tested separately and never merged into one zero-shot number.", "boundary": "Questions about negation, location or waveform shape need signal-level ground truth. Writing more prompts does not supply it.", "origin": { "cite": "Yu & Yao, 2026 · Visual Jev: Accurate and Efficient Decisions from Shared Visual Context", "url": "https://arxiv.org/abs/2609.25845", "arxiv": "2609.25845", "note": "The decision-research plan started from Visual Jev (Yu & Yao, 2026), which applies the same encode-once, answer-many idea to images." }, "jev_style": { "en": "“Jev-style” here means the interface of a decision model like TypeSafe’s Jev, applied to EEG: encode the recording once, then answer several explicit, typed questions about it, each with a probability.", "zh": "“Jev-style”（Jev 式）在这里指把 TypeSafe 的 Jev 这类决策模型的接口用在 EEG 上：一段记录只编码一次，再回答关于它的多个明确的、规定输出类型的问题，每个回答都带一个概率。" }, "jev_source": { "company": "TypeSafe AI", "url": "https://typesafe.ai/blog/introducing-system-one-models-and-jev" } }, "design": { "unseen_definition": "Unseen by the EEG head: no training row carries the concept's label, rows whose label is that concept are not used for training, scaling or batch sampling, and its name, synonyms and wordings appear in no training text. The frozen text encoder has read the words. For SSVEP, every text token of a held-out frequency occurs in that rotation's training texts, so only the combination is new.", "question_forms": { "ID": { "vector": "one-hot over the K seen predicates (full rank)", "can_ask_unseen": false }, "TPL": { "vector": "standardised frozen 384-d text embedding of a training frame filled with a label name", "can_ask_unseen": true }, "DESC": { "vector": "standardised frozen text embedding of a description", "can_ask_unseen": true }, "SHUF": { "vector": "TPL embeddings attached to the wrong seen predicates by declared derangements; unseen predicates get their true TPL embedding", "role": "alignment control for TPL only", "can_ask_unseen": true }, "NUM": { "vector": "23-d smooth code: (f-11.9)/2.309 (mean, SD of the 40 grid values) plus 22 Gaussian bumps sigma 0.4 Hz centred 7.9, 8.3, ..., 16.3 Hz", "role": "non-text control", "can_ask_unseen": true }, "B-sh": { "vector": "none: one output per seen question", "role": "reference: shared hidden layer, one output per seen predicate, BCE per predicate (route 2's design, not route 2's numbers)", "can_ask_unseen": false } }, "head": "One head for every arm: standardised features → Linear(d→64) + LayerNorm → FiLM from the question vector → ReLU, dropout 0.25 → one output; binary cross-entropy per (window, seen predicate); AdamW 1e-3, batch 256, 30 epochs (60 for every SSVEP arm by the declared fit-check fallback); no validation set, no early stopping.", "text_encoders": { "bge": { "repo": "BAAI/bge-small-en-v1.5", "commit": "5c38ec7c405ec4b44b94cc5a9bb96e735b38267a", "licence": "mit" }, "minilm": { "repo": "sentence-transformers/all-MiniLM-L6-v2", "commit": "1110a243fdf4706b3f48f1d95db1a4f5529b4d41", "licence": "apache-2.0" }, "e5": { "repo": "intfloat/multilingual-e5-small", "commit": "614241f622f53c4eeff9890bdc4f31cfecc418b3", "licence": "mit" } }, "text_encoders_note": "Frozen at pinned commits and never redistributed.", "margins": { "delta_pp": 2.0, "p3_ratio": 1.25, "p3_definition": "R = mean over people (1-AUROC) language arm / same for prior-corrected read-off; non-inferior if upper bound of log R < log 1.25", "fixed": "at freeze, before any fit; no pilot", "basis": "absolute (user-facing); chance-corrected printed beside" }, "flags": { "difference": "interval excludes 0 (P3: log R = 0)", "margin": "equivalent within delta if interval inside (-delta,+delta); non-inferior if the bound in the language arm's disfavour clears the margin; else margin not met", "inconclusive": "no difference and margin not met" }, "route_sentence_rules": { "always": [ "name domain and level ('frozen CBraMod features' or 'a plain spectrum')", "unseen sentences carry 'unseen by the EEG head, not by the text encoder'", "never pool rewording and unseen results", "print both flags" ], "margin_tested_entries": [ "P1", "P2", "P3", "P4 vs CCA" ], "first_match_order": [ { "rule": 0, "if": "a gate fails", "then": "the gate's sentence" }, { "rule": 1, "if": "difference against the language arm and margin not met", "then": "'{subject} {cost / fell short by} {x} [{lo}, {hi}].'" }, { "rule": 2, "if": "difference against and margin met", "then": "'{subject} was lower by {x} [{lo}, {hi}], within the {margin}.'" }, { "rule": 3, "if": "difference for the language arm", "then": "'{subject} did better by {x} [{lo}, {hi}]' + ', within the margin of equivalence' if equivalent" }, { "rule": 4, "if": "no difference and equivalent", "then": "'{subject} showed no measurable difference (equivalent within the {margin}).'" }, { "rule": 5, "if": "no difference and non-inferior", "then": "'{subject} was not worse by more than the {margin}.'" }, { "rule": 6, "if": "otherwise", "then": "'{subject}: inconclusive at this sample size.'" } ], "difference_only_entries": { "entries": [ "P4 vs NN(TPL)", "P4 vs NUM", "P5" ], "words": [ "better", "worse", "no difference shown" ], "never": "no differently" }, "conjunctions": "a summary over several entries only when all reach the same rule; otherwise one sentence per entry" }, "uncertainty": "Hierarchical paired bootstrap: 2,000 draws, seed 20261007; people (BOAS by person), then seed indices within each outer fold (shared across SSVEP rotations), then held-out wordings within kind for a1 (12 frames), a3 (10 descriptions) and the unseen descriptions; the three a2 synonym indices are fixed. Percentile 95% intervals. Single-seed entries resample people only and carry no training-run variance.", "multiplicity": "35 pre-declared comparisons, not corrected for multiplicity; each interval is a 95% interval for its own comparison.", "expected_inconclusive": [ "every EESM19 entry", "every single-seed S entry" ], "a2_note": "a2 entries rest on three fixed synonyms; worded 'for these three synonyms'", "metric_note": "Per-person accuracy is averaged equally over people; it compares set-ups and is not a deployment error rate.", "freeze": "Protocol frozen 2026-10-07 13:37 UTC; amended at stage 0b (wording hashes, canary revisions 4 and 5) and once for engineering (EA-1) before the first stage-1 fit; every key that defines the 35 entries is identical throughout. Test labels were masked until each block's frozen-predictions manifest was logged. Nothing was selected on test results; no fit failed and nothing was re-run." }, "key_findings": [ { "id": "seen-sleep", "en": "Sleep (BOAS, 100 people, frozen CBraMod features, 3 seeds): asking the seen five-stage question with a label template or a description instead of a question number made no measurable difference. TPL - ID -0.05 pp [-0.70, +0.61], DESC - ID +0.02 pp [-0.59, +0.62]; both equivalent within the 2-pp margin.", "zh": "睡眠（BOAS，100 人，冻结 CBraMod 特征，3 个种子）：问已见过的五期问题时，用标签模板或描述代替题号，没有可测差别。TPL − ID -0.05 个百分点 [-0.70, +0.61]，DESC − ID +0.02 个百分点 [-0.59, +0.62]；两者都在 2 个百分点的界值内等效。", "cites": [ "P-sleep-L1/0", "P-sleep-L1/1" ] }, { "id": "seen-ssvep", "en": "SSVEP (BETA, 70 people, 3 seeds, 5 rotations): on the seen 32-way task, language cost accuracy at both levels. TPL - ID -5.96 pp [-6.84, -5.08] on a plain spectrum (L0) and -3.49 pp [-4.10, -2.93] on frozen CBraMod features (L1); DESC - ID -5.46 and -3.37 pp. Margin not met in all four.", "zh": "SSVEP（BETA，70 人，3 个种子，5 种留出轮换）：在已见过的 32 选 1 任务上，用语言提问在两个层级上都有代价。TPL − ID 在普通频谱（L0）上为 -5.96 个百分点 [-6.84, -5.08]，在冻结 CBraMod 特征（L1）上为 -3.49 个百分点 [-4.10, -2.93]；DESC − ID 为 -5.46 和 -3.37 个百分点。四项都未达界值。", "cites": [ "P-ssvep-L0/0", "P-ssvep-L1/0" ] }, { "id": "rewording", "en": "Rewording a seen question (BOAS): a new sentence frame cost nothing measurable (-0.46 pp [-1.33, +0.16], equivalent), but the three synonyms per stage cost 12.60 pp [11.43, 13.74] and new descriptions 20.87 pp [13.71, 29.49]. On BETA rewording lowered accuracy by 1.33 to 4.13 pp; only the L1 synonyms stayed within the 2-pp margin. Rewording and unseen questions are separate rows, never pooled.", "zh": "改写已见过的问题（BOAS）：换一个新句式没有可测代价（-0.46 个百分点 [-1.33, +0.16]，等效），但每期 3 个同义词让准确率下降 12.60 个百分点 [11.43, 13.74]，新描述下降 20.87 个百分点 [13.71, 29.49]。BETA 上改写的代价为 1.33 到 4.13 个百分点，只有 L1 的同义词留在 2 个百分点界值内。改写与未见问题分开列出，从不合并。", "cites": [ "P-sleep-L1/2", "P-sleep-L1/3", "P-sleep-L1/4" ] }, { "id": "unseen-sleep", "en": "Unseen sleep questions (asleep; N1, N2 or N3 sleep; light sleep), never trained by the EEG head: in all 8 wording conditions the head fell short of adding up its own stage answers, by a ratio of remaining error R from 2.42 [2.07, 2.85] (N1, N2 or N3 sleep, explicit wording) to 33.80 [23.09, 52.82] (\"asleep\").", "zh": "未见过的睡眠问题（睡着了吗；是否 N1、N2 或 N3 期；是否浅睡），EEG 分类头从未训练过：8 种问法下，分类头都不如把它自己的各期答案加起来，剩余误差之比 R 从 2.42 [2.07, 2.85]（N1、N2 或 N3 期，显式问法）到 33.80 [23.09, 52.82]（“睡着”）。", "cites": [ "P-sleep-L1/5", "P-sleep-L1/7" ] }, { "id": "unseen-ssvep", "en": "Unseen flicker frequencies (BETA): asked about a frequency it was never trained on, the head fell far short of CCA, which needs no training: 52.21 pp and 61.94 pp in the 8-way at L0 and L1 (head 28.7% and 19.0%, CCA 80.9%; chance 12.5%). It also did worse than averaging its own answers for the two neighbouring seen frequencies, and worse than a numeric code of the frequency, at both levels.", "zh": "未见过的闪烁频率（BETA）：问到从未训练过的频率时，分类头远不如无需训练的 CCA：8 选 1 在 L0 和 L1 上分别低 52.21 和 61.94 个百分点（分类头 28.7% 和 19.0%，CCA 80.9%；随机水平 12.5%）。在两个层级上，它也不如把相邻两个已见频率的答案取平均，也不如用数字编码表示频率。", "cites": [ "P-ssvep-L0/5", "P-ssvep-L1/5" ] }, { "id": "alignment", "en": "Is the wording-stage link used? For explicit wordings (\"N1 or N2 sleep\") yes: attaching the same templates to the wrong stages lowered unseen AUROC by 0.629 [0.603, 0.654]. For implicit names (\"asleep\", \"light sleep\") it was not shown: the wrong-stage control scored higher by 0.378 [0.253, 0.501], and the head ranked windows in reverse (AUROC 0.317 for \"asleep\", 0.244 for \"light sleep\").", "zh": "分类头是否用到了“问法—分期”的对应关系？显式问法（如“N1 或 N2 期睡眠”）用到了：把同样的模板接到错误的分期上，未见问题的 AUROC 降低 0.629 [0.603, 0.654]。隐式名称（“睡着”“浅睡”）没有显示出用到：错配对照反而高 0.378 [0.253, 0.501]，分类头的排序是反的（“睡着”AUROC 0.317，“浅睡” 0.244）。", "cites": [ "P-sleep-L1/14", "P-sleep-L1/13" ] }, { "id": "negation", "en": "Negation (boundary probe, BOAS): \"not X\" wordings and \"non-REM sleep\" were answered as if they asked for X (AUROC 0.021 to 0.135 against the truth of the negated question; \"non-REM sleep\" 0.048). Writing more prompts does not supply signal-level ground truth.", "zh": "否定（边界探针，BOAS）：“不是 X”类问法和“非快速眼动睡眠”被当成在问 X 本身（对被否定问题真值的 AUROC 为 0.021 到 0.135；“非快速眼动睡眠” 0.048）。多写提示词并不能提供信号层面的真值。", "cites": [ "secondary/S6" ] }, { "id": "secondary", "en": "Secondary (single seed unless stated; declared likely inconclusive): EESM19 (20 people, 3 seeds), REVE-L features, the MiniLM encoder and Chinese wordings all repeat the primary pattern. On motor imagery (OpenBMI, 51 people, 3 seeds) the unseen \"imagining a hand movement\" came closest: R 1.08 [1.05, 1.11], a difference but within the 25% margin. On Wearable-102 (102 people, off-grid frequencies) the head fell short of CCA by 48.18 pp (dry) and 62.92 pp (wet).", "zh": "次要分析（除注明外为单种子；预先声明多半不确定）：EESM19（20 人，3 个种子）、REVE-L 特征、MiniLM 文本编码器和中文问法都重复了主分析的模式。运动想象（OpenBMI，51 人，3 个种子）上的未见问题“想象一个手部动作”最接近读出：R 1.08 [1.05, 1.11]，有差别但在 25% 界值内。Wearable-102（102 人，频率不在 BETA 网格上）上分类头比 CCA 低 48.18 个百分点（干电极）和 62.92 个百分点（湿电极）。", "cites": [ "secondary/S2/5", "secondary/S12", "secondary/S8-L0" ] }, { "id": "pre-run-checks", "en": "Pre-run checks, disclosed: the training-label permutation canary failed twice before any result (revision 3: 1 of 38 intervals; revision 4: 3 of 38, all on BOAS, one below chance). The owner approved revision 5, overriding revision 4's \"no further revision\": 20 heads per configuration, exact per-person chance and a two-level bootstrap; it passed (0 of 38). The failure pattern fits an interval that was too narrow for a single head rather than a leak: single shuffled-label heads missed chance in both directions (66 above, 59 below of 760). The token rule for the Chinese encoder was reported instead of stopping the run (S0b-6).", "zh": "运行前检查（已披露）：训练标签置换金丝雀在任何结果出来之前失败过两次（修订 3：38 个区间中 1 个；修订 4：38 个中 3 个，全在 BOAS 上，其中 1 个低于随机水平）。所有者批准了修订 5，并推翻了修订 4 写的“不再修订”：每个配置训练 20 个头，用每人的精确随机水平，做两层 bootstrap；结果通过（38 个中 0 个）。失败更像是检查本身的区间太窄，而不是泄漏：单个打乱标签训练出的头在两个方向上都会偏离随机水平（760 次中 66 次偏高、59 次偏低）。中文编码器的分词规则改为报告而不停止运行（S0b-6）。", "cites": [ "pre_run_checks" ] }, { "id": "numeracy", "en": "The declared numeracy prediction did not activate: its premise (Spearman rho below 0.3 for every training frame of bge-small-en-v1.5) was false (0.53 to 0.59). The 8-way TPL - NUM flag reads \"NUM higher\" at both levels anyway.", "zh": "预先声明的“数感”预测没有触发：它的前提（bge-small-en-v1.5 的每个训练句式 Spearman rho 都低于 0.3）不成立（0.53 到 0.59）。尽管如此，8 选 1 的 TPL − NUM 差别标记在两个层级上都是“NUM 更高”。", "cites": [ "pre_run_checks" ] } ], "route_answer": { "rule": "boundaries_for_any_page: rewording and unseen results are separate rows and never pooled; 35 pre-declared comparisons, not corrected for multiplicity", "seen_training_wording_P1": { "sleep_BOAS_L1": "equivalent within 2 pp (TPL and DESC against ID)", "ssvep_BETA": "language cost accuracy at both levels (TPL 6.0 / 3.5 pp, DESC 5.5 / 3.4 pp at L0 / L1; margin not met)" }, "rewording_P2": { "new_sentence_frame_a1": "free on BOAS (equivalent); cost 4.0 / 2.7 pp on BETA L0 / L1", "synonym_a2_for_these_three_synonyms": "cost 12.6 pp on BOAS; 1.8 pp on BETA L0; within the 2-pp margin on BETA L1 (lower by 1.3 pp)", "new_description_a3": "cost 20.9 pp on BOAS; 4.1 / 2.9 pp on BETA L0 / L1" }, "unseen_P3_P4_P5": { "sleep_compositional_P3": "every unseen question in every wording fell short of the read-off from the head's own stage answers (8 of 8 at rule 1; R 2.42 to 33.80)", "ssvep_unseen_frequency_P4": "far below training-free CCA (8-way 52.2 / 61.9 pp, neighbour 3-way 62.4 / 54.7 pp short at L0 / L1); worse than the head's own neighbour average and than a numeric code in the 8-way at both levels; neighbour 3-way against the numeric code: no difference shown at L0, better at L1 (frequency and stimulus phase)", "alignment_P5": "explicit template wordings use the wording-stage link (TPL - SHUF +0.629 AUROC); for the implicit wordings (\"asleep\", \"light sleep\") it was not shown to be used (SHUF higher by 0.378 AUROC)" } }, "primary": { "entries_count": 35, "multiplicity": "35 pre-declared comparisons, not corrected for multiplicity; each interval is a 95% interval for its own comparison.", "blocks": [ { "block": "P-ssvep-L0", "dataset": "BETA", "domain": "ssvep", "level": "L0", "level_words": "a plain spectrum (L0)", "people": 70, "seeds": 3, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.7075 } }, "entries": [ { "key": "P-ssvep-L0/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.059593253968253976, "interval_95": [ -0.0683783482142857, -0.050755828373015864 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Asking a seen question with a template instead of a question number cost 6.0 pp [5.1, 6.8].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.061515616999487974, "interval_95": [ -0.07058410138248847, -0.05239311315924218 ] } }, { "key": "P-ssvep-L0/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.054605654761904765, "interval_95": [ -0.0626162574404762, -0.0465581287202381 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Asking a seen question with a description instead of a question number cost 5.5 pp [4.7, 6.3].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.056367127496159755, "interval_95": [ -0.06463613671274963, -0.04806000384024579 ] } }, { "key": "P-ssvep-L0/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.04013516865079365, "interval_95": [ -0.06527120535714286, -0.021193157862103174 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "SSVEP (BETA), a plain spectrum: Rewording by a new sentence frame cost 4.0 pp [2.1, 6.5].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.04142985151049667, "interval_95": [ -0.06737672811059908, -0.021876808115719407 ] } }, { "key": "P-ssvep-L0/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.01835069444444443, "interval_95": [ -0.02097827794312168, -0.015709821428571424 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Rewording by a synonym, for these three synonyms cost 1.8 pp [1.6, 2.1].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.01894265232974909, "interval_95": [ -0.021654996586448188, -0.016216589861751146 ] } }, { "key": "P-ssvep-L0/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.04129761904761905, "interval_95": [ -0.05581729910714286, -0.03105386904761904 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "SSVEP (BETA), a plain spectrum: Rewording by a new description cost 4.1 pp [3.1, 5.6].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.04262980030721966, "interval_95": [ -0.057617857142857144, -0.03205560675883255 ] } }, { "key": "P-ssvep-L0/5", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, 8-way unseen", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.5221130952380951, "interval_95": [ -0.5512760416666668, -0.489888640873016 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 52.2 pp [49.0, 55.1].", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.5967006802721088, "interval_95": [ -0.630029761904762, -0.559872732426304 ] } }, { "key": "P-ssvep-L0/6", "id": "P4", "type": "P4_nn", "kind": "unseen", "label": "TPL - NN(TPL), 8-way unseen", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.10975198412698413, "interval_95": [ -0.1223635912698413, -0.09666443452380954 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies.", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.12543083900226756, "interval_95": [ -0.13984410430839006, -0.11047363945578234 ] } }, { "key": "P-ssvep-L0/7", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, 8-way unseen", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.1277380952380952, "interval_95": [ -0.14517881944444447, -0.10989980158730159 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number.", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.14598639455782308, "interval_95": [ -0.1659186507936508, -0.12559977324263039 ] } }, { "key": "P-ssvep-L0/8", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, neighbour 3-way", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.6236309523809523, "interval_95": [ -0.6586743551587303, -0.5868829365079365 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors), the head, compared with CCA, which needs no training, fell short by 62.4 pp [58.7, 65.9].", "display_edits": [], "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.9472875226039783, "interval_95": [ -1.000518007836046, -0.8914677516576252 ] } }, { "key": "P-ssvep-L0/9", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, neighbour 3-way", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.012410714285714282, "interval_95": [ -0.031682043650793655, 0.00850223214285713 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder): no difference shown against giving it as a number.", "display_edits": [ "W2a" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder) did no difference shown against giving it as a number.", "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.01885171790235081, "interval_95": [ -0.048124623267028335, 0.012914783001808301 ] } } ] }, { "block": "P-ssvep-L1", "dataset": "BETA", "domain": "ssvep", "level": "L1", "level_words": "frozen CBraMod features (L1)", "people": 70, "seeds": 3, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.7075 } }, "entries": [ { "key": "P-ssvep-L1/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.03493551587301588, "interval_95": [ -0.040996589781746036, -0.029288132440476188 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Asking a seen question with a template instead of a question number cost 3.5 pp [2.9, 4.1].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.03606246799795188, "interval_95": [ -0.042319060419866876, -0.030232910906298 ] } }, { "key": "P-ssvep-L1/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.03369419642857143, "interval_95": [ -0.039774460565476184, -0.028036365327380967 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Asking a seen question with a description instead of a question number cost 3.4 pp [2.8, 4.0].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.03478110599078341, "interval_95": [ -0.041057507680491545, -0.028940764208909384 ] } }, { "key": "P-ssvep-L1/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.02715649801587301, "interval_95": [ -0.04429645647321429, -0.014337394593253979 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "SSVEP (BETA), frozen CBraMod features: Rewording by a new sentence frame cost 2.7 pp [1.4, 4.4].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.02803251408090117, "interval_95": [ -0.04572537442396314, -0.014799891193036365 ] } }, { "key": "P-ssvep-L1/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.013297784391534388, "interval_95": [ -0.01533279596560847, -0.01137398726851852 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "equivalent within the margin", "rule": 2, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Rewording by a synonym, for these three synonyms was lower by 1.3 pp [1.1, 1.5], within the 2-pp margin.", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.013726745178358079, "interval_95": [ -0.01582740228707971, -0.01174089008363202 ] } }, { "key": "P-ssvep-L1/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.029495535714285717, "interval_95": [ -0.038063206845238076, -0.022550539434523814 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "SSVEP (BETA), frozen CBraMod features: Rewording by a new description cost 2.9 pp [2.3, 3.8].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.030447004608294935, "interval_95": [ -0.039291052227342534, -0.023277976190476196 ] } }, { "key": "P-ssvep-L1/5", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, 8-way unseen", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.6194146825396826, "interval_95": [ -0.6565200892857143, -0.5806168154761905 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 61.9 pp [58.1, 65.7].", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.7079024943310658, "interval_95": [ -0.7503086734693877, -0.663562074829932 ] } }, { "key": "P-ssvep-L1/6", "id": "P4", "type": "P4_nn", "kind": "unseen", "label": "TPL - NN(TPL), 8-way unseen", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.011587301587301587, "interval_95": [ -0.01973313492063492, -0.003729910714285721 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies.", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.013242630385487527, "interval_95": [ -0.022552154195011335, -0.004262755102040825 ] } }, { "key": "P-ssvep-L1/7", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, 8-way unseen", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.02462301587301587, "interval_95": [ -0.03234201388888888, -0.016884672619047615 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number.", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.028140589569160993, "interval_95": [ -0.03696230158730158, -0.019296768707482988 ] } }, { "key": "P-ssvep-L1/8", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, neighbour 3-way", "people": 70, "seeds": 3, "single_seed": false, "estimate": -0.5466765873015872, "interval_95": [ -0.5776631944444446, -0.5146626984126985 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors (frequency and stimulus phase)), the head, compared with CCA, which needs no training, fell short by 54.7 pp [51.5, 57.8].", "display_edits": [], "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.8303948161543097, "interval_95": [ -0.8774630801687766, -0.7817661241711876 ] } }, { "key": "P-ssvep-L1/9", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, neighbour 3-way", "people": 70, "seeds": 3, "single_seed": false, "estimate": 0.15606150793650794, "interval_95": [ 0.1425096726190476, 0.16931820436507936 ], "unit": "difference of proportions", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features: Stating the unseen frequency in words (neighbour 3-way, frequency and stimulus phase) (unseen by the EEG head, not by the text encoder) did better than giving it as a number.", "display_edits": [], "chance_corrected": { "chance": 0.3416666666666667, "estimate": 0.2370554550934298, "interval_95": [ 0.21647038878842673, 0.25719220916214586 ] } } ] }, { "block": "P-sleep-L1", "dataset": "BOAS", "domain": "sleep", "level": "L1", "level_words": "frozen CBraMod features (L1)", "people": 100, "seeds": 3, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.76 } }, "entries": [ { "key": "P-sleep-L1/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 3, "single_seed": false, "estimate": -0.0004557702451403065, "interval_95": [ -0.007005544594304308, 0.006055389355442171 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.0005697128064253831, "interval_95": [ -0.008756930742880384, 0.007569236694302713 ] } }, { "key": "P-sleep-L1/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 3, "single_seed": false, "estimate": 0.00020790134365992086, "interval_95": [ -0.005919418974507645, 0.006155216335181857 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": 0.00025987667957490105, "interval_95": [ -0.007399273718134556, 0.007694020418977321 ] } }, { "key": "P-sleep-L1/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 100, "seeds": 3, "single_seed": false, "estimate": -0.004584179169633986, "interval_95": [ -0.013257442253035902, 0.001626046243712444 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "Sleep (BOAS), frozen CBraMod features: Rewording by a new sentence frame showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.005730223962042483, "interval_95": [ -0.016571802816294876, 0.002032557804640555 ] } }, { "key": "P-sleep-L1/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 100, "seeds": 3, "single_seed": false, "estimate": -0.1260229011733984, "interval_95": [ -0.13743705557965524, -0.1143482469202063 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Rewording by a synonym, for these three synonyms cost 12.6 pp [11.4, 13.7].", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.15752862646674798, "interval_95": [ -0.17179631947456905, -0.14293530865025786 ] } }, { "key": "P-sleep-L1/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 100, "seeds": 3, "single_seed": false, "estimate": -0.2086946839227036, "interval_95": [ -0.2949034556756221, -0.13707409386710195 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features: Rewording by a new description cost 20.9 pp [13.7, 29.5].", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.2608683549033795, "interval_95": [ -0.3686293195945276, -0.17134261733387743 ] } }, { "key": "P-sleep-L1/5", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 3.5204734043887447, "interval_95": [ 3.1391948457215944, 3.966869070170286 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 33.80 [23.09, 52.82] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 33.80042594689614, "interval_95": [ 23.08527216827499, 52.81889941704095 ] } }, { "key": "P-sleep-L1/6", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 1.9509611023946816, "interval_95": [ 1.6226998636046974, 2.356697493734992 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 7.04 [5.07, 10.56] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 7.035446116356673, "interval_95": [ 5.066751404439514, 10.556032460741271 ] } }, { "key": "P-sleep-L1/7", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL NREM (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 0.8830636662420241, "interval_95": [ 0.7263246931564886, 1.0471839916429677 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.42 [2.07, 2.85] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 2.418297224437661, "interval_95": [ 2.0674680475322207, 2.8496152684533356 ] } }, { "key": "P-sleep-L1/8", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 2.267133170986099, "interval_95": [ 2.072465199342213, 2.4658786289125794 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 9.65 [7.94, 11.77] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 9.651691365086023, "interval_95": [ 7.944383486785048, 11.773822429479882 ] } }, { "key": "P-sleep-L1/9", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 1.5194671184716142, "interval_95": [ 1.3427618149230514, 1.6979654495979608 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 4.57 [3.83, 5.46] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 4.569789389847512, "interval_95": [ 3.829605575238713, 5.462821691597262 ] } }, { "key": "P-sleep-L1/10", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC asleep (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 3.4437091648492375, "interval_95": [ 2.912999206022445, 3.956268243130789 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 31.30 [18.41, 52.26] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 31.302850536381328, "interval_95": [ 18.411937032309563, 52.26193276932236 ] } }, { "key": "P-sleep-L1/11", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC NREM (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 1.2989735597978829, "interval_95": [ 0.79317175217099, 1.7442873060959232 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 3.67 [2.21, 5.72] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 3.6655322862927764, "interval_95": [ 2.210396148478437, 5.721822114185041 ] } }, { "key": "P-sleep-L1/12", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC light (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 3, "single_seed": false, "estimate": 1.4166472904115102, "interval_95": [ 0.9025167262672134, 1.8563915293782929 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 4.12 [2.47, 6.40] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 4.123273102904075, "interval_95": [ 2.4658010549062013, 6.40059867768049 ] } }, { "key": "P-sleep-L1/13", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the implicit cells", "people": 100, "seeds": 3, "single_seed": false, "estimate": -0.37793583276713855, "interval_95": [ -0.5012919288739838, -0.25335907907909777 ], "unit": "difference of AUROC", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: for the implicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages raised the unseen answers by 0.378 AUROC [0.253, 0.501], so the link between wording and stage was not shown to be used.", "display_edits": [] }, { "key": "P-sleep-L1/14", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the explicit cells", "people": 100, "seeds": 3, "single_seed": false, "estimate": 0.6288699994116821, "interval_95": [ 0.6025033523039122, 0.6538525063023334 ], "unit": "difference of AUROC", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features: for the explicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages lowered the unseen answers by 0.629 AUROC [0.603, 0.654], so the link between wording and stage was shown to be used.", "display_edits": [] } ] } ], "family_sentences": { "P1": { "summary_allowed": false, "rule": null, "entries": [ "P-ssvep-L0/0", "P-ssvep-L0/1", "P-ssvep-L1/0", "P-ssvep-L1/1", "P-sleep-L1/0", "P-sleep-L1/1" ], "summary_not_written": "rules differ (1 on SSVEP, 4 on BOAS): one sentence per entry" }, "P2": { "summary_allowed": false, "rule": null, "entries": [ "P-ssvep-L0/2", "P-ssvep-L0/3", "P-ssvep-L0/4", "P-ssvep-L1/2", "P-ssvep-L1/3", "P-ssvep-L1/4", "P-sleep-L1/2", "P-sleep-L1/3", "P-sleep-L1/4" ], "summary_not_written": "rules differ (1, 2 and 4): one sentence per entry" }, "P3": { "summary_allowed": true, "rule": 1, "entries": [ "P-sleep-L1/5", "P-sleep-L1/6", "P-sleep-L1/7", "P-sleep-L1/8", "P-sleep-L1/9", "P-sleep-L1/10", "P-sleep-L1/11", "P-sleep-L1/12" ], "summary": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, in N1, N2 or N3 sleep, or in light sleep, none of which it was trained on (unseen by the EEG head, not by the text encoder), the head fell short of adding up its own stage answers in all 8 wording conditions (templates with implicit and explicit names; new descriptions): R from 2.42 [2.07, 2.85] to 33.80 [23.09, 52.82] (ratio of remaining error; 100 people, 3 seeds)." }, "P4": { "summary_allowed": false, "rule": null, "entries": [ "P-ssvep-L0/5", "P-ssvep-L0/6", "P-ssvep-L0/7", "P-ssvep-L0/8", "P-ssvep-L0/9", "P-ssvep-L1/5", "P-ssvep-L1/6", "P-ssvep-L1/7", "P-ssvep-L1/8", "P-ssvep-L1/9" ], "summary_not_written": "rules differ (1 and -1): one sentence per entry" }, "P5": { "summary_allowed": true, "rule": -1, "entries": [ "P-sleep-L1/13", "P-sleep-L1/14" ], "summary_not_written": "both entries are difference-only (rule -1), so a summary is allowed, but they point in opposite directions (explicit: TPL higher; implicit: SHUF higher); a summary would hide that, so none is written (W2d)" } }, "conjunction_rule": "route_sentence_rules.conjunctions: a summary over several entries only when all reach the same rule; otherwise one sentence per entry", "display_edits": { "W2a": "literal template fill \"did no difference shown against\" rewritten as \": no difference shown against\" (A2 D-1, A4 D-4)", "W2b-minilm": "prefix names the text encoder (A4 O-1): \", text encoder all-MiniLM-L6-v2\"", "W2b-zh": "prefix names the language and encoder (A4 O-1): \", Chinese wordings with multilingual-e5-small\"", "W2c": "REVE-L neighbour 3-way carries the phase caveat the template gives L1 (A4 O-2)" } }, "descriptive": { "P-ssvep-L0": { "levels": { "ID seen_ba": { "estimate": 0.32056547619047615, "interval_95": [ 0.2878588169642858, 0.3518826264880952 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.29864823348694314, "interval_95": [ 0.26488652073732727, 0.3309756144393241 ] } }, "ID nn8": { "estimate": 0.4432142857142857, "interval_95": [ 0.4084174107142857, 0.4757150297619048 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.36367346938775513, "interval_95": [ 0.32390561224489794, 0.4008171768707483 ] } }, "ID m8": { "estimate": 0.5858482142857143, "interval_95": [ 0.5474590773809522, 0.6232898065476189 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.5266836734693878, "interval_95": [ 0.4828103741496597, 0.5694740646258502 ] } }, "ID m3": { "estimate": 0.5093201754385965, "interval_95": [ 0.4881097274436089, 0.5301221804511277 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.2546635576282479, "interval_95": [ 0.22244515561054518, 0.2862615399257636 ] } }, "TPL seen_ba": { "estimate": 0.26097222222222227, "interval_95": [ 0.23491108630952384, 0.28585627480158726 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.23713261648745526, "interval_95": [ 0.21023079877112139, 0.26281938044034814 ] } }, "TPL u8": { "estimate": 0.28690476190476194, "interval_95": [ 0.2694930555555555, 0.30545684523809513 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.18503401360544222, "interval_95": [ 0.1651349206349206, 0.206236394557823 ] } }, "TPL u3": { "estimate": 0.19333333333333336, "interval_95": [ 0.18231125992063493, 0.20467435515873011 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.2253164556962025, "interval_95": [ -0.2420588456901748, -0.20808958710066314 ] } }, "TPL nn8": { "estimate": 0.3966567460317461, "interval_95": [ 0.36779489087301587, 0.42474528769841263 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.31046485260770984, "interval_95": [ 0.2774798752834467, 0.34256604308390015 ] } }, "TPL m8": { "estimate": 0.5312921626984127, "interval_95": [ 0.49567832341269835, 0.5666230778769841 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.4643339002267574, "interval_95": [ 0.4236323696145124, 0.5047120890022675 ] } }, "TPL m3": { "estimate": 0.4716008771929824, "interval_95": [ 0.4554053101503759, 0.4883676639515455 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.19736842105263144, "interval_95": [ 0.17276755972208993, 0.2228369579010818 ] } }, "DESC seen_ba": { "estimate": 0.26595982142857144, "interval_95": [ 0.23886021205357144, 0.29154445684523805 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.24228110599078342, "interval_95": [ 0.21430731566820277, 0.2686910522273425 ] } }, "DESC u8": { "estimate": 0.28607142857142853, "interval_95": [ 0.2683604910714285, 0.3043656994047619 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.18408163265306118, "interval_95": [ 0.16384056122448973, 0.2049893707482993 ] } }, "DESC u3": { "estimate": 0.18973214285714285, "interval_95": [ 0.17877938988095235, 0.20129538690476192 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.23078661844484633, "interval_95": [ -0.24742371157323695, -0.21322219710669077 ] } }, "DESC nn8": { "estimate": 0.39409226190476193, "interval_95": [ 0.36490959821428565, 0.4220126488095237 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.3075340136054422, "interval_95": [ 0.2741823979591836, 0.33944302721088426 ] } }, "DESC m8": { "estimate": 0.5339657738095239, "interval_95": [ 0.4975864025297616, 0.5696023995535713 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.46738945578231306, "interval_95": [ 0.4258130314625847, 0.5081170280612243 ] } }, "DESC m3": { "estimate": 0.4791823308270676, "interval_95": [ 0.4619319783834586, 0.49617030858396 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.20888455315503943, "interval_95": [ 0.18268148615208904, 0.2346890763300658 ] } }, "SHUF seen_ba": { "estimate": 0.2510639880952381, "interval_95": [ 0.22546155753968256, 0.27576934523809515 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.22690476190476191, "interval_95": [ 0.20047644649257554, 0.25240706605222724 ] } }, "SHUF u8": { "estimate": 0.09185515873015875, "interval_95": [ 0.08643799603174604, 0.09742063492063491 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": -0.03787981859410429, "interval_95": [ -0.04407086167800452, -0.03151927437641725 ] } }, "SHUF u3": { "estimate": 0.14539682539682536, "interval_95": [ 0.13317336309523808, 0.1579677579365079 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.29813140446051845, "interval_95": [ -0.3166986889692586, -0.279036317058469 ] } }, "SHUF nn8": { "estimate": 0.38248015873015867, "interval_95": [ 0.3550297619047618, 0.40930729166666663 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.29426303854875274, "interval_95": [ 0.26289115646258493, 0.324922619047619 ] } }, "SHUF m8": { "estimate": 0.49814236111111104, "interval_95": [ 0.4645541914682539, 0.5307104414682539 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.4264484126984126, "interval_95": [ 0.3880619331065759, 0.4636690759637187 ] } }, "SHUF m3": { "estimate": 0.4950083542188804, "interval_95": [ 0.4806280022974101, 0.5097509398496239 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.232924082357793, "interval_95": [ 0.21108050981885076, 0.25531788331588445 ] } }, "NUM seen_ba": { "estimate": 0.23673363095238098, "interval_95": [ 0.21219475446428568, 0.2609082961309524 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.21211213517665134, "interval_95": [ 0.18678168202764972, 0.23706662826420893 ] } }, "NUM u8": { "estimate": 0.4146428571428571, "interval_95": [ 0.38151413690476194, 0.4467604166666666 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.3310204081632652, "interval_95": [ 0.2931590136054422, 0.36772619047619043 ] } }, "NUM u3": { "estimate": 0.2057440476190476, "interval_95": [ 0.19333333333333336, 0.217828125 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.20646473779385174, "interval_95": [ -0.2253164556962025, -0.18810917721518988 ] } }, "NUM nn8": { "estimate": 0.40958333333333335, "interval_95": [ 0.3765052083333333, 0.44131622023809525 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.32523809523809527, "interval_95": [ 0.2874345238095238, 0.3615042517006803 ] } }, "NUM m8": { "estimate": 0.5376339285714286, "interval_95": [ 0.49873772321428567, 0.5748898809523809 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.4715816326530612, "interval_95": [ 0.4271288265306122, 0.5141598639455782 ] } }, "NUM m3": { "estimate": 0.2784461152882206, "interval_95": [ 0.2579248903508772, 0.29758771929824557 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.09603121728371554, "interval_95": [ -0.12720269820119923, -0.06695536309127256 ] } }, "B-sh seen_ba": { "estimate": 0.3216443452380953, "interval_95": [ 0.2893370535714286, 0.35292559523809525 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.29976190476190484, "interval_95": [ 0.2664124423963134, 0.3320522273425499 ] } }, "B-sh nn8": { "estimate": 0.44279761904761905, "interval_95": [ 0.40925223214285716, 0.475061755952381 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.3631972789115646, "interval_95": [ 0.32485969387755104, 0.40007057823129255 ] } }, "B-sh m8": { "estimate": 0.5709226190476189, "interval_95": [ 0.5338521205357143, 0.6074970238095239 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.5096258503401359, "interval_95": [ 0.4672595663265306, 0.5514251700680273 ] } }, "B-sh m3": { "estimate": 0.5590068922305765, "interval_95": [ 0.538016134085213, 0.5790926535087717 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.33013705148948336, "interval_95": [ 0.2982523555724754, 0.36064706862091905 ] } }, "CCA seen_ba": { "estimate": 0.6545982142857143, "interval_95": [ 0.5965619419642857, 0.7108253348214284 ], "people": 70, "seeds": 1, "chance": 0.03125, "chance_corrected": { "estimate": 0.6434562211981567, "interval_95": [ 0.5835478110599078, 0.701497119815668 ] } }, "CCA u8": { "estimate": 0.809017857142857, "interval_95": [ 0.7658928571428572, 0.848484375 ], "people": 70, "seeds": 1, "chance": 0.125, "chance_corrected": { "estimate": 0.7817346938775509, "interval_95": [ 0.7324489795918367, 0.8268392857142857 ] } }, "CCA u3": { "estimate": 0.8169642857142855, "interval_95": [ 0.78490625, 0.8457165178571429 ], "people": 70, "seeds": 1, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.7219710669077753, "interval_95": [ 0.6732753164556963, 0.7656453435804702 ] } }, "CCA m8": { "estimate": 0.809017857142857, "interval_95": [ 0.7658928571428572, 0.848484375 ], "people": 70, "seeds": 1, "chance": 0.125, "chance_corrected": { "estimate": 0.7817346938775509, "interval_95": [ 0.7324489795918367, 0.8268392857142857 ] } }, "CCA m3": { "estimate": 0.8128759398496239, "interval_95": [ 0.7805404135338347, 0.8422932330827068 ], "people": 70, "seeds": 1, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.715760921290568, "interval_95": [ 0.6666436661273439, 0.7604454173408204 ] } } }, "contrasts": { "TPL(a1) - ID": { "estimate": -0.09972842261904762, "interval_95": [ -0.12738885788690477, -0.07693934461805559 ], "people": 70, "seeds": 3 }, "TPL(a2) - ID": { "estimate": -0.0779439484126984, "interval_95": [ -0.08793597056878306, -0.06744339037698413 ], "people": 70, "seeds": 3 }, "DESC(a3) - ID": { "estimate": -0.0959032738095238, "interval_95": [ -0.11345007440476189, -0.08003748139880949 ], "people": 70, "seeds": 3 }, "TPL - NN(ID), 8-way": { "estimate": -0.15630952380952381, "interval_95": [ -0.17475248015873016, -0.13622991071428572 ], "people": 70, "seeds": 3 }, "TPL(a2, synonym 1) - TPL(train)": { "estimate": -0.01447172619047618, "interval_95": [ -0.01674355158730158, -0.012249317956349204 ], "people": 70, "seeds": 3 }, "TPL(a2, synonym 2) - TPL(train)": { "estimate": -0.008722718253968252, "interval_95": [ -0.010406870039682537, -0.00695436507936508 ], "people": 70, "seeds": 3 }, "TPL(a2, synonym 3) - TPL(train)": { "estimate": -0.03185763888888887, "interval_95": [ -0.0369422123015873, -0.026860119047619042 ], "people": 70, "seeds": 3 }, "S14 ID - B-sh seen": { "estimate": -0.0010788690476190473, "interval_95": [ -0.00787239583333334, 0.005365327380952377 ], "people": 70, "seeds": 3 }, "S14 SHUF seen": { "estimate": 0.2510639880952381, "interval_95": [ 0.22546155753968256, 0.27576934523809515 ], "people": 70, "seeds": 3 } }, "generalisation_costs": { "ID 8-way (unseen NN - matched seen)": { "estimate": -0.1426339285714286, "interval_95": [ -0.15501060267857145, -0.13064099702380955 ], "people": 70, "seeds": 3 }, "TPL 8-way (unseen - matched seen)": { "estimate": -0.24438740079365082, "interval_95": [ -0.26438250248015865, -0.22262617807539686 ], "people": 70, "seeds": 3 }, "TPL 3-way (unseen - matched seen)": { "estimate": -0.2782675438596491, "interval_95": [ -0.30181007205513793, -0.2536702694235589 ], "people": 70, "seeds": 3 }, "DESC 8-way (unseen - matched seen)": { "estimate": -0.24789434523809523, "interval_95": [ -0.2688450520833333, -0.22548707217261904 ], "people": 70, "seeds": 3 }, "DESC 3-way (unseen - matched seen)": { "estimate": -0.2894501879699248, "interval_95": [ -0.31330930842731824, -0.2646081218671679 ], "people": 70, "seeds": 3 }, "SHUF 8-way (unseen - matched seen)": { "estimate": -0.4062872023809524, "interval_95": [ -0.4416324404761904, -0.36984058779761914 ], "people": 70, "seeds": 3 }, "SHUF 3-way (unseen - matched seen)": { "estimate": -0.3496115288220552, "interval_95": [ -0.3749890481411862, -0.32433864609440266 ], "people": 70, "seeds": 3 }, "NUM 8-way (unseen - matched seen)": { "estimate": -0.12299107142857142, "interval_95": [ -0.13418191964285717, -0.11167280505952383 ], "people": 70, "seeds": 3 }, "NUM 3-way (unseen - matched seen)": { "estimate": -0.07270206766917293, "interval_95": [ -0.08490836466165412, -0.06020978226817042 ], "people": 70, "seeds": 3 }, "B-sh 8-way (unseen NN - matched seen)": { "estimate": -0.128125, "interval_95": [ -0.14090829613095238, -0.11560193452380953 ], "people": 70, "seeds": 3 }, "CCA 8-way (unseen - matched seen)": { "estimate": 0.0, "interval_95": [ 0.0, 0.0 ], "people": 70, "seeds": 1 }, "CCA 3-way (unseen - matched seen)": { "estimate": 0.004088345864661656, "interval_95": [ 0.0025703712406014963, 0.005559915413533832 ], "people": 70, "seeds": 1 } }, "chance_3way_mean": { "stored": 0.34166667610406876, "exact": "41/120", "note": "A display value stored as a float32 mean; the exact mean per-trial chance of the neighbour 3-way is 41/120 (2-way at the two grid edges)." } }, "P-ssvep-L1": { "levels": { "ID seen_ba": { "estimate": 0.24241071428571423, "interval_95": [ 0.21608072916666665, 0.2691519717261905 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.2179723502304147, "interval_95": [ 0.19079301075268815, 0.24557622887864825 ] } }, "ID nn8": { "estimate": 0.18282738095238094, "interval_95": [ 0.17303497023809522, 0.19303571428571425 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.06608843537414964, "interval_95": [ 0.05489710884353739, 0.07775510204081629 ] } }, "ID m8": { "estimate": 0.46078869047619053, "interval_95": [ 0.43037574404761914, 0.4915775669642858 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.3837585034013606, "interval_95": [ 0.34900085034013617, 0.4189457908163266 ] } }, "ID m3": { "estimate": 0.631516290726817, "interval_95": [ 0.6102126409774438, 0.6537762374686716 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.44027790996478533, "interval_95": [ 0.4079179356619399, 0.47409048729418474 ] } }, "TPL seen_ba": { "estimate": 0.20747519841269843, "interval_95": [ 0.18494047619047618, 0.230875744047619 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.1819098822324629, "interval_95": [ 0.15864823348694315, 0.20606528417818737 ] } }, "TPL u8": { "estimate": 0.18960317460317458, "interval_95": [ 0.18022817460317464, 0.19881051587301588 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.07383219954648523, "interval_95": [ 0.06311791383219958, 0.08435487528344672 ] } }, "TPL u3": { "estimate": 0.2702876984126984, "interval_95": [ 0.25964186507936504, 0.28048635912698416 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.1084237492465341, "interval_95": [ -0.1245946353224835, -0.09293211271850509 ] } }, "TPL nn8": { "estimate": 0.20119047619047623, "interval_95": [ 0.18980158730158733, 0.2133340773809524 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.08707482993197282, "interval_95": [ 0.0740589569160998, 0.10095323129251701 ] } }, "TPL m8": { "estimate": 0.4322668650793651, "interval_95": [ 0.4019912574404762, 0.46293074156746034 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.35116213151927445, "interval_95": [ 0.3165614370748299, 0.38620656179138324 ] } }, "TPL m3": { "estimate": 0.5669695071010861, "interval_95": [ 0.5462385129490392, 0.5874284670008354 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.342232162685194, "interval_95": [ 0.31074204498588237, 0.3733090637987373 ] } }, "DESC seen_ba": { "estimate": 0.20871651785714282, "interval_95": [ 0.18666657366071426, 0.23201255580357144 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.1831912442396313, "interval_95": [ 0.1604300115207373, 0.207238767281106 ] } }, "DESC u8": { "estimate": 0.18638392857142855, "interval_95": [ 0.17645647321428568, 0.19613206845238096 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.07015306122448976, "interval_95": [ 0.058807397959183626, 0.08129379251700682 ] } }, "DESC u3": { "estimate": 0.26767857142857143, "interval_95": [ 0.25901599702380956, 0.2768165922619047 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.1123869801084991, "interval_95": [ -0.12554532097649182, -0.09850644213381567 ] } }, "DESC nn8": { "estimate": 0.19904761904761908, "interval_95": [ 0.18751078869047616, 0.21076153273809523 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.0846258503401361, "interval_95": [ 0.07144090136054418, 0.09801318027210883 ] } }, "DESC m8": { "estimate": 0.43446800595238094, "interval_95": [ 0.40435630580357135, 0.46467420014880945 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.35367772108843537, "interval_95": [ 0.3192643494897958, 0.38819908588435365 ] } }, "DESC m3": { "estimate": 0.565029761904762, "interval_95": [ 0.5459267308897242, 0.5850744047619046 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.33928571428571436, "interval_95": [ 0.3102684519843912, 0.3697332730560577 ] } }, "SHUF seen_ba": { "estimate": 0.20210565476190478, "interval_95": [ 0.18110695684523806, 0.22339285714285706 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.17636712749615976, "interval_95": [ 0.1546910522273425, 0.1983410138248847 ] } }, "SHUF u8": { "estimate": 0.10986111111111112, "interval_95": [ 0.10500967261904759, 0.11517881944444444 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": -0.017301587301587294, "interval_95": [ -0.022846088435374185, -0.011224206349206354 ] } }, "SHUF u3": { "estimate": 0.2715476190476191, "interval_95": [ 0.26019766865079363, 0.28361656746031744 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.10650994575045203, "interval_95": [ -0.1237503767329717, -0.08817736588306213 ] } }, "SHUF nn8": { "estimate": 0.1691964285714286, "interval_95": [ 0.15994766865079363, 0.17881001984126987 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.0505102040816327, "interval_95": [ 0.03994019274376415, 0.06149716553287986 ] } }, "SHUF m8": { "estimate": 0.41075644841269837, "interval_95": [ 0.3838407738095238, 0.4384526909722222 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.326578798185941, "interval_95": [ 0.29581802721088435, 0.3582316468253968 ] } }, "SHUF m3": { "estimate": 0.5967575187969926, "interval_95": [ 0.5763340643274854, 0.6172501566416039 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.3874797753878368, "interval_95": [ 0.35645680657339546, 0.4186078328733223 ] } }, "NUM seen_ba": { "estimate": 0.13857142857142857, "interval_95": [ 0.12519977678571428, 0.15218098958333337 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.11078341013824884, "interval_95": [ 0.09698041474654377, 0.12483198924731187 ] } }, "NUM u8": { "estimate": 0.21422619047619051, "interval_95": [ 0.20157068452380952, 0.22624999999999995 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.1019727891156463, "interval_95": [ 0.0875093537414966, 0.11571428571428566 ] } }, "NUM u3": { "estimate": 0.11422619047619047, "interval_95": [ 0.1073809523809524, 0.12116071428571427 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.3454792043399638, "interval_95": [ -0.35587703435804696, -0.3349457504520796 ] } }, "NUM nn8": { "estimate": 0.21464285714285714, "interval_95": [ 0.2019940476190476, 0.2266674107142857 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.10244897959183673, "interval_95": [ 0.08799319727891154, 0.11619132653061223 ] } }, "NUM m8": { "estimate": 0.37580357142857146, "interval_95": [ 0.3500736607142857, 0.401719494047619 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.28663265306122454, "interval_95": [ 0.2572270408163265, 0.31625085034013595 ] } }, "NUM m3": { "estimate": 0.19052318295739346, "interval_95": [ 0.17914982769423554, 0.2031175595238095 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": -0.22958503854573148, "interval_95": [ -0.24686102122394601, -0.2104543399638337 ] } }, "B-sh seen_ba": { "estimate": 0.27037202380952385, "interval_95": [ 0.24168098958333334, 0.3008590029761905 ], "people": 70, "seeds": 3, "chance": 0.03125, "chance_corrected": { "estimate": 0.24683563748079881, "interval_95": [ 0.2172190860215054, 0.27830606758832566 ] } }, "B-sh nn8": { "estimate": 0.17583333333333337, "interval_95": [ 0.16571354166666666, 0.18809523809523807 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.05809523809523814, "interval_95": [ 0.04652976190476189, 0.07210884353741494 ] } }, "B-sh m8": { "estimate": 0.48773809523809525, "interval_95": [ 0.4556463913690475, 0.5203989955357142 ], "people": 70, "seeds": 3, "chance": 0.125, "chance_corrected": { "estimate": 0.4145578231292517, "interval_95": [ 0.37788159013605427, 0.45188456632653057 ] } }, "B-sh m3": { "estimate": 0.6578320802005013, "interval_95": [ 0.6349447838345864, 0.6810463659147868 ], "people": 70, "seeds": 3, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.48025126106405264, "interval_95": [ 0.4454857475968401, 0.5155134672123345 ] } }, "CCA seen_ba": { "estimate": 0.6545982142857143, "interval_95": [ 0.5965619419642857, 0.7108253348214284 ], "people": 70, "seeds": 1, "chance": 0.03125, "chance_corrected": { "estimate": 0.6434562211981567, "interval_95": [ 0.5835478110599078, 0.701497119815668 ] } }, "CCA u8": { "estimate": 0.809017857142857, "interval_95": [ 0.7658928571428572, 0.848484375 ], "people": 70, "seeds": 1, "chance": 0.125, "chance_corrected": { "estimate": 0.7817346938775509, "interval_95": [ 0.7324489795918367, 0.8268392857142857 ] } }, "CCA u3": { "estimate": 0.8169642857142855, "interval_95": [ 0.78490625, 0.8457165178571429 ], "people": 70, "seeds": 1, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.7219710669077753, "interval_95": [ 0.6732753164556963, 0.7656453435804702 ] } }, "CCA m8": { "estimate": 0.809017857142857, "interval_95": [ 0.7658928571428572, 0.848484375 ], "people": 70, "seeds": 1, "chance": 0.125, "chance_corrected": { "estimate": 0.7817346938775509, "interval_95": [ 0.7324489795918367, 0.8268392857142857 ] } }, "CCA m3": { "estimate": 0.8128759398496239, "interval_95": [ 0.7805404135338347, 0.8422932330827068 ], "people": 70, "seeds": 1, "chance": 0.3416666666666667, "chance_corrected": { "estimate": 0.715760921290568, "interval_95": [ 0.6666436661273439, 0.7604454173408204 ] } } }, "contrasts": { "TPL(a1) - ID": { "estimate": -0.0620920138888889, "interval_95": [ -0.08049152095734126, -0.04749215649801589 ], "people": 70, "seeds": 3 }, "TPL(a2) - ID": { "estimate": -0.04823330026455027, "interval_95": [ -0.05495463376322751, -0.04183232060185185 ], "people": 70, "seeds": 3 }, "DESC(a3) - ID": { "estimate": -0.06318973214285713, "interval_95": [ -0.07399066220238096, -0.05291041666666667 ], "people": 70, "seeds": 3 }, "TPL - NN(ID), 8-way": { "estimate": 0.006775793650793656, "interval_95": [ -0.0024518849206349226, 0.01611111111111111 ], "people": 70, "seeds": 3 }, "TPL(a2, synonym 1) - TPL(train)": { "estimate": -0.010949900793650795, "interval_95": [ -0.01269115823412698, -0.009045014880952386 ], "people": 70, "seeds": 3 }, "TPL(a2, synonym 2) - TPL(train)": { "estimate": -0.008377976190476191, "interval_95": [ -0.009985119047619046, -0.0067980530753968225 ], "people": 70, "seeds": 3 }, "TPL(a2, synonym 3) - TPL(train)": { "estimate": -0.020565476190476193, "interval_95": [ -0.024442150297619033, -0.01725638640873017 ], "people": 70, "seeds": 3 }, "S14 ID - B-sh seen": { "estimate": -0.02796130952380953, "interval_95": [ -0.03346837797619047, -0.022640252976190486 ], "people": 70, "seeds": 3 }, "S14 SHUF seen": { "estimate": 0.20210565476190478, "interval_95": [ 0.18110695684523806, 0.22339285714285706 ], "people": 70, "seeds": 3 } }, "generalisation_costs": { "ID 8-way (unseen NN - matched seen)": { "estimate": -0.2779613095238096, "interval_95": [ -0.3061244419642857, -0.2502587425595238 ], "people": 70, "seeds": 3 }, "TPL 8-way (unseen - matched seen)": { "estimate": -0.24266369047619044, "interval_95": [ -0.26718346974206353, -0.21847340029761908 ], "people": 70, "seeds": 3 }, "TPL 3-way (unseen - matched seen)": { "estimate": -0.29668180868838767, "interval_95": [ -0.31764202172096895, -0.2763724023600668 ], "people": 70, "seeds": 3 }, "DESC 8-way (unseen - matched seen)": { "estimate": -0.2480840773809524, "interval_95": [ -0.2731459263392857, -0.22388225446428575 ], "people": 70, "seeds": 3 }, "DESC 3-way (unseen - matched seen)": { "estimate": -0.29735119047619046, "interval_95": [ -0.31755731124686715, -0.2776477521929825 ], "people": 70, "seeds": 3 }, "SHUF 8-way (unseen - matched seen)": { "estimate": -0.3008953373015873, "interval_95": [ -0.33020039682539676, -0.2727214781746032 ], "people": 70, "seeds": 3 }, "SHUF 3-way (unseen - matched seen)": { "estimate": -0.3252098997493734, "interval_95": [ -0.35179406850459477, -0.29837917710944034 ], "people": 70, "seeds": 3 }, "NUM 8-way (unseen - matched seen)": { "estimate": -0.16157738095238092, "interval_95": [ -0.1788344494047619, -0.14474683779761902 ], "people": 70, "seeds": 3 }, "NUM 3-way (unseen - matched seen)": { "estimate": -0.07629699248120302, "interval_95": [ -0.0874048402255639, -0.06532902568922305 ], "people": 70, "seeds": 3 }, "B-sh 8-way (unseen NN - matched seen)": { "estimate": -0.3119047619047619, "interval_95": [ -0.3408962053571428, -0.28323549107142854 ], "people": 70, "seeds": 3 }, "CCA 8-way (unseen - matched seen)": { "estimate": 0.0, "interval_95": [ 0.0, 0.0 ], "people": 70, "seeds": 1 }, "CCA 3-way (unseen - matched seen)": { "estimate": 0.004088345864661656, "interval_95": [ 0.0025703712406014963, 0.005559915413533832 ], "people": 70, "seeds": 1 } }, "chance_3way_mean": { "stored": 0.34166667610406876, "exact": "41/120", "note": "A display value stored as a float32 mean; the exact mean per-trial chance of the neighbour 3-way is 41/120 (2-way at the two grid edges)." } }, "P-sleep-L1": { "levels": { "ID seen_ba": { "estimate": 0.7145802372582131, "interval_95": [ 0.6913674343036235, 0.7345371188063007 ], "people": 100, "seeds": 3, "chance": 0.2, "chance_corrected": { "estimate": 0.6432252965727664, "interval_95": [ 0.6142092928795293, 0.6681713985078758 ] } }, "TPL seen_ba": { "estimate": 0.7141244670130728, "interval_95": [ 0.6908765671260783, 0.7342262656845189 ], "people": 100, "seeds": 3, "chance": 0.2, "chance_corrected": { "estimate": 0.6426555837663408, "interval_95": [ 0.6135957089075977, 0.6677828321056486 ] } }, "DESC seen_ba": { "estimate": 0.7147881386018728, "interval_95": [ 0.6912818199730705, 0.7347236710526416 ], "people": 100, "seeds": 3, "chance": 0.2, "chance_corrected": { "estimate": 0.6434851732523411, "interval_95": [ 0.6141022749663381, 0.6684045888158019 ] } }, "SHUF seen_ba": { "estimate": 0.7149265217993174, "interval_95": [ 0.6917257116048581, 0.7346367371932621 ], "people": 100, "seeds": 3, "chance": 0.2, "chance_corrected": { "estimate": 0.6436581522491466, "interval_95": [ 0.6146571395060726, 0.6682959214915776 ] } }, "B-sh seen_ba": { "estimate": 0.7140161062371334, "interval_95": [ 0.6913549210007606, 0.7338052110312374 ], "people": 100, "seeds": 3, "chance": 0.2, "chance_corrected": { "estimate": 0.6425201327964166, "interval_95": [ 0.6141936512509507, 0.6672565137890468 ] } }, "TPL unseen asleep implicit AUROC": { "estimate": 0.31722381684092604, "interval_95": [ 0.2944946102984704, 0.34128994234146554 ], "people": 100, "seeds": 3 }, "SHUF unseen asleep implicit AUROC": { "estimate": 0.6762070494823804, "interval_95": [ 0.5291548976584539, 0.8122473130480133 ], "people": 100, "seeds": 3 }, "TPL unseen asleep explicit AUROC": { "estimate": 0.8578824108993678, "interval_95": [ 0.838926827335656, 0.8751458421742866 ], "people": 100, "seeds": 3 }, "SHUF unseen asleep explicit AUROC": { "estimate": 0.13694541434944557, "interval_95": [ 0.10881012003112213, 0.167937601987345 ], "people": 100, "seeds": 3 }, "TPL unseen NREM explicit AUROC": { "estimate": 0.8711263248200034, "interval_95": [ 0.854384111327175, 0.8873709705770991 ], "people": 100, "seeds": 3 }, "SHUF unseen NREM explicit AUROC": { "estimate": 0.17134517960867474, "interval_95": [ 0.1453770870010597, 0.20056198651427945 ], "people": 100, "seeds": 3 }, "TPL unseen light implicit AUROC": { "estimate": 0.24397603144778995, "interval_95": [ 0.22161945943505393, 0.2672802368054641 ], "people": 100, "seeds": 3 }, "SHUF unseen light implicit AUROC": { "estimate": 0.6408644643406126, "interval_95": [ 0.5371539109719549, 0.7378653729519237 ], "people": 100, "seeds": 3 }, "TPL unseen light explicit AUROC": { "estimate": 0.6420450904120362, "interval_95": [ 0.6164022876775221, 0.6671199334610118 ], "people": 100, "seeds": 3 }, "SHUF unseen light explicit AUROC": { "estimate": 0.17615323393824053, "interval_95": [ 0.15489302125357943, 0.199451146021832 ], "people": 100, "seeds": 3 } }, "contrasts": { "TPL(a1) - ID": { "estimate": -0.0050399494147742915, "interval_95": [ -0.015491950430263563, 0.004261601739624585 ], "people": 100, "seeds": 3 }, "TPL(a2) - ID": { "estimate": -0.12647867141853872, "interval_95": [ -0.1392540583728555, -0.1130977140918392 ], "people": 100, "seeds": 3 }, "DESC(a3) - ID": { "estimate": -0.2084867825790437, "interval_95": [ -0.2943601279645254, -0.1373384018824331 ], "people": 100, "seeds": 3 }, "TPL(a2, synonym 1) - TPL(train)": { "estimate": -0.23531196096027288, "interval_95": [ -0.2528836913180727, -0.21644905051782928 ], "people": 100, "seeds": 3 }, "TPL(a2, synonym 2) - TPL(train)": { "estimate": -0.1085356913971758, "interval_95": [ -0.12594212889981293, -0.09172460817850121 ], "people": 100, "seeds": 3 }, "TPL(a2, synonym 3) - TPL(train)": { "estimate": -0.034221051162746624, "interval_95": [ -0.0449297209822384, -0.0240885750443153 ], "people": 100, "seeds": 3 }, "S14 ID - B-sh seen": { "estimate": 0.0005641310210794814, "interval_95": [ -0.006020593013313189, 0.006787005388288528 ], "people": 100, "seeds": 3 }, "S14 SHUF seen": { "estimate": 0.7149265217993174, "interval_95": [ 0.6917257116048581, 0.7346367371932621 ], "people": 100, "seeds": 3 } }, "S15": { "asleep balanced read-off (TPL)": { "estimate": 0.9786074962250565, "interval_95": [ 0.9688159178619072, 0.9861149014223396 ], "people": 100, "seeds": 3 }, "asleep prior-corrected read-off (ID)": { "estimate": 0.9807293882927298, "interval_95": [ 0.9717106259437823, 0.9873541991621027 ], "people": 100, "seeds": 3 }, "asleep prior-corrected read-off (TPL)": { "estimate": 0.9797997757711162, "interval_95": [ 0.9709102490252292, 0.9868154012348229 ], "people": 100, "seeds": 3 }, "NREM balanced read-off (TPL)": { "estimate": 0.9391199566413279, "interval_95": [ 0.9244135070728828, 0.9518513576089164 ], "people": 100, "seeds": 3 }, "NREM prior-corrected read-off (ID)": { "estimate": 0.946611637581319, "interval_95": [ 0.9320252653461589, 0.9584443707409355 ], "people": 100, "seeds": 3 }, "NREM prior-corrected read-off (TPL)": { "estimate": 0.946708918210017, "interval_95": [ 0.9340654315799207, 0.9576852086599629 ], "people": 100, "seeds": 3 }, "light balanced read-off (TPL)": { "estimate": 0.9067910750822276, "interval_95": [ 0.8903237284805565, 0.9212455287430164 ], "people": 100, "seeds": 3 }, "light prior-corrected read-off (ID)": { "estimate": 0.9221924096084559, "interval_95": [ 0.9050725381007412, 0.9362176481883719 ], "people": 100, "seeds": 3 }, "light prior-corrected read-off (TPL)": { "estimate": 0.9216692764040253, "interval_95": [ 0.906643341431724, 0.9348304669942242 ], "people": 100, "seeds": 3 } }, "seen_auroc_per_predicate": { "ID W": { "estimate": 0.9817440590322881, "interval_95": [ 0.9756804906134985, 0.9868799205840441 ], "people": 100, "seeds": 3 }, "ID N1": { "estimate": 0.8757895642194026, "interval_95": [ 0.8590406036574977, 0.8900668075074987 ], "people": 100, "seeds": 3 }, "ID N2": { "estimate": 0.9221049573142459, "interval_95": [ 0.9038739665906927, 0.9368063850639256 ], "people": 100, "seeds": 3 }, "ID N3": { "estimate": 0.9778322081931536, "interval_95": [ 0.9711731421752442, 0.982963259964409 ], "people": 71, "seeds": 3 }, "ID REM": { "estimate": 0.9571396364615122, "interval_95": [ 0.945487695518251, 0.9671531251838394 ], "people": 100, "seeds": 3 }, "TPL W": { "estimate": 0.9801445674548448, "interval_95": [ 0.9720031817823664, 0.9867598451065624 ], "people": 100, "seeds": 3 }, "TPL N1": { "estimate": 0.871907152480864, "interval_95": [ 0.8547408737005391, 0.8867714949126292 ], "people": 100, "seeds": 3 }, "TPL N2": { "estimate": 0.9236456935443291, "interval_95": [ 0.9090000634370796, 0.9368066675889611 ], "people": 100, "seeds": 3 }, "TPL N3": { "estimate": 0.9789817414117379, "interval_95": [ 0.9742015478774562, 0.9830091828676588 ], "people": 71, "seeds": 3 }, "TPL REM": { "estimate": 0.9562969425742252, "interval_95": [ 0.9442535066012538, 0.9669642900273482 ], "people": 100, "seeds": 3 }, "DESC W": { "estimate": 0.981291345428251, "interval_95": [ 0.9737086679076432, 0.9872493295040984 ], "people": 100, "seeds": 3 }, "DESC N1": { "estimate": 0.8691133571143486, "interval_95": [ 0.8508933396031924, 0.8844301999580023 ], "people": 100, "seeds": 3 }, "DESC N2": { "estimate": 0.9231467736582308, "interval_95": [ 0.9068345216034488, 0.9371964430069675 ], "people": 100, "seeds": 3 }, "DESC N3": { "estimate": 0.9789790045779804, "interval_95": [ 0.9741983876116702, 0.9828953097635579 ], "people": 71, "seeds": 3 }, "DESC REM": { "estimate": 0.9563518531698254, "interval_95": [ 0.9444157894430644, 0.9668827718114689 ], "people": 100, "seeds": 3 } }, "log_loss": { "ID": { "estimate": 0.2668462795698636, "interval_95": [ 0.24233640773702825, 0.29618307517499326 ], "people": 100, "seeds": 3 }, "TPL": { "estimate": 0.2756870711038614, "interval_95": [ 0.2494492173422629, 0.30742902298218794 ], "people": 100, "seeds": 3 }, "DESC": { "estimate": 0.2779532719567373, "interval_95": [ 0.25099992693930245, 0.3099468804425251 ], "people": 100, "seeds": 3 }, "B-sh": { "estimate": 0.2589737215142191, "interval_95": [ 0.23595227554776096, 0.2865310430419868 ], "people": 100, "seeds": 3 } } } }, "pre_run_checks": { "disclosure": "Two failed pre-run checks and two owner-approved revisions, disclosed. (1) The first training-label permutation canary (protocol revision 3; labels permuted within each training person; one head per cell) failed at stage 0b on 2026-10-07: one of its 38 99.9% person-bootstrap intervals excluded chance (sleep, L1, SHUF arm, unseen \"light sleep\", explicit wording). The run stopped. (2) On 2026-10-08 the owner approved revision 4 before any new fit: a gate canary with labels permuted across all training windows (one head per cell). It failed too: 3 of 38 intervals excluded chance, all on BOAS (sleep, L1: DESC seen balanced accuracy; TPL unseen \"light sleep\" with the implicit wording, below chance, and with the explicit wording, above chance); all 20 SSVEP intervals included chance both times. Revision 4 said \"no further revision\"; the owner overrode that on 2026-10-08 after seeing how it failed, and approved revision 5 before any new fit. Its per-person centring diagnostic was not run. (3) The revision-5 gate canary refits each of the 17 configurations 20 times (training seeds 20261007 + 1..20, labels permuted across all training windows) and compares each metric with its exact per-person chance; every 99.9% two-level (replicate, then person) bootstrap interval for the mean difference had to include 0. Its result is in the record. These changed a check after seeing it fail; both failed records are kept (decision log S0b-11 to S0b-15).", "why_they_failed": { "en": "A permutation canary refits the heads on shuffled training labels and checks that test scores sit at chance. Revisions 3 and 4 trained one head per cell and put a person-only bootstrap interval around it. A head trained on shuffled labels is still a random function of sleep features that are strongly organised by stage, so on new people its scores lean one way for everyone, up or down; the person bootstrap measures spread between people, not between heads, so its 99.9% interval was too narrow for this question. Revision 3 also permuted within each training person, which keeps each person's stage mix. The failures fit this reading: all were on BOAS, one of them (revision 4, TPL light implicit) was below chance, which a leak cannot produce, and of 760 single-head intervals in revision 5, 125 excluded the exact chance, 66 above and 59 below, with 18 cells failing in both directions; averaged over 20 heads with a two-level bootstrap, every cell included 0. This is a reading the revision-5 design tested, not a proof that no leak exists; the asserts (person-disjoint folds, no held-out label or wording in training, training-only scalers) and both conformance audits cover leakage directly.", "zh": "置换金丝雀（canary）的做法是：把训练标签打乱后重新训练分类头，测试分数应当落在随机水平。修订 3 和 4 每个格子只训练 1 个头，再按人做 bootstrap 给区间。用打乱标签训练出的头仍是睡眠特征的某个随机函数，而这些特征按睡眠分期分得很开，所以它在新的人身上会整体偏向同一方向，可能偏高也可能偏低；按人的 bootstrap 只量了人与人之间的波动，没量“换一个头”的波动，99.9% 区间因此偏窄。修订 3 还是在每个训练者内部打乱，保留了每个人的分期比例。失败的样子符合这个解释：全部发生在 BOAS 上；修订 4 有一个格子（TPL，浅睡，隐式问法）明显低于随机水平，而泄漏只会把分数往上推；修订 5 的 760 个单头区间里有 125 个排除了精确随机水平，66 个偏高、59 个偏低，18 个格子两个方向都有；对 20 个头取平均并做两层 bootstrap 后，38 个区间全部包含 0。这是修订 5 的设计检验过的解释，不是“绝无泄漏”的证明；泄漏本身由断言（按人互斥的折、训练中无留出标签或问法、只在训练行上拟合的标准化）和两次一致性审计直接覆盖。" }, "owner_override": "revision 4 said \"no further revision\"; the owner overrode it on 2026-10-08 (\"批准方案二\") after seeing how the revision-4 canary failed", "owner_decisions": { "revision_4": { "decided": "owner decision in chat on 2026-10-08 (\"两个都批准，按方案二继续跑\")", "approved": "APPROVED: revision 4 of the permutation canary: the failed revision-3 canary is kept and disclosed; a gate canary with labels permuted across all training windows replaces it as the stopping rule; a per-person centring diagnostic tests the between-person explanation.", "S0b-6": "APPROVED: the token rule for multilingual-e5-small is reported, not a stop, for number pieces; every S5 SSVEP unseen entry states that the frequency is a new token (as for S12)." }, "revision_5": { "decided": "owner decision in chat on 2026-10-08 (\"批准方案二\")", "approved": "APPROVED: revision 5 of the permutation canary, overriding revision 4's \"no further revision\": the failed revision-3 and revision-4 canaries are kept and disclosed; the revision-4 centring diagnostic is not run; a gate canary of 20 replicate heads per configuration (D = mean of metric - exact per-person chance, 99.9% two-level bootstrap) is the stopping rule." } }, "canaries": { "what": "A training-label permutation canary refits the heads on shuffled training labels and checks that test scores sit at chance (17 configurations, the first fold, 38 intervals at 99.9%).", "revision_3": { "pass": false, "status": "failed pre-run check, superseded by revision 4, kept and disclosed", "permutation": "within training person", "intervals": 38, "intervals_excluding_chance": 1, "cells_excluding_chance": [ { "cell": "sleep/L1/SHUF unseen light explicit", "side": "above" } ], "decision": "decision log S0b-11" }, "revision_4": { "pass": false, "status": "failed pre-run check, superseded by revision 5 (owner override), kept and disclosed", "permutation": "one permutation across all training windows of fold 0 (seed 20261007, RNG stream 99), whole label rows", "fits": 17, "intervals": 38, "intervals_excluding_chance": 3, "cells_excluding_chance": [ { "cell": "sleep/L1/DESC seen_ba", "side": "above" }, { "cell": "sleep/L1/TPL unseen light implicit", "side": "below" }, { "cell": "sleep/L1/TPL unseen light explicit", "side": "above" } ], "centring_diagnostic": "not run (revision 5 part A.2: with labels permuted across all windows it no longer tests anything; no fit exists)", "decision": "decision log S0b-14" }, "revision_5": { "pass": true, "stopping_rule": "revision-5 gate canary (17 configurations x 20 replicate heads; labels permuted across all training windows of fold 0 per replicate; D = mean of metric - exact per-person chance; 99.9% two-level bootstrap)", "permutation": "per replicate k = 1..20: one permutation across all training windows of fold 0 (RNG stream 99 of training seed 20261007 + k), whole label rows", "configurations": 17, "replicates": 20, "fits": 340, "statistic": "D = mean over replicates and fold-0 test people of (metric - exact per-person chance)", "interval": "two-level bootstrap (replicates, then people)", "level": 0.999, "bootstrap_draws": 2000, "intervals": 38, "intervals_excluding_zero": 0, "cells_interval_includes_zero": { "sleep/L1/B-sh": { "seen_ba": true }, "sleep/L1/DESC": { "seen_ba": true, "unseen NREM description": true, "unseen asleep description": true, "unseen light description": true }, "sleep/L1/ID": { "seen_ba": true }, "sleep/L1/SHUF": { "seen_ba": true, "unseen NREM explicit": true, "unseen asleep explicit": true, "unseen asleep implicit": true, "unseen light explicit": true, "unseen light implicit": true }, "sleep/L1/TPL": { "seen_ba": true, "unseen NREM explicit": true, "unseen asleep explicit": true, "unseen asleep implicit": true, "unseen light explicit": true, "unseen light implicit": true }, "ssvep/L0/B-sh": { "seen_ba": true }, "ssvep/L0/DESC": { "seen_ba": true, "unseen8": true }, "ssvep/L0/ID": { "seen_ba": true }, "ssvep/L0/NUM": { "seen_ba": true, "unseen8": true }, "ssvep/L0/SHUF": { "seen_ba": true, "unseen8": true }, "ssvep/L0/TPL": { "seen_ba": true, "unseen8": true }, "ssvep/L1/B-sh": { "seen_ba": true }, "ssvep/L1/DESC": { "seen_ba": true, "unseen8": true }, "ssvep/L1/ID": { "seen_ba": true }, "ssvep/L1/NUM": { "seen_ba": true, "unseen8": true }, "ssvep/L1/SHUF": { "seen_ba": true, "unseen8": true }, "ssvep/L1/TPL": { "seen_ba": true, "unseen8": true } }, "single_head_tests": 760, "single_head_intervals_excluding_exact_chance": { "above": 66, "below": 59, "total": 125 }, "cells_with_single_head_failures_in_both_directions": 18, "single_head_role": "descriptive (declared; no choice depends on it)" }, "boas": "BOAS cells: pass or fail and direction counts only; no BOAS canary value left the private run root.", "audit_reproduction": "audit A4 recomputed all three canaries from their kept fits: revision 3 fails on 1 of 38 cells, revision 4 on 3 of 38 (same cells and directions), revision 5 passes (38 of 38 D intervals include 0; single-head totals 66 above / 59 below of 760; 18 cells with single-head failures in both directions); A4 D-3: three SSVEP NUM replicate-direction counts count a float residue (about 1e-18) as above chance" }, "centring_diagnostic": "revision 4's per-person centring diagnostic was not run (revision 5 part A.2: with labels permuted across all windows it no longer tests anything); disclosed as not run", "decision_s0b6": { "decision": "decision log S0b-6 (2026-10-07), approved by the owner on 2026-10-08 (\"两个都批准，按方案二继续跑\")", "what": "the SSVEP token rule (every tokenizer token of every held-out frequency occurs in that rotation's training texts) cannot hold for multilingual-e5-small: its tokenizer keeps one-decimal number pieces such as \"8.2\" or \"2.6\" (of \"12.6\") as single tokens, and the held-out number itself is barred from training texts. The rule stops the run for bge-small-en-v1.5 and all-MiniLM-L6-v2 (both pass all five primary rotations); for e5 it is computed and reported, and stops the run only for a missing token that is not a number piece (none). S5 ran as declared; every S5 SSVEP unseen-frequency entry states that the frequency is a new token for this encoder, as S12 does.", "owner_approval": "APPROVED: the token rule for multilingual-e5-small is reported, not a stop, for number pieces; every S5 SSVEP unseen entry states that the frequency is a new token (as for S12).", "token_rule": { "stops_the_run_for": [ "bge-small-en-v1.5", "all-MiniLM-L6-v2" ], "rotations_passed": 5, "multilingual_e5_small": { "rotations": 5, "number_pieces_only_rule_passed": 5, "full_rule_passed": 0 } }, "statement_on_entries": "For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6)." }, "engineering_amendment_ea1": { "decision": "EA-1 (the decision log), code-only engineering amendment adopted 2026-10-08 before any stage-1 fit; it changed code.files_sha256 and decisions_file only: no question, partition, wording, arm, metric, margin or gate", "s8_nn": "not defined in S8: NN(arm) averages the answers for the two neighbouring SEEN frequencies, but the S8 held-out band 14.4-15.8 Hz is contiguous, so every held-out frequency has a held-out (untrained) neighbour (14.4 Hz: 14.6 Hz; 15.8 Hz: its only neighbour 15.6 Hz); score.nn_defined, engineering amendment EA-1", "s8_three_way": "In S8 the neighbour 3-way competitors are themselves held-out questions the head was never trained on (14 of the 15 neighbour slots; the one trained neighbour is 14.2 Hz, below 14.4 Hz; 15.8 Hz is a 2-way at the grid edge), not trained detectors as in the primary rotations, so this 3-way tells adjacent unseen frequencies apart.", "failed_fits": "A NaN/inf training loss writes fit.json status \"failed\" and is never re-tuned; every entry that needs a failed (block, arm, encoder, language, seed) cell is reported as \"failed\" (no numbers, failed units listed) and still counted. Failed fits in this run: none (see fits).", "resumable_secondary_scoring": "The secondary scoring is split into 19 parts, each kept with a done marker, plus the per-(block, arm, seed) value caches, so each foreground segment resumes after the last finished part. It changes no number." }, "numeracy_probe": { "declared": "stage 0, text only: Spearman rho between |f_i-f_j| and cosine distance of the 40 filled templates (780 pairs) per training frame and encoder. Declared prediction: if rho < 0.3 for every training frame of the primary encoder, the 8-way TPL-NUM difference flag will not read 'TPL higher' at either level. Scored by that flag; not a gate.", "spearman_rho": { "bge/frame0": 0.5931, "bge/frame1": 0.5628, "bge/frame2": 0.5256, "minilm/frame0": 0.7349, "minilm/frame1": 0.7374, "minilm/frame2": 0.7233 }, "pairs": 780, "activated": false, "premise": "false: Spearman rho for bge-small-en-v1.5 was 0.593, 0.563 and 0.526 for the three training frames (all at or above 0.3), so the declared prediction was not activated and is not tested by this run", "flag_anyway": { "L0": { "entry": "P4 TPL - NUM, 8-way unseen", "difference": "difference: reference higher" }, "L1": { "entry": "P4 TPL - NUM, 8-way unseen", "difference": "difference: reference higher" } }, "reading": "the flag does not read \"TPL higher\" at either level (the numeric code scored higher in the 8-way), which is what the prediction would have said; since its premise failed, this is a description, not a confirmed prediction. MiniLM (S4) had higher rho (0.72-0.74) and its 8-way TPL - NUM flag also reads \"reference higher\" at both levels." }, "gates": { "rule": { "headroom_gate": "unseen tests of a domain and level count only if the ID arm's seen balanced-accuracy interval is above chance+5pp and below 95%", "triviality_canary": "route 2 BOAS result carried; BETA ridge on per-channel mean and log-variance, fold-0 training people, 4 inner folds; >= chance + 0.7*(1-chance) excludes the seen task from route sentences", "readoff_floor_gate": "P3 cell reads 'read-off at floor: not informative' unless the read-off AUROC interval lies above 0.55" }, "per_entry_rule": "decision log B14: headroom gates the unseen entries (P3, P4, P5) of its domain and level; the triviality canary gates the seen task (P1, P2); the read-off floor applies per P3 cell", "headroom": { "P-ssvep-L0": { "arm": "ID", "metric": "seen balanced accuracy", "pass": true, "chance": 0.03125, "floor": 0.08125, "ceiling": 0.95, "estimate": 0.32056547619047615, "interval_95": [ 0.2878588169642858, 0.3518826264880952 ], "people": 70, "seeds": 3 }, "P-ssvep-L1": { "arm": "ID", "metric": "seen balanced accuracy", "pass": true, "chance": 0.03125, "floor": 0.08125, "ceiling": 0.95, "estimate": 0.24241071428571423, "interval_95": [ 0.21608072916666665, 0.2691519717261905 ], "people": 70, "seeds": 3 }, "P-sleep-L1": { "arm": "ID", "metric": "seen balanced accuracy", "pass": true, "chance": 0.2, "floor": 0.25, "ceiling": 0.95, "estimate": 0.7145802372582131, "interval_95": [ 0.6913674343036235, 0.7345371188063007 ], "people": 100, "seeds": 3 } }, "triviality": { "BETA": { "pass": true, "threshold": 0.7075, "training_people": 60, "origin": "stage 0: the first fold's training people, four inner folds; recomputed by audit A2" }, "BOAS": { "pass": true, "threshold": 0.76, "training_people": 80, "origin": "route 2's canary, carried" }, "EESM19": { "pass": true, "threshold": 0.76, "training_people": 16, "origin": "route 2's canary, carried" }, "OpenBMI MI": { "pass": true, "label": "not run for this task (no declared canary)" } }, "readoff_floor": { "rule": "P3 cell informative only if the read-off AUROC interval lies above 0.55", "primary_cells": 8, "primary_cells_passing": 8, "secondary_cells": 34, "secondary_cells_passing": 34 }, "fit_check": { "rule": "two-sided vs B-sh, fold 0, rotation 0, seed 20261007, training rows only; 60 epochs for every arm of a domain/level if any conditioned arm is > 5 pp below B-sh", "fits": 38, "by_domain_and_level": { "ssvep/L0": { "training_epochs": 60, "fallback_to_60_epochs": true, "arms_more_than_5_pp_below_b_sh": [ "ID", "TPL", "DESC", "SHUF", "NUM" ], "failed_fits": 0 }, "ssvep/L1": { "training_epochs": 60, "fallback_to_60_epochs": true, "arms_more_than_5_pp_below_b_sh": [ "ID", "TPL", "DESC", "SHUF", "NUM" ], "failed_fits": 0 }, "ssvep/REVE": { "training_epochs": 60, "fallback_to_60_epochs": true, "arms_more_than_5_pp_below_b_sh": [ "ID", "TPL", "DESC", "SHUF", "NUM" ], "failed_fits": 0 }, "sleep/L1": { "training_epochs": 30, "fallback_to_60_epochs": false, "arms_more_than_5_pp_below_b_sh": [], "failed_fits": 0 }, "sleep/REVE": { "training_epochs": 30, "fallback_to_60_epochs": false, "arms_more_than_5_pp_below_b_sh": [], "failed_fits": 0 }, "eesm19/L1": { "training_epochs": 30, "fallback_to_60_epochs": false, "arms_more_than_5_pp_below_b_sh": [], "failed_fits": 0 }, "mi/L1": { "training_epochs": 30, "fallback_to_60_epochs": false, "arms_more_than_5_pp_below_b_sh": [], "failed_fits": 0 } }, "boas": "pass or fail only; BOAS training scores never left the private run root" }, "shuffled_eeg_scorer_check": { "rule": "each primary aggregate recomputed 1000 times with test features permuted within person; the mean must lie within 3 Monte-Carlo SE of analytic chance (1/K or 0.5), else stop; text-only baseline at chance by construction (stated, not tested)", "pass": true, "cells": 51, "cells_within_3_se": 51, "ssvep_cells": 26, "boas_cells": 25 } } }, "secondary": { "multiplicity_note": "secondary entries are not part of the 35 primary comparisons", "interval_note": "every interval is a 95% hierarchical person bootstrap (2000 draws, seed 20261007) for its own comparison; flags use the frozen P1 rule (2.0 pp) unless the row says otherwise (S13 uses P3, ratio 1.25)", "none_inconclusive": "no secondary entry reached the \"inconclusive\" flag combination and no entry reached route-sentence rule 0 or 6, although every EESM19 and single-seed entry was declared likely inconclusive", "expected_inconclusive": [ "every EESM19 entry", "every single-seed S entry" ], "parts": { "S1": { "part": "S1", "what": "S1 EESM19 replication (sleep, CBraMod L1 features, 3 seeds): the primary sleep entries on a second sleep set", "dataset": "EESM19", "label": "EESM19, 20 people, 3 seeds", "status_label": "declared likely inconclusive (protocol decision_rules.expected_inconclusive: every EESM19 entry)", "single_seed": false, "declared_likely_inconclusive": true, "boas": false, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.76 } }, "entries": [ { "key": "secondary/S1/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (5-way), training-wording score", "people": 20, "seeds": 3, "single_seed": false, "estimate": -0.00038139231232486306, "interval_95": [ -0.004687959525959226, 0.003919951341635743 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.0004767403904060788, "interval_95": [ -0.0058599494074490326, 0.004899939177044678 ] } }, { "key": "secondary/S1/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (5-way), training-wording score", "people": 20, "seeds": 3, "single_seed": false, "estimate": -0.0002749495852206019, "interval_95": [ -0.004311349853700936, 0.004045183916648495 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.00034368698152575233, "interval_95": [ -0.00538918731712617, 0.005056479895810618 ] } }, { "key": "secondary/S1/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 20, "seeds": 3, "single_seed": false, "estimate": -0.001077883815172137, "interval_95": [ -0.0055673500228540325, 0.0019816586276110693 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "Sleep (EESM19), frozen CBraMod features: Rewording by a new sentence frame showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.001347354768965171, "interval_95": [ -0.00695918752856754, 0.0024770732845138363 ] } }, { "key": "secondary/S1/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 20, "seeds": 3, "single_seed": false, "estimate": -0.1527352210018774, "interval_95": [ -0.16240279640835076, -0.14287661927735912 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Rewording by a synonym, for these three synonyms cost 15.3 pp [14.3, 16.2].", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.19091902625234672, "interval_95": [ -0.20300349551043845, -0.17859577409669888 ] } }, { "key": "secondary/S1/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 20, "seeds": 3, "single_seed": false, "estimate": -0.18544622107999909, "interval_95": [ -0.27300913920726205, -0.1080400019845493 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "Sleep (EESM19), frozen CBraMod features: Rewording by a new description cost 18.5 pp [10.8, 27.3].", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.23180777634999886, "interval_95": [ -0.34126142400907755, -0.1350500024806866 ] } }, { "key": "secondary/S1/5", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (implicit) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 4.720046976401097, "interval_95": [ 4.409994671054623, 5.1129006627430655 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 112.17 [82.27, 166.15] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 112.17352205240425, "interval_95": [ 82.26902509589257, 166.15160633640508 ] } }, { "key": "secondary/S1/6", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (explicit) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 2.566402379208406, "interval_95": [ 2.179427602152041, 3.010478998848366 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 13.02 [8.84, 20.30] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 13.018903012627733, "interval_95": [ 8.841244100978317, 20.297119894182575 ] } }, { "key": "secondary/S1/7", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL NREM (explicit) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 1.0802639666684264, "interval_95": [ 0.8903866579665419, 1.267472267018982 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.95 [2.44, 3.55] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 2.945456950915512, "interval_95": [ 2.4360713956238498, 3.55186304423033 ] } }, { "key": "secondary/S1/8", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (implicit) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 2.4088074814921923, "interval_95": [ 2.2081196898317876, 2.6085574310886908 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 11.12 [9.10, 13.58] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 11.120691603993812, "interval_95": [ 9.098592123330494, 13.579447426894015 ] } }, { "key": "secondary/S1/9", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (explicit) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 0.8879996264741563, "interval_95": [ 0.7687329108249816, 1.0242306815706583 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.43 [2.16, 2.78] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 2.4302633512350424, "interval_95": [ 2.1570313703806603, 2.784952121224642 ] } }, { "key": "secondary/S1/10", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC asleep (new descriptions) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 4.706997859904129, "interval_95": [ 4.278746875546386, 5.154206935829425 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 110.72 [72.15, 173.16] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 110.71926571124455, "interval_95": [ 72.1499704420635, 173.1584265464963 ] } }, { "key": "secondary/S1/11", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC NREM (new descriptions) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 2.2230726848815214, "interval_95": [ 1.6221844444863622, 2.6493925177491136 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 9.24 [5.06, 14.15] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 9.23566560179707, "interval_95": [ 5.064140576790855, 14.145442929257207 ] } }, { "key": "secondary/S1/12", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC light (new descriptions) vs prior-corrected read-off, log R", "people": 20, "seeds": 3, "single_seed": false, "estimate": 1.7071133948613189, "interval_95": [ 1.2400502623244567, 2.0555083865835178 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (EESM19), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 5.51 [3.46, 7.81] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 5.513024561868762, "interval_95": [ 3.455787156292872, 7.810807773306661 ] } }, { "key": "secondary/S1/13", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the implicit cells", "people": 20, "seeds": 3, "single_seed": false, "estimate": -0.3906391473351739, "interval_95": [ -0.4696921758438683, -0.31089444797724264 ], "unit": "difference of AUROC", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: for the implicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages raised the unseen answers by 0.391 AUROC [0.311, 0.470], so the link between wording and stage was not shown to be used.", "display_edits": [] }, { "key": "secondary/S1/14", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the explicit cells", "people": 20, "seeds": 3, "single_seed": false, "estimate": 0.6669643660615514, "interval_95": [ 0.6336234744712421, 0.698439126586989 ], "unit": "difference of AUROC", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (EESM19), frozen CBraMod features: for the explicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages lowered the unseen answers by 0.667 AUROC [0.634, 0.698], so the link between wording and stage was shown to be used.", "display_edits": [] } ] }, "S2": { "part": "S2", "what": "S2 OpenBMI MI (L1, 3 seeds): the primary composition entries on motor imagery", "dataset": "OpenBMI", "label": "OpenBMI motor imagery, 51 people, 3 seeds", "status_label": "secondary", "single_seed": false, "declared_likely_inconclusive": false, "boas": false, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "not run for this task (no declared canary)", "threshold": 0.8 } }, "entries": [ { "key": "secondary/S2/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (3-way), training-wording score", "people": 51, "seeds": 3, "single_seed": false, "estimate": -0.004895110755368586, "interval_95": [ -0.01067214847976669, 0.00022938734106555856 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "not run for this task (no declared canary)" }, "resampled_wording_kinds": {}, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.3333333333333333, "estimate": -0.007342666133052878, "interval_95": [ -0.016008222719650033, 0.0003440810115983378 ] } }, { "key": "secondary/S2/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (3-way), training-wording score", "people": 51, "seeds": 3, "single_seed": false, "estimate": -0.005372772156004935, "interval_95": [ -0.010757251751406221, 0.00046322838391852706 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "not run for this task (no declared canary)" }, "resampled_wording_kinds": {}, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.3333333333333333, "estimate": -0.008059158234007402, "interval_95": [ -0.01613587762710933, 0.0006948425758777905 ] } }, { "key": "secondary/S2/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 51, "seeds": 3, "single_seed": false, "estimate": 0.000518829764654003, "interval_95": [ -0.001030429391524374, 0.0020002286168972635 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "not run for this task (no declared canary)" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Rewording by a new sentence frame showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.3333333333333333, "estimate": 0.0007782446469810045, "interval_95": [ -0.0015456440872865607, 0.003000342925345895 ] } }, { "key": "secondary/S2/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 51, "seeds": 3, "single_seed": false, "estimate": 0.0021616711075627573, "interval_95": [ -0.0010797744354605005, 0.0055159701985307225 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "not run for this task (no declared canary)" }, "resampled_wording_kinds": {}, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Rewording by a synonym, for these three synonyms showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.3333333333333333, "estimate": 0.0032425066613441357, "interval_95": [ -0.0016196616531907505, 0.008273955297796082 ] } }, { "key": "secondary/S2/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 51, "seeds": 3, "single_seed": false, "estimate": -0.03489259142595047, "interval_95": [ -0.0662884023790045, -0.014196715731192518 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "not run for this task (no declared canary)" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Rewording by a new description cost 3.5 pp [1.4, 6.6].", "display_edits": [], "chance_corrected": { "chance": 0.3333333333333333, "estimate": -0.0523388871389257, "interval_95": [ -0.09943260356850675, -0.021295073596788776 ] } }, { "key": "secondary/S2/5", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL hand_movement (implicit) vs prior-corrected read-off, log R", "people": 51, "seeds": 3, "single_seed": false, "estimate": 0.07595080718522022, "interval_95": [ 0.0503677810603777, 0.1074622311965319 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "equivalent within the margin", "rule": 2, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Asked whether the person was imagining a hand movement, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, was lower by R = 1.08 [1.05, 1.11] (ratio of remaining error), within the margin of 25% more remaining error.", "display_edits": [], "r": { "estimate": 1.0789094982567062, "interval_95": [ 1.0516578050823038, 1.1134488063232306 ] } }, { "key": "secondary/S2/6", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC hand_movement (new descriptions) vs prior-corrected read-off, log R", "people": 51, "seeds": 3, "single_seed": false, "estimate": 0.08845012380997347, "interval_95": [ 0.04843254074587318, 0.13849638742145606 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "equivalent within the margin", "rule": 2, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Motor imagery (OpenBMI), frozen CBraMod features: Asked whether the person was imagining a hand movement, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, was lower by R = 1.09 [1.05, 1.15] (ratio of remaining error), within the margin of 25% more remaining error (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 1.092479762523376, "interval_95": [ 1.0496245625418792, 1.148545532351737 ] } } ], "gate_note": "not gated by a triviality canary: none is declared for MI (A4 O-3); the gate label reads \"not run for this task (no declared canary)\" and excludes nothing", "shuf_derangements_note": "MI has only two 3-cycles, so the declared SHUF derangements repeat: seeds 20261007 and 20261009 share left->right, right->rest, rest->left; seed 20261008 uses left->rest, right->left, rest->right (protocol arms.SHUF.mi \"stated in output\"; A3 discrepancy)", "sentence_note": "the two P3 rule-2 sentences say \"within the margin of 25% more remaining error\"; the literal template fill is \"within the 25% more remaining error\" (A4 D-2); kept as produced" }, "S3-sleep": { "part": "S3-sleep", "what": "S3 REVE-L features, BOAS sleep (1 seed)", "dataset": "BOAS", "label": "REVE-L features, BOAS, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive", "single_seed": true, "declared_likely_inconclusive": true, "boas": true, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.76 } }, "entries": [ { "key": "secondary/S3-sleep/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.002676904127798577, "interval_95": [ -0.003623282375526954, 0.008608006495747708 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": 0.0033461301597482213, "interval_95": [ -0.004529102969408692, 0.010760008119684635 ] } }, { "key": "secondary/S3-sleep/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.0012284755782464175, "interval_95": [ -0.004788946574837976, 0.007101952133274993 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": 0.0015355944728080218, "interval_95": [ -0.00598618321854747, 0.008877440166593741 ] } }, { "key": "secondary/S3-sleep/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.0038952823296079507, "interval_95": [ -0.010357858590381751, 0.00054459724526653 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "Sleep (BOAS), frozen REVE-L features: Rewording by a new sentence frame showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.004869102912009938, "interval_95": [ -0.012947323237977188, 0.0006807465565831625 ] } }, { "key": "secondary/S3-sleep/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.13540504448674962, "interval_95": [ -0.14566141659903548, -0.12489738925675795 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Rewording by a synonym, for these three synonyms cost 13.5 pp [12.5, 14.6].", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.16925630560843702, "interval_95": [ -0.18207677074879433, -0.1561217365709474 ] } }, { "key": "secondary/S3-sleep/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.21858852336574258, "interval_95": [ -0.30621487812323556, -0.14119029397506266 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "Sleep (BOAS), frozen REVE-L features: Rewording by a new description cost 21.9 pp [14.1, 30.6].", "display_edits": [], "chance_corrected": { "chance": 0.2, "estimate": -0.2732356542071782, "interval_95": [ -0.3827685976540444, -0.17648786746882833 ] } }, { "key": "secondary/S3-sleep/5", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 4.149333046580226, "interval_95": [ 3.7533550201533563, 4.613635076182565 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 63.39 [42.66, 100.85] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 63.39170688013221, "interval_95": [ 42.663980667028454, 100.8500818682826 ] } }, { "key": "secondary/S3-sleep/6", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 2.647022812726814, "interval_95": [ 2.2739638815135415, 3.07027719611665 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 14.11 [9.72, 21.55] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 14.111962087632012, "interval_95": [ 9.717844955695718, 21.547874834460643 ] } }, { "key": "secondary/S3-sleep/7", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL NREM (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.3170816258219011, "interval_95": [ 1.162698165141484, 1.4830984993855754 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 3.73 [3.20, 4.41] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 3.7325125986670233, "interval_95": [ 3.198551864844621, 4.406578329882257 ] } }, { "key": "secondary/S3-sleep/8", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 2.7295391481006224, "interval_95": [ 2.5395194949639115, 2.929251368941949 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 15.33 [12.67, 18.71] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 15.325822457784598, "interval_95": [ 12.673579788615507, 18.7136156574659 ] } }, { "key": "secondary/S3-sleep/9", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.8718690895272136, "interval_95": [ 1.6912621184272383, 2.0572597142455535 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 6.50 [5.43, 7.82] (ratio of remaining error).", "display_edits": [], "r": { "estimate": 6.500434946618093, "interval_95": [ 5.426325049873471, 7.824499042472677 ] } }, { "key": "secondary/S3-sleep/10", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC asleep (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 4.091712584267312, "interval_95": [ 3.5654224331479005, 4.6095164937200215 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 59.84 [35.35, 100.44] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 59.84228895974601, "interval_95": [ 35.35438511442896, 100.43557666261707 ] } }, { "key": "secondary/S3-sleep/11", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC NREM (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.828720455416431, "interval_95": [ 1.249615485433395, 2.2923610021012553 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 6.23 [3.49, 9.90] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 6.225915223606997, "interval_95": [ 3.4890011277453183, 9.898279974546911 ] } }, { "key": "secondary/S3-sleep/12", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC light (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.773906155450779, "interval_95": [ 1.162455494415868, 2.235593471012737 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen REVE-L features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 5.89 [3.20, 9.35] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [], "r": { "estimate": 5.893830674503593, "interval_95": [ 3.197775764114931, 9.352030364524747 ] } }, { "key": "secondary/S3-sleep/13", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the implicit cells", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.5847179371696831, "interval_95": [ -0.6118583677676402, -0.5553946105819667 ], "unit": "difference of AUROC", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: for the implicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages raised the unseen answers by 0.585 AUROC [0.555, 0.612], so the link between wording and stage was not shown to be used.", "display_edits": [] }, { "key": "secondary/S3-sleep/14", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the explicit cells", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.6262736828554746, "interval_95": [ 0.6021952354064467, 0.6475084181150963 ], "unit": "difference of AUROC", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen REVE-L features: for the explicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages lowered the unseen answers by 0.626 AUROC [0.602, 0.648], so the link between wording and stage was shown to be used.", "display_edits": [] } ] }, "S3-ssvep": { "part": "S3-ssvep", "what": "S3 REVE-L features, BETA SSVEP (1 seed)", "dataset": "BETA", "label": "REVE-L features, BETA, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive", "single_seed": true, "declared_likely_inconclusive": true, "boas": false, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.7075 } }, "entries": [ { "key": "secondary/S3-ssvep/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.027626488095238096, "interval_95": [ -0.035053013392857144, -0.020719680059523814 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Asking a seen question with a template instead of a question number cost 2.8 pp [2.1, 3.5].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.028517665130568356, "interval_95": [ -0.03618375576036866, -0.021388056835637485 ] } }, { "key": "secondary/S3-ssvep/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.025580357142857134, "interval_95": [ -0.03240011160714285, -0.01895005580357143 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Asking a seen question with a description instead of a question number cost 2.6 pp [1.9, 3.2].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.026405529953917043, "interval_95": [ -0.033445276497695846, -0.01956134792626728 ] } }, { "key": "secondary/S3-ssvep/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.04484747023809524, "interval_95": [ -0.0765708705357143, -0.021649972098214308 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "SSVEP (BETA), frozen REVE-L features: Rewording by a new sentence frame cost 4.5 pp [2.2, 7.7].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.04629416282642089, "interval_95": [ -0.07904089861751154, -0.022348358294930898 ] } }, { "key": "secondary/S3-ssvep/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.02158978174603175, "interval_95": [ -0.024399863591269836, -0.01876984126984127 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Rewording by a synonym, for these three synonyms cost 2.2 pp [1.9, 2.4].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.022286226318484387, "interval_95": [ -0.025186955965181765, -0.01937532002048131 ] } }, { "key": "secondary/S3-ssvep/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.05053794642857143, "interval_95": [ -0.06496227678571426, -0.03772293526785714 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "SSVEP (BETA), frozen REVE-L features: Rewording by a new description cost 5.1 pp [3.8, 6.5].", "display_edits": [], "chance_corrected": { "chance": 0.03125, "estimate": -0.05216820276497696, "interval_95": [ -0.06705783410138245, -0.03893980414746543 ] } }, { "key": "secondary/S3-ssvep/5", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.6532142857142859, "interval_95": [ -0.6917626488095238, -0.611656994047619 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 65.3 pp [61.2, 69.2].", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": -0.7465306122448981, "interval_95": [ -0.7905858843537414, -0.6990365646258503 ] } }, { "key": "secondary/S3-ssvep/6", "id": "P4", "type": "P4_nn", "kind": "unseen", "label": "TPL - NN(TPL), 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": 0.05806547619047619, "interval_95": [ 0.04541517857142857, 0.07041815476190474 ], "unit": "difference of proportions", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did better than averaging its own answers for the two neighbouring seen frequencies.", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": 0.06636054421768707, "interval_95": [ 0.051903061224489795, 0.08047789115646256 ] } }, { "key": "secondary/S3-ssvep/7", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": 0.01919642857142857, "interval_95": [ 0.008511160714285714, 0.029170386904761886 ], "unit": "difference of proportions", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did better than giving it as a number.", "display_edits": [], "chance_corrected": { "chance": 0.125, "estimate": 0.021938775510204078, "interval_95": [ 0.00972704081632653, 0.033337585034013587 ] } }, { "key": "secondary/S3-ssvep/8", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.3875297619047619, "interval_95": [ -0.4161934523809524, -0.3559813988095238 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors (frequency and stimulus phase)), the head, compared with CCA, which needs no training, fell short by 38.8 pp [35.6, 41.6].", "display_edits": [ "W2c" ], "sentence_as_produced": "SSVEP (BETA), frozen REVE-L features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors), the head, compared with CCA, which needs no training, fell short by 38.8 pp [35.6, 41.6].", "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.5886528028933092, "interval_95": [ -0.6321925858951176, -0.5407312386980109 ] } }, { "key": "secondary/S3-ssvep/9", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": 0.3522023809523809, "interval_95": [ 0.32913392857142854, 0.37419717261904756 ], "unit": "difference of proportions", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen REVE-L features: Stating the unseen frequency in words (neighbour 3-way, frequency and stimulus phase) (unseen by the EEG head, not by the text encoder) did better than giving it as a number.", "display_edits": [ "W2c" ], "sentence_as_produced": "SSVEP (BETA), frozen REVE-L features: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder) did better than giving it as a number.", "chance_corrected": { "chance": 0.3416666666666667, "estimate": 0.5349909584086798, "interval_95": [ 0.49995027124773955, 0.5684007685352621 ] } } ] }, "S4-sleep-L1": { "part": "S4-sleep-L1", "what": "S4 all-MiniLM-L6-v2 text encoder, BOAS sleep L1 (1 seed; non-text arms from the primary block, same seed)", "dataset": "BOAS", "label": "all-MiniLM-L6-v2, BOAS L1, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive", "single_seed": true, "declared_likely_inconclusive": true, "boas": true, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.76 } }, "entries": [ { "key": "secondary/S4-sleep-L1/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.001900212303481441, "interval_95": [ -0.0037282389285810953, 0.007489040005533212 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "chance_corrected": { "chance": 0.2, "estimate": 0.002375265379351801, "interval_95": [ -0.004660298660726369, 0.009361300006916514 ] } }, { "key": "secondary/S4-sleep-L1/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.00012084876520722932, "interval_95": [ -0.0064273731949877145, 0.007201621438844179 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "chance_corrected": { "chance": 0.2, "estimate": 0.00015106095650903664, "interval_95": [ -0.008034216493734642, 0.009002026798555223 ] } }, { "key": "secondary/S4-sleep-L1/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.01716431748127351, "interval_95": [ -0.043055469630105564, -0.0008916059069849953 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Rewording by a new sentence frame cost 1.7 pp [0.1, 4.3].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Rewording by a new sentence frame cost 1.7 pp [0.1, 4.3].", "chance_corrected": { "chance": 0.2, "estimate": -0.021455396851591885, "interval_95": [ -0.053819337037631955, -0.001114507383731244 ] } }, { "key": "secondary/S4-sleep-L1/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.14043879399162776, "interval_95": [ -0.15288855542821628, -0.1268429718857462 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Rewording by a synonym, for these three synonyms cost 14.0 pp [12.7, 15.3].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Rewording by a synonym, for these three synonyms cost 14.0 pp [12.7, 15.3].", "chance_corrected": { "chance": 0.2, "estimate": -0.1755484924895347, "interval_95": [ -0.19111069428527033, -0.15855371485718273 ] } }, { "key": "secondary/S4-sleep-L1/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.2573985056694924, "interval_95": [ -0.34287611467236107, -0.16256041120565853 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Rewording by a new description cost 25.7 pp [16.3, 34.3].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Rewording by a new description cost 25.7 pp [16.3, 34.3].", "chance_corrected": { "chance": 0.2, "estimate": -0.3217481320868655, "interval_95": [ -0.4285951433404513, -0.20320051400707315 ] } }, { "key": "secondary/S4-sleep-L1/5", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 3.797715139627106, "interval_95": [ 3.3854559687069488, 4.273337655196738 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 44.60 [29.53, 71.76] (ratio of remaining error).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 44.60 [29.53, 71.76] (ratio of remaining error).", "r": { "estimate": 44.599165122622736, "interval_95": [ 29.531455066093905, 71.76074899504793 ] } }, { "key": "secondary/S4-sleep-L1/6", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.3252615631135052, "interval_95": [ 0.9985491819024604, 1.714819062653399 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 3.76 [2.71, 5.56] (ratio of remaining error).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 3.76 [2.71, 5.56] (ratio of remaining error).", "r": { "estimate": 3.7631695326216184, "interval_95": [ 2.7143409554243165, 5.555670193190924 ] } }, { "key": "secondary/S4-sleep-L1/7", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL NREM (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.881695266502562, "interval_95": [ 0.7171855363845893, 1.047026038432296 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.41 [2.05, 2.85] (ratio of remaining error).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.41 [2.05, 2.85] (ratio of remaining error).", "r": { "estimate": 2.4149902902657523, "interval_95": [ 2.048659212287802, 2.849165198118467 ] } }, { "key": "secondary/S4-sleep-L1/8", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 2.338053260493132, "interval_95": [ 2.1338266703199658, 2.551730041214566 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 10.36 [8.45, 12.83] (ratio of remaining error).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 10.36 [8.45, 12.83] (ratio of remaining error).", "r": { "estimate": 10.361046658003916, "interval_95": [ 8.447129422332173, 12.82927977720739 ] } }, { "key": "secondary/S4-sleep-L1/9", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.7962606359815592, "interval_95": [ 1.597824065880024, 2.000337607864331 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 6.03 [4.94, 7.39] (ratio of remaining error).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 6.03 [4.94, 7.39] (ratio of remaining error).", "r": { "estimate": 6.027067873326066, "interval_95": [ 4.942266669180285, 7.391551123525896 ] } }, { "key": "secondary/S4-sleep-L1/10", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC asleep (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 3.0687995485343182, "interval_95": [ 2.5081458482094523, 3.601328976091376 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 21.52 [12.28, 36.65] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 21.52 [12.28, 36.65] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "r": { "estimate": 21.516058181996065, "interval_95": [ 12.282135990769035, 36.646904956050456 ] } }, { "key": "secondary/S4-sleep-L1/11", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC NREM (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.1179313148116743, "interval_95": [ 0.7933050241080076, 1.449103249247085 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 3.06 [2.21, 4.26] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 3.06 [2.21, 4.26] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "r": { "estimate": 3.058520538236538, "interval_95": [ 2.210690751885468, 4.259293277622475 ] } }, { "key": "secondary/S4-sleep-L1/12", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC light (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.0704374687304536, "interval_95": [ 0.8395734875604675, 1.3325087171375543 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 2.92 [2.32, 3.79] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 2.92 [2.32, 3.79] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "r": { "estimate": 2.9166551663573737, "interval_95": [ 2.3153792281094936, 3.7905408641730127 ] } }, { "key": "secondary/S4-sleep-L1/13", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the implicit cells", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.6255149244061795, "interval_95": [ -0.6533152522508616, -0.5962162952014464 ], "unit": "difference of AUROC", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: for the implicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages raised the unseen answers by 0.626 AUROC [0.596, 0.653], so the link between wording and stage was not shown to be used.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: for the implicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages raised the unseen answers by 0.626 AUROC [0.596, 0.653], so the link between wording and stage was not shown to be used." }, { "key": "secondary/S4-sleep-L1/14", "id": "P5", "type": "P5", "kind": "unseen", "label": "TPL - SHUF unseen AUROC, mean of the explicit cells", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.5720501446110335, "interval_95": [ 0.5500913783397637, 0.5921855676956208 ], "unit": "difference of AUROC", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, text encoder all-MiniLM-L6-v2: for the explicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages lowered the unseen answers by 0.572 AUROC [0.550, 0.592], so the link between wording and stage was shown to be used.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: for the explicit template wordings of the unseen questions (unseen by the EEG head, not by the text encoder), attaching the same template wordings to the wrong stages lowered the unseen answers by 0.572 AUROC [0.550, 0.592], so the link between wording and stage was shown to be used." } ] }, "S4-ssvep-L0": { "part": "S4-ssvep-L0", "what": "S4 all-MiniLM-L6-v2, BETA SSVEP L0 (1 seed)", "dataset": "BETA", "label": "all-MiniLM-L6-v2, BETA L0, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive", "single_seed": true, "declared_likely_inconclusive": true, "boas": false, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.7075 } }, "entries": [ { "key": "secondary/S4-ssvep-L0/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.06733630952380953, "interval_95": [ -0.07753087797619047, -0.056812127976190493 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Asking a seen question with a template instead of a question number cost 6.7 pp [5.7, 7.8].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asking a seen question with a template instead of a question number cost 6.7 pp [5.7, 7.8].", "chance_corrected": { "chance": 0.03125, "estimate": -0.06950844854070662, "interval_95": [ -0.08003187403993856, -0.058644777265745025 ] } }, { "key": "secondary/S4-ssvep-L0/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.11583705357142861, "interval_95": [ -0.13155217633928573, -0.09973688616071433 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Asking a seen question with a description instead of a question number cost 11.6 pp [10.0, 13.2].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asking a seen question with a description instead of a question number cost 11.6 pp [10.0, 13.2].", "chance_corrected": { "chance": 0.03125, "estimate": -0.11957373271889404, "interval_95": [ -0.1357957949308756, -0.10295420506912446 ] } }, { "key": "secondary/S4-ssvep-L0/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.03954613095238096, "interval_95": [ -0.051111979166666675, -0.02862095424107145 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Rewording by a new sentence frame cost 4.0 pp [2.9, 5.1].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Rewording by a new sentence frame cost 4.0 pp [2.9, 5.1].", "chance_corrected": { "chance": 0.03125, "estimate": -0.04082181259600615, "interval_95": [ -0.052760752688172055, -0.02954421082949311 ] } }, { "key": "secondary/S4-ssvep-L0/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.04895833333333333, "interval_95": [ -0.05530016121031746, -0.04254873511904762 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Rewording by a synonym, for these three synonyms cost 4.9 pp [4.3, 5.5].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Rewording by a synonym, for these three synonyms cost 4.9 pp [4.3, 5.5].", "chance_corrected": { "chance": 0.03125, "estimate": -0.05053763440860215, "interval_95": [ -0.05708403737839222, -0.04392127496159755 ] } }, { "key": "secondary/S4-ssvep-L0/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.05494419642857142, "interval_95": [ -0.06483370535714286, -0.04597527901785713 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Rewording by a new description cost 5.5 pp [4.6, 6.5].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Rewording by a new description cost 5.5 pp [4.6, 6.5].", "chance_corrected": { "chance": 0.03125, "estimate": -0.05671658986175115, "interval_95": [ -0.06692511520737328, -0.0474583525345622 ] } }, { "key": "secondary/S4-ssvep-L0/5", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.5009523809523809, "interval_95": [ -0.5305736607142857, -0.46734821428571427 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 50.1 pp [46.7, 53.1].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 50.1 pp [46.7, 53.1].", "chance_corrected": { "chance": 0.125, "estimate": -0.5725170068027211, "interval_95": [ -0.6063698979591836, -0.5341122448979592 ] } }, { "key": "secondary/S4-ssvep-L0/6", "id": "P4", "type": "P4_nn", "kind": "unseen", "label": "TPL - NN(TPL), 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.07681547619047617, "interval_95": [ -0.08824925595238096, -0.06496651785714283 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies.", "chance_corrected": { "chance": 0.125, "estimate": -0.08778911564625848, "interval_95": [ -0.10085629251700681, -0.0742474489795918 ] } }, { "key": "secondary/S4-ssvep-L0/7", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.10836309523809523, "interval_95": [ -0.12526785714285713, -0.09083184523809523 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number.", "chance_corrected": { "chance": 0.125, "estimate": -0.12384353741496598, "interval_95": [ -0.14316326530612242, -0.1038078231292517 ] } }, { "key": "secondary/S4-ssvep-L0/8", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.618452380952381, "interval_95": [ -0.651265625, -0.5831220238095238 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors), the head, compared with CCA, which needs no training, fell short by 61.8 pp [58.3, 65.1].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors), the head, compared with CCA, which needs no training, fell short by 61.8 pp [58.3, 65.1].", "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.9394213381555154, "interval_95": [ -0.9892642405063291, -0.8857549728752261 ] } }, { "key": "secondary/S4-ssvep-L0/9", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.0017559523809523823, "interval_95": [ -0.020210565476190472, 0.017326636904761872 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, text encoder all-MiniLM-L6-v2: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder): no difference shown against giving it as a number.", "display_edits": [ "W2a", "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder) did no difference shown against giving it as a number.", "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.0026672694394213404, "interval_95": [ -0.03069959312839059, 0.026318942133815502 ] } } ] }, "S4-ssvep-L1": { "part": "S4-ssvep-L1", "what": "S4 all-MiniLM-L6-v2, BETA SSVEP L1 (1 seed)", "dataset": "BETA", "label": "all-MiniLM-L6-v2, BETA L1, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive", "single_seed": true, "declared_likely_inconclusive": true, "boas": false, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.7075 } }, "entries": [ { "key": "secondary/S4-ssvep-L1/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.04729910714285713, "interval_95": [ -0.05428776041666668, -0.0407289806547619 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asking a seen question with a template instead of a question number cost 4.7 pp [4.1, 5.4].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Asking a seen question with a template instead of a question number cost 4.7 pp [4.1, 5.4].", "chance_corrected": { "chance": 0.03125, "estimate": -0.04882488479262671, "interval_95": [ -0.056038978494623665, -0.04204281874039938 ] } }, { "key": "secondary/S4-ssvep-L1/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.08840401785714286, "interval_95": [ -0.10118331473214284, -0.07600446428571428 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asking a seen question with a description instead of a question number cost 8.8 pp [7.6, 10.1].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Asking a seen question with a description instead of a question number cost 8.8 pp [7.6, 10.1].", "chance_corrected": { "chance": 0.03125, "estimate": -0.09125576036866359, "interval_95": [ -0.1044472926267281, -0.07845622119815668 ] } }, { "key": "secondary/S4-ssvep-L1/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.028921130952380945, "interval_95": [ -0.03731036086309525, -0.020754650297619054 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Rewording by a new sentence frame cost 2.9 pp [2.1, 3.7].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Rewording by a new sentence frame cost 2.9 pp [2.1, 3.7].", "chance_corrected": { "chance": 0.03125, "estimate": -0.029854070660522265, "interval_95": [ -0.03851392089093703, -0.021424155145929345 ] } }, { "key": "secondary/S4-ssvep-L1/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.030947420634920637, "interval_95": [ -0.03573970734126984, -0.026298673115079368 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Rewording by a synonym, for these three synonyms cost 3.1 pp [2.6, 3.6].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Rewording by a synonym, for these three synonyms cost 3.1 pp [2.6, 3.6].", "chance_corrected": { "chance": 0.03125, "estimate": -0.03194572452636969, "interval_95": [ -0.036892601126472094, -0.027147017409114186 ] } }, { "key": "secondary/S4-ssvep-L1/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.029808035714285714, "interval_95": [ -0.03669949776785714, -0.023390011160714284 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Rewording by a new description cost 3.0 pp [2.3, 3.7].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Rewording by a new description cost 3.0 pp [2.3, 3.7].", "chance_corrected": { "chance": 0.03125, "estimate": -0.03076958525345622, "interval_95": [ -0.0378833525345622, -0.024144527649769584 ] } }, { "key": "secondary/S4-ssvep-L1/5", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.6155357142857143, "interval_95": [ -0.6521183035714285, -0.5749293154761906 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 61.6 pp [57.5, 65.2].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 61.6 pp [57.5, 65.2].", "chance_corrected": { "chance": 0.125, "estimate": -0.703469387755102, "interval_95": [ -0.7452780612244897, -0.6570620748299322 ] } }, { "key": "secondary/S4-ssvep-L1/6", "id": "P4", "type": "P4_nn", "kind": "unseen", "label": "TPL - NN(TPL), 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.010863095238095236, "interval_95": [ -0.01693526785714286, -0.004761904761904763 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies.", "chance_corrected": { "chance": 0.125, "estimate": -0.012414965986394555, "interval_95": [ -0.019354591836734696, -0.005442176870748301 ] } }, { "key": "secondary/S4-ssvep-L1/7", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.02026785714285714, "interval_95": [ -0.028602678571428574, -0.011873511904761914 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number.", "chance_corrected": { "chance": 0.125, "estimate": -0.023163265306122446, "interval_95": [ -0.032688775510204084, -0.013569727891156473 ] } }, { "key": "secondary/S4-ssvep-L1/8", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.54625, "interval_95": [ -0.5751197916666669, -0.5129352678571429 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors (frequency and stimulus phase)), the head, compared with CCA, which needs no training, fell short by 54.6 pp [51.3, 57.5].", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors (frequency and stimulus phase)), the head, compared with CCA, which needs no training, fell short by 54.6 pp [51.3, 57.5].", "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.829746835443038, "interval_95": [ -0.8735996835443042, -0.7791421790235082 ] } }, { "key": "secondary/S4-ssvep-L1/9", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": 0.15571428571428572, "interval_95": [ 0.14273660714285713, 0.16747247023809522 ], "unit": "difference of proportions", "difference": "difference: language arm higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), frozen CBraMod features, text encoder all-MiniLM-L6-v2: Stating the unseen frequency in words (neighbour 3-way, frequency and stimulus phase) (unseen by the EEG head, not by the text encoder) did better than giving it as a number.", "display_edits": [ "W2b-minilm" ], "sentence_as_produced": "SSVEP (BETA), frozen CBraMod features: Stating the unseen frequency in words (neighbour 3-way, frequency and stimulus phase) (unseen by the EEG head, not by the text encoder) did better than giving it as a number.", "chance_corrected": { "chance": 0.3416666666666667, "estimate": 0.23652802893309224, "interval_95": [ 0.21681509945750452, 0.2543885623869801 ] } } ] }, "S5-sleep-L1": { "part": "S5-sleep-L1", "what": "S5 Chinese wordings, multilingual-e5-small, BOAS sleep L1 (1 seed)", "dataset": "BOAS", "label": "Chinese wordings, multilingual-e5-small, BOAS L1, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive", "single_seed": true, "declared_likely_inconclusive": true, "boas": true, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.76 } }, "entries": [ { "key": "secondary/S5-sleep-L1/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.0033649485932037426, "interval_95": [ -0.00293495148631439, 0.009893287198633293 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asking a seen question with a template instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "chance_corrected": { "chance": 0.2, "estimate": 0.004206185741504678, "interval_95": [ -0.0036686893578929875, 0.012366608998291615 ] } }, { "key": "secondary/S5-sleep-L1/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (5-way), training-wording score", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.0028123987870505985, "interval_95": [ -0.009282212148267226, 0.004176293881541657 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asking a seen question with a description instead of a question number showed no measurable difference (equivalent within the 2-pp margin).", "chance_corrected": { "chance": 0.2, "estimate": -0.003515498483813248, "interval_95": [ -0.011602765185334032, 0.005220367351927071 ] } }, { "key": "secondary/S5-sleep-L1/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.006951294482272536, "interval_95": [ -0.018331569448406363, 0.00076505697663534 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "equivalent within the margin", "rule": 4, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Rewording by a new sentence frame showed no measurable difference (equivalent within the 2-pp margin).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Rewording by a new sentence frame showed no measurable difference (equivalent within the 2-pp margin).", "chance_corrected": { "chance": 0.2, "estimate": -0.00868911810284067, "interval_95": [ -0.022914461810507953, 0.000956321220794175 ] } }, { "key": "secondary/S5-sleep-L1/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.12508242522424884, "interval_95": [ -0.13732718039106037, -0.11271970002659513 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Rewording by a synonym, for these three synonyms cost 12.5 pp [11.3, 13.7].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Rewording by a synonym, for these three synonyms cost 12.5 pp [11.3, 13.7].", "chance_corrected": { "chance": 0.2, "estimate": -0.15635303153031105, "interval_95": [ -0.17165897548882544, -0.1408996250332439 ] } }, { "key": "secondary/S5-sleep-L1/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 100, "seeds": 1, "single_seed": true, "estimate": -0.2580569102445444, "interval_95": [ -0.3396922249327728, -0.17726934282360934 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Rewording by a new description cost 25.8 pp [17.7, 34.0].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Rewording by a new description cost 25.8 pp [17.7, 34.0].", "chance_corrected": { "chance": 0.2, "estimate": -0.3225711378056805, "interval_95": [ -0.424615281165966, -0.22158667852951167 ] } }, { "key": "secondary/S5-sleep-L1/5", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 3.8258221359197804, "interval_95": [ 3.423961312890943, 4.277278723491485 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 45.87 [30.69, 72.04] (ratio of remaining error).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 45.87 [30.69, 72.04] (ratio of remaining error).", "r": { "estimate": 45.87049665143049, "interval_95": [ 30.690750203486914, 72.04412103522296 ] } }, { "key": "secondary/S5-sleep-L1/6", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL asleep (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.5583646240468414, "interval_95": [ 1.2157267494464048, 1.9500927187754504 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 4.75 [3.37, 7.03] (ratio of remaining error).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 4.75 [3.37, 7.03] (ratio of remaining error).", "r": { "estimate": 4.7510451434660865, "interval_95": [ 3.3727443139179982, 7.029339302107733 ] } }, { "key": "secondary/S5-sleep-L1/7", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL NREM (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.7367309413127218, "interval_95": [ 0.5839851992862739, 0.8975547029266139 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.09 [1.79, 2.45] (ratio of remaining error).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 2.09 [1.79, 2.45] (ratio of remaining error).", "r": { "estimate": 2.089094965462783, "interval_95": [ 1.7931703514532207, 2.453595998448392 ] } }, { "key": "secondary/S5-sleep-L1/8", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (implicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 2.321527048159614, "interval_95": [ 2.1216022018953753, 2.5316053779465775 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 10.19 [8.34, 12.57] (ratio of remaining error).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (implicit wording), compared with adding up its own stage answers, fell short by R = 10.19 [8.34, 12.57] (ratio of remaining error).", "r": { "estimate": 10.1912249210424, "interval_95": [ 8.344496350901338, 12.573675443921035 ] } }, { "key": "secondary/S5-sleep-L1/9", "id": "P3", "type": "P3", "kind": "unseen", "label": "TPL light (explicit) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 1.6146031376266599, "interval_95": [ 1.4236696412911896, 1.8093484776252653 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 5.03 [4.15, 6.11] (ratio of remaining error).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (explicit wording), compared with adding up its own stage answers, fell short by R = 5.03 [4.15, 6.11] (ratio of remaining error).", "r": { "estimate": 5.025892939827709, "interval_95": [ 4.152330078135081, 6.106467635612922 ] } }, { "key": "secondary/S5-sleep-L1/10", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC asleep (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 2.7151803229096982, "interval_95": [ 2.1268274314431737, 3.2436787440905155 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 15.11 [8.39, 25.63] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was asleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 15.11 [8.39, 25.63] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "r": { "estimate": 15.107334024721592, "interval_95": [ 8.388212373457026, 25.62782676334605 ] } }, { "key": "secondary/S5-sleep-L1/11", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC NREM (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.7976184988122573, "interval_95": [ 0.5985525017146048, 1.0388204999054387 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 2.22 [1.82, 2.83] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in N1, N2 or N3 sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 2.22 [1.82, 2.83] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "r": { "estimate": 2.2202471062518834, "interval_95": [ 1.8194831945288283, 2.8258819198743774 ] } }, { "key": "secondary/S5-sleep-L1/12", "id": "P3", "type": "P3", "kind": "unseen", "label": "DESC light (new descriptions) vs prior-corrected read-off, log R", "people": 100, "seeds": 1, "single_seed": true, "estimate": 0.9433342152896088, "interval_95": [ 0.7731192388010031, 1.1232487826303803 ], "unit": "log R", "difference": "difference: the read-off leaves less error", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": { "ud": 10 }, "sentence": "Sleep (BOAS), frozen CBraMod features, Chinese wordings with multilingual-e5-small: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 2.57 [2.17, 3.07] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "Sleep (BOAS), frozen CBraMod features: Asked whether the person was in light sleep, which it was never trained on (unseen by the EEG head, not by the text encoder), the head (a new description), compared with adding up its own stage answers, fell short by R = 2.57 [2.17, 3.07] (ratio of remaining error) (no alignment control exists for descriptions: the shuffled control is built from template vectors).", "r": { "estimate": 2.5685311929894303, "interval_95": [ 2.1665135983034562, 3.0748274400605484 ] } } ] }, "S5-ssvep-L0": { "part": "S5-ssvep-L0", "what": "S5 Chinese wordings, multilingual-e5-small, BETA SSVEP L0 (1 seed)", "dataset": "BETA", "label": "Chinese wordings, multilingual-e5-small, BETA L0, 1 seed", "status_label": "single seed: no training-run variance; declared likely inconclusive; every unseen-frequency entry states that each held-out frequency is a new token for this encoder (S0b-6)", "single_seed": true, "declared_likely_inconclusive": true, "boas": false, "gates": { "headroom": { "pass": true, "label": "pass" }, "triviality": { "pass": true, "label": "passes canary", "threshold": 0.7075 } }, "entries": [ { "key": "secondary/S5-ssvep-L0/0", "id": "P1", "type": "P1", "kind": "seen", "label": "TPL - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.03148809523809523, "interval_95": [ -0.037954985119047616, -0.02478404017857143 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Asking a seen question with a template instead of a question number cost 3.1 pp [2.5, 3.8].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asking a seen question with a template instead of a question number cost 3.1 pp [2.5, 3.8].", "chance_corrected": { "chance": 0.03125, "estimate": -0.032503840245775725, "interval_95": [ -0.039179339477726574, -0.02558352534562212 ] } }, { "key": "secondary/S5-ssvep-L0/1", "id": "P1", "type": "P1", "kind": "seen", "label": "DESC - ID, seen task (32-way), training-wording score", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.049408482142857146, "interval_95": [ -0.05685323660714286, -0.04182952008928573 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Asking a seen question with a description instead of a question number cost 4.9 pp [4.2, 5.7].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asking a seen question with a description instead of a question number cost 4.9 pp [4.2, 5.7].", "chance_corrected": { "chance": 0.03125, "estimate": -0.05100230414746544, "interval_95": [ -0.058687211981566824, -0.04317885944700462 ] } }, { "key": "secondary/S5-ssvep-L0/2", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a1) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.046690848214285714, "interval_95": [ -0.06567875744047619, -0.032364164806547635 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a1": 12 }, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Rewording by a new sentence frame cost 4.7 pp [3.2, 6.6].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Rewording by a new sentence frame cost 4.7 pp [3.2, 6.6].", "chance_corrected": { "chance": 0.03125, "estimate": -0.04819700460829493, "interval_95": [ -0.06779742703533026, -0.03340817012288788 ] } }, { "key": "secondary/S5-ssvep-L0/3", "id": "P2", "type": "P2", "kind": "rewording", "label": "TPL(a2) - TPL(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.08698412698412698, "interval_95": [ -0.09764180307539683, -0.07599392361111111 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Rewording by a synonym, for these three synonyms cost 8.7 pp [7.6, 9.8].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Rewording by a synonym, for these three synonyms cost 8.7 pp [7.6, 9.8].", "chance_corrected": { "chance": 0.03125, "estimate": -0.08979006656426011, "interval_95": [ -0.10079153865847415, -0.07844534050179211 ] } }, { "key": "secondary/S5-ssvep-L0/4", "id": "P2", "type": "P2", "kind": "rewording", "label": "DESC(a3) - DESC(train), seen task", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.07351785714285715, "interval_95": [ -0.08578348214285714, -0.061747712053571414 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "triviality canary", "label": "passes canary" }, "resampled_wording_kinds": { "a3": 10 }, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Rewording by a new description cost 7.4 pp [6.2, 8.6].", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Rewording by a new description cost 7.4 pp [6.2, 8.6].", "chance_corrected": { "chance": 0.03125, "estimate": -0.075889400921659, "interval_95": [ -0.08855069124423963, -0.06373957373271888 ] } }, { "key": "secondary/S5-ssvep-L0/5", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.5476488095238097, "interval_95": [ -0.5776815476190476, -0.51246875 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 54.8 pp [51.2, 57.8]. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head, compared with CCA, which needs no training, fell short by 54.8 pp [51.2, 57.8]. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "chance_corrected": { "chance": 0.125, "estimate": -0.6258843537414968, "interval_95": [ -0.6602074829931972, -0.5856785714285715 ] } }, { "key": "secondary/S5-ssvep-L0/6", "id": "P4", "type": "P4_nn", "kind": "unseen", "label": "TPL - NN(TPL), 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.15806547619047617, "interval_95": [ -0.17675595238095235, -0.1393720238095238 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (8-way among unseen frequencies), the head did worse than averaging its own answers for the two neighbouring seen frequencies. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "chance_corrected": { "chance": 0.125, "estimate": -0.18064625850340135, "interval_95": [ -0.2020068027210884, -0.15928231292517006 ] } }, { "key": "secondary/S5-ssvep-L0/7", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, 8-way unseen", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.15505952380952379, "interval_95": [ -0.1758355654761905, -0.13443229166666665 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (8-way) (unseen by the EEG head, not by the text encoder) did worse than giving it as a number. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "chance_corrected": { "chance": 0.125, "estimate": -0.17721088435374147, "interval_95": [ -0.20095493197278916, -0.15363690476190475 ] } }, { "key": "secondary/S5-ssvep-L0/8", "id": "P4", "type": "P4_cca", "kind": "unseen", "label": "TPL - CCA, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": -0.5997619047619046, "interval_95": [ -0.6374479166666668, -0.5586160714285715 ], "unit": "difference of proportions", "difference": "difference: reference higher", "margin": "margin not met", "rule": 1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors), the head, compared with CCA, which needs no training, fell short by 60.0 pp [55.9, 63.7]. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "display_edits": [ "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Asked about a flicker frequency it was never trained on (unseen by the EEG head, not by the text encoder) (telling it from its 0.2 Hz neighbours, which are trained detectors), the head, compared with CCA, which needs no training, fell short by 60.0 pp [55.9, 63.7]. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "chance_corrected": { "chance": 0.3416666666666667, "estimate": -0.911030741410488, "interval_95": [ -0.9682753164556964, -0.8485307414104883 ] } }, { "key": "secondary/S5-ssvep-L0/9", "id": "P4", "type": "P4_num", "kind": "unseen", "label": "TPL - NUM, neighbour 3-way", "people": 70, "seeds": 1, "single_seed": true, "estimate": 0.016934523809523806, "interval_95": [ -0.005953869047619047, 0.041822916666666626 ], "unit": "difference of proportions", "difference": "no difference shown", "margin": "not applicable (difference only)", "rule": -1, "gate": { "gate": "headroom", "label": "pass" }, "resampled_wording_kinds": {}, "sentence": "SSVEP (BETA), a plain spectrum, Chinese wordings with multilingual-e5-small: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder): no difference shown against giving it as a number. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "display_edits": [ "W2a", "W2b-zh" ], "sentence_as_produced": "SSVEP (BETA), a plain spectrum: Stating the unseen frequency in words (neighbour 3-way) (unseen by the EEG head, not by the text encoder) did no difference shown against giving it as a number. For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6).", "chance_corrected": { "chance": 0.3416666666666667, "estimate": 0.02572332730560578, "interval_95": [ -0.00904385171790235, 0.06352848101265816 ] } } ], "token_note": "For multilingual-e5-small each held-out frequency is a new token, not only a new combination of seen tokens: its tokenizer keeps one-decimal number fragments such as \"8.2\" or \"2.6\" (of \"12.6\") as single pieces, so the token rule cannot hold for this encoder (reported, not a stop; decision log S0b-6)." }, "S6": { "part": "S6", "what": "S6 negation probe, BOAS sleep L1 (boundary probe; never a route sentence)", "dataset": "BOAS", "boas": true, "boundary_probe": true, "never_a_route_sentence": true, "cells": { "not W": { "negated_wording_auroc": { "estimate": 0.021506780843323376, "interval_95": [ 0.014449931422547406, 0.030214131454444806 ], "people": 100, "seeds": 3 }, "one_minus_p_auroc": { "estimate": 0.9801445674548448, "interval_95": [ 0.9720031817823664, 0.9867598451065624 ], "people": 100, "seeds": 3 } }, "not N1": { "negated_wording_auroc": { "estimate": 0.13452029355979675, "interval_95": [ 0.11963987861703639, 0.15162795972729975 ], "people": 100, "seeds": 3 }, "one_minus_p_auroc": { "estimate": 0.871907152480864, "interval_95": [ 0.8547408737005391, 0.8867714949126292 ], "people": 100, "seeds": 3 } }, "not N2": { "negated_wording_auroc": { "estimate": 0.12371052137575671, "interval_95": [ 0.11047383043735164, 0.13886062039791092 ], "people": 100, "seeds": 3 }, "one_minus_p_auroc": { "estimate": 0.9236456935443291, "interval_95": [ 0.9090000634370796, 0.9368066675889611 ], "people": 100, "seeds": 3 } }, "not N3": { "negated_wording_auroc": { "estimate": 0.021400423411069444, "interval_95": [ 0.017384423473401668, 0.02624049991797009 ], "people": 71, "seeds": 3 }, "one_minus_p_auroc": { "estimate": 0.9789817414117379, "interval_95": [ 0.9742015478774562, 0.9830091828676588 ], "people": 71, "seeds": 3 } }, "not REM": { "negated_wording_auroc": { "estimate": 0.049008707586919685, "interval_95": [ 0.03848277490226961, 0.060816208025616665 ], "people": 100, "seeds": 3 }, "one_minus_p_auroc": { "estimate": 0.9562969425742252, "interval_95": [ 0.9442535066012538, 0.9669642900273482 ], "people": 100, "seeds": 3 } }, "non-REM sleep (implicit)": { "auroc": { "estimate": 0.04811863748438766, "interval_95": [ 0.037408265431429354, 0.060484157808071065 ], "people": 100, "seeds": 3 }, "one_minus_p_REM_auroc": { "estimate": 0.9562969425742252, "interval_95": [ 0.9442535066012538, 0.9669642900273482 ], "people": 100, "seeds": 3 } } }, "note": "Each \"not X\" wording is scored against the truth of the negated question, beside the AUROC of 1 - p(X); \"non-REM sleep\" (an implicit name) is scored against the truth of \"not REM\"." }, "S13-sleep": { "part": "S13-sleep", "what": "S13 held-out template frames vs the read-off, BOAS sleep L1 (log ratio, P3 flags)", "dataset": "BOAS", "boas": true, "rows": { "TPL asleep implicit (held-out frames)": { "estimate": 3.5654807950963625, "interval_95": [ 3.1617244149307924, 4.005851625813937 ], "people": 100, "difference": "difference: the read-off leaves less error", "margin": "margin not met", "unit": "log R" }, "TPL asleep explicit (held-out frames)": { "estimate": 1.9762639222030514, "interval_95": [ 1.5938624445868963, 2.392425343744512 ], "people": 100, "difference": "difference: the read-off leaves less error", "margin": "margin not met", "unit": "log R" }, "TPL NREM explicit (held-out frames)": { "estimate": 0.9199064993319938, "interval_95": [ 0.7425565943575476, 1.1003410147513628 ], "people": 100, "difference": "difference: the read-off leaves less error", "margin": "margin not met", "unit": "log R" }, "TPL light implicit (held-out frames)": { "estimate": 2.2333469563517983, "interval_95": [ 2.0220627983765898, 2.443423956036792 ], "people": 100, "difference": "difference: the read-off leaves less error", "margin": "margin not met", "unit": "log R" }, "TPL light explicit (held-out frames)": { "estimate": 1.4840724678814725, "interval_95": [ 1.2402178997482416, 1.718935330426296 ], "people": 100, "difference": "difference: the read-off leaves less error", "margin": "margin not met", "unit": "log R" } } }, "SSVEP-L0-secondary": { "part": "SSVEP-L0-secondary", "what": "S9 seen/unseen levels, S10 contrasts and per-rotation TPL - CCA, S11 FBCCA, S13 held-out wordings; BETA L0", "dataset": "BETA", "boas": false, "rows": { "S10 DESC - CCA (8-way)": { "estimate": -0.5229464285714286, "interval_95": [ -0.5541030505952382, -0.4900290178571428 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC - CCA 8-way" }, "S10 DESC - CCA (3-way)": { "estimate": -0.6272321428571429, "interval_95": [ -0.6633939732142857, -0.5897976190476191 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC - CCA 3-way" }, "S10 DESC - NN(DESC) (8-way)": { "estimate": -0.10802083333333332, "interval_95": [ -0.1215208333333333, -0.09394196428571427 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC - NN(DESC)" }, "S10 NUM - CCA (8-way)": { "estimate": -0.39437500000000003, "interval_95": [ -0.4224806547619047, -0.3647619047619048 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "NUM - CCA 8-way" }, "S10 TPL - NN(ID) (8-way)": { "estimate": -0.15630952380952381, "interval_95": [ -0.17475248015873016, -0.13622991071428572 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL - NN(ID)" }, "S10 TPL - SHUF (8-way)": { "estimate": 0.19504960317460318, "interval_95": [ 0.1745334821428571, 0.2161612103174603 ], "people": 70, "seeds": 3, "difference": "difference: language arm higher", "margin": "non-inferior", "unit": "difference of proportions", "label": "TPL - SHUF" }, "S11 TPL - FBCCA (8-way)": { "estimate": -0.6168452380952381, "interval_95": [ -0.6371634424603174, -0.5969129464285713 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL - FBCCA 8-way" }, "S11 TPL - FBCCA (3-way)": { "estimate": -0.7335416666666669, "interval_95": [ -0.7561225198412699, -0.7093152281746031 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL - FBCCA 3-way" }, "S13 TPL held-out frames - CCA (8-way)": { "estimate": -0.537093253968254, "interval_95": [ -0.5673372395833334, -0.5027199900793651 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL(a1) - CCA" }, "S13 DESC held-out descriptions - CCA (8-way)": { "estimate": -0.5400654761904762, "interval_95": [ -0.5706662946428571, -0.5083139880952383 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC(a3) - CCA" } }, "levels": { "S9 TPL": { "seen_truth": { "estimate": 0.23429811507936504, "interval_95": [ 0.2096971106150794, 0.25738504464285716 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.04413690476190476, "interval_95": [ 0.04009920634920635, 0.04809598214285713 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.07428084008288212 }, "S9 DESC": { "seen_truth": { "estimate": 0.24031249999999998, "interval_95": [ 0.2147739025297619, 0.2646489955357143 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.04232142857142857, "interval_95": [ 0.0377827380952381, 0.04651785714285714 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.07196848839045963 }, "S9 SHUF": { "seen_truth": { "estimate": 0.23375248015873012, "interval_95": [ 0.20938963293650792, 0.25713752480158725 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.011359126984126984, "interval_95": [ 0.00951364087301587, 0.0135032242063492 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.021665429360349375 }, "S9 NUM": { "seen_truth": { "estimate": 0.2070907738095238, "interval_95": [ 0.18478404017857142, 0.22859840029761902 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.11464285714285714, "interval_95": [ 0.10190327380952381, 0.12708333333333333 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.1475846831875832 }, "S10 NN(B-sh) (8-way)": { "estimate": 0.44279761904761905, "interval_95": [ 0.40925223214285716, 0.475061755952381 ], "people": 70, "seeds": 3 }, "S11 FBCCA levels": { "seen_ba": { "estimate": 0.8110937499999998, "interval_95": [ 0.7698565848214285, 0.8480033482142858 ], "people": 70, "seeds": 1 }, "u8": { "estimate": 0.9037499999999998, "interval_95": [ 0.8772321428571429, 0.9275892857142858 ], "people": 70, "seeds": 1 }, "u3": { "estimate": 0.9268749999999999, "interval_95": [ 0.9086607142857144, 0.9437499999999998 ], "people": 70, "seeds": 1 } } }, "per_rotation_tpl_minus_cca_8_way": { "rotation 0": { "estimate": -0.498809523809524, "interval_95": [ -0.5316046626984128, -0.46164682539682544 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 0" }, "rotation 1": { "estimate": -0.5296130952380952, "interval_95": [ -0.5702393353174604, -0.4898809523809525 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 1" }, "rotation 2": { "estimate": -0.4873511904761905, "interval_95": [ -0.5196937003968254, -0.4541629464285715 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 2" }, "rotation 3": { "estimate": -0.5733630952380953, "interval_95": [ -0.6084337797619048, -0.5373487103174603 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 3" }, "rotation 4": { "estimate": -0.5214285714285715, "interval_95": [ -0.5579873511904762, -0.48199032738095243 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 4" } } }, "SSVEP-L1-secondary": { "part": "SSVEP-L1-secondary", "what": "the same rows on BETA L1", "dataset": "BETA", "boas": false, "rows": { "S10 DESC - CCA (8-way)": { "estimate": -0.6226339285714286, "interval_95": [ -0.6601488095238094, -0.5841454613095236 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC - CCA 8-way" }, "S10 DESC - CCA (3-way)": { "estimate": -0.5492857142857144, "interval_95": [ -0.5800982142857144, -0.5171774553571429 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC - CCA 3-way" }, "S10 DESC - NN(DESC) (8-way)": { "estimate": -0.01266369047619047, "interval_95": [ -0.020848586309523808, -0.004582961309523817 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC - NN(DESC)" }, "S10 NUM - CCA (8-way)": { "estimate": -0.5947916666666666, "interval_95": [ -0.6291994047619048, -0.5586599702380952 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "NUM - CCA 8-way" }, "S10 TPL - NN(ID) (8-way)": { "estimate": 0.006775793650793656, "interval_95": [ -0.0024518849206349226, 0.01611111111111111 ], "people": 70, "seeds": 3, "difference": "no difference shown", "margin": "equivalent within the margin", "unit": "difference of proportions", "label": "TPL - NN(ID)" }, "S10 TPL - SHUF (8-way)": { "estimate": 0.0797420634920635, "interval_95": [ 0.06879935515873016, 0.09089781746031741 ], "people": 70, "seeds": 3, "difference": "difference: language arm higher", "margin": "non-inferior", "unit": "difference of proportions", "label": "TPL - SHUF" }, "S11 TPL - FBCCA (8-way)": { "estimate": -0.7141468253968254, "interval_95": [ -0.7353184523809523, -0.6916542658730159 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL - FBCCA 8-way" }, "S11 TPL - FBCCA (3-way)": { "estimate": -0.6565873015873015, "interval_95": [ -0.6761016865079366, -0.636249007936508 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL - FBCCA 3-way" }, "S13 TPL held-out frames - CCA (8-way)": { "estimate": -0.6231249999999999, "interval_95": [ -0.66060013640873, -0.5827109995039682 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "TPL(a1) - CCA" }, "S13 DESC held-out descriptions - CCA (8-way)": { "estimate": -0.6258452380952382, "interval_95": [ -0.661523288690476, -0.5858965029761906 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "DESC(a3) - CCA" } }, "levels": { "S9 TPL": { "seen_truth": { "estimate": 0.18836061507936508, "interval_95": [ 0.16715885416666668, 0.21077170138888884 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.02513888888888889, "interval_95": [ 0.021795634920634924, 0.028393105158730156 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.04435772903928638 }, "S9 DESC": { "seen_truth": { "estimate": 0.18938244047619046, "interval_95": [ 0.16850399925595239, 0.21161058407738093 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.02517857142857143, "interval_95": [ 0.0225, 0.028020833333333328 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.044447770473449284 }, "S9 SHUF": { "seen_truth": { "estimate": 0.18339533730158733, "interval_95": [ 0.16362338789682543, 0.2035539434523809 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.015634920634920637, "interval_95": [ 0.013015873015873017, 0.018582341269841266 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.02881342337846469 }, "S9 NUM": { "seen_truth": { "estimate": 0.11541666666666668, "interval_95": [ 0.10401655505952381, 0.12701711309523805 ], "people": 70, "seeds": 3 }, "unseen_truth": { "estimate": 0.042886904761904765, "interval_95": [ 0.0380952380952381, 0.04800669642857143 ], "people": 70, "seeds": 3 }, "harmonic_mean": 0.06253634768440185 }, "S10 NN(B-sh) (8-way)": { "estimate": 0.17583333333333337, "interval_95": [ 0.16571354166666666, 0.18809523809523807 ], "people": 70, "seeds": 3 }, "S11 FBCCA levels": { "seen_ba": { "estimate": 0.8110937499999998, "interval_95": [ 0.7698565848214285, 0.8480033482142858 ], "people": 70, "seeds": 1 }, "u8": { "estimate": 0.9037499999999998, "interval_95": [ 0.8772321428571429, 0.9275892857142858 ], "people": 70, "seeds": 1 }, "u3": { "estimate": 0.9268749999999999, "interval_95": [ 0.9086607142857144, 0.9437499999999998 ], "people": 70, "seeds": 1 } } }, "per_rotation_tpl_minus_cca_8_way": { "rotation 0": { "estimate": -0.6300099206349206, "interval_95": [ -0.6722730654761906, -0.5854117063492064 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 0" }, "rotation 1": { "estimate": -0.6175595238095237, "interval_95": [ -0.6581473214285714, -0.5774193948412699 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 1" }, "rotation 2": { "estimate": -0.5973710317460318, "interval_95": [ -0.6374516369047621, -0.5551351686507937 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 2" }, "rotation 3": { "estimate": -0.6274801587301587, "interval_95": [ -0.6673139880952381, -0.5842249503968252 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 3" }, "rotation 4": { "estimate": -0.6246527777777778, "interval_95": [ -0.6652827380952381, -0.5822867063492062 ], "people": 70, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "rotation 4" } } }, "S8-L0": { "part": "S8-L0", "what": "S8 extrapolation to a held-out contiguous band 14.4-15.8 Hz (1 seed), BETA L0", "dataset": "BETA", "boas": false, "single_seed": true, "status_label": "single seed: no training-run variance; declared likely inconclusive", "declared_likely_inconclusive": true, "rows": { "TPL - CCA (8-way)": { "estimate": -0.653720238095238, "interval_95": [ -0.6921205357142858, -0.6139806547619048 ], "people": 70, "seeds": 1, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S8 TPL - CCA" }, "TPL - NUM (8-way)": { "estimate": 0.03511904761904762, "interval_95": [ 0.016365327380952387, 0.05491071428571428 ], "people": 70, "seeds": 1, "difference": "difference: language arm higher", "margin": "non-inferior", "unit": "difference of proportions", "label": "S8 TPL - NUM" }, "TPL - NN(TPL) (8-way)": { "label": "S8 TPL - NN(TPL)", "status": "not defined", "reason": "not defined in S8: NN(arm) averages the answers for the two neighbouring SEEN frequencies, but the S8 held-out band 14.4-15.8 Hz is contiguous, so every held-out frequency has a held-out (untrained) neighbour (14.4 Hz: 14.6 Hz; 15.8 Hz: its only neighbour 15.6 Hz); score.nn_defined, engineering amendment EA-1" }, "TPL - CCA (3-way)": { "estimate": -0.5019345238095237, "interval_95": [ -0.5404873511904761, -0.46248511904761913 ], "people": 70, "seeds": 1, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S8 TPL - CCA 3-way (competitors: held-out questions)", "note": "In S8 the neighbour 3-way competitors are themselves held-out questions the head was never trained on (14 of the 15 neighbour slots; the one trained neighbour is 14.2 Hz, below 14.4 Hz; 15.8 Hz is a 2-way at the grid edge), not trained detectors as in the primary rotations, so this 3-way tells adjacent unseen frequencies apart." } }, "notes": { "nn": "not defined in S8: NN(arm) averages the answers for the two neighbouring SEEN frequencies, but the S8 held-out band 14.4-15.8 Hz is contiguous, so every held-out frequency has a held-out (untrained) neighbour (14.4 Hz: 14.6 Hz; 15.8 Hz: its only neighbour 15.6 Hz); score.nn_defined, engineering amendment EA-1", "three_way": "In S8 the neighbour 3-way competitors are themselves held-out questions the head was never trained on (14 of the 15 neighbour slots; the one trained neighbour is 14.2 Hz, below 14.4 Hz; 15.8 Hz is a 2-way at the grid edge), not trained detectors as in the primary rotations, so this 3-way tells adjacent unseen frequencies apart.", "tokens": "For bge-small-en-v1.5, 25 of the 40 S8 unseen-frequency texts contain the token \"15\", which never occurs in the training texts (8.0-14.2 Hz): those questions use a new token, not only a new combination of seen tokens (token coverage reported, not a stop; decision log B18, S0b-10)." } }, "S8-L1": { "part": "S8-L1", "what": "S8 extrapolation, BETA L1", "dataset": "BETA", "boas": false, "single_seed": true, "status_label": "single seed: no training-run variance; declared likely inconclusive", "declared_likely_inconclusive": true, "rows": { "TPL - CCA (8-way)": { "estimate": -0.6747023809523809, "interval_95": [ -0.7122023809523809, -0.6355580357142857 ], "people": 70, "seeds": 1, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S8 TPL - CCA" }, "TPL - NUM (8-way)": { "estimate": 0.008779761904761907, "interval_95": [ -0.006994047619047618, 0.02470238095238095 ], "people": 70, "seeds": 1, "difference": "no difference shown", "margin": "non-inferior", "unit": "difference of proportions", "label": "S8 TPL - NUM" }, "TPL - NN(TPL) (8-way)": { "label": "S8 TPL - NN(TPL)", "status": "not defined", "reason": "not defined in S8: NN(arm) averages the answers for the two neighbouring SEEN frequencies, but the S8 held-out band 14.4-15.8 Hz is contiguous, so every held-out frequency has a held-out (untrained) neighbour (14.4 Hz: 14.6 Hz; 15.8 Hz: its only neighbour 15.6 Hz); score.nn_defined, engineering amendment EA-1" }, "TPL - CCA (3-way)": { "estimate": -0.48407738095238084, "interval_95": [ -0.520982142857143, -0.44404017857142863 ], "people": 70, "seeds": 1, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S8 TPL - CCA 3-way (competitors: held-out questions)", "note": "In S8 the neighbour 3-way competitors are themselves held-out questions the head was never trained on (14 of the 15 neighbour slots; the one trained neighbour is 14.2 Hz, below 14.4 Hz; 15.8 Hz is a 2-way at the grid edge), not trained detectors as in the primary rotations, so this 3-way tells adjacent unseen frequencies apart." } }, "notes": { "nn": "not defined in S8: NN(arm) averages the answers for the two neighbouring SEEN frequencies, but the S8 held-out band 14.4-15.8 Hz is contiguous, so every held-out frequency has a held-out (untrained) neighbour (14.4 Hz: 14.6 Hz; 15.8 Hz: its only neighbour 15.6 Hz); score.nn_defined, engineering amendment EA-1", "three_way": "In S8 the neighbour 3-way competitors are themselves held-out questions the head was never trained on (14 of the 15 neighbour slots; the one trained neighbour is 14.2 Hz, below 14.4 Hz; 15.8 Hz is a 2-way at the grid edge), not trained detectors as in the primary rotations, so this 3-way tells adjacent unseen frequencies apart.", "tokens": "For bge-small-en-v1.5, 25 of the 40 S8 unseen-frequency texts contain the token \"15\", which never occurs in the training texts (8.0-14.2 Hz): those questions use a new token, not only a new combination of seen tokens (token coverage reported, not a stop; decision log B18, S0b-10)." } }, "S12": { "part": "S12", "what": "S12 Wearable-102 (L0 heads trained on all 40 BETA frequencies; 12 off-grid frequencies; per sensor)", "dataset": "Wearable-102", "boas": false, "note": "two-decimal off-grid frequencies: the hundredths tokens never occur in BETA training texts, so the text arms face new tokens; NUM is the clean comparison", "by_sensor": { "dry": { "TPL - CCA (12-way)": { "estimate": -0.4818355119825709, "interval_95": [ -0.523639251997095, -0.4390241466957154 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 TPL - CCA" }, "NUM - CCA (12-way)": { "estimate": -0.4215958605664489, "interval_95": [ -0.46052287581699336, -0.3838167211328976 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 NUM - CCA" }, "TPL - NUM (12-way)": { "estimate": -0.060239651416122024, "interval_95": [ -0.07320284132171388, -0.047276234567901226 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 TPL - NUM" }, "DESC - CCA (12-way)": { "estimate": -0.48518518518518533, "interval_95": [ -0.5298648556644879, -0.44017497276688455 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 DESC - CCA" } }, "wet": { "TPL - CCA (12-way)": { "estimate": -0.6292302106027596, "interval_95": [ -0.6615774782135077, -0.5966401143790848 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 TPL - CCA" }, "NUM - CCA (12-way)": { "estimate": -0.5610566448801743, "interval_95": [ -0.5916401143790849, -0.528291802832244 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 NUM - CCA" }, "TPL - NUM (12-way)": { "estimate": -0.06817356572258533, "interval_95": [ -0.08314406318082788, -0.05333968772694264 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 TPL - NUM" }, "DESC - CCA (12-way)": { "estimate": -0.6356481481481481, "interval_95": [ -0.6668031726579521, -0.6031396377995643 ], "people": 102, "seeds": 3, "difference": "difference: reference higher", "margin": "margin not met", "unit": "difference of proportions", "label": "S12 DESC - CCA" } } } } }, "flip_rates": { "flip-ssvep-L0": { "what": "answer flip rates between wordings (descriptive), BETA L0", "boas": false, "TPL": { "a1": { "rate": 0.39722900132275124, "people": 70 }, "a2": { "rate": 0.2926247795414462, "people": 70 }, "ref_training": { "rate": 0.19406746031746036, "people": 70 } }, "DESC": { "a3": { "rate": 0.41233005952380947, "people": 70 }, "ref_training": { "rate": 0.1928214285714286, "people": 70 } }, "between_ID_seeds": { "rate": 0.568184523809524, "people": 70 } }, "flip-ssvep-L1": { "what": "answer flip rates between wordings (descriptive), BETA L1", "boas": false, "TPL": { "a1": { "rate": 0.4069102182539683, "people": 70 }, "a2": { "rate": 0.31960295414462075, "people": 70 }, "ref_training": { "rate": 0.24333333333333332, "people": 70 } }, "DESC": { "a3": { "rate": 0.4266196428571429, "people": 70 }, "ref_training": { "rate": 0.2331785714285714, "people": 70 } }, "between_ID_seeds": { "rate": 0.6222142857142862, "people": 70 } }, "flip-sleep-L1": { "what": "answer flip rates between wordings (descriptive), BOAS sleep L1", "boas": true, "TPL": { "a1": { "rate": 0.0639417460968405, "people": 100 }, "a2": { "rate": 0.2002501240243287, "people": 100 }, "ref_training": { "rate": 0.03286092251469089, "people": 100 } }, "DESC": { "a3": { "rate": 0.45384295822393866, "people": 100 }, "ref_training": { "rate": 0.030892700490100792, "people": 100 } }, "between_ID_seeds": { "rate": 0.07370298867512261, "people": 100 }, "people_note": "cohort mean over the BOAS test people of P-sleep-L1 (per-person means first); each BOAS person is a test person in exactly one outer fold; n is the largest n of the P-sleep-L1 primary entries" } } }, "boundaries": { "protocol": [ "'Unseen' means unseen by the EEG head; the text encoder has read these words, and every token of a held-out frequency also occurs in the training texts.", "Held-out sleep questions are combinations of seen stages ('asleep' is the complement of one), reported as compositional beside the read-off.", "Held-out SSVEP questions name a frequency, which CCA answers without training; interpolation tested, extrapolation secondary.", "In the neighbour 3-way the unseen frequency competes against trained detectors for its neighbours; at L1 that test also involves stimulus phase.", "Rewording and unseen results are separate rows and never pooled.", "Wordings were written by agents from a fixed brief before scoring, not by users and not tuned.", "Negation is a boundary probe; location and waveform-shape questions are not tested (no signal-level ground truth held).", "Frozen features and a small head; one recipe; not a ranking of text or EEG encoders.", "35 pre-declared comparisons, not corrected for multiplicity." ], "route_boundary": "Questions about negation, location or waveform shape need signal-level ground truth. Writing more prompts does not supply it.", "added_by_results": [ "Negation was not understood: the negated wordings (\"not X\", \"non-REM sleep\") scored as if they asked for X (AUROC 0.02-0.13 against the truth of the negated question; boundary probe S6, never a route sentence).", "Implicit names of the unseen sleep unions (\"asleep\", \"light sleep\") ranked windows in reverse (TPL AUROC 0.317 and 0.244, below 0.5); only the explicit names (\"N1, N2, N3 or REM sleep\") carried the composition.", "The neighbour 3-way at L1 involves stimulus phase as well as frequency; at L0 (a phase-free spectrum) it is the clean frequency test.", "Unseen SSVEP questions compete against trained detectors for their neighbours (seen bias, stated)." ], "multiplicity": "35 pre-declared comparisons, not corrected for multiplicity; each interval is a 95% interval for its own comparison." }, "limitations": [ "'Unseen' means unseen by the EEG head; the text encoder has read these words, and every token of a held-out frequency also occurs in the training texts.", "Held-out sleep questions are combinations of seen stages ('asleep' is the complement of one), reported as compositional beside the read-off.", "Held-out SSVEP questions name a frequency, which CCA answers without training; interpolation tested, extrapolation secondary.", "In the neighbour 3-way the unseen frequency competes against trained detectors for its neighbours; at L1 that test also involves stimulus phase.", "Rewording and unseen results are separate rows and never pooled.", "Wordings were written by agents from a fixed brief before scoring, not by users and not tuned.", "Negation is a boundary probe; location and waveform-shape questions are not tested (no signal-level ground truth held).", "Frozen features and a small head; one recipe; not a ranking of text or EEG encoders.", "35 pre-declared comparisons, not corrected for multiplicity.", "Questions about negation, location or waveform shape need signal-level ground truth. Writing more prompts does not supply it.", "Negation was not understood: the negated wordings (\"not X\", \"non-REM sleep\") scored as if they asked for X (AUROC 0.02-0.13 against the truth of the negated question; boundary probe S6, never a route sentence).", "Implicit names of the unseen sleep unions (\"asleep\", \"light sleep\") ranked windows in reverse (TPL AUROC 0.317 and 0.244, below 0.5); only the explicit names (\"N1, N2, N3 or REM sleep\") carried the composition.", "The neighbour 3-way at L1 involves stimulus phase as well as frequency; at L0 (a phase-free spectrum) it is the clean frequency test.", "Unseen SSVEP questions compete against trained detectors for their neighbours (seen bias, stated).", "The pre-run permutation canary failed twice (revisions 3 and 4) and was revised by the owner, who overrode revision 4's \"no further revision\"; revision 5 passed. Both failed records are kept and disclosed.", "For multilingual-e5-small each held-out frequency is a new token (S0b-6); S8 and S12 also involve new tokens.", "Secondary single-seed entries carry no training-run variance; EESM19 has 20 people; every EESM19 and single-seed entry was declared likely inconclusive.", "BETA numbers use the published 2-s windows and folds; Wearable-102 numbers are computed from the 2026-09-20 Tsinghua author mirror and claim no physical unit.", "BOAS: the three gaps and the attribution above; participants are pseudonymised in the public release; no result evaluates Bitbrain's headband or its automatic scoring." ], "not_run": [ { "item": "revision-4 per-person centring diagnostic", "status": "not run (revision 5 part A.2; no fit exists)" }, { "item": "Ying scorer agreement; factorial cognitive sets", "status": "deferred (D10)" }, { "item": "E1 EEGNet trained jointly with the question vector", "status": "deferred (D6)" }, { "item": "third text encoder; LEAF-style Q-Former heads; route-1 calibration on language heads; user-written wordings", "status": "deferred (protocol.deferred)" }, { "item": "S8 TPL - NN(TPL)", "status": "not defined (EA-1): the held-out band is contiguous, so no held-out frequency has two seen neighbours" }, { "item": "location and waveform-shape questions", "status": "not tested: no signal-level ground truth held" } ], "not_reported": [ { "item": "per-union paired contrasts beyond the P3 entries and the per-union levels", "status": "not computed (W1)" }, { "item": "counts of people above or below zero on paired contrasts", "status": "allowed for BOAS but not computed by stage 2; not added" }, { "item": "per-person distributions, per-fold values, per-window scores", "status": "private by design" } ], "deviations": [ { "id": "canary revisions 4 and 5", "what": "the pre-run permutation canary was revised twice after failing (owner decisions 2026-10-08; revision 5 overrode revision 4's \"no further revision\")" }, { "id": "S0b-6", "what": "token rule reported, not a stop, for multilingual-e5-small number pieces; S5 SSVEP unseen entries say the frequency is a new token" }, { "id": "EA-1", "what": "engineering amendment before the first stage-1 fit: S8 NN(arm) not defined (contiguous held-out band), S8 3-way relabelled, failed-fit handling, resumable secondary scoring; no question, partition, wording, arm, metric, margin or gate changed" }, { "id": "B1-B31, S0b-1 to S0b-18", "what": "build and stage-0b judgment calls recorded in the decision log before the freeze" }, { "id": "S12 fit count", "what": "9 fits (TPL, DESC, NUM x 3 seeds) against the JSON fit_grid's 20 (decision log B16, before the freeze; ID and B-sh cannot answer off-grid questions)" }, { "id": "W1", "what": "S10 per-frequency and interior-only breakdowns computed descriptively at stage 3" }, { "id": "W2", "what": "display sentences: W2a literal-fill fix, W2b encoder/language in S4/S5 prefixes, W2c REVE-L phase caveat, W2d P3 summary written and P5 summary not written; stage-2 sentences kept beside them" } ], "audits": { "primary_conformance": { "sha256": "36f3b1b37aeaceaf5a3d05fc0b7873c094f9f4bc289d5a28d47f06fab07801c4", "verdict": "pass", "checks": 106, "failed": 0 }, "primary_numeric": { "sha256": "e21e6ed5835faadd24fd464c493b414b7bbb7cb7dfdc29a7d71a4f0af472d01f", "verdict": "pass", "comparisons": 2270, "numeric_values": 725, "mismatches": 0, "entries_recomputed": 35 }, "secondary_conformance": { "sha256": "8dc1934179f067a482beddb5a3b3849b2092a23ee933b07861f870ecd8569179", "verdict": "pass", "checks": 231, "failed": 0 }, "secondary_numeric": { "sha256": "73c09588ffeda8bb56bf0c31db1c6251f500726a7886a0a09c1657a981d86068", "verdict": "pass", "comparisons": 18619, "outside_tolerance": 29, "unexplained": 0, "entries_recomputed": 105 }, "summary": "four independent audits passed: A1 conformance of the primary run (106 checks, 0 failed); A2 numerical audit of the primary (2,270 comparisons, 0 mismatches; 718 of 725 numeric values bit-identical, the rest within 1.4e-16 or a float32 display value); A3 conformance of the secondary run (231 checks, 0 failed); A4 numerical audit of the secondary (18,619 comparisons, 29 outside the 1e-9 tolerance, all explained, 0 unexplained). None of the listed discrepancies changes a number, flag, gate, rule or D11 condition.", "independence": "The audits re-implement the definitions without importing the study's scoring modules.", "not_audited": "The S10 per-frequency and interior-only breakdowns were self-checked at stage 3 and are not covered by an independent audit; they are not published." }, "references": [ { "id": "yu-yao-2026", "cite": "Yu & Yao, 2026 · Visual Jev: Accurate and Efficient Decisions from Shared Visual Context", "url": "https://arxiv.org/abs/2609.25845", "role": "the vision paper the decision-research plan started from; it applies the same encode-once, answer-many idea to images" }, { "id": "film", "cite": "Perez et al., 2017 · FiLM: Visual Reasoning with a General Conditioning Layer", "url": "https://arxiv.org/abs/1709.07871", "role": "the conditioning layer of the head" }, { "id": "cca", "cite": "Lin, Zhang, Wu & Gao, 2006 · Frequency Recognition Based on CCA for SSVEP-Based BCIs, IEEE TBME 53:2610–2614", "url": "https://doi.org/10.1109/TBME.2006.886577", "role": "CCA, the training-free reference for unseen frequencies" }, { "id": "fbcca", "cite": "Chen, Wang, Gao, Jung & Gao, 2015 · Filter bank CCA for implementing a high-speed SSVEP-based BCI, J. Neural Eng. 12:046008", "url": "https://doi.org/10.1088/1741-2560/12/4/046008", "role": "FBCCA, the second training-free reference (S11)" }, { "id": "numeracy", "cite": "Wallace et al., 2019 · Do NLP Models Know Numbers? Probing Numeracy in Embeddings, EMNLP 2019", "url": "https://aclanthology.org/D19-1534/", "role": "numeracy in text embeddings (the declared numeracy prediction)" }, { "id": "nevir", "cite": "Weller, Lawrie & Van Durme, 2024 · NevIR: Negation in Neural Information Retrieval, EACL 2024", "url": "https://aclanthology.org/2024.eacl-long.139/", "role": "negation in neural text representations (the S6 probe)" }, { "id": "zero-shot", "cite": "Xian, Lampert, Schiele & Akata · Zero-Shot Learning — A Comprehensive Evaluation of the Good, the Bad and the Ugly, TPAMI", "url": "https://arxiv.org/abs/1707.00600", "role": "why unseen questions are reported apart from rewordings, never as one zero-shot number" } ], "required_disclosures": [ { "id": "revision-3-failed", "where": "pre_run_checks.canaries.revision_3" }, { "id": "revision-4-failed", "where": "pre_run_checks.canaries.revision_4" }, { "id": "revision-5-passed", "where": "pre_run_checks.canaries.revision_5" }, { "id": "owner-override", "where": "pre_run_checks.owner_override" }, { "id": "centring-diagnostic-not-run", "where": "pre_run_checks.centring_diagnostic" }, { "id": "revision-5-direction-counts", "where": "pre_run_checks.canaries.revision_5.single_head_intervals_excluding_exact_chance" }, { "id": "decision-s0b6", "where": "pre_run_checks.decision_s0b6" }, { "id": "ea-1", "where": "pre_run_checks.engineering_amendment_ea1" }, { "id": "numeracy-not-activated", "where": "pre_run_checks.numeracy_probe" }, { "id": "multiplicity", "where": "design.multiplicity" }, { "id": "secondary-single-seed-likely-inconclusive", "where": "secondary.expected_inconclusive" } ] } }, "datasets": { "beta": { "name": "BETA", "role": "primary: SSVEP, 70 people, 3 seeds, 5 held-out-frequency rotations", "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "source": "https://figshare.com/articles/dataset/The_BETA_database/12264401", "attribution": "Bingchuan Liu et al. · BETA: A Large Benchmark Database Toward SSVEP-BCI Application (2020), doi:10.3389/fnins.2020.00627. Figshare 12264401 v3; mirror Bingchuan/BETA.", "rights_reviewed": "2026-09-20", "rights_record": "reused unchanged from the released record", "published_here": "Cohort aggregates over the 70 people, with the published 2-s windows and participant folds: paired contrasts with their intervals and both flags, per-arm levels with chance-corrected values, generalisation costs, flip rates and the per-rotation TPL - CCA rows. No per-person, per-fold or per-frequency value." }, "boas": { "name": "BOAS, the Bitbrain Open Access Sleep dataset", "role": "primary: sleep, 100 people, 3 seeds", "license": "CC0-1.0", "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "source": "https://openneuro.org/datasets/ds005555/versions/1.1.3", "attribution": "Eduardo López-Larraz, María Sierra-Torralba, Sergio Clemente, Galit Fierro, David Oriol, Javier Minguez, Luis Montesano and Jens G. Klinzing · The Bitbrain Open Access Sleep (BOAS) dataset, OpenNeuro ds005555, version 1.1.3 (2026), doi:10.18112/openneuro.ds005555.v1.1.3. Polysomnography EEG and the human-consensus stage labels only; the headband recordings and the publisher's automatic labels are not used.", "rights_reviewed": "2026-10-07", "rights_record": "reused unchanged from the released record", "published_here": "Cohort aggregates only, each stating its n: 100 people, or 71 where the N3 stage needs at least five N3 windows per person (the per-stage AUROC for N3 and the \"not N3\" negation cell). No per-person, per-night, per-recording or per-fold value, no demographic, date or clock time; every pre-run check on BOAS as pass or fail and counts only. Read from route 2's frozen CBraMod and REVE-L features." }, "eesm19": { "name": "EESM19", "role": "secondary (S1): sleep, 20 people, 3 seeds, declared likely inconclusive", "license": "CC0-1.0 declared by upstream and mirror", "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "source": "https://doi.org/10.18112/openneuro.ds005185.v1.0.2", "attribution": "Kaare B. Mikkelsen et al. · Accurate whole-night sleep monitoring with dry-contact ear-EEG (2019), doi:10.1038/s41598-019-53115-3; OpenNeuro ds005185 v1.0.2. Processed mirror: Zachary1150/EESM19-Processed.", "rights_reviewed": "2026-09-20", "rights_record": "reused unchanged from the released record", "published_here": "Cohort aggregates over 20 people (S1, a replication of the primary sleep entries), each with its flags; declared likely inconclusive before the run." }, "openbmi": { "name": "OpenBMI motor imagery", "role": "secondary (S2): motor imagery, 51 people, 3 seeds", "license": "CC0-1.0", "licenseUrl": "https://creativecommons.org/publicdomain/zero/1.0/", "source": "https://doi.org/10.5524/100542", "attribution": "Min-Ho Lee, O-Yeon Kwon, Yong-Jeong Kim, Hong-Kyung Kim, Young-Eun Lee, John Williamson, Siamac Fazli and Seong-Whan Lee · EEG dataset and OpenBMI toolbox for three BCI paradigms: an investigation into BCI illiteracy, GigaScience (2019), giz002, doi:10.1093/gigascience/giz002. Data: Supporting data, GigaScience Database, doi:10.5524/100542.", "rights_reviewed": "2026-10-03", "rights_record": "reused unchanged from the released record", "published_here": "Cohort aggregates over the published 51-person cohort (S2), each with its flags. Not gated by a triviality canary: none is declared for motor imagery." }, "wearable-ssvep-102": { "name": "Wearable SSVEP BCI dataset (dry and wet electrodes)", "role": "secondary (S12): off-grid frequencies, 102 people, 3 seeds", "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "source": "https://figshare.com/articles/dataset/An_Open_Dataset_for_Wearable_SSVEP-Based_Brain-Computer_Interfaces/13560281", "attribution": "Zhu, F., Jiang, L., Dong, G., Gao, X., & Wang, Y. (2021). An Open Dataset for Wearable SSVEP-Based Brain-Computer Interfaces (Version 4) [Data set]. Figshare. https://doi.org/10.6084/m9.figshare.13560281.v4", "rights_reviewed": "2026-09-20", "rights_record": "reused unchanged from the released record", "published_here": "Cohort aggregates over 102 people for each sensor type (S12), computed from the official Tsinghua author mirror acquired on 2026-09-20; no physical unit is claimed and no participant row is published.", "paper": "Zhu, F., Jiang, L., Dong, G., Gao, X., & Wang, Y. (2021). An Open Dataset for Wearable SSVEP-Based Brain-Computer Interfaces. Sensors, 21(4), 1256. https://doi.org/10.3390/s21041256", "computed_from": "Official Tsinghua BCI Lab author mirror snapshot acquired 2026-09-20; the mirror itself has no separate DOI or version number." } }, "boas_conditions": { "owner_approval": { "approved_on": "2026-10-07", "decision": "publishable with stated gaps", "extended_to_route_3": "D11, approved by the owner on 2026-10-07" }, "gaps": [ "The consent statement does not say whether participants agreed to public sharing or secondary use.", "The ethics and consent statements come from the publisher's dataset description and README. No peer-reviewed paper describes BOAS.", "The ethics reference was added to the release in version 1.1.1 (May 2025), and the release does not say when it was granted relative to the recordings." ], "gaps_zh": [ "知情同意声明没有说明被试是否同意公开共享或二次使用。", "伦理与知情同意的陈述只来自发布者的数据集说明与 README，没有任何同行评审论文描述 BOAS。", "伦理批件编号是在 1.1.1 版（2025 年 5 月）才加入发布的，发布中没有说明它是在记录之前还是之后批准的。" ], "attribution": "Eduardo López-Larraz, María Sierra-Torralba, Sergio Clemente, Galit Fierro, David Oriol, Javier Minguez, Luis Montesano and Jens G. Klinzing · The Bitbrain Open Access Sleep (BOAS) dataset, OpenNeuro ds005555, version 1.1.3 (2026), doi:10.18112/openneuro.ds005555.v1.1.3. Polysomnography EEG and the human-consensus stage labels only; the headband recordings and the publisher's automatic labels are not used.", "participants": "pseudonymised in the public release", "not_an_evaluation_of": "No result here evaluates Bitbrain's headband or its automatic sleep scoring; neither is used.", "minimum_cell_people": 20, "smallest_cell_people": 71, "smallest_cell_note": "n = 71 appears only where the N3 stage needs at least 5 N3 windows per person (AUROC inclusion rule): per-stage seen AUROC for N3 and the S6 \"not N3\" cell", "pre_run_checks": "pass or fail and direction counts only" }, "status_only": [], "holds": [], "not_published": [ "Per-person, per-night, per-recording and per-fold values of any kind, per-window scores and predictions, features, head weights, text-encoder files and fold assignments: none of them left the private run root.", "The S10 per-frequency and interior-only BETA breakdowns the release candidate carries: computed at stage 3 and not covered by an independent audit. The per-rotation rows, which an independent audit recomputed, are published.", "Values of the pre-run checks: the permutation canaries' cell values and per-cell direction counts, the fit check's fold-0 training scores, the shuffled-EEG scorer check's Monte-Carlo means and the triviality canaries' own scores. Each check is published as pass or fail and as counts; the BOAS ones never left the private run root.", "The secondary parts' descriptive blocks (the S1 to S5 and S8 levels, read-offs, per-stage AUROC and log loss) and the nearest-training-wording cosines: kept in the release candidate, not served.", "Demographics, recording dates and clock times, nights per person, and any image of a real recording.", "Measured compute: GPU-hours, wall-clock times, waits for other users' jobs, per-step timings and the cost ledger, all measured on a shared GPU.", "The text of the audits' discrepancy lists, which name private files; what must travel with the results is carried as the results' own labels and notes.", "The handoff, the release candidate, the aggregates, the protocol, the stage-0 report, the canary declarations, the BOAS scope extension, the reference check and the four independent audits: pinned by SHA-256 in this manifest and checked, never copied, because they carry private storage paths." ], "provenance": { "manifest_sha256": "c85dbadaab3d08b9ae079015e664cec4ed7e16438c6e1ca00689c1ef341ade5e", "release_candidate_sha256": "9729fda20a9556b34ffb3117b81529aec5c014e01457035445c5c5127fa2d678", "wording_lists_sha256": "b903f4f49138aefdba9385e82d5288f58eccc24ab228d9b45ab5bd53bf63912a", "wordings_file": "questions-in-language-wordings.json", "included": [ "questions-in-language" ], "inputs": [ "beta", "boas", "eesm19", "openbmi", "wearable-ssvep-102" ], "holds": [] } }