{ "schema_version": "bci-report-pilot-studies-update-v1", "release_id": "pilot-studies-update-20261012", "generated_at": "2026-10-12", "status": "aggregate_preview", "title": "Pilot studies", "question": "What can a small pilot show, and what can it not?", "placement": "small pilots outside every leaderboard and model ranking", "metric_units": "balanced accuracy, AUROC, average precision, top-3 accuracy, shares and chance levels are proportions in [0,1]; differences of these are differences of proportions; the REFED error is a mean absolute error in raw label units on the published 1–255 scale (lower is better); counts of people, fields, folders, clips, runs, events, trials and channels are whole numbers; null means not computed or not stated, never zero", "scope": "Small pilots on four sources, all kept outside every leaderboard and model ranking. Published, each from its own approved aggregate export: one Orthosis cohort in three configurations, counted once; a cross-person, cross-language inner-speech baseline on BINGO; a mask-aware EEG analysis of a handwriting dataset; and a stored-clip computational diagnostic on REFED. No dataset, cohort, species, task or metric is pooled, and the results share no ranking.", "pilots": [ { "id": "orthosis", "dataset": "orthosis-error-detection", "label": "Orthosis error events", "publication": "released", "results": [ "orthosis-64ch-classical", "orthosis-58ch-classical", "orthosis-eegpt-vs-58ch" ] }, { "id": "bingo", "dataset": "bingo-imagined-speech", "label": "Inner speech in Greek and English", "publication": "released", "results": [ "bingo-cross-language" ] }, { "id": "handwriting", "dataset": "eeg-fnirs-handwriting", "label": "Mask-aware EEG on a handwriting dataset", "publication": "released", "results": [ "handwriting-mask-aware" ] }, { "id": "refed", "dataset": "refed", "label": "REFED stored clips", "publication": "released", "results": [ "refed-stored-clip" ] } ], "results": { "orthosis-64ch-classical": { "id": "orthosis-64ch-classical", "dataset": "orthosis-error-detection", "pilot": "orthosis", "title": "Orthosis error events: the 64-channel classical baseline", "question": "Can a fixed EEG baseline tell injected orthosis errors from standard device events in a person it was not trained on?", "configuration": { "label": "64-channel configuration", "values_per_event": 576, "note": "the first configuration of this cohort; it stays beside the 58-channel one" }, "model_family": "classical", "foundation_model": false, "fine_tuning": false, "leaderboard": false, "task": "Recorded injected orthosis device errors against standard device events with no error introduced, from the 0–0.5 second window after each event. The labels describe recorded device events, not whether the person noticed the error.", "cohort": { "people": 8, "runs": 83, "events": 2407, "standard_events": 1933, "error_events": 474, "error_share": 0.19692563356875778, "counted_once": "the same 8 people, 83 runs and 2,407 events in all three Orthosis results: one cohort, never counted twice in coverage", "split": "leave one whole person out: eight folds, all of a person’s runs held out together; the scaler and the probe are fitted on the other seven people", "exclusions": { "baseline_runs": 8, "targets_without_following_movement": 82, "windows_crossing_boundary": 1, "note": "baseline runs are not main-task runs; a target needs an observed following movement and a complete window" } }, "signal": { "eeg_channels": 64, "sampling_hz": 500, "window_seconds": [ 0, 0.5 ], "window_samples": 250 }, "metric": "balanced_accuracy", "secondary_metrics": [ "auroc", "average_precision" ], "chance_level": 0.5, "weighting": "macro-person: the unweighted mean of the eight held-out people’s metrics, not pooled trial accuracy; balanced accuracy is not ordinary accuracy", "arms": { "eeg": { "label": "Fixed EEG features + logistic regression, 64 channels", "kind": "EEG", "description": "576 fixed amplitude and log band-power values per event from 64 EEG channels; a training-only StandardScaler and a balanced L2 logistic regression with C=1, no tuning", "balanced_accuracy": { "mean": 0.7297338046181997, "interval_95": [ 0.6797889807861512, 0.776835762640067 ] }, "auroc": { "mean": 0.8115223447072195, "interval_95": [ 0.7511392171747198, 0.8630552782256828 ] }, "average_precision": { "mean": 0.5845123403187809, "interval_95": [ 0.4922951099262258, 0.6721480977682927 ] } }, "confound": { "label": "Direction/time confound probe", "kind": "control", "description": "the same fixed probe on movement direction, run order within the person, position in the run and time in the run: no EEG", "balanced_accuracy": { "mean": 0.5135810219563451, "interval_95": [ 0.5043607539197512, 0.5224723963791478 ] }, "auroc": { "mean": 0.5241804457997877, "interval_95": [ 0.5066233272841131, 0.5408112594938129 ] }, "average_precision": { "mean": 0.22826059095832624, "interval_95": [ 0.20795685868251862, 0.24911750776840863 ] } }, "majority": { "label": "Training majority", "kind": "control", "description": "predicts the most frequent class of the seven training people", "balanced_accuracy": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "auroc": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "average_precision": { "mean": 0.19683117115553578, "interval_95": [ 0.19377474482324739, 0.1998432601880878 ] } }, "prior": { "label": "Training class prior", "kind": "control", "description": "a constant probability, the error share of the seven training people", "balanced_accuracy": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "auroc": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "average_precision": { "mean": 0.19683117115553578, "interval_95": [ 0.19373040752351095, 0.1998432601880878 ] } } }, "eeg_minus_confound": { "comparison": "EEG minus the direction/time confound probe, matched person by person", "balanced_accuracy_difference": { "mean": 0.21615278266185456, "interval_95": null }, "interval_note": "no interval was computed for this difference", "people_with_eeg_above_confound": 8, "people": 8 }, "permutation_control": { "balanced_accuracy": { "replicates": 20, "balanced_accuracy_mean": 0.5008648055589445, "observed_range": [ 0.4800148102554569, 0.5259876753378205 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "design": "training labels shuffled within person and run, held-out labels untouched; a diagnostic control, not a full permutation significance test" }, "response_clean_subset": { "status": "secondary: counts only, never a headline", "events": 451, "error_events": 446, "standard_events": 5, "people_with_both_classes": 3, "note": "strongly label-selected and dominated by errors, so its high average precision is not comparable with the primary cohort and its scores are not published" }, "method": { "signal": "64 EEG channels at 500 Hz with the declared header scaling; three acceleration channels excluded; the recorded reference kept", "filter": "continuous forward-only fourth-order Butterworth 1–30 Hz filtering that uses no future samples, zero state at each run start and a five-second warm-up", "window": "the complete half-open 0–0.5 second window after each event", "probe": "a training-only StandardScaler and a balanced L2 logistic regression; C=1 and the 0.5 decision rule fixed before execution; no tuning", "eligibility": "main-task runs only; inclusion does not depend on whether the person responded correctly", "sealing": "every fold and control prediction was sealed before a separate scorer read the held-out labels" }, "uncertainty": { "kind": "pointwise 95% percentile interval from 10,000 person-level bootstrap draws", "limited_by": "eight people give limited uncertainty estimation", "conditional_on": "the fixed sealed predictions; refitting is not included" }, "reading": "The fixed EEG baseline is above the direction/time confound probe, the class prior and every permutation control, with its interval clear of all three. It separates recorded device events; it does not show that the person recognised the error, and movement, muscle or device artifacts may contribute.", "limitations": [ "Eight people in one dataset: person-bootstrap uncertainty is limited, and this is not a population-level ranking.", "The labels are recorded injected device events. The result does not show subjective error recognition, a mental state or isolated neural activity: movement, muscle and device-injection artifacts may contribute, and beating the direction/time confound probe does not isolate a neural origin.", "The permutation controls shuffle training labels within person and run: diagnostic controls, not a full permutation significance test.", "A response-clean subset of 451 events has only 5 standard events and 446 errors, and only three people have both classes. Its high average precision reflects that prevalence, so its scores are not published and it never replaces the primary result.", "EEG only: no EMG model, fusion or paired EEG–EMG clock claim.", "Offline, and a reanalysis by this site: not a reproduction of the authors’ competition score, and no real-time, clinical or state-of-the-art claim." ], "release_limits": [ "Eight-person single-dataset pilot; person-bootstrap uncertainty is limited and is not a population-level ranking.", "Labels describe recorded injected orthosis device events, not subjective recognition, mental states, clinical performance or isolated neural cognition.", "EEG-only: no EMG model, multimodal fusion or paired EEG/EMG clock claim.", "Movement, muscle, device-injection and other acquisition artifacts may contribute; the direction/time confound control does not isolate their origin.", "Train-only within-person/run permutations are diagnostic controls, not a full dataset label-permutation significance test.", "Response-clean secondary subset is strongly label-selected (446 error versus 5 correct events) and only three people have both classes; high AUPRC is not comparable to the primary population." ], "claims_not_supported": [ "not subjective error recognition or isolated neural decoding", "no real-time, online, clinical or deployment claim", "not a population ranking and no state-of-the-art claim", "no EMG, fusion or foundation-model result in this configuration" ], "independent_audit": { "status": "pass", "people": 8, "checked": "all 384 score rows and 144 aggregate and interval rows recomputed from the sealed predictions, agreeing within 1e-12; twelve synthetic checks passed before execution", "not_checked": "no raw signal was read and no model was refitted" } }, "orthosis-58ch-classical": { "id": "orthosis-58ch-classical", "dataset": "orthosis-error-detection", "pilot": "orthosis", "title": "Orthosis error events: the matched 58-channel classical configuration", "question": "The same baseline on the 58 channels EEGPT supports: does it still separate the events?", "configuration": { "label": "58-channel configuration", "values_per_event": 522, "omitted_channels": [ "TP9", "TP10", "PO9", "PO10", "FT9", "FT10" ], "rule": "the native-supported 58-channel intersection in source order, fixed before scoring; no substitution or score-based selection", "note": "a second configuration of the 64-channel baseline’s cohort, not a new cohort" }, "same_cohort_as": "orthosis-64ch-classical", "model_family": "classical", "foundation_model": false, "fine_tuning": false, "leaderboard": false, "task": "Recorded injected orthosis device errors against standard device events with no error introduced, from the 0–0.5 second window after each event. The labels describe recorded device events, not whether the person noticed the error.", "cohort": { "people": 8, "runs": 83, "events": 2407, "standard_events": 1933, "error_events": 474, "error_share": 0.19692563356875778, "counted_once": "the same 8 people, 83 runs and 2,407 events in all three Orthosis results: one cohort, never counted twice in coverage", "split": "leave one whole person out: eight folds, all of a person’s runs held out together; the scaler and the probe are fitted on the other seven people" }, "signal": { "eeg_channels": 58, "sampling_hz": 500, "window_seconds": [ 0, 0.5 ], "window_samples": 250 }, "metric": "balanced_accuracy", "secondary_metrics": [ "auroc", "average_precision" ], "chance_level": 0.5, "weighting": "macro-person: the unweighted mean of the eight held-out people’s metrics, not pooled trial accuracy; balanced accuracy is not ordinary accuracy", "arms": { "classical58": { "label": "Matched classical EEG features + the same probe, 58 channels", "kind": "EEG", "description": "522 fixed amplitude and log band-power values per event from the 58 channels EEGPT supports, in source order; the same scaler and probe", "balanced_accuracy": { "mean": 0.7097113365795216, "interval_95": [ 0.6481218415815692, 0.7689555045503014 ] }, "auroc": { "mean": 0.7934577007551802, "interval_95": [ 0.7190084215562214, 0.8542417824673231 ] }, "average_precision": { "mean": 0.5442331679969993, "interval_95": [ 0.4382040652213037, 0.6541596513028163 ] } }, "confound": { "label": "Direction/time confound probe", "kind": "control", "description": "the same fixed probe on movement direction, run order within the person, position in the run and time in the run: no EEG", "balanced_accuracy": { "mean": 0.5135810219563451, "interval_95": [ 0.5044985313612883, 0.5226282428980794 ] }, "auroc": { "mean": 0.5241804457997877, "interval_95": [ 0.5064116398272049, 0.5408592388203259 ] }, "average_precision": { "mean": 0.22826059095832624, "interval_95": [ 0.20833927064920651, 0.2488889512126497 ] } }, "majority": { "label": "Training majority", "kind": "control", "description": "predicts the most frequent class of the seven training people", "balanced_accuracy": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "auroc": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "average_precision": { "mean": 0.19683117115553578, "interval_95": [ 0.1937407122170277, 0.19992678244080225 ] } }, "prior": { "label": "Training class prior", "kind": "control", "description": "a constant probability, the error share of the seven training people", "balanced_accuracy": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "auroc": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "average_precision": { "mean": 0.19683117115553578, "interval_95": [ 0.1936912225705329, 0.1998432601880878 ] } } }, "permutation_control": { "balanced_accuracy": { "replicates": 20, "balanced_accuracy_mean": 0.5005315788674973, "observed_range": [ 0.4816502731392335, 0.517332735977331 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "design": "training labels shuffled within person and run, preserving run class counts, held-out labels untouched; a diagnostic control, not a full permutation significance test" }, "prior_exposure": "The 64-channel scores were already seen before this configuration was frozen: an additional prospective diagnostic, not a blind confirmation.", "method": { "signal": "all 64 EEG channels filtered with continuous forward-only fourth-order 1–30 Hz filtering (zero state per run, five-second warm-up), then the fixed 58-channel intersection selected; the recorded FCz reference kept and acceleration excluded", "window": "exactly the complete 0–0.5 second window after each event", "probe": "a training-only StandardScaler and a balanced L2 logistic regression, C=1, liblinear, maximum 2,000 iterations and the 0.5 decision rule; no hyperparameter search", "sealing": "all predictions were sealed before separate scoring" }, "uncertainty": { "kind": "pointwise 95% percentile interval from 10,000 person-level bootstrap draws", "limited_by": "eight people give limited uncertainty estimation", "conditional_on": "the fixed sealed predictions; refitting is not included" }, "reading": "The 58-channel configuration of the same baseline is also above the confound probe, the prior and every permutation control. It is a second configuration on the same people, not a replication, and the change from 64 to 58 channels says nothing about whether more channels help.", "limitations": [ "The same eight people as the 64-channel result: not new coverage and not an independent replication.", "This configuration does not replace the 64-channel baseline, and the change in channel count was not the question: no conclusion about channel count follows.", "The 64-channel scores were seen before this configuration was frozen.", "The labels are recorded injected device events, not subjective error recognition or isolated neural activity; movement, muscle and device artifacts may contribute.", "The permutation controls are diagnostic, not a full permutation significance test." ], "release_limits": [ "Eight-person single-dataset pilot; person-bootstrap uncertainty is limited and is not a population-level ranking.", "Labels describe recorded injected orthosis device events, not subjective recognition, mental states, clinical performance or isolated neural cognition.", "EEG-only: no EMG model, multimodal fusion or paired EEG/EMG clock claim.", "Movement, muscle, device-injection and other acquisition artifacts may contribute; the direction/time confound control does not isolate their origin.", "Train-only within-person/run permutations are diagnostic controls, not a full dataset label-permutation significance test.", "Response-clean secondary subset is strongly label-selected (446 error versus 5 correct events) and only three people have both classes; high AUPRC is not comparable to the primary population.", "This 58-channel configuration does not replace the 64-channel baseline or establish a causal effect of channel count.", "Previously observed 64-channel scores and the narrow single-dataset setting limit confirmatory interpretation." ], "claims_not_supported": [ "not new dataset or person coverage", "no conclusion about how many channels are needed", "not subjective error recognition or isolated neural decoding", "no real-time, online or clinical claim" ], "independent_audit": { "status": "pass", "people": 8, "checked": "all 720 metric rows and 270 aggregate and interval rows recomputed within 1e-12; 24 source-only refits reproduced the sealed probabilities exactly", "not_checked": "the waveform features were not all re-extracted and the permutation probes were not all refitted" } }, "orthosis-eegpt-vs-58ch": { "id": "orthosis-eegpt-vs-58ch", "dataset": "orthosis-error-detection", "pilot": "orthosis", "title": "Orthosis error events: frozen EEGPT against the matched classical features", "question": "On the same people and channels, does a frozen EEG foundation model with a fixed supervised probe do better than fixed classical features?", "configuration": { "label": "58-channel configuration", "note": "the same channels, people, folds and probe as the 58-channel classical result" }, "same_cohort_as": "orthosis-64ch-classical", "classical_rows_same_as": "orthosis-58ch-classical", "model_family": "foundation", "foundation_model": true, "encoder": "frozen", "fine_tuning": false, "lora": false, "leaderboard": false, "model": { "name": "EEGPT", "method_slug": "eegpt", "paper": "https://openreview.net/forum?id=XNiFuoqTuD", "paper_title": "EEGPT: Pretrained Transformer for Universal and Reliable Representation of EEG Signals", "repository": "https://github.com/BINE022/EEGPT", "source_revision": "a0e0a8fad729e2ecf4eedb3a81548a6e6d48a705", "checkpoint": { "file": "eegpt_mcae_58chs_4s_large4E.ckpt", "sha256": "9d63ebc80bba1ad5e3277df4eecfa05d2745e6243a12ff743c4c0a93930dfac0", "link": "https://figshare.com/s/e37df4f8a907a866df4b", "tensor_conversion_sha256": "9ce130abd9f66cd6141b15918c35760da62ffc86b90f611cdf7b4cf4cb0585df" }, "encoder_parameters": 25287168, "code_licence": "Apache-2.0", "code_licence_scope": "the repository’s source code only", "weights_licence": null, "weights_licence_status": "unresolved: the checkpoint’s own licence and terms could not be read from its hosting record, and the repository’s code licence is not evidence that it covers the weights", "weights_distributed": false, "weights_note": "no original or converted weights are distributed; the hashes identify the checkpoint used and grant nothing" }, "pretraining_exposure": { "status": "not established", "statement": "Overlap between this dataset and EEGPT’s pretraining data has not been excluded; nothing here certifies that these recordings were unseen." }, "transfer": "supervised: the frozen encoder’s outputs feed a probe trained on the other seven people’s labels; not zero-shot", "task": "Recorded injected orthosis device errors against standard device events with no error introduced, from the 0–0.5 second window after each event. The labels describe recorded device events, not whether the person noticed the error.", "cohort": { "people": 8, "runs": 83, "events": 2407, "standard_events": 1933, "error_events": 474, "error_share": 0.19692563356875778, "counted_once": "the same 8 people, 83 runs and 2,407 events in all three Orthosis results: one cohort, never counted twice in coverage", "split": "leave one whole person out: eight folds, all of a person’s runs held out together; the scaler and the probe are fitted on the other seven people" }, "signal": { "eeg_channels": 58, "sampling_hz": 500, "window_seconds": [ 0, 0.5 ], "window_samples": 250 }, "encoder_input": { "channels": 58, "samples": 128, "sampling_hz": 256 }, "metric": "balanced_accuracy", "secondary_metrics": [ "auroc", "average_precision" ], "chance_level": 0.5, "weighting": "macro-person: the unweighted mean of the eight held-out people’s metrics, not pooled trial accuracy; balanced accuracy is not ordinary accuracy", "arms": { "eegpt2048": { "label": "Frozen EEGPT + the same fixed supervised probe, 58 channels", "kind": "EEG", "description": "the frozen 25,287,168-parameter EEGPT target encoder at its native 58-by-128 input (resampled from 500 to 256 Hz, microvolts divided by 1000); two temporal patches averaged and four 512-dimensional summary tokens flattened to 2,048 values; the same scaler and probe; the encoder’s weights unchanged, with no LoRA or fine-tuning", "balanced_accuracy": { "mean": 0.7033691494901184, "interval_95": [ 0.661588955420624, 0.7387206981266006 ] }, "auroc": { "mean": 0.7804495137416887, "interval_95": [ 0.7209373794570128, 0.82683754739618 ] }, "average_precision": { "mean": 0.5639283246314877, "interval_95": [ 0.4891844480422808, 0.6303624255527341 ] } }, "classical58": { "label": "Matched classical EEG features + the same probe, 58 channels", "kind": "EEG", "description": "522 fixed amplitude and log band-power values per event from the 58 channels EEGPT supports, in source order; the same scaler and probe", "balanced_accuracy": { "mean": 0.7097113365795216, "interval_95": [ 0.6481218415815692, 0.7689555045503014 ] }, "auroc": { "mean": 0.7934577007551802, "interval_95": [ 0.7190084215562214, 0.8542417824673231 ] }, "average_precision": { "mean": 0.5442331679969993, "interval_95": [ 0.4382040652213037, 0.6541596513028163 ] } }, "confound": { "label": "Direction/time confound probe", "kind": "control", "description": "the same fixed probe on movement direction, run order within the person, position in the run and time in the run: no EEG", "balanced_accuracy": { "mean": 0.5135810219563451, "interval_95": [ 0.5044985313612883, 0.5226282428980794 ] }, "auroc": { "mean": 0.5241804457997877, "interval_95": [ 0.5064116398272049, 0.5408592388203259 ] }, "average_precision": { "mean": 0.22826059095832624, "interval_95": [ 0.20833927064920651, 0.2488889512126497 ] } }, "majority": { "label": "Training majority", "kind": "control", "description": "predicts the most frequent class of the seven training people", "balanced_accuracy": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "auroc": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "average_precision": { "mean": 0.19683117115553578, "interval_95": [ 0.1937407122170277, 0.19992678244080225 ] } }, "prior": { "label": "Training class prior", "kind": "control", "description": "a constant probability, the error share of the seven training people", "balanced_accuracy": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "auroc": { "mean": 0.5, "interval_95": [ 0.5, 0.5 ] }, "average_precision": { "mean": 0.19683117115553578, "interval_95": [ 0.1936912225705329, 0.1998432601880878 ] } } }, "paired": { "comparison": "frozen EEGPT minus the matched 58-channel classical features, the same people and folds", "balanced_accuracy_difference": { "mean": -0.006342187089403081, "interval_95": [ -0.03957220706708949, 0.032334271929074954 ] }, "interval_excludes_zero": false, "people": 8, "note": "the two arms and their difference share the same 10,000 person-bootstrap draws" }, "permutation_control": { "eegpt2048": { "replicates": 20, "balanced_accuracy_mean": 0.4975929219984277, "observed_range": [ 0.4758285966855188, 0.521545904703516 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "classical58": { "replicates": 20, "balanced_accuracy_mean": 0.5005315788674973, "observed_range": [ 0.4816502731392335, 0.517332735977331 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "design": "twenty training-label shuffles within person and run per fold, preserving run class counts; each shuffled label vector is used for both representations and held-out labels stay original; diagnostic controls, not a full permutation significance test" }, "fits": { "folds": 8, "sealed_arms_per_fold": 45, "fitted_probes": 344 }, "prior_exposure": "The 64-channel classical scores were seen before this comparison was frozen: a prospective diagnostic, not a blind confirmatory comparison.", "method": { "signal": "header scales applied once to obtain microvolts; the recorded FCz reference kept and acceleration excluded; continuous forward-only fourth-order 1–30 Hz filtering of all 64 channels, then the fixed 58-channel intersection", "window": "exactly the complete 0–0.5 second window after each event: 250 samples at 500 Hz", "encoder": "fixed 64/125 polyphase resampling to 256 Hz with numerical boundary extension that adds no observed time; microvolts divided by 1000; the encoder frozen", "probe": "a training-only StandardScaler and a balanced L2 logistic regression, C=1, liblinear, maximum 2,000 iterations and the 0.5 decision rule, the same for both representations; no hyperparameter search, change to the encoder’s weights, LoRA or fine-tuning", "sealing": "all eight folds sealed before separate scoring" }, "uncertainty": { "kind": "pointwise 95% percentile interval from 10,000 person-level bootstrap draws", "limited_by": "eight people give limited uncertainty estimation", "conditional_on": "the fixed sealed predictions; refitting is not included", "paired": "the paired interval resamples matched per-person differences, not the gap between two separate intervals" }, "reading": "Frozen EEGPT with the fixed supervised probe is slightly below the matched 58-channel classical features on balanced accuracy, and the paired interval spans zero. This pilot establishes no advantage for either arm, and neither equivalence nor a general inferiority of EEG foundation models.", "limitations": [ "One short-window supervised transfer setup on eight people: the frozen encoder feeds a probe trained on the other seven people’s labels. Not zero-shot and not a verified unseen-dataset test.", "The 64-channel classical scores were seen before this comparison was frozen: a prospective diagnostic, not a blind confirmatory comparison.", "Montage, filtering, reference, scaling, population and the half-second, two-patch context differ from EEGPT’s pretraining; running it natively does not establish that the transfer suits it.", "Overlap between this dataset and EEGPT’s pretraining data has not been excluded.", "The labels are recorded injected device events; movement, muscle and device artifacts may contribute. There is no subjective-recognition, isolated-neural, online, clinical or population claim.", "The response-clean subset (5 standard events, 446 errors) stays out of the comparison.", "Audit scope: the independent audit recomputed all 720 metric rows, 270 aggregate and interval rows and both paired intervals, and 24 source-only refits reproduced the sealed probabilities exactly; it did not re-extract the encoder features or refit every permutation probe." ], "release_limits": [ "Eight-person single-dataset pilot; person-bootstrap uncertainty is limited and is not a population-level ranking.", "Labels describe recorded injected orthosis device events, not subjective recognition, mental states, clinical performance or isolated neural cognition.", "EEG-only: no EMG model, multimodal fusion or paired EEG/EMG clock claim.", "Movement, muscle, device-injection and other acquisition artifacts may contribute; the direction/time confound control does not isolate their origin.", "Train-only within-person/run permutations are diagnostic controls, not a full dataset label-permutation significance test.", "Response-clean secondary subset is strongly label-selected (446 error versus 5 correct events) and only three people have both classes; high AUPRC is not comparable to the primary population.", "This 58-channel configuration does not replace the 64-channel baseline or establish a causal effect of channel count.", "Previously observed 64-channel scores and the narrow single-dataset setting limit confirmatory interpretation.", "Prior CPU score exposure; prospective diagnostic rather than pristine blind confirmatory evaluation.", "Pretraining overlap is not fully excluded; no verified unseen-dataset or zero-shot claim. Only encoder is frozen; supervised probes use seven source people.", "Montage, filter, reference, scaling, population and 0.5-second/two-patch context differ from pretraining; native engineering compatibility does not establish transfer suitability.", "No real-time/online deployment validation, subjective error recognition, mental-state, isolated neural decoding, broad foundation-model ranking or clinical claim.", "The paired primary interval spans zero: no established advantage, equivalence, or general inferiority of EEG foundation models.", "Independent audit recomputed metrics/intervals and 24 source-only main/confound fits; it did not re-extract raw encoder features or refit every permutation probe.", "Original external checkpoint licensing and any acquisition-specific usage terms remain unverified; no accessible author prohibition on aggregate benchmark publication was found. Source-code Apache-2.0 is not certified weight clearance." ], "claims_not_supported": [ "no advantage for either arm, and not equivalence", "not zero-shot transfer and not a verified unseen-dataset test", "not evidence about EEG foundation models in general", "no online, clinical or isolated neural decoding claim", "not new dataset or person coverage" ], "independent_audit": { "status": "pass", "people": 8, "metric_rows_recomputed": 720, "aggregate_rows_recomputed": 270, "source_only_refits": 24, "largest_refit_difference": 0.0, "not_checked": "the encoder features were not re-extracted and the permutation probes were not all refitted" } }, "bingo-cross-language": { "id": "bingo-cross-language", "dataset": "bingo-imagined-speech", "pilot": "bingo", "title": "Inner speech in Greek and English: across people and across languages", "question": "Can a fixed spectral baseline trained on other people decode which of 15 nouns someone imagined saying, and does training in one language carry over to the other?", "kind": "negative result", "model_family": "classical", "foundation_model": false, "fine_tuning": false, "leaderboard": false, "task": "15-class classification of inner speech of 15 matched nouns in Greek and in English, from the 0–1.5 second inner-speech window that starts when the fixation cross appears; each source field is held out whole, both of its languages excluded from every fitted operation", "cohort": { "source_fields": 17, "trials": 5100, "trials_per_held_out_field": { "greek": 150, "english": 150 }, "trials_per_noun_and_language": 10, "training_rows": 2400, "counting_unit": "source-defined subject fields: the operational groups; biological identity and cross-day grouping were not independently certified", "incomplete_field": "one incomplete field excluded by the prospective complete-record rule; no exclusion depended on a score" }, "signal": { "channels": 19, "sampling_hz": 300, "window_seconds": [ 0, 1.5 ], "window_samples": 450, "values_per_trial": 76 }, "classes": 15, "chance": { "balanced_accuracy": 0.06666666666666667, "top3_accuracy": 0.2, "note": "balanced 15-class prior; top-3 prior is three in fifteen" }, "metric": "balanced_accuracy", "secondary_metrics": [ "top3_accuracy" ], "weighting": "field-equal: the mean over the 17 held-out fields", "directions": [ { "id": "train_gr_test_gr", "label": "Greek → Greek", "train_language": "Greek", "test_language": "Greek", "kind": "within-language", "balanced_accuracy": { "mean": 0.06352941176470589, "interval_95": [ 0.0584313725490196, 0.06901960784313725 ] }, "top3_accuracy": { "mean": 0.19058823529411767, "interval_95": [ 0.18039215686274512, 0.19960784313725494 ] }, "controls": { "order_only": { "label": "Order-only control", "balanced_accuracy": { "mean": 0.07098039215686275, "interval_95": [ 0.06392156862745096, 0.0792254901960783 ] } }, "high_frequency_ratio": { "label": "High-frequency ratio control", "balanced_accuracy": { "mean": 0.07098039215686273, "interval_95": [ 0.06431372549019608, 0.0776470588235294 ] } }, "class_prior": { "label": "Class prior", "balanced_accuracy": { "mean": 0.06666666666666667 } }, "label_shuffle": { "label": "Training-label shuffles", "balanced_accuracy": { "replicates": 20, "balanced_accuracy_mean": 0.0656078431372549, "observed_range": [ 0.05529411764705883, 0.07490196078431371 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" } } } }, { "id": "train_gr_test_eng", "label": "Greek → English", "train_language": "Greek", "test_language": "English", "kind": "cross-language", "balanced_accuracy": { "mean": 0.06588235294117646, "interval_95": [ 0.054901960784313725, 0.07647058823529412 ] }, "top3_accuracy": { "mean": 0.20980392156862743, "interval_95": [ 0.19568627450980391, 0.2243137254901961 ] }, "controls": { "order_only": { "label": "Order-only control", "balanced_accuracy": { "mean": 0.06235294117647059, "interval_95": [ 0.05411764705882354, 0.06980392156862746 ] } }, "high_frequency_ratio": { "label": "High-frequency ratio control", "balanced_accuracy": { "mean": 0.06392156862745099, "interval_95": [ 0.056862745098039215, 0.07098039215686275 ] } }, "class_prior": { "label": "Class prior", "balanced_accuracy": { "mean": 0.06666666666666667 } }, "label_shuffle": { "label": "Training-label shuffles", "balanced_accuracy": { "replicates": 20, "balanced_accuracy_mean": 0.06507843137254901, "observed_range": [ 0.054117647058823534, 0.07333333333333335 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" } } } }, { "id": "train_eng_test_eng", "label": "English → English", "train_language": "English", "test_language": "English", "kind": "within-language", "balanced_accuracy": { "mean": 0.06705882352941177, "interval_95": [ 0.0584313725490196, 0.07568627450980393 ] }, "top3_accuracy": { "mean": 0.19921568627450983, "interval_95": [ 0.18196078431372548, 0.216078431372549 ] }, "controls": { "order_only": { "label": "Order-only control", "balanced_accuracy": { "mean": 0.07411764705882352, "interval_95": [ 0.06705882352941177, 0.08078431372549019 ] } }, "high_frequency_ratio": { "label": "High-frequency ratio control", "balanced_accuracy": { "mean": 0.06549019607843137, "interval_95": [ 0.05647058823529412, 0.07490196078431373 ] } }, "class_prior": { "label": "Class prior", "balanced_accuracy": { "mean": 0.06666666666666667 } }, "label_shuffle": { "label": "Training-label shuffles", "balanced_accuracy": { "replicates": 20, "balanced_accuracy_mean": 0.06809803921568627, "observed_range": [ 0.0611764705882353, 0.0784313725490196 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" } } } }, { "id": "train_eng_test_gr", "label": "English → Greek", "train_language": "English", "test_language": "Greek", "kind": "cross-language", "balanced_accuracy": { "mean": 0.07058823529411766, "interval_95": [ 0.062352941176470576, 0.07921568627450981 ] }, "top3_accuracy": { "mean": 0.19607843137254902, "interval_95": [ 0.18274509803921568, 0.20941176470588235 ] }, "controls": { "order_only": { "label": "Order-only control", "balanced_accuracy": { "mean": 0.06549019607843137, "interval_95": [ 0.05843137254901961, 0.07294117647058823 ] } }, "high_frequency_ratio": { "label": "High-frequency ratio control", "balanced_accuracy": { "mean": 0.07215686274509804, "interval_95": [ 0.06509803921568627, 0.08 ] } }, "class_prior": { "label": "Class prior", "balanced_accuracy": { "mean": 0.06666666666666667 } }, "label_shuffle": { "label": "Training-label shuffles", "balanced_accuracy": { "replicates": 20, "balanced_accuracy_mean": 0.06754901960784314, "observed_range": [ 0.05450980392156863, 0.07725490196078433 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" } } } } ], "contrast": { "comparison": "cross-language minus within-language balanced accuracy, the mean over the two training languages; the same held-out fields", "balanced_accuracy_difference": { "mean": 0.002941176470588233, "interval_95": [ -0.0062745098039215744, 0.012745098039215686 ] }, "interval_excludes_zero": false, "by_training_language": { "greek": { "comparison": "trained on Greek: tested on English minus tested on Greek", "balanced_accuracy_difference": { "mean": 0.0023529411764705867, "interval_95": [ -0.01254901960784314, 0.016862745098039218 ] } }, "english": { "comparison": "trained on English: tested on Greek minus tested on English", "balanced_accuracy_difference": { "mean": 0.0035294117647058807, "interval_95": [ -0.008637254901960786, 0.017254901960784316 ] } } } }, "fits": { "main_models": 34, "order_and_high_frequency_controls": 68, "shuffled_label_fits": 680, "class_prior_outputs": 34, "sealed_checkpoints": 816 }, "method": { "window": "the stored time vectors verify the half-open 0–1.5 second action window at 300 Hz", "inputs": "per trial and channel, max-absolute scaling and demeaning, then Hann periodograms; four relative bands partition 1–30 Hz across 19 source-order channels; no guessed voltage calibration, montage interpolation or re-referencing", "model": "a source-only StandardScaler and RidgeClassifier with fixed alpha 1 and the Cholesky solver; no tuning against held-out scores; ties broken by a fixed noun order", "controls": "an order-only control, a high-frequency ratio control, the class prior and 20 training-label shuffles per direction", "sealing": "all 816 checkpoints sealed before any scoring" }, "uncertainty": { "kind": "approximate descriptive 2.5–97.5% intervals from 10,000 field bootstrap draws", "conditional_on": "fixed predictions from overlapping leave-one-field-out fits: training refit variability is omitted, and they are not calibrated population confidence intervals" }, "protocol": { "nouns": 15, "languages": [ "Greek", "English" ], "timeline_seconds": { "word_shown": -2.5, "word_removed_countdown": -1.0, "fixation_inner_speech_onset": 0.0, "end": 1.5 }, "repetitions_per_noun": 10, "order": "fully randomized across the session", "source": "BINGO deliverable D3.1, version 2.0, the cross-linguistic protocol (https://bingo-project.gr/download/397)" }, "delivered_signal": { "statement": "The project documents the same preprocessing for both of its released datasets: band-pass filtering from 1 to 145 Hz, notch filtering at 50 Hz and its harmonics, Artifact Subspace Reconstruction (ASR), FORCe removal of ocular and muscle artifacts, and then segmentation into trials. The signals analysed here were therefore artifact-cleaned on the continuous recordings before any trial was cut.", "source": "BINGO deliverable D3.1, version 2.0, pre-processing and denoising (https://bingo-project.gr/download/397), read 2026-10-12", "verified_against_file": false, "note": "as the project states it; not re-verified against the delivered file" }, "project_reported": { "kind": "reported by the BINGO project on its other dataset; not run here and not comparable with the results above", "dataset": "the project’s NATO-alphabet inner-speech dataset: 20 participants, 26 words, three sessions", "evaluation": "EEGNet, leave-one-subject-out, 26 classes", "mean_accuracy_percent": 3.85, "classes": 26, "chance_level": 0.038461538461538464, "reading": "the deliverable reports performance at chance level for people the model was not trained on", "source": "BINGO deliverable D3.1, version 3.0, section 4 (https://bingo-project.gr/download/402)" }, "reading": "Every direction is close to the 15-class prior of 6.67%, with each interval holding it, and top-3 accuracy is close to its 20% prior. The cross-language-minus-within-language contrast is small and its interval spans zero. A negative result for this baseline in this setting: it does not show that inner speech cannot be decoded by other methods.", "limitations": [ "One fixed spectral baseline: no EEG foundation model, PEFT or deep model was evaluated.", "Source-defined subject fields are the operational groups; biological identity and cross-day grouping were not independently certified.", "The delivered signals were already cleaned by the project (ASR and FORCe on the continuous recordings, by its own account); physical amplitude units, the named montage and the measured display latency are not established.", "The nominal window was verified against the stored time vectors; that does not certify causal or online processing, brain-only signal origin, full BINGO coverage or cross-day performance.", "The intervals are approximate and descriptive: fixed overlapping leave-one-field-out fits, no refit variability, not calibrated population confidence intervals. The shuffles measure sensitivity to the training labels and are not a calibrated randomization test.", "The contrast is not superiority, equivalence, a significance test or a reliable transfer gain." ], "release_limits": [ "Source-defined subject fields are the operational groups; biological identity and cross-day grouping were not independently certified.", "Delivered signal preprocessing, physical amplitude units, named montage and measured display latency are not established.", "The nominal action window was verified against stored time vectors; this does not certify causal or online processing.", "The partial nine-trial field is excluded by the prospective complete-record rule; no score-driven exclusions.", "No EEG foundation model, PEFT or deep model was evaluated in this pilot.", "Approximate descriptive bootstrap intervals condition on overlapping leave-one-field-out fits and omit training-refit variability." ], "claims_not_supported": [ "not evidence that inner speech cannot be decoded", "no superiority or equivalence between languages, and no significance test", "no online, real-time or brain-only decoding claim", "not a foundation-model ranking, and no medical or diagnostic claim" ], "independent_audit": { "status": "pass", "source_fields": 17, "refitted_models": 102, "reconstructed_models": 782, "refitted_shuffle_models": 0, "largest_refit_difference": 0.0, "not_checked": "the 680 shuffle models were reconstructed, not refitted, and the raw file was not re-extracted: the audit used cached features and verified lineage receipts" } }, "handwriting-mask-aware": { "id": "handwriting-mask-aware", "dataset": "eeg-fnirs-handwriting", "pilot": "handwriting", "title": "Mask-aware EEG: a five-person development diagnostic", "question": "When channels that sit flat at a digital rail are masked, what is left, and does it predict four released labels better than the masks and the clock do?", "kind": "data-quality and methods case study", "model_family": "classical", "foundation_model": false, "fine_tuning": false, "leaderboard": false, "task": "four released coded labels, 0 to 3 (events 200 to 203), from a fixed window 0.5–4.5 seconds after the released EEG-relative onset; glyph identity and continuous trajectories are not established, so this is not handwriting recognition", "cohort": { "people": 5, "sessions": 45, "events": 1620, "trials": 1618, "quarantined_events": 2, "classes": 4, "reuse": "a reused development cohort: the same five training people as the earlier analysis, whose scores were seen before this design; not a new or independent cohort", "split": "five leave-one-person-out folds; every session of the held-out person excluded from fitting", "official_test": "the official ten-person test signals and reference labels were not used; earlier intake inspected unlabeled test metadata, and this analysis made no further test access" }, "metric": "balanced_accuracy", "chance_level": 0.25, "weighting": "the unweighted mean of the five held-out people’s balanced accuracy", "arms": { "relative_eeg": { "label": "Mask-aware relative EEG", "kind": "EEG", "description": "295 channel-major relative Welch band-power slots from the available channels, zero where a channel is unavailable", "balanced_accuracy": { "mean": 0.26808641975308645, "interval_95": [ 0.26296296296296295, 0.2719753086419753 ] } }, "mask_only": { "label": "Missingness mask only", "kind": "control", "description": "59 availability bits plus the available-channel fraction", "balanced_accuracy": { "mean": 0.24993055555555554, "interval_95": null }, "interval_note": "not published: with five people, this control’s bootstrap bounds would show individual people’s scores" }, "time_only": { "label": "Elapsed time / trial order", "kind": "control", "description": "onset, log1p(onset), trial rank and rank over count", "balanced_accuracy": { "mean": 0.2543672839506173, "interval_95": [ 0.24026234567901233, 0.26790123456790116 ] } }, "mask_plus_time": { "label": "Mask + time", "kind": "control", "description": "the mask and time controls together", "balanced_accuracy": { "mean": 0.2500771604938271, "interval_95": [ 0.2390432098765432, 0.26118827160493824 ] } }, "majority": { "label": "Training-majority control", "kind": "control", "description": "the most frequent class of the four training people", "balanced_accuracy": { "mean": 0.25, "interval_95": [ 0.25, 0.25 ] } } }, "contrasts": { "relative_eeg_minus_mask_only": { "label": "Relative EEG minus missingness mask only", "balanced_accuracy_difference": { "mean": 0.018155864197530847, "interval_95": [ 0.012962962962962944, 0.022114197530864188 ] }, "interval_excludes_zero": true }, "relative_eeg_minus_time_only": { "label": "Relative EEG minus elapsed time / trial order", "balanced_accuracy_difference": { "mean": 0.01371913580246914, "interval_95": [ -0.0012345679012345622, 0.03066358024691359 ] }, "interval_excludes_zero": false }, "relative_eeg_minus_mask_plus_time": { "label": "Relative EEG minus mask + time", "balanced_accuracy_difference": { "mean": 0.018009259259259253, "interval_95": [ 0.004320987654320985, 0.03046296296296296 ] }, "interval_excludes_zero": true } }, "label_shuffle": { "representations": { "relative_eeg": { "replicates": 5, "balanced_accuracy_mean": 0.2520123456790123, "observed_range": [ 0.23853395061728394, 0.2668904320987654 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "mask_only": { "replicates": 5, "balanced_accuracy_mean": 0.2494783950617284, "observed_range": [ 0.24876543209876542, 0.25 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "time_only": { "replicates": 5, "balanced_accuracy_mean": 0.24701388888888887, "observed_range": [ 0.24301697530864197, 0.2515586419753086 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" }, "mask_plus_time": { "replicates": 5, "balanced_accuracy_mean": 0.24714197530864196, "observed_range": [ 0.24114969135802466, 0.257770061728395 ], "range_is": "the lowest and highest replicate means: an observed range, not a confidence interval" } }, "design": "five fixed training-label shuffles per representation, preserving source person and session; sensitivity controls, not a full permutation significance test" }, "availability": { "channels_recorded": 59, "available_channels_per_trial": { "fewest": 27, "most": 38 }, "trials_by_available_channels": [ { "channels": 27, "trials": 1429 }, { "channels": 28, "trials": 65 }, { "channels": 29, "trials": 16 }, { "channels": 30, "trials": 36 }, { "channels": 37, "trials": 36 }, { "channels": 38, "trials": 36 } ], "channel_trials": 95462, "flags": { "constant": 50811, "lower_rail": 50815, "upper_rail": 0, "nonfinite": 0, "invalid_denominator": 0 }, "flags_note": "the constant and lower-rail flags overlap and must not be summed", "available_means": "nonconstant, not touching either header-declared digital rail and numerically valid under a fixed rule: not certified clean physiological EEG", "scope": "counts for the selected cohort only, not the whole source dataset" }, "earlier_analysis": { "relation": "the same five people, sessions and trials, analysed earlier with all-channel common-average referencing and log band power: context, not a second cohort", "arms": { "logistic": { "label": "EEG spectral logistic regression", "balanced_accuracy": { "mean": 0.2704783950617284, "interval_95": [ 0.2538117283950617, 0.28714506172839505 ] } }, "shrinkage_lda": { "label": "EEG spectral shrinkage LDA", "balanced_accuracy": { "mean": 0.2742283950617284, "interval_95": [ 0.26049382716049385, 0.29259259259259257 ] } } }, "quality_findings": { "windows": 1618, "windows_with_a_rail_sample": 1618, "windows_over_500_microvolts": 721, "inspected_windows": 5, "channel_windows": 295, "rail_flat_channel_windows": 148 }, "not_attributable": "masking, referencing and the spectral representation changed together, so the difference between the two analyses cannot be attributed to masking, and no best model is selected from them" }, "method": { "window": "0.5–4.5 seconds after the released EEG-relative onset: 4,000 samples at 1,000 Hz; only the 59 header-identified EEG channels read", "masking": "constant, rail-touching or nonfinite channel-windows masked before any transform; available channels demeaned individually; no common-average reference and no interpolation", "inputs": "Welch estimates (1,000-sample Hann window, 500 overlap); five half-open bands from 1 to 45 Hz, each divided by the sum of the five", "model": "source-only standardization and balanced multinomial logistic regression, C=1, lbfgs, max_iter 2000, tolerance 1e-4, seed 20261012", "sealing": "all 125 checkpoints sealed before the separate scorer opened held-out truth" }, "uncertainty": { "kind": "descriptive 2.5–97.5% intervals from 10,000 fixed-seed person-bootstrap draws over five people", "conditional_on": "the already fitted overlapping folds: not calibrated population confidence intervals or evidence of superiority" }, "reading": "Mask-aware relative EEG is slightly above the 25% majority control, and above the missingness-only and mask-plus-time controls with paired intervals above zero; against the elapsed-time control the paired interval crosses zero. Five people and four coded labels: a data-quality and methods case study, not handwriting recognition.", "limitations": [ "Five reused development people whose earlier scores were seen before this design: no independent confirmation and no population inference.", "Four released coded labels: glyph identity and trajectories are not established, so this is not handwriting recognition.", "Availability is a numerical rule, not certified clean EEG; why other channels sit flat at a digital rail is unknown.", "Masking, referencing and the spectral representation changed together, so no difference from the earlier analysis can be attributed to masking.", "The contrast with the elapsed-time control crosses zero.", "Optical species, units and the clock between modalities are unresolved, so no fNIRS or fusion conclusion follows; no online, physiological-origin, competition-standing or medical claim.", "Audit scope: all 120 stored models reconstructed and the 20 true-label models refitted independently, with no discrepancy; the shuffle models were not refitted, and the audit used cached derived artifacts, not fresh raw-feature extraction." ], "claims_not_supported": [ "not handwriting recognition and not glyph or trajectory decoding", "no superiority over the elapsed-time control", "no fNIRS, fusion, online, competition-standing or medical claim", "not a leaderboard entry or a model recommendation" ], "independent_audit": { "status": "pass", "people": 5, "reconstructed_models": 120, "refitted_models": 20, "refitted_shuffle_models": 0, "largest_refit_difference": 0.0, "not_checked": "the raw samples were not reread and the features were not re-extracted" } }, "refed-stored-clip": { "id": "refed-stored-clip", "dataset": "refed", "pilot": "refed", "title": "REFED: a stored-clip computational diagnostic", "question": "Can whole-clip EEG band power predict a clip’s mean self-reported valence and arousal better than simple source-only controls?", "kind": "computational diagnostic", "model_family": "classical", "foundation_model": false, "fine_tuning": false, "leaderboard": false, "cohort": { "nominal_folders": 2, "videos": 15, "clips": 30, "note": "two source-declared nominal participant folders with the same 15 videos; folder identity is the grouping unit, the 30 clips are not 30 independent people, and two folders support no population claim" }, "target": { "axes": [ "valence", "arousal" ], "scale": "the published raw self-report scale from 1 to 255", "definition": "per axis, the mean of each clip’s complete stored label array", "meaning": "descriptive self-report means: not SAM scores and not calibrated physical or clinical distances" }, "metric": { "id": "equal_axis_mae", "name": "Equal-axis mean absolute error", "unit": "raw label units on the 1–255 scale", "lower_is_better": true, "aggregation": "the mean of the valence and arousal absolute errors, the 30 clips weighted equally" }, "arms": { "eeg_ridge": { "label": "EEG Ridge", "kind": "EEG", "description": "320 relative Welch band powers per clip (five bands at each of 64 channel positions), source-only standardization, Ridge with alpha 1" }, "source_global_mean": { "label": "Source global mean", "kind": "control", "description": "the training clips’ mean target" }, "source_video_lookup": { "label": "Source video lookup", "kind": "control", "description": "the training target of the same video ID, falling back to the source global mean when the video is absent" }, "duration_video_id_ridge": { "label": "Duration / numeric-video-ID Ridge", "kind": "control", "description": "Ridge on stored duration and the numeric video ID, a key-index proxy and not established presentation order" } }, "conditions": [ { "id": "shared_videos_held_folder", "label": "Shared videos, held folder", "design": "two folds: train on all 15 clips of one folder and predict the other folder; familiar stimuli can support prediction through stimulus identity", "folds": 2, "training_clips_per_fold": 15, "held_clips_per_fold": 15, "clips_scored": 30, "lookup_matched_videos": 30, "arms": { "eeg_ridge": { "label": "EEG Ridge", "kind": "EEG", "equal_axis_mae": 107.18211992384872 }, "source_global_mean": { "label": "Source global mean", "kind": "control", "equal_axis_mae": 40.2870293539505 }, "source_video_lookup": { "label": "Source video lookup", "kind": "control", "equal_axis_mae": 27.324569639648146 }, "duration_video_id_ridge": { "label": "Duration / numeric-video-ID Ridge", "kind": "control", "equal_axis_mae": 37.909675455075345 } } }, { "id": "held_folder_and_videos", "label": "Jointly held folder and video IDs", "design": "ten folds: each folder held out with one of five fixed three-video groups, training on the other folder’s 12 remaining videos", "folds": 10, "training_clips_per_fold": 12, "held_clips_per_fold": 3, "clips_scored": 30, "lookup_matched_videos": 0, "arms": { "eeg_ridge": { "label": "EEG Ridge", "kind": "EEG", "equal_axis_mae": 91.98446456118164 }, "source_global_mean": { "label": "Source global mean", "kind": "control", "equal_axis_mae": 41.032302303312534 }, "source_video_lookup": { "label": "Source video lookup", "kind": "control", "equal_axis_mae": 41.032302303312534 }, "duration_video_id_ridge": { "label": "Duration / numeric-video-ID Ridge", "kind": "control", "equal_axis_mae": 47.097702856735566 } } } ], "controls_lesson": "With shared videos, the lookup of the same video’s training target has the smallest error: familiar stimuli can carry the prediction through stimulus identity, which is why that control is there. When folder and video are held out together the lookup has no matching video and falls back to the global mean, so the two are equal. This illustrates the confound; it does not establish a mechanism.", "quality": { "clips_checked": 30, "channel_clip_rows": 1920, "clips_or_channels_removed": 0, "note": "finite, nonconstant values and valid spectral denominators in every clip: numerical eligibility only, not acquisition quality, physical units or physiological specificity" }, "uncertainty": { "kind": "none: no confidence interval or significance test, because two nominal folders support no population inference" }, "method": { "inputs": "five relative Welch band powers, [1,4), [4,8), [8,13), [13,30) and [30,45) Hz, each divided by that channel’s [1,45) Hz total, for each of the 64 original channel positions: 320 per clip; Welch at the nominal 1,000 Hz with one-second Hann windows and 50% overlap", "model": "source-only standardization and Ridge with fixed alpha 1, no tuning; predictions finite but unclipped, no calibration after the fact", "channels": "no channel selection, common-average reference, cross-channel mixing or invented trim", "sealing": "all fold predictions and fitted parameters sealed before separate scoring" }, "reading": "EEG Ridge has a larger error than every source-only control in both conditions. With shared videos, the same-video lookup has the smallest error; when folder and video are held out together, the lookup has no matching video and equals the global mean. No physiological or population claim follows.", "limitations": [ "Two nominal folders and 15 shared video keys: a descriptive computational diagnostic, not a population, cross-day or leaderboard result.", "Stored EEG units, calibration and sensor validity, the precise EEG-to-label clock and the annotation polling are unresolved; relative band power removes a constant channel gain, which is a property of the representation only.", "Published video durations exceed the stored arrays by two seconds and the crop end points are unknown: pairing whole arrays establishes no precise physiological alignment.", "The numeric video ID is a key-index proxy, not established presentation order, and the shared-video condition carries familiar-stimulus confounding.", "The label vectors had been inspected, all 30 means computed privately and two arrays quality-checked before this design: these are not untouched targets.", "No confidence interval, significance test, clinical affect, neural specificity or population claim." ], "release_limits": [ "Two nominal source groups and shared15video keys support descriptive computational diagnostics only; clips are not30independent people.", "Stored EEG units/calibration/sensor validity, precise EEG-label/fNIRS clocks and native annotation polling/aggregation remain unresolved. Constant-gain invariance is a technical representation property only.", "Published video durations exceed stored arrays by2s; crop endpoints remain unknown. Whole-array pairing establishes no precise physiological alignment.", "Numeric video ID is a key-index proxy, not established presentation order. Shared-video results have familiar-stimulus confounding.", "Prior label/target/quality exposure is disclosed; no untouched holdout claim. Fixed protocol had no score-informed tuning, clipping, replacement or exclusions.", "No confidence interval, significance test, population/crossday inference, clinical affect, neural specificity or leaderboard claim." ], "claims_not_supported": [ "no physiological or neural decoding claim", "not a population, cross-day or leaderboard result", "no causal mechanism for the lookup’s advantage", "no fNIRS or fusion result" ], "independent_audit": { "status": "pass", "assertions": 2838, "categories": 58, "largest_metric_difference": 2.842170943040401e-14, "checked": "sealed predictions reconstructed exactly and every metric recomputed independently; all 30 preparation records verified", "not_checked": "no raw signal or original annotation was reread and no model was refitted or tuned" } } }, "datasets": { "orthosis-error-detection": { "publication": "released", "rights": { "name": "Orthosis error-detection EEG and EMG dataset (Kueper et al. 2024)", "task": "Recorded injected orthosis device errors against standard device events, while an active orthosis moved the right arm in elbow flexion and extension; EEG only here", "source": "https://doi.org/10.5281/zenodo.8345429", "version": "Zenodo record 8345429 (doi:10.5281/zenodo.8345429), the third and latest version of concept record 7951043: the complete EEG and EMG dataset, read from the Zenodo API on 2026-10-12. EEG only is used: 64 channels at 500 Hz from 8 people.", "license": "CC BY 4.0", "licenseUrl": "https://creativecommons.org/licenses/by/4.0/", "attribution": "Niklas Kueper, Kartik Chari, Judith Bütefür, Julia Habenicht, Tobias Rossol, Su Kyoung Kim, Marc Tabie, Frank Kirchner and Elsa Andrea Kirchner · EEG and EMG dataset for the detection of errors introduced by an active orthosis device (IJCAI’23 CC6 Competition), Zenodo, doi:10.5281/zenodo.8345429, CC BY 4.0. Paper: Niklas Kueper et al., EEG and EMG dataset for the detection of errors introduced by an active orthosis device, Frontiers in Human Neuroscience 18 (2024), doi:10.3389/fnhum.2024.1304311. Derived analysis by BCI Report; not endorsed by the authors.", "reviewedAt": "2026-10-12", "reviewBasis": [ "https://zenodo.org/api/records/8345429", "https://doi.org/10.3389/fnhum.2024.1304311", "https://europepmc.org/article/PMC/PMC10839100" ], "privacyReview": "The data paper states that the study was approved by the Ethics Committee of the Department of Computer Science and Applied Cognitive Science of the Faculty of Engineering at the University of Duisburg-Essen, with no approval number given, and that the participants gave written informed consent. Published here: means over the 8 people with person-bootstrap intervals for three configurations of one cohort, the paired EEGPT-minus-classical difference with its interval, permutation-control means with their observed ranges, and cohort counts. No per-person value, participant or run identifier, response-clean subset score or demographic is published." }, "consent_and_ethics": { "ethics_approval": { "stated": true, "statement": "Approved by the Ethics Committee of the Department of Computer Science and Applied Cognitive Science of the Faculty of Engineering at the University of Duisburg-Essen; no approval number is given.", "quote": "The studies involving humans were approved by Ethics Committee" }, "informed_consent": { "stated": true, "statement": "The participants gave written informed consent and were told they could stop at any time.", "quote": "The participants provided their written informed consent to participate in this study" }, "read_from": "https://europepmc.org/article/PMC/PMC10839100, Ethics statement; Methods, Participants, read 2026-10-12" }, "results": [ "orthosis-64ch-classical", "orthosis-58ch-classical", "orthosis-eegpt-vs-58ch" ] }, "bingo-imagined-speech": { "publication": "released", "rights": { "name": "BINGO cross-linguistic inner-speech EEG dataset (Greek and English)", "task": "Inner speech of 15 matched nouns in Greek and in English, 15-class word classification on people the model was not trained on, within and across the two languages", "source": "https://bingo-project.gr/datasets/", "version": "Dataset 2, cross-linguistic inner speech elicitation, on the BINGO project’s datasets page (read on 2026-10-12; its access link leads to a sign-in, and the file analysed was supplied by the site owner, who confirmed this page as its source). Documented by the project’s public deliverable D3.1, version 2.0, linked from that page. 17 complete source-defined subject fields, 19 channels at 300 Hz.", "license": "No licence stated", "licenseUrl": "https://bingo-project.gr/datasets/", "attribution": "BINGO (Brain Imagined-Speech Communication) project, CERTH · cross-linguistic inner-speech EEG dataset, https://bingo-project.gr/datasets/; no licence is stated by the source. Documentation: Fotis P. Kalaganis, Kostas Georgiadis, Maria Kyrou, Spiros Nikolopoulos and Ioannis Kompatsiaris, D3.1 – The BINGO Benchmarking Framework, version 2.0, public deliverable (2025), https://bingo-project.gr/download/397. Derived analysis by BCI Report; not endorsed by the authors.", "reviewedAt": "2026-10-12", "reviewBasis": [ "https://bingo-project.gr/datasets/", "https://bingo-project.gr/download/397", "https://bingo-project.gr/download/402" ], "privacyReview": "The source states no licence, and the cross-language dataset’s own documentation has no ethics-approval or consent statement. The project’s public deliverable D3.1, version 3.0, reports that participants signed a consent form; it says so for the project’s NATO-alphabet recordings in the same CERTH laboratory. No ethics-approval reference is published. Aggregates are published on that basis by the owner’s decision. Published here: means over the 17 source-defined subject fields with descriptive intervals, the cross-language contrast, controls and counts. No per-field value, field identifier, confusion matrix, prediction or source file is published, and no right to redistribute the data is implied." }, "consent_and_ethics": { "ethics_approval": { "stated": false, "statement": null, "note": "No ethics-approval reference is published by the source." }, "informed_consent": { "stated": true, "statement": "The project’s public deliverable D3.1, version 3.0, states that each participant signed a consent form before the sessions. It says so for the project’s 20-participant NATO-alphabet dataset, recorded in the same CERTH laboratory; the cross-language dataset’s own documentation, D3.1 version 2.0, has no ethics or consent statement.", "quote": "After signing a consent form, each participant attended three experimental sessions", "scope": "stated for the project’s NATO-alphabet dataset, not for the cross-language dataset" }, "basis": "owner decision: aggregates published with these rights as they stand", "read_from": "D3.1 – The BINGO Benchmarking Framework, version 3.0 (https://bingo-project.gr/download/402), Section 2, BINGO dataset, read 2026-10-12" }, "documentation": [ { "title": "D3.1 – The BINGO Benchmarking Framework, version 2.0", "url": "https://bingo-project.gr/download/397", "dissemination": "Public (PU)" }, { "title": "D3.1 – The BINGO Benchmarking Framework, version 3.0", "url": "https://bingo-project.gr/download/402", "dissemination": "Public (PU)" } ], "results": [ "bingo-cross-language" ] }, "eeg-fnirs-handwriting": { "publication": "released", "rights": { "name": "EEG-fNIRS-based Handwriting Trajectory Dataset (Hugging Face)", "task": "Four released coded labels in an imagined-handwriting competition dataset (the ACM MM 2026 Multimodal Brain-Computer Interface Grand Challenge, as a participating team’s paper identifies it); EEG only here", "source": "https://huggingface.co/datasets/lasfk/EEG-fNIRS-based-Handwriting-Trajectory-Dataset", "version": "Revision 4036680b5fb0015472d55589da2f34c5a87a3c10, read from the Hugging Face API on 2026-10-12; gated: its files require accepting the uploader’s conditions. Five training participants, all nine sessions each.", "license": "CC BY-NC-SA 4.0", "licenseUrl": "https://creativecommons.org/licenses/by-nc-sa/4.0/", "attribution": "lasfk on Hugging Face · EEG-fNIRS-based Handwriting Trajectory Dataset, revision 4036680b5fb0015472d55589da2f34c5a87a3c10, CC BY-NC-SA 4.0 as declared on the dataset card, gated access. Derived analysis by BCI Report; not endorsed by the uploader.", "reviewedAt": "2026-10-12", "reviewBasis": [ "https://huggingface.co/api/datasets/lasfk/EEG-fNIRS-based-Handwriting-Trajectory-Dataset/revision/4036680b5fb0015472d55589da2f34c5a87a3c10", "https://huggingface.co/datasets/lasfk/EEG-fNIRS-based-Handwriting-Trajectory-Dataset", "https://arxiv.org/abs/2608.03176v2" ], "privacyReview": "No ethics-approval or informed-consent statement was found; the aggregates are published with that gap stated, by the owner’s decision. Published here: means over the 5 people with descriptive person-bootstrap intervals (none for the mask-only control, whose bounds would show individual people’s scores), three paired contrasts, label-shuffle means with observed ranges, the channel-availability distribution over trials and cohort counts. No per-person value, identifier, label, prediction or source file is published, and no right to redistribute the data is implied." }, "consent_and_ethics": { "ethics_approval": { "stated": false, "statement": null, "note": "No ethics-approval statement was found: the dataset card has none, and a participating team’s challenge paper has none." }, "informed_consent": { "stated": false, "statement": null, "note": "No informed-consent statement was found: the dataset card has none, and a participating team’s challenge paper has none." }, "read_from": "https://huggingface.co/datasets/lasfk/EEG-fNIRS-based-Handwriting-Trajectory-Dataset, the dataset card, read 2026-10-12", "searched": [ { "source": "the dataset card", "url": "https://huggingface.co/datasets/lasfk/EEG-fNIRS-based-Handwriting-Trajectory-Dataset", "statement_found": false }, { "source": "a participating team’s challenge paper", "title": "FRED: Frequency-Decorrelated Temporal Ensembles for EEG–fNIRS Imagined-Handwriting Decoding", "authors": "Xiao Fan, Hongbin Guo, Yubo Han and Yi Zhang (Xidian University)", "url": "https://arxiv.org/abs/2608.03176v2", "identifies": "the ACM MM 2026 Multimodal Brain-Computer Interface Grand Challenge", "statement_found": false }, { "source": "the organisers’ paper", "citation": "B. Chen, Z. Jia, R. Mark, J. Wang, T. Jiang, H. Huang, X. Zhou and J. Ma, “Multimodal brain–computer interface grand challenge: EEG–fNIRS-based handwriting-trajectory classification”, Proceedings of the ACM International Conference on Multimedia (2026)", "status": "listed as forthcoming in the challenge paper; not found published when checked on 2026-10-12" } ], "gap": "No ethics-approval or informed-consent statement was found. The dataset card has none; a participating team’s challenge paper, which identifies the competition as the ACM MM 2026 Multimodal Brain-Computer Interface Grand Challenge, has none; and the organisers’ paper is listed there as forthcoming. The aggregates are published with this gap stated, by the owner’s decision, to be rechecked when the organisers’ paper appears.", "basis": "owner decision: aggregates published with this gap stated", "recheck": "when the organisers’ paper appears" }, "results": [ "handwriting-mask-aware" ] }, "refed": { "publication": "released", "rights": { "name": "REFED: real-time dynamic labelled EEG-fNIRS emotion dataset (Ning et al. 2025)", "task": "Continuous self-reported valence and arousal while watching videos; here a whole-clip mean-target regression diagnostic on stored EEG clips", "source": "https://huggingface.co/datasets/REFED2025/REFED-dataset", "version": "Hugging Face revision 0298355c2823946ac5800b2ec12a082bf2dcd01b, read from the Hugging Face API on 2026-10-12; gated: users agree to non-commercial use under CC BY-NC-SA and not to cause harm to human subjects. Two nominal participant folders with the same 15 videos: 30 stored EEG clips.", "license": "CC BY-NC-SA 4.0", "licenseUrl": "https://creativecommons.org/licenses/by-nc-sa/4.0/", "attribution": "Xiaojun Ning, Jing Wang, Zhiyang Feng, Tianzuo Xin, Shuo Zhang, Shaoqi Zhang, Zheng Lian, Yi Ding, Youfang Lin and Ziyu Jia · REFED: A Subject Real-time Dynamic Labeled EEG-fNIRS Synchronized Recorded Emotion Dataset, NeurIPS 2025 Datasets and Benchmarks Track. Data: REFED2025/REFED-dataset on Hugging Face, revision 0298355c2823946ac5800b2ec12a082bf2dcd01b, CC BY-NC-SA 4.0, gated. Derived analysis by BCI Report; not endorsed by the authors.", "reviewedAt": "2026-10-12", "reviewBasis": [ "https://huggingface.co/api/datasets/REFED2025/REFED-dataset/revision/0298355c2823946ac5800b2ec12a082bf2dcd01b", "https://papers.neurips.cc/paper_files/paper/2025/file/2bf0ed7c35d9d84128e7f7c72ab76402-Paper-Datasets_and_Benchmarks_Track.pdf" ], "privacyReview": "The paper states that the local Institutional Review Board approved the study (its datasheet names IRB-CASIA) and that all participants gave informed consent to the recording of their physiological signals and ratings, for non-commercial research use. Published here: equal-axis mean absolute errors over the 30 stored clips for four fixed arms in two conditions, counts and the method. No per-folder, per-clip or per-video value, target, prediction or demographic is published." }, "consent_and_ethics": { "ethics_approval": { "stated": true, "statement": "Approved by the local Institutional Review Board; the paper’s datasheet names the approval IRB-CASIA.", "quote": "approved by the local Institutional Review Board" }, "informed_consent": { "stated": true, "statement": "All participants gave informed consent to the recording of their physiological signals and rating scales, and agreed that the data would be used only for non-commercial scientific research.", "quote": "All participants provided informed consent for the recording of their physiological signals" }, "read_from": "https://papers.neurips.cc/paper_files/paper/2025/file/2bf0ed7c35d9d84128e7f7c72ab76402-Paper-Datasets_and_Benchmarks_Track.pdf, Section 5, Ethical Considerations and Dataset Accessibility; datasheet; Appendix C, read 2026-10-12" }, "results": [ "refed-stored-clip" ] } }, "status_only": [ { "id": "orthosis-emg-tinymyo-fusion", "statement": "No EMG, TinyMyo or EEG–EMG fusion result exists for the Orthosis dataset.", "reason": "The exact clock anchors between the EMG sample rows and the EEG events, the latency and end points, and the meaning and units of the exported columns are not established. A nominal sampling rate alone does not establish the labels.", "scope": "Orthosis error-detection dataset: EMG and paired EEG–EMG analyses" }, { "id": "handwriting-fnirs-fusion", "statement": "No fNIRS or EEG–fNIRS fusion result exists for the handwriting dataset.", "reason": "The optical species and column meanings, their units and a verified clock between the two modalities are not established.", "scope": "EEG-fNIRS handwriting dataset: optical and fused analyses" }, { "id": "refed-physiological-fusion", "statement": "No physiological or precisely aligned fusion evaluation exists for REFED beyond the stored-clip computational diagnostic.", "reason": "Physical calibration, the crop boundaries, the mapping of the six optical components and paired timing are not established.", "scope": "REFED: physiological, pointwise and fused analyses" }, { "id": "meditation-eeg-phase", "statement": "No meditation EEG phase prediction or paired analysis exists.", "reason": "An explicit association between events and records, and clock anchors, are missing. One inspected record has a constant marker column, and that finding is not generalised to the other records. In this dataset RR means heartbeat RR intervals, not respiratory rate.", "scope": "Synchronized multimodal meditation and rest dataset (Zenodo): EEG phase labels and paired analyses" } ], "holds": [], "not_published": [ "Per-person, per-field, per-folder, per-clip and per-record values of any kind: the Orthosis matched EEG-minus-confound range, every per-person minimum and maximum, per-fold scores. How many people or fields did better is published where an audited flag implies it; which ones is not.", "The Orthosis response-clean subset scores (only its counts are published), and every secondary paired difference.", "The individual permutation and label-shuffle replicates: their mean and observed range are published.", "BINGO mean confusion matrices and per-replicate shuffle intervals; handwriting pooled metrics, per-class recall and confusion matrices, and the mask-only control’s bootstrap interval, whose bounds would show individual people’s scores; REFED per-axis errors, RMSE and combined RMSE.", "Participant, field, folder, session, clip, video, file and run identifiers, labels, targets, features, embeddings, predictions, fitted parameters and checkpoints, and every source file. EEGPT weights, original or converted, are not distributed.", "Measured compute: wall times, memory and process counts from every handoff and audit.", "Demographics from the source publications.", "The handoffs, approval and finalization records, QA records, release decisions and independent audits themselves: each is pinned by SHA-256 in this manifest and its verdict and binding are checked by the export. They carry private storage paths and are not copied. Candidate exports named *-candidate.json were not opened." ], "provenance": { "manifest_sha256": "89dcd9295ad0a85abea68deb180fcbed25165d4f523d5e02b402937eeb512c29", "included": [ "orthosis-64ch-classical", "orthosis-58ch-classical", "orthosis-eegpt-vs-58ch", "bingo-cross-language", "handwriting-mask-aware", "refed-stored-clip" ], "holds": [], "held_sources": [], "source_export_sha256": { "orthosis-64ch-classical": "c0aeaabdfe530679a5c825447bdb78e5c7c9c9528d199aa35c7b4b1d7c31afa4", "orthosis-58ch-classical": "9ee443d5424ea7a40b86d3dfb3bb5d15a5e53e5ad7ad9e80c618abc0c213b4d7", "orthosis-eegpt-vs-58ch": "201ac453a0f3985b727aca340dc0f6f69261c0e8425db81b90c39b4b99101591", "bingo-cross-language": "df3dd3ce0acb218411f2d48c502f0e0795b1b58e85a97f3975a2de7d8ed43a0a", "handwriting-mask-aware": "caccae63e5cacfed7199c13f977c1740b36634a427c96342d5871ecade10ddbe", "refed-stored-clip": "f9ecab0bf86303f1ed5e437e15519540898057d5b5e75296081fc2cba084b613" } } }