{ "protocol": { "lockedAtUTC": "2026-10-13T17:06:08.644563+00:00", "candidate": "M4 v6: absolute-error boosted trees; remove fixed ridge blend; equal total training weight per study. Existing 52 geometry features.", "selection": "12 configurations each family. Same gene-disjoint inner 80/20 split. Equal-study mean validation MAE. Outer leave-one-study-out diagnostics; all three datasets are development. Final configuration minimizes mean inner MAE across outer folds.", "sourceHashes": { "docs/validation/2026-10-13-response-reconciliation-v4/m4-aligned.parquet": "08527f163c57d28fc248a78eb7f9c262dafed686300727a4257df2e90d2ff68b", "docs/validation/2026-10-13-public-only-benchmarks-v8/evaluated-predictions.csv": "a199a705ce31e8bb27e0f4e9c9fe207dbbac9251990cf94041cefd846b55b357" }, "configurations": { "M4_v6": [ { "max_depth": 2, "min_child_weight": 20, "reg_lambda": 10 }, { "max_depth": 2, "min_child_weight": 20, "reg_lambda": 100 }, { "max_depth": 2, "min_child_weight": 80, "reg_lambda": 10 }, { "max_depth": 2, "min_child_weight": 80, "reg_lambda": 100 }, { "max_depth": 4, "min_child_weight": 20, "reg_lambda": 10 }, { "max_depth": 4, "min_child_weight": 20, "reg_lambda": 100 }, { "max_depth": 4, "min_child_weight": 80, "reg_lambda": 10 }, { "max_depth": 4, "min_child_weight": 80, "reg_lambda": 100 }, { "max_depth": 6, "min_child_weight": 20, "reg_lambda": 10 }, { "max_depth": 6, "min_child_weight": 20, "reg_lambda": 100 }, { "max_depth": 6, "min_child_weight": 80, "reg_lambda": 10 }, { "max_depth": 6, "min_child_weight": 80, "reg_lambda": 100 } ], "ExtraTrees": [ { "min_samples_leaf": 2, "max_features": 0.5, "max_depth": 18 }, { "min_samples_leaf": 2, "max_features": 0.5, "max_depth": null }, { "min_samples_leaf": 2, "max_features": 1.0, "max_depth": 18 }, { "min_samples_leaf": 2, "max_features": 1.0, "max_depth": null }, { "min_samples_leaf": 5, "max_features": 0.5, "max_depth": 18 }, { "min_samples_leaf": 5, "max_features": 0.5, "max_depth": null }, { "min_samples_leaf": 5, "max_features": 1.0, "max_depth": 18 }, { "min_samples_leaf": 5, "max_features": 1.0, "max_depth": null }, { "min_samples_leaf": 10, "max_features": 0.5, "max_depth": 18 }, { "min_samples_leaf": 10, "max_features": 0.5, "max_depth": null }, { "min_samples_leaf": 10, "max_features": 1.0, "max_depth": 18 }, { "min_samples_leaf": 10, "max_features": 1.0, "max_depth": null } ], "ridge": [ { "alpha": 0.01 }, { "alpha": 0.1 }, { "alpha": 1.0 }, { "alpha": 10.0 }, { "alpha": 100.0 }, { "alpha": 1000.0 }, { "alpha": 10000.0 }, { "alpha": 100000.0 }, { "alpha": 1000000.0 }, { "alpha": 10000000.0 }, { "alpha": 100000000.0 }, { "alpha": 1000000000.0 } ] }, "studies": [ "NMDRHT", "SMG1_HEK293", "Tani2012" ], "rows": 29703, "genes": 11277, "freshHoldout": false, "primaryMetric": "MAE, lower better", "limitations": [ "GSE289489 was already evaluated for v5 and is now explicitly development data.", "Study-balanced weighting is used by all families; validation scores equally weight studies.", "Outer folds have gene overlap. Inner splits separate genes across all training studies.", "Repeated development on known studies cannot establish independent superiority.", "Different UPF1/SMG1 perturbations, cells and decay assays remain; no assay offset learned from held-study outcomes.", "Gene bootstrap intervals condition on measured estimates and do not capture laboratory or replicate uncertainty." ] }, "folds": [ { "study": "NMDRHT", "genes": 10990, "overlappingGenes": 10813, "metrics": { "M4_v6": { "spearman": 0.5449911919932277, "mae": 0.5047684174371222, "r2": 0.09955716303896855 }, "ExtraTrees": { "spearman": 0.5959547409063429, "mae": 0.4768180392190265, "r2": 0.2129883714588905 }, "ridge": { "spearman": 0.4785093673131193, "mae": 0.5128689886365398, "r2": 0.07413050161577561 }, "training_median": { "spearman": null, "mae": 0.5675473432855375, "r2": -0.10189963958838 } }, "comparisons": [ { "baseline": "ExtraTrees", "baselineMinusM4MAE": -0.02795037821809568, "adjustedPairedCi": [ -0.03162399776883855, -0.024748848719041282 ] }, { "baseline": "ridge", "baselineMinusM4MAE": 0.008100571199417605, "adjustedPairedCi": [ 0.005454319911139905, 0.010613929626040469 ] } ], "selected": { "M4_v6": { "configurationId": 5, "innerMAE": 0.43449779872877725 }, "ExtraTrees": { "configurationId": 8, "innerMAE": 0.4414123405414055 }, "ridge": { "configurationId": 6, "innerMAE": 0.4458262141362694 } } }, { "study": "SMG1_HEK293", "genes": 11026, "overlappingGenes": 11026, "metrics": { "M4_v6": { "spearman": 0.47822452371907853, "mae": 0.3152805587954828, "r2": -0.02752815878337067 }, "ExtraTrees": { "spearman": 0.5799844713366191, "mae": 0.29401334552029457, "r2": 0.16520040797102054 }, "ridge": { "spearman": 0.41404510466266237, "mae": 0.32589989456892, "r2": -0.07730212523684488 }, "training_median": { "spearman": null, "mae": 0.30850374408515713, "r2": -0.0719493901606727 } }, "comparisons": [ { "baseline": "ExtraTrees", "baselineMinusM4MAE": -0.02126721327518823, "adjustedPairedCi": [ -0.02517303705846681, -0.017194030190013403 ] }, { "baseline": "ridge", "baselineMinusM4MAE": 0.01061933577343725, "adjustedPairedCi": [ 0.007098758539254039, 0.014176102420650441 ] } ], "selected": { "M4_v6": { "configurationId": 5, "innerMAE": 0.5324814945544156 }, "ExtraTrees": { "configurationId": 8, "innerMAE": 0.5377755894494545 }, "ridge": { "configurationId": 5, "innerMAE": 0.5464300183334476 } } }, { "study": "Tani2012", "genes": 7687, "overlappingGenes": 7643, "metrics": { "M4_v6": { "spearman": 0.4535900512104125, "mae": 0.6162641094906888, "r2": 0.03260251909473166 }, "ExtraTrees": { "spearman": 0.4963067837862868, "mae": 0.6235053269034547, "r2": 0.03609311693973649 }, "ridge": { "spearman": 0.3939429105211656, "mae": 0.6648507723026432, "r2": -0.05557351723207127 }, "training_median": { "spearman": null, "mae": 0.6450195364304797, "r2": -0.07579048338070993 } }, "comparisons": [ { "baseline": "ExtraTrees", "baselineMinusM4MAE": 0.0072412174127659545, "adjustedPairedCi": [ 0.003189029276690932, 0.011310650838710124 ] }, { "baseline": "ridge", "baselineMinusM4MAE": 0.04858666281195438, "adjustedPairedCi": [ 0.044355605970871126, 0.052842166288156925 ] } ], "selected": { "M4_v6": { "configurationId": 10, "innerMAE": 0.3796205621736407 }, "ExtraTrees": { "configurationId": 9, "innerMAE": 0.3856745007661165 }, "ridge": { "configurationId": 4, "innerMAE": 0.39624963744023656 } } } ], "finalModels": { "M4_v6": { "sha256": "a90b9b696265038df0dc6f89df0fcb962b22928d517928a33be6e801778231ea", "configurationId": 5 }, "ExtraTrees": { "sha256": "34ad4c06f4cb16d6d3a8fd703d2899a078cef83318e6b3f13c263caa9fd5be89", "configurationId": 8 }, "ridge": { "sha256": "edf4a8924c0622e0a1e77b5e11da72882cbce0a22a2f32ff457bec37a5f3e22e", "configurationId": 4 } }, "independentSuperiority": false, "freshTestStatus": "Pending qualification of another public study.", "search": [ { "name": "2015 BRIC study", "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC4359779/", "decision": "UPF1 RNA/BRIC measurements explicitly reuse DRA000591. New EXOSC5/STAU1 perturbations are not an independent NMD response endpoint." }, { "name": "2026 EZbakR final publication", "url": "https://link.springer.com/article/10.1186/s13059-026-04204-w", "decision": "GSE289489 is the evaluated SMG1 cohort, now development. GSE291379 measures cleavage sites, not decay-rate responses or RPF." }, { "name": "GSE207924", "url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE207924", "decision": "Subcellular RNA kinetics in K562 and mouse cells; no matched NMD perturbation response established." }, { "name": "GSE143662", "url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE143662", "decision": "Human DCP2 knockout stability assay; different perturbation from the requested NMD response test." }, { "name": "TDP43 endogenous stability study", "url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC11848802/", "decision": "Selected SRSF2/TARDBP isoform qPCR decay measurements. Does not qualify as a broad gene-level M4 benchmark." }, { "name": "MDC human UPF1 Ribo-seq deposit", "url": "https://edoc.mdc-berlin.de/id/eprint/25705/", "decision": "External deposit resolves to E-MTAB-13837, already-used NMDRHT. A separate catalogue entry is not a fresh study." }, { "name": "GSE308938", "url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE308938", "decision": "Public September 2026, matched UPF1 RNA/RPF in yeast. Wrong species for the requested human M5 holdout." }, { "name": "GSE134059", "url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE134059", "decision": "Human UPF1 isoform RNA-seq and RIP-seq. RIP is an RNA-binding assay, not RPF; no matched M5 endpoint." } ], "searchScope": "Targeted primary-study, supplementary lineage and public repository review, not an exhaustive proof that no qualifying dataset exists. No author contact.", "decision": "M4 v6 beats ExtraTrees on Tani MAE only, and beats ridge on all three development folds. ExtraTrees wins MAE on NMDRHT and SMG1 and rank correlation on all three. No independent superiority established. Research checkpoints only; no serving promotion. Another fresh human M4 study and a fresh matched human M5 RNA/RPF study remain unqualified." }