{ "date": "2026-10-13", "version": "m5-pathway-response-v5", "evaluation": "Retrospective three-study development; no fresh independent test", "freshHoldout": false, "superiorityEstablished": false, "folds": [ { "heldStudy": "Aliouat2019", "trainStudies": [ "NMDRHT", "SMG1_K562" ], "trainRows": 8348, "evaluatedGenes": 5268, "overlappingGenes": 4742, "metrics": { "M5_v5": { "spearman": -0.3122084098213585, "mae": 0.4548342500886957, "r2": -0.5956244139223363 }, "ExtraTrees": { "spearman": -0.2912279323736916, "mae": 0.466557613971459, "r2": -0.6540069907056798 }, "ridge": { "spearman": -0.32077460453088036, "mae": 0.39385338734761605, "r2": -0.2757348382472318 }, "training_median": { "spearman": null, "mae": 0.3317865568961469, "r2": -0.01804351556927819 } }, "comparisons": [ { "baseline": "ExtraTrees", "deltaSpearman": -0.0209804774476669, "adjustedPairedCi": [ -0.03453440794289578, -0.006302742679042417 ], "positiveDevelopmentInterval": false }, { "baseline": "ridge", "deltaSpearman": 0.008566194709521857, "adjustedPairedCi": [ -0.011163773179011733, 0.02925182086855254 ], "positiveDevelopmentInterval": false } ], "selections": { "M5_v5": { "configurationId": 4, "configuration": { "max_depth": 4, "min_child_weight": 10, "reg_lambda": 10 }, "innerMeanSpearman": 0.41133217567114333 }, "ExtraTrees": { "configurationId": 5, "configuration": { "min_samples_leaf": 10, "max_features": 0.5, "max_depth": null }, "innerMeanSpearman": 0.4382299706000121 }, "ridge": { "configurationId": 6, "configuration": { "alpha": 10000.0 }, "innerMeanSpearman": 0.35176907216973274 } } }, { "heldStudy": "NMDRHT", "trainStudies": [ "Aliouat2019", "SMG1_K562" ], "trainRows": 8730, "evaluatedGenes": 4886, "overlappingGenes": 4745, "metrics": { "M5_v5": { "spearman": -0.2701988289045821, "mae": 0.5059331659600207, "r2": -0.27514061116864696 }, "ExtraTrees": { "spearman": -0.3983103659230929, "mae": 0.5119341028358948, "r2": -0.29713269012890087 }, "ridge": { "spearman": -0.1865460983538332, "mae": 0.46566881505127367, "r2": -0.12074880659544562 }, "training_median": { "spearman": null, "mae": 0.44597790895356826, "r2": -0.04712286841346813 } }, "comparisons": [ { "baseline": "ExtraTrees", "deltaSpearman": 0.1281115370185108, "adjustedPairedCi": [ 0.11090596437752516, 0.1480949582948676 ], "positiveDevelopmentInterval": true }, { "baseline": "ridge", "deltaSpearman": -0.08365273055074887, "adjustedPairedCi": [ -0.11458961022273116, -0.050090883865364366 ], "positiveDevelopmentInterval": false } ], "selections": { "M5_v5": { "configurationId": 9, "configuration": { "max_depth": 6, "min_child_weight": 10, "reg_lambda": 100 }, "innerMeanSpearman": 0.3559536112297865 }, "ExtraTrees": { "configurationId": 11, "configuration": { "min_samples_leaf": 30, "max_features": 1.0, "max_depth": null }, "innerMeanSpearman": 0.4062482019182362 }, "ridge": { "configurationId": 5, "configuration": { "alpha": 1000.0 }, "innerMeanSpearman": 0.23910561885910236 } } }, { "heldStudy": "SMG1_K562", "trainStudies": [ "Aliouat2019", "NMDRHT" ], "trainRows": 10154, "evaluatedGenes": 3462, "overlappingGenes": 3397, "metrics": { "M5_v5": { "spearman": 0.31214948526182323, "mae": 0.25698601349106986, "r2": 0.0553026543233065 }, "ExtraTrees": { "spearman": 0.289743415461677, "mae": 0.269155002055328, "r2": 0.02172457176285192 }, "ridge": { "spearman": 0.25773110265577776, "mae": 0.26332067966401895, "r2": 0.03204837517422776 }, "training_median": { "spearman": null, "mae": 0.2706529413686233, "r2": -0.0007745234327523054 } }, "comparisons": [ { "baseline": "ExtraTrees", "deltaSpearman": 0.022406069800146255, "adjustedPairedCi": [ -0.009556729483243697, 0.05635928429091126 ], "positiveDevelopmentInterval": false }, { "baseline": "ridge", "deltaSpearman": 0.05441838260604548, "adjustedPairedCi": [ 0.02047159835441282, 0.09156337795945238 ], "positiveDevelopmentInterval": true } ], "selections": { "M5_v5": { "configurationId": 7, "configuration": { "max_depth": 4, "min_child_weight": 50, "reg_lambda": 100 }, "innerMeanSpearman": 0.10356162895782793 }, "ExtraTrees": { "configurationId": 7, "configuration": { "min_samples_leaf": 10, "max_features": 1.0, "max_depth": null }, "innerMeanSpearman": 0.12013902964620052 }, "ridge": { "configurationId": 3, "configuration": { "alpha": 10.0 }, "innerMeanSpearman": 0.10364523071674779 } } } ], "finalResearchFit": { "M5_v5": { "configurationId": 9, "configuration": { "max_depth": 6, "min_child_weight": 10, "reg_lambda": 100 }, "innerMeanSpearman": 0.28379275411687166, "artifactSha256": "78a597d9486b763ddef7d908d50a461d87530659f818852dc9f1158b6079ccbc", "artifactBytes": 604197 }, "ExtraTrees": { "configurationId": 7, "configuration": { "min_samples_leaf": 10, "max_features": 1.0, "max_depth": null }, "innerMeanSpearman": 0.32001479806190963, "artifactSha256": "a921bed210f2eb379cd138c447398889898cee256971a77dee92d6a0ed2908eb", "artifactBytes": 12874210 }, "ridge": { "configurationId": 3, "configuration": { "alpha": 10.0 }, "innerMeanSpearman": 0.22791050349522757, "artifactSha256": "03273f2d1e9e39e2adf922d5963557c4dab27fbeaf0c826323ef82a9e988ad80", "artifactBytes": 7892 } }, "featureCount": 157, "rowsByStudy": { "Aliouat2019": 5268, "NMDRHT": 4886, "SMG1_K562": 3462 }, "trainingRows": 13616, "trainingGenes": 5591, "protocol": { "date": "2026-10-13", "lockedAtUTC": "2026-10-13T14:27:10.403563+00:00", "status": "Retrospective development only; all three studies previously examined", "sources": { "docs/validation/2026-10-13-nmd-response-models/m5-frozen-inputs.parquet": "55d4c68114559046b464fc6834d00d3c985f6728c97957ccc325c9affc071e56", "docs/validation/2026-10-13-nmd-response-models/m4-frozen-inputs.parquet": "b773969e5f13b1fd0ab134f185c9a75cd7311dc531fde0556d6a0a1c9f4546f7", "docs/validation/2026-10-13-response-reconciliation-v4/m5-aligned.parquet": "a823c00856fc0cd0583171168c5624cdafc69fa0bcba5e8963bb3a76861fe1b9", "docs/validation/2026-10-13-m5-smg1-transfer-v8/evaluated-predictions.csv": "6289fe80ab162326c7f977740286a42d232d98f5d2ab8430592281ff3298759a" }, "features": [ "n_uorfs_aug", "n_uorfs_near", "max_uorf_len", "dist_cap_to_first_uorf", "first_uorf_is_aug", "uorf_overlaps_main_orf", "min_dist_stop_to_cds", "kozak_max", "kozak_first", "utr_len", "utr_gc", "utr_dg", "kozak_x_dist", "n_uorfs_x_utrlen", "gc_x_dg", "main_kozak", "cds_len", "utr3_len", "dg_per_nt", "utr_kmer_A", "utr_kmer_C", "utr_kmer_G", "utr_kmer_T", "utr_kmer_AA", "utr_kmer_AC", "utr_kmer_AG", "utr_kmer_AT", "utr_kmer_CA", "utr_kmer_CC", "utr_kmer_CG", "utr_kmer_CT", "utr_kmer_GA", "utr_kmer_GC", "utr_kmer_GG", "utr_kmer_GT", "utr_kmer_TA", "utr_kmer_TC", "utr_kmer_TG", "utr_kmer_TT", "utr_kmer_AAA", "utr_kmer_AAC", "utr_kmer_AAG", "utr_kmer_AAT", "utr_kmer_ACA", "utr_kmer_ACC", "utr_kmer_ACG", "utr_kmer_ACT", "utr_kmer_AGA", "utr_kmer_AGC", "utr_kmer_AGG", "utr_kmer_AGT", "utr_kmer_ATA", "utr_kmer_ATC", "utr_kmer_ATG", "utr_kmer_ATT", "utr_kmer_CAA", "utr_kmer_CAC", "utr_kmer_CAG", "utr_kmer_CAT", "utr_kmer_CCA", "utr_kmer_CCC", "utr_kmer_CCG", "utr_kmer_CCT", "utr_kmer_CGA", "utr_kmer_CGC", "utr_kmer_CGG", "utr_kmer_CGT", "utr_kmer_CTA", "utr_kmer_CTC", "utr_kmer_CTG", "utr_kmer_CTT", "utr_kmer_GAA", "utr_kmer_GAC", "utr_kmer_GAG", "utr_kmer_GAT", "utr_kmer_GCA", "utr_kmer_GCC", "utr_kmer_GCG", "utr_kmer_GCT", "utr_kmer_GGA", "utr_kmer_GGC", "utr_kmer_GGG", "utr_kmer_GGT", "utr_kmer_GTA", "utr_kmer_GTC", "utr_kmer_GTG", "utr_kmer_GTT", "utr_kmer_TAA", "utr_kmer_TAC", "utr_kmer_TAG", "utr_kmer_TAT", "utr_kmer_TCA", "utr_kmer_TCC", "utr_kmer_TCG", "utr_kmer_TCT", "utr_kmer_TGA", "utr_kmer_TGC", "utr_kmer_TGG", "utr_kmer_TGT", "utr_kmer_TTA", "utr_kmer_TTC", "utr_kmer_TTG", "utr_kmer_TTT", "geometry_nexon_mean", "geometry_nexon_max", "geometry_nexon_std", "geometry_tx_len_mean", "geometry_tx_len_max", "geometry_tx_len_std", "geometry_cds_len_mean", "geometry_cds_len_max", "geometry_cds_len_std", "geometry_utr5_len_mean", "geometry_utr5_len_max", "geometry_utr5_len_std", "geometry_utr3_len_mean", "geometry_utr3_len_max", "geometry_utr3_len_std", "geometry_stop_to_lastEJ_mean", "geometry_stop_to_lastEJ_max", "geometry_stop_to_lastEJ_std", "geometry_downstream_junctions_mean", "geometry_downstream_junctions_max", "geometry_downstream_junctions_std", "geometry_stop_upstream_50nt_mean", "geometry_stop_upstream_50nt_max", "geometry_stop_upstream_50nt_std", "geometry_max_exon_len_mean", "geometry_max_exon_len_max", "geometry_max_exon_len_std", "geometry_mean_exon_len_mean", "geometry_mean_exon_len_max", "geometry_mean_exon_len_std", "geometry_last_exon_len_mean", "geometry_last_exon_len_max", "geometry_last_exon_len_std", "geometry_cds_fraction_mean", "geometry_cds_fraction_max", "geometry_cds_fraction_std", "geometry_GC_tx_mean", "geometry_GC_tx_max", "geometry_GC_tx_std", "geometry_GC_utr5_mean", "geometry_GC_utr5_max", "geometry_GC_utr5_std", "geometry_GC_cds_mean", "geometry_GC_cds_max", "geometry_GC_cds_std", "geometry_GC_utr3_mean", "geometry_GC_utr3_max", "geometry_GC_utr3_std", "geometry_utr3_mfe_nt_mean", "geometry_utr3_mfe_nt_max", "geometry_utr3_mfe_nt_std", "geometry_isoform_count", "geometry_available", "perturbation_smg1" ], "changes": [ "Add gene-level annotation geometry means/maxima/standard deviations and isoform count, derived without measured responses.", "Add explicit SMG1 perturbation indicator so pooled models distinguish endpoint contexts.", "Use three previously examined studies, equal total study weights, and 12 inner-validation configurations per model family." ], "evaluation": "Leave one study out for retrospective transfer diagnostics. Within remaining studies use deterministic gene-disjoint 80/20 inner split, keeping every occurrence of a gene on one side. Select by mean per-study inner Spearman; never select by outer study scores.", "geneSplitSeed": "m5-v5-development", "families": { "M5_v5": [ { "max_depth": 2, "min_child_weight": 10, "reg_lambda": 10 }, { "max_depth": 2, "min_child_weight": 10, "reg_lambda": 100 }, { "max_depth": 2, "min_child_weight": 50, "reg_lambda": 10 }, { "max_depth": 2, "min_child_weight": 50, "reg_lambda": 100 }, { "max_depth": 4, "min_child_weight": 10, "reg_lambda": 10 }, { "max_depth": 4, "min_child_weight": 10, "reg_lambda": 100 }, { "max_depth": 4, "min_child_weight": 50, "reg_lambda": 10 }, { "max_depth": 4, "min_child_weight": 50, "reg_lambda": 100 }, { "max_depth": 6, "min_child_weight": 10, "reg_lambda": 10 }, { "max_depth": 6, "min_child_weight": 10, "reg_lambda": 100 }, { "max_depth": 6, "min_child_weight": 50, "reg_lambda": 10 }, { "max_depth": 6, "min_child_weight": 50, "reg_lambda": 100 } ], "ExtraTrees": [ { "min_samples_leaf": 3, "max_features": 0.5, "max_depth": 12 }, { "min_samples_leaf": 3, "max_features": 0.5, "max_depth": null }, { "min_samples_leaf": 3, "max_features": 1.0, "max_depth": 12 }, { "min_samples_leaf": 3, "max_features": 1.0, "max_depth": null }, { "min_samples_leaf": 10, "max_features": 0.5, "max_depth": 12 }, { "min_samples_leaf": 10, "max_features": 0.5, "max_depth": null }, { "min_samples_leaf": 10, "max_features": 1.0, "max_depth": 12 }, { "min_samples_leaf": 10, "max_features": 1.0, "max_depth": null }, { "min_samples_leaf": 30, "max_features": 0.5, "max_depth": 12 }, { "min_samples_leaf": 30, "max_features": 0.5, "max_depth": null }, { "min_samples_leaf": 30, "max_features": 1.0, "max_depth": 12 }, { "min_samples_leaf": 30, "max_features": 1.0, "max_depth": null } ], "ridge": [ { "alpha": 0.01 }, { "alpha": 0.1 }, { "alpha": 1.0 }, { "alpha": 10.0 }, { "alpha": 100.0 }, { "alpha": 1000.0 }, { "alpha": 10000.0 }, { "alpha": 100000.0 }, { "alpha": 1000000.0 }, { "alpha": 10000000.0 }, { "alpha": 100000000.0 }, { "alpha": 1000000000.0 } ] }, "finalSelection": "Mean inner-validation Spearman across the three outer folds, then refit all development data with equal study weights.", "uncertainty": "1000 paired gene bootstrap resamples within each outer development study; Bonferroni adjustment for six candidate/baseline comparisons. Descriptive retrospective intervals, not confirmatory.", "freshIndependentTest": false, "endpoint": "UPF1-depletion or SMG1-inhibition gene-level TE change with explicit perturbation flag; research-only pathway-response model", "trainingRows": 13616, "trainingGenes": 5591, "rowsByStudy": { "Aliouat2019": 5268, "NMDRHT": 4886, "SMG1_K562": 3462 }, "limitations": [ "SMG1 is now development data and cannot be reused as a fresh final test.", "Only three source studies, two UPF1 and one SMG1; perturbation and cell/study context are confounded.", "A held-SMG1 fold trains on UPF1 only; its perturbation indicator is unseen in training.", "Gene overlap between outer studies is reported; study holdout does not imply unseen genes.", "Geometry comes from the prior NMDRHT annotation and inherits its discovery/response-availability ascertainment.", "No fresh independent study has yet qualified; no model is promoted or superiority guaranteed." ] }, "freshStudySearch": { "status": "No fresh qualified human study found; independent test not run. Search is not exhaustive.", "criteria": "Human, direct UPF1 perturbation (or separately labelled SMG1 transfer), control and treated RNA plus RPF, biological replication, compatible gene-level TE endpoint, accessible sample provenance, and no overlap with development experiments. Qualify before opening numerical outcomes.", "newlyReviewed": [ { "study": "Palbociclib / GSE297939", "url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE297939", "decision": "Not qualified: deposited RNA-seq and ATAC-seq are palbociclib versus DMSO. SMG1 appears in targeted RT-PCR validation, with no matched SMG1 RNA/RPF cohort." }, { "study": "Hidden stop codons / GSE303542", "url": "https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE303542", "decision": "Not qualified: human Ribo-seq compares eRF1 depletion with WT. UPF1 knockout is in fungal SLAM-seq; this is not a human UPF1 RNA/RPF contrast." }, { "study": "NMDRHT / E-MTAB-13837", "url": "https://www.ebi.ac.uk/biostudies/arrayexpress/studies/E-MTAB-13837", "decision": "Already used: a newly indexed deposit or publication date does not make this existing NMDRHT experiment independent." } ], "nextStep": "Obtain an unused study with matched RNA/RPF and sample metadata. Freeze these model hashes, cohort rules, primary metric and paired comparisons before testing once." } }