{ "protocol": { "lockedAtUTC": "2026-10-13T17:54:38.274701+00:00", "scope": "Research-only M3 calibration on the public MFASS cohort with independently published, provenance-tracked SpliceAI S0 and Pangolin P0 predictions. Not a new independent study, not fresh neural inference, not clinical validation.", "endpoint": "MFASS mutant strong_lof: v2_dpsi <= -0.50; author QC and reference inclusion >=0.5. Do not relabel ClinVar pathogenicity as this assay endpoint.", "inputs": "Single unmasked max SpliceAI delta, distance 50, GENCODE44 canonical annotation; Pangolin same-annotation P0 baseline. No measured inclusion or assay-derived predictors.", "eligibility": "All author-QC mutants with defined strong_lof and matching public S0/P0 score IDs; missing scores excluded with reasons, never imputed as zero. Exclude any chromosome/position matching the 137 ClinVar calibration variants, and all their gene symbols. Verify ref/alt and reporter pairs.", "split": "Connected components over gene, exon and identical genomic variant identities. Deterministic SHA256 m3-mfass-v1|component modulo100: <30 test, otherwise development. No label balancing, retries or reseeding. Five GroupKFold development splits; untouched local test labels used once after candidate/checkpoint lock.", "candidate": "M3 MFASS v1: logistic regression on cubic spline of clipped-logit SpliceAI score. n_knots [3,5,7], C [.01,.1,1,10,100], 15 configurations selected by development grouped-CV Brier.", "baselines": [ "raw SpliceAI", "legacy M3 maxDelta calibrator (transfer diagnostic; input/endpoint differ)", "development prevalence", "Platt raw-score logistic", "Platt logit-score logistic", "beta calibration", "isotonic regression", "ExtraTrees calibration", "Pangolin Platt", "Pangolin isotonic" ], "baselineTuning": "Logistic C [.01,.1,1,10,100], ExtraTrees 200 trees min_samples_leaf [20,50,100], max_depth [3,None], random_state 20261013. Same grouped folds and Brier selection. Isotonic fixed. No test-driven selection.", "primaryMetric": "Brier score (lower better)", "secondaryMetrics": [ "log loss, predictions clipped to [1e-6,1-1e-6]", "ECE with fixed ten bins including probability 1", "AUROC", "average precision" ], "success": "Positive paired Brier improvement versus every baseline with Bonferroni-adjusted 99.5% cluster-bootstrap intervals excluding zero; 5000 whole-component resamples. Report each comparison and all failures. Secondary metrics cannot replace a failed primary outcome.", "limitations": [ "One previously published reporter study, subset determined by published score availability; no independent study claim.", "Base SpliceAI/Pangolin training exposure to these genes/exons is not ruled out; only local calibrator separation can be audited.", "Published score provenance/hashes can be checked; neural computations are not reproduced in this run.", "Current M3 sequence-window deltas differ from annotation-based published variant scores, so research calibration is not a drop-in serving replacement.", "Experimental exon-disruption labels do not mean clinical pathogenicity." ], "sourceHashes": { "MFASS-tree.json": "713c87fc379ccaf9a9ee421f2f1bbe1eaf9814cf70910823f9188353e6367715", "rewire-provenance.json": "6304e9625e793f24cccdc2669d092e6cdcc7a508a7f789967e726816fffde062", "spliceconsensus-tree.json": "d892c1066eecf078896c4da1f14c05969acf53ec7eb5fad5a4efe33a655c7a3e", "S0-predictions.tsv": "fa30a4dc6a05e6e762fbac9411e486869aec5b248cab80146aeab7a42c27320d", "mfass-annotation.tsv": "71a857fe647c4e68acbb41ca61e959c47e1176de89b1442bd6ca1772aa60d5a1", "spliceconsensus-score_one_tool.py": "2e627c70c5e73aeb7c3227fffdf69cdcac9cf730d89114aa79d014efdd4f8308", "P0-unscored.tsv": "f8f274522c28f428e97bf84458294e235979466b7c9433746a6ad1adef37fe5a", "rewire-manifest-v1.json": "3fce70dd0bd96cb92d9d0e41287f31e62ddc4ecb8d188615093a873e952433e8", "rewire-tree.json": "ace69a582a3f61ff62f402f592a8a5cdcafb96be497784faca131880206dc725", "mfass-source.tsv": "a637ca0e307e66ff48811ec7efa22b9ce453bc7883b04f0cacb867f7283132d8", "S0-unscored.tsv": "cd22dae59572ae7bfeee7d5843f487e093ba5706235cb149ee5fd3704c83efc7", "rewire-build_dataset.py": "44af1b37a5e5958a3f12d307bf54f1f38a1dcb783a4a5ec5b6b6fc4714094a24", "MFASS-snv_data_clean.R": "a2992699d516ce18ff23ecdcd7dbe40819cacee9b9dd268bd39f10b3a0c3433c", "rewire-split-v2.manifest.json": "3a6e0754cab398f1bfc7ba6080067f065c6b70ab8af53cd27e016d11b0b8f618", "P0-predictions.tsv": "fbd8861a37c2d041038cfdfb0dd4e9d1f572d72b0eb7e41406739ef740eaf4a1" } }, "qualification": { "status": "Qualified for research-only calibration reanalysis; not independent base-model validation", "primaryMFASSRows": 27733, "missingGeneMetadataRows": 77, "publishedScoreSubset": 8324, "scores": { "S0": { "available": 8297, "unscored": 27, "reasons": { "no annotated gene overlapping the variant": 4, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x134169750>": 3, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x1336c4210>": 3, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x134180f50>": 2, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x1332b4650>": 2, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x13381c290>": 2, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x133401f90>": 2, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x13368ead0>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x132cb1190>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x133974050>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x13339ab50>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x133932d90>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x133679790>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x1336e28d0>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x133496950>": 1, "skipped by SpliceAI: Skipping record (ref issue): <__main__._Record object at 0x1336c6f50>": 1 } }, "P0": { "available": 8297, "unscored": 27, "reasons": { "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 1974] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4225] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 7480] WARNING, skipping variant: Variant not contained in a gene body. Do GTF/FASTA chromosome names match?": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 7479] WARNING, skipping variant: Variant not contained in a gene body. Do GTF/FASTA chromosome names match?": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 7478] WARNING, skipping variant: Variant not contained in a gene body. Do GTF/FASTA chromosome names match?": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4233] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4232] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4231] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4230] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4229] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4228] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4227] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4226] WARNING, skipping variant: Mismatch between FASTA (ref base: T) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4224] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 1975] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4223] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4222] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4221] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4220] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4219] WARNING, skipping variant: Mismatch between FASTA (ref base: T) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4218] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4217] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4216] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4215] WARNING, skipping variant: Mismatch between FASTA (ref base: C) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4214] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 4213] WARNING, skipping variant: Mismatch between FASTA (ref base: G) and variant file (ref base: ).": 1, "skipped by Pangolin (no gene, ref mismatch or unsupported): [Line 7481] WARNING, skipping variant: Variant not contained in a gene body. Do GTF/FASTA chromosome names match?": 1 } } }, "localClinVarRowsAudited": 137, "exactVariantOverlaps": 0, "positionOverlaps": 0, "geneOverlapRows": 20, "geneOverlapSymbols": [ "DMD" ], "eligibleRows": 8262, "uniqueVariants": 8262, "groups": 460, "partitions": { "development": { "rows": 6128, "groups": 327 }, "test": { "rows": 2134, "groups": 133 } }, "reporterAlleleChecksPassed": 27733, "baseModelTrainingOverlap": "Not ruled out: local site calibration uses GENCODE chr22; base networks trained on genomic splicing. Gene/variant overlap audit covers the local 137-SNV delta calibrator, not all upstream training.", "scoreInputContract": "Published GENCODE44 canonical unmasked delta within 50nt; differs from deployed M3 sequence-window contract.", "sources": { "assay": "https://github.com/KosuriLab/MFASS", "scores": "https://github.com/rewire-bio/rewire-benchmarks/tree/779eeb5c88712442fbc9682c3d417cb7a7427fe3/benchmarks/mfass/results/matched-annotation-v1" }, "hashes": { "qualified-inputs.csv": "08557265d924fd384e97eaa7ccf22b77122777e7924ff1a85411c721d6e6b7bf", "outcomes.csv": "1ad6ce084b69d601b2e4221efec95a57cfc3c193715092b2e35e3e26462ef013", "delta_calibration_data_clinvar.parquet": "448f251f86a0974098da86b822c1ac76c71464d589c290a5e68a3944d4891371", "delta_calibration_models.pkl": "205b8bfc7d3830194c27da9e8ddd18aefaa0643f1da446135656a7fcc39be775" } }, "testRows": 2134, "testGroups": 133, "testPositives": 79, "metrics": { "M3_spline": { "brier": 0.030367831279679833, "logLoss": 0.1270667626511454, "ece": 0.00457408531268662, "auroc": 0.8000153993039515, "averagePrecision": 0.2797628994986362 }, "Platt_raw": { "brier": 0.031039340608679595, "logLoss": 0.13488942520493502, "ece": 0.0025397158823611495, "auroc": 0.7999722812528873, "averagePrecision": 0.26561766482101135 }, "Platt_logit": { "brier": 0.03211506258895258, "logLoss": 0.13497854737144885, "ece": 0.008723886155095176, "auroc": 0.7999722812528873, "averagePrecision": 0.26561766482101135 }, "Beta": { "brier": 0.03135639179969099, "logLoss": 0.1336226310996804, "ece": 0.004445512474646056, "auroc": 0.7999722812528873, "averagePrecision": 0.26561766482101135 }, "Isotonic": { "brier": 0.030754293104725442, "logLoss": 0.12811358173944706, "ece": 0.007496964089794795, "auroc": 0.7987064584680773, "averagePrecision": 0.24834505413756702 }, "ExtraTrees": { "brier": 0.030765060337343077, "logLoss": 0.13126163790617068, "ece": 0.002908029628156563, "auroc": 0.8001077951276603, "averagePrecision": 0.2691500489815351 }, "Pangolin_Platt": { "brier": 0.028885496887032267, "logLoss": 0.12151200275632267, "ece": 0.005138753550415727, "auroc": 0.871012350241769, "averagePrecision": 0.35588286234590827 }, "Pangolin_Isotonic": { "brier": 0.02913138414908382, "logLoss": 0.11522440155459082, "ece": 0.009047876979946384, "auroc": 0.8690504789183527, "averagePrecision": 0.3357549052918668 }, "Legacy_M3": { "brier": 0.07240381282091873, "logLoss": 0.3048588769838911, "ece": 0.20127729615144113, "auroc": 0.7999722812528873, "averagePrecision": 0.26561766482101135 }, "Raw_SpliceAI": { "brier": 0.031858575445173384, "logLoss": 0.18868187378726245, "ece": 0.01951265229615745, "auroc": 0.7999722812528873, "averagePrecision": 0.26561766482101135 }, "Prevalence": { "brier": 0.03565022957238722, "logLoss": 0.1583678756715688, "ece": 0.0010025118619099993, "auroc": 0.5, "averagePrecision": 0.037019681349578254 } }, "comparisons": [ { "baseline": "Platt_raw", "baselineMinusM3Brier": 0.000671509328999757, "adjustedCI": [ -0.0002933804834880031, 0.0016945814589976467 ], "superioritySupported": false }, { "baseline": "Platt_logit", "baselineMinusM3Brier": 0.001747231309272753, "adjustedCI": [ -0.00048151350275583006, 0.00439100222976843 ], "superioritySupported": false }, { "baseline": "Beta", "baselineMinusM3Brier": 0.0009885605200111573, "adjustedCI": [ -0.0007145466182724771, 0.0028977201189878533 ], "superioritySupported": false }, { "baseline": "Isotonic", "baselineMinusM3Brier": 0.0003864618250456058, "adjustedCI": [ -0.0008181056895784593, 0.0015720852471509514 ], "superioritySupported": false }, { "baseline": "ExtraTrees", "baselineMinusM3Brier": 0.0003972290576632361, "adjustedCI": [ -0.00045908964920883186, 0.0013697015358451 ], "superioritySupported": false }, { "baseline": "Pangolin_Platt", "baselineMinusM3Brier": -0.0014823343926475675, "adjustedCI": [ -0.0043339523243703425, 0.0002928702114735968 ], "superioritySupported": false }, { "baseline": "Pangolin_Isotonic", "baselineMinusM3Brier": -0.0012364471305960159, "adjustedCI": [ -0.005054179093496308, 0.0011964216488351512 ], "superioritySupported": false }, { "baseline": "Legacy_M3", "baselineMinusM3Brier": 0.0420359815412389, "adjustedCI": [ 0.03518740344318099, 0.047394939877516355 ], "superioritySupported": true }, { "baseline": "Raw_SpliceAI", "baselineMinusM3Brier": 0.0014907441654935457, "adjustedCI": [ -0.0005598332006355965, 0.0039003146080320646 ], "superioritySupported": false }, { "baseline": "Prevalence", "baselineMinusM3Brier": 0.005282398292707377, "adjustedCI": [ 0.00038173292105418764, 0.01142000944128719 ], "superioritySupported": true } ], "allBaselineSuperiority": false, "selected": { "M3_spline": { "knots": 5, "C": 10 }, "Platt_raw": { "C": 1 }, "Platt_logit": { "C": 100 }, "Beta": { "C": 100 }, "Isotonic": {}, "ExtraTrees": { "leaf": 20, "depth": 3 }, "Pangolin_Platt": { "C": 100 }, "Pangolin_Isotonic": {} }, "checkpoint": "backend/models/m3-mfass-calibration-v1/calibrators.joblib", "checkpointSHA256": "b4b15cf61ba3201035e54d2bb327ba36bf606e987aa411c8f5e9d623d313fa26", "freshIndependentStudy": false, "fits": 215, "calibrationBins": [ { "model": "M3_spline", "bin": 0, "count": 2022, "predicted": 0.02303521070568672, "observed": 0.025222551928783383 }, { "model": "M3_spline", "bin": 1, "count": 70, "predicted": 0.13301729347033367, "observed": 0.12857142857142856 }, { "model": "M3_spline", "bin": 2, "count": 18, "predicted": 0.25162116389888606, "observed": 0.2777777777777778 }, { "model": "M3_spline", "bin": 3, "count": 4, "predicted": 0.33618484168977647, "observed": 0.25 }, { "model": "M3_spline", "bin": 4, "count": 6, "predicted": 0.43566591183668146, "observed": 0.3333333333333333 }, { "model": "M3_spline", "bin": 5, "count": 2, "predicted": 0.5580876676417341, "observed": 0.5 }, { "model": "M3_spline", "bin": 6, "count": 6, "predicted": 0.6528844780995543, "observed": 1.0 }, { "model": "M3_spline", "bin": 7, "count": 4, "predicted": 0.7516403870929647, "observed": 0.5 }, { "model": "M3_spline", "bin": 8, "count": 2, "predicted": 0.8039501707044044, "observed": 1.0 }, { "model": "Raw_SpliceAI", "bin": 0, "count": 2002, "predicted": 0.010764235764235765, "observed": 0.024475524475524476 }, { "model": "Raw_SpliceAI", "bin": 1, "count": 64, "predicted": 0.13656249999999998, "observed": 0.140625 }, { "model": "Raw_SpliceAI", "bin": 2, "count": 24, "predicted": 0.23416666666666666, "observed": 0.041666666666666664 }, { "model": "Raw_SpliceAI", "bin": 3, "count": 7, "predicted": 0.34714285714285714, "observed": 0.2857142857142857 }, { "model": "Raw_SpliceAI", "bin": 4, "count": 8, "predicted": 0.44125000000000003, "observed": 0.375 }, { "model": "Raw_SpliceAI", "bin": 5, "count": 8, "predicted": 0.54125, "observed": 0.25 }, { "model": "Raw_SpliceAI", "bin": 6, "count": 1, "predicted": 0.69, "observed": 0.0 }, { "model": "Raw_SpliceAI", "bin": 7, "count": 3, "predicted": 0.7433333333333333, "observed": 0.3333333333333333 }, { "model": "Raw_SpliceAI", "bin": 8, "count": 3, "predicted": 0.86, "observed": 0.3333333333333333 }, { "model": "Raw_SpliceAI", "bin": 9, "count": 14, "predicted": 0.9657142857142856, "observed": 0.7857142857142857 }, { "model": "Platt_logit", "bin": 0, "count": 2027, "predicted": 0.02848524410857918, "observed": 0.025160335471139616 }, { "model": "Platt_logit", "bin": 1, "count": 88, "predicted": 0.1234132248500011, "observed": 0.17045454545454544 }, { "model": "Platt_logit", "bin": 2, "count": 10, "predicted": 0.25429365173159624, "observed": 0.7 }, { "model": "Platt_logit", "bin": 3, "count": 5, "predicted": 0.34139048706416425, "observed": 0.6 }, { "model": "Platt_logit", "bin": 4, "count": 2, "predicted": 0.4007827713127126, "observed": 1.0 }, { "model": "Platt_logit", "bin": 8, "count": 2, "predicted": 0.8945007644727092, "observed": 0.5 }, { "model": "Pangolin_Platt", "bin": 0, "count": 2024, "predicted": 0.020062057977644734, "observed": 0.020256916996047432 }, { "model": "Pangolin_Platt", "bin": 1, "count": 62, "predicted": 0.14686758875459371, "observed": 0.24193548387096775 }, { "model": "Pangolin_Platt", "bin": 2, "count": 11, "predicted": 0.22941096613801254, "observed": 0.36363636363636365 }, { "model": "Pangolin_Platt", "bin": 3, "count": 8, "predicted": 0.3341175635149851, "observed": 0.25 }, { "model": "Pangolin_Platt", "bin": 4, "count": 8, "predicted": 0.4346886390589447, "observed": 0.375 }, { "model": "Pangolin_Platt", "bin": 5, "count": 10, "predicted": 0.5556013868808256, "observed": 0.5 }, { "model": "Pangolin_Platt", "bin": 6, "count": 3, "predicted": 0.6425814251943195, "observed": 1.0 }, { "model": "Pangolin_Platt", "bin": 7, "count": 7, "predicted": 0.7493742338226494, "observed": 0.7142857142857143 }, { "model": "Pangolin_Platt", "bin": 8, "count": 1, "predicted": 0.823322296932234, "observed": 1.0 } ] }