diff --git a/rust/crates/du-db/src/ystr.rs b/rust/crates/du-db/src/ystr.rs index 8aa46a1..d7dc29d 100644 --- a/rust/crates/du-db/src/ystr.rs +++ b/rust/crates/du-db/src/ystr.rs @@ -1110,6 +1110,12 @@ pub struct MarkerStat { pub null_alleles: i64, /// Partial-repeat / footnoted values (`{type:complex}`), counted but unscored. pub complex_count: i64, + /// Observations reported in the *minority* shape for this marker — a single + /// repeat count where the marker is normally a copy vector, or vice versa. + /// DYS413 has 866 multi-copy calls and one bare `23`. The range is taken from + /// the majority shape, so these are excluded from it and surfaced here rather + /// than silently overriding it. + pub mixed_shape_count: i64, pub motif: Option, pub period: Option, pub coordinates: Option, @@ -1173,18 +1179,24 @@ const MARKER_STATS_AGGREGATION: &str = "WITH obs AS ( \ count(DISTINCT COALESCE(repeats::text, array_to_string(copies, '-'), raw)) \ AS distinct_values, \ count(*) FILTER (WHERE is_null_allele) AS null_alleles, \ - count(*) FILTER (WHERE vtype = 'complex') AS complex_count \ + count(*) FILTER (WHERE vtype = 'complex') AS complex_count, \ + count(*) FILTER (WHERE vtype = 'simple') AS simple_obs, \ + count(*) FILTER (WHERE vtype = 'multiCopy') AS multi_obs \ FROM scored \ GROUP BY marker \ ) \ SELECT a.marker AS marker_name, \ (a.observed_multi OR COALESCE(m.multi_copy, false)) AS multi_copy, \ a.observations, a.samples, \ - a.min_value, a.modal_value, a.max_value, \ - a.min_copies AS min_combination, \ - array_to_string(a.modal_copies, '-') AS modal_combination, \ - a.max_copies AS max_combination, \ + CASE WHEN a.multi_obs <= a.simple_obs THEN a.min_value END AS min_value, \ + CASE WHEN a.multi_obs <= a.simple_obs THEN a.modal_value END AS modal_value, \ + CASE WHEN a.multi_obs <= a.simple_obs THEN a.max_value END AS max_value, \ + CASE WHEN a.multi_obs > a.simple_obs THEN a.min_copies END AS min_combination, \ + CASE WHEN a.multi_obs > a.simple_obs \ + THEN array_to_string(a.modal_copies, '-') END AS modal_combination, \ + CASE WHEN a.multi_obs > a.simple_obs THEN a.max_copies END AS max_combination, \ a.distinct_values, a.null_alleles, a.complex_count, \ + LEAST(a.simple_obs, a.multi_obs) AS mixed_shape_count, \ m.motif, m.period, NULLIF(m.coordinates, '{}'::jsonb) AS coordinates, \ r.mutation_rate::float8 AS mutation_rate, \ r.mutation_rate_lower::float8 AS rate_ci_low, \ @@ -1211,7 +1223,8 @@ const MARKER_STATS_AGGREGATION: &str = "WITH obs AS ( \ /// Column list shared by the refresh insert and the read path, in table order. const MARKER_STAT_COLUMNS: &str = "marker_name, multi_copy, observations, samples, \ min_value, modal_value, max_value, min_combination, modal_combination, max_combination, \ - distinct_values, null_alleles, complex_count, motif, period, coordinates, \ + distinct_values, null_alleles, complex_count, mixed_shape_count, \ + motif, period, coordinates, \ mutation_rate, rate_ci_low, rate_ci_high, rate_method, rate_source, age_model_status"; /// Recompute `genomics.str_marker_stat` from every stored profile. Returns the diff --git a/rust/crates/du-db/tests/str_marker_stats.rs b/rust/crates/du-db/tests/str_marker_stats.rs index d6e8904..7c566e1 100644 --- a/rust/crates/du-db/tests/str_marker_stats.rs +++ b/rust/crates/du-db/tests/str_marker_stats.rs @@ -184,6 +184,51 @@ async fn marker_stats_reports_range_motif_and_rate_basis() { assert_eq!(dys442.modal_value, None); } +/// A marker the vendor reports inconsistently — DYS413 in the real corpus has 866 +/// copy-vector calls and one bare `23`. The range must come from the majority +/// shape: preferring the scalar let a single observation replace the whole range, +/// which showed min=modal=max=23 for a marker with 25 distinct values. +#[tokio::test] +async fn minority_value_shape_does_not_hijack_the_range() { + let Some(url) = database_url() else { + eprintln!("DATABASE_URL unset — skipping mixed-shape test"); + return; + }; + let db = du_db::testing::ephemeral_db(&url).await.expect("ephemeral db"); + let pool = db.pool().clone(); + + // Copy-vector calls plus one stray single count, on both marker shapes so the + // mirror case (mostly-simple with a stray vector) is covered too. 23-23 is + // repeated so it is a genuine mode rather than an arbitrary tie-break. + for (n, dys413, dys390) in [ + (1, multi("DYS413", &[23, 23]), simple("DYS390", 24)), + (2, multi("DYS413", &[23, 23]), simple("DYS390", 24)), + (3, multi("DYS413", &[21, 23]), simple("DYS390", 25)), + (4, multi("DYS413", &[23, 25]), simple("DYS390", 24)), + (5, simple("DYS413", 23), multi("DYS390", &[9, 9])), + ] { + seed_profile(&pool, &format!("MIXED-{n}"), serde_json::json!([dys413, dys390])).await; + } + + let stats = ystr::marker_stats(&pool).await.expect("marker_stats"); + + // Majority multi-copy ⇒ the copy vectors win and the lone scalar is set aside. + let m = find(&stats, "DYS413"); + assert_eq!(m.min_combination.as_deref(), Some("21-23")); + assert_eq!(m.modal_combination.as_deref(), Some("23-23")); + assert_eq!(m.max_combination.as_deref(), Some("23-25")); + assert_eq!(m.min_value, None, "the stray single count must not be shown as the range"); + assert_eq!(m.max_value, None); + assert_eq!(m.mixed_shape_count, 1, "the odd observation is surfaced, not discarded"); + assert_eq!(m.observations, 5, "it still counts as an observation"); + + // Mirror case: majority simple ⇒ scalars win over the stray vector. + let s = find(&stats, "DYS390"); + assert_eq!((s.min_value, s.modal_value, s.max_value), (Some(24), Some(24), Some(25))); + assert_eq!(s.min_combination, None); + assert_eq!(s.mixed_shape_count, 1); +} + #[tokio::test] async fn load_marker_models_honors_the_active_rate() { let Some(url) = database_url() else { diff --git a/rust/crates/du-web/src/api/dto.rs b/rust/crates/du-web/src/api/dto.rs index c5afd98..86d68cb 100644 --- a/rust/crates/du-web/src/api/dto.rs +++ b/rust/crates/du-web/src/api/dto.rs @@ -694,6 +694,10 @@ pub struct StrMarkerStatDto { pub null_alleles: i64, /// Partial-repeat / footnoted values, preserved but unscored. pub complex_count: i64, + /// Observations in this marker's minority value shape (a bare repeat count + /// where it is normally a copy vector, or vice versa). Excluded from the + /// reported range, which follows the majority shape. + pub mixed_shape_count: i64, /// Repeat unit, where known — we hold one for a minority of markers. pub motif: Option, pub period: Option, @@ -726,6 +730,7 @@ impl From for StrMarkerStatDto { distinct_values: m.distinct_values, null_alleles: m.null_alleles, complex_count: m.complex_count, + mixed_shape_count: m.mixed_shape_count, motif: m.motif, period: m.period, coordinates: m.coordinates, diff --git a/rust/crates/du-web/src/routes/str_markers.rs b/rust/crates/du-web/src/routes/str_markers.rs index 2b9dab4..ec25ff2 100644 --- a/rust/crates/du-web/src/routes/str_markers.rs +++ b/rust/crates/du-web/src/routes/str_markers.rs @@ -129,7 +129,7 @@ fn range_cell(value: Option, combination: &Option) -> String { /// Null-allele and partial-repeat counts folded into one cell, so two columns /// that are empty for nearly every marker don't widen the table. -fn notes(null_alleles: i64, complex: i64, t: &T) -> String { +fn notes(null_alleles: i64, complex: i64, mixed_shape: i64, t: &T) -> String { let mut parts = Vec::new(); if null_alleles > 0 { parts.push(format!("{null_alleles} {}", t.get("str.markers.note.null"))); @@ -137,6 +137,10 @@ fn notes(null_alleles: i64, complex: i64, t: &T) -> String { if complex > 0 { parts.push(format!("{complex} {}", t.get("str.markers.note.partial"))); } + // Explains why `distinct` can exceed what the shown range accounts for. + if mixed_shape > 0 { + parts.push(format!("{mixed_shape} {}", t.get("str.markers.note.mixedShape"))); + } parts.join(" · ") } @@ -168,7 +172,7 @@ fn to_row(m: du_db::ystr::MarkerStat, t: &T) -> MarkerRow { observations: fmt_count(m.observations), samples: fmt_count(m.samples), distinct_values: fmt_count(m.distinct_values), - notes: notes(m.null_alleles, m.complex_count, t), + notes: notes(m.null_alleles, m.complex_count, m.mixed_shape_count, t), rate: fmt_rate(m.mutation_rate), rate_ci: ci, rate_source: m.rate_source.unwrap_or_default(), diff --git a/rust/locales/en.txt b/rust/locales/en.txt index 9f77712..efed123 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -800,6 +800,7 @@ str.markers.multiCopy=multi-copy str.markers.multiCopy.help=Palindromic or duplicated locus, reported as a copy vector (e.g. 11-15) and not used for age estimation. str.markers.note.null=null str.markers.note.partial=partial +str.markers.note.mixedShape=other shape str.markers.status.measured=Measured rate str.markers.status.default=Default rate str.markers.status.excluded=Not scored diff --git a/rust/locales/es.txt b/rust/locales/es.txt index ea7be9b..d102b4e 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -794,6 +794,7 @@ str.markers.multiCopy=multicopia str.markers.multiCopy.help=Locus palindrómico o duplicado, notificado como vector de copias (p. ej. 11-15) y no utilizado para estimar la edad. str.markers.note.null=nulo str.markers.note.partial=parcial +str.markers.note.mixedShape=otra forma str.markers.status.measured=Tasa medida str.markers.status.default=Tasa por defecto str.markers.status.excluded=No puntuado diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index 1c4e302..4e86381 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -794,6 +794,7 @@ str.markers.multiCopy=multicopie str.markers.multiCopy.help=Locus palindromique ou dupliqué, rapporté sous forme de vecteur de copies (p. ex. 11-15) et non utilisé pour l'estimation de l'âge. str.markers.note.null=nul str.markers.note.partial=partiel +str.markers.note.mixedShape=autre forme str.markers.status.measured=Taux mesuré str.markers.status.default=Taux par défaut str.markers.status.excluded=Non évalué diff --git a/rust/migrations/0072_str_marker_stat_mixed_shape.sql b/rust/migrations/0072_str_marker_stat_mixed_shape.sql new file mode 100644 index 0000000..25d7de4 --- /dev/null +++ b/rust/migrations/0072_str_marker_stat_mixed_shape.sql @@ -0,0 +1,23 @@ +-- Record observations reported in a marker's minority value shape. +-- +-- Some markers are reported inconsistently by the vendor export: DYS413 has 866 +-- multi-copy calls (23-23, 21-23, …) and exactly one bare `23`. The report picked +-- whichever shape was non-null with the scalar winning ties, so that single +-- observation replaced the whole copy-vector range — the page showed +-- min=modal=max=23 for a marker with 25 distinct observed values, presenting one +-- sample's value as the corpus range. +-- +-- The range is now taken from whichever shape the majority of observations use +-- (which also fixes the mirror case: a mostly-simple marker with a stray vector), +-- and the minority count is kept here so the discrepancy is visible in the report +-- rather than silently discarded — the same reasoning as null_alleles and +-- complex_count. + +ALTER TABLE genomics.str_marker_stat + ADD COLUMN mixed_shape_count BIGINT NOT NULL DEFAULT 0; + +COMMENT ON COLUMN genomics.str_marker_stat.mixed_shape_count IS + 'Observations in the marker''s minority value shape (single count vs copy vector); excluded from the reported range'; + +-- Existing rows carry the old shape-precedence result; the next +-- `du-jobs run-once str-marker-stats` recomputes them.