Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions rust/crates/du-db/src/ystr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1110,6 +1110,12 @@ pub struct MarkerStat {
pub null_alleles: i64,
/// Partial-repeat / footnoted values (`{type:complex}`), counted but unscored.
pub complex_count: i64,
/// Observations reported in the *minority* shape for this marker — a single
/// repeat count where the marker is normally a copy vector, or vice versa.
/// DYS413 has 866 multi-copy calls and one bare `23`. The range is taken from
/// the majority shape, so these are excluded from it and surfaced here rather
/// than silently overriding it.
pub mixed_shape_count: i64,
pub motif: Option<String>,
pub period: Option<i16>,
pub coordinates: Option<Value>,
Expand Down Expand Up @@ -1173,18 +1179,24 @@ const MARKER_STATS_AGGREGATION: &str = "WITH obs AS ( \
count(DISTINCT COALESCE(repeats::text, array_to_string(copies, '-'), raw)) \
AS distinct_values, \
count(*) FILTER (WHERE is_null_allele) AS null_alleles, \
count(*) FILTER (WHERE vtype = 'complex') AS complex_count \
count(*) FILTER (WHERE vtype = 'complex') AS complex_count, \
count(*) FILTER (WHERE vtype = 'simple') AS simple_obs, \
count(*) FILTER (WHERE vtype = 'multiCopy') AS multi_obs \
FROM scored \
GROUP BY marker \
) \
SELECT a.marker AS marker_name, \
(a.observed_multi OR COALESCE(m.multi_copy, false)) AS multi_copy, \
a.observations, a.samples, \
a.min_value, a.modal_value, a.max_value, \
a.min_copies AS min_combination, \
array_to_string(a.modal_copies, '-') AS modal_combination, \
a.max_copies AS max_combination, \
CASE WHEN a.multi_obs <= a.simple_obs THEN a.min_value END AS min_value, \
CASE WHEN a.multi_obs <= a.simple_obs THEN a.modal_value END AS modal_value, \
CASE WHEN a.multi_obs <= a.simple_obs THEN a.max_value END AS max_value, \
CASE WHEN a.multi_obs > a.simple_obs THEN a.min_copies END AS min_combination, \
CASE WHEN a.multi_obs > a.simple_obs \
THEN array_to_string(a.modal_copies, '-') END AS modal_combination, \
CASE WHEN a.multi_obs > a.simple_obs THEN a.max_copies END AS max_combination, \
a.distinct_values, a.null_alleles, a.complex_count, \
LEAST(a.simple_obs, a.multi_obs) AS mixed_shape_count, \
m.motif, m.period, NULLIF(m.coordinates, '{}'::jsonb) AS coordinates, \
r.mutation_rate::float8 AS mutation_rate, \
r.mutation_rate_lower::float8 AS rate_ci_low, \
Expand All @@ -1211,7 +1223,8 @@ const MARKER_STATS_AGGREGATION: &str = "WITH obs AS ( \
/// Column list shared by the refresh insert and the read path, in table order.
const MARKER_STAT_COLUMNS: &str = "marker_name, multi_copy, observations, samples, \
min_value, modal_value, max_value, min_combination, modal_combination, max_combination, \
distinct_values, null_alleles, complex_count, motif, period, coordinates, \
distinct_values, null_alleles, complex_count, mixed_shape_count, \
motif, period, coordinates, \
mutation_rate, rate_ci_low, rate_ci_high, rate_method, rate_source, age_model_status";

/// Recompute `genomics.str_marker_stat` from every stored profile. Returns the
Expand Down
45 changes: 45 additions & 0 deletions rust/crates/du-db/tests/str_marker_stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,51 @@ async fn marker_stats_reports_range_motif_and_rate_basis() {
assert_eq!(dys442.modal_value, None);
}

/// A marker the vendor reports inconsistently — DYS413 in the real corpus has 866
/// copy-vector calls and one bare `23`. The range must come from the majority
/// shape: preferring the scalar let a single observation replace the whole range,
/// which showed min=modal=max=23 for a marker with 25 distinct values.
#[tokio::test]
async fn minority_value_shape_does_not_hijack_the_range() {
let Some(url) = database_url() else {
eprintln!("DATABASE_URL unset — skipping mixed-shape test");
return;
};
let db = du_db::testing::ephemeral_db(&url).await.expect("ephemeral db");
let pool = db.pool().clone();

// Copy-vector calls plus one stray single count, on both marker shapes so the
// mirror case (mostly-simple with a stray vector) is covered too. 23-23 is
// repeated so it is a genuine mode rather than an arbitrary tie-break.
for (n, dys413, dys390) in [
(1, multi("DYS413", &[23, 23]), simple("DYS390", 24)),
(2, multi("DYS413", &[23, 23]), simple("DYS390", 24)),
(3, multi("DYS413", &[21, 23]), simple("DYS390", 25)),
(4, multi("DYS413", &[23, 25]), simple("DYS390", 24)),
(5, simple("DYS413", 23), multi("DYS390", &[9, 9])),
] {
seed_profile(&pool, &format!("MIXED-{n}"), serde_json::json!([dys413, dys390])).await;
}

let stats = ystr::marker_stats(&pool).await.expect("marker_stats");

// Majority multi-copy ⇒ the copy vectors win and the lone scalar is set aside.
let m = find(&stats, "DYS413");
assert_eq!(m.min_combination.as_deref(), Some("21-23"));
assert_eq!(m.modal_combination.as_deref(), Some("23-23"));
assert_eq!(m.max_combination.as_deref(), Some("23-25"));
assert_eq!(m.min_value, None, "the stray single count must not be shown as the range");
assert_eq!(m.max_value, None);
assert_eq!(m.mixed_shape_count, 1, "the odd observation is surfaced, not discarded");
assert_eq!(m.observations, 5, "it still counts as an observation");

// Mirror case: majority simple ⇒ scalars win over the stray vector.
let s = find(&stats, "DYS390");
assert_eq!((s.min_value, s.modal_value, s.max_value), (Some(24), Some(24), Some(25)));
assert_eq!(s.min_combination, None);
assert_eq!(s.mixed_shape_count, 1);
}

#[tokio::test]
async fn load_marker_models_honors_the_active_rate() {
let Some(url) = database_url() else {
Expand Down
5 changes: 5 additions & 0 deletions rust/crates/du-web/src/api/dto.rs
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,10 @@ pub struct StrMarkerStatDto {
pub null_alleles: i64,
/// Partial-repeat / footnoted values, preserved but unscored.
pub complex_count: i64,
/// Observations in this marker's minority value shape (a bare repeat count
/// where it is normally a copy vector, or vice versa). Excluded from the
/// reported range, which follows the majority shape.
pub mixed_shape_count: i64,
/// Repeat unit, where known — we hold one for a minority of markers.
pub motif: Option<String>,
pub period: Option<i16>,
Expand Down Expand Up @@ -726,6 +730,7 @@ impl From<du_db::ystr::MarkerStat> for StrMarkerStatDto {
distinct_values: m.distinct_values,
null_alleles: m.null_alleles,
complex_count: m.complex_count,
mixed_shape_count: m.mixed_shape_count,
motif: m.motif,
period: m.period,
coordinates: m.coordinates,
Expand Down
8 changes: 6 additions & 2 deletions rust/crates/du-web/src/routes/str_markers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,14 +129,18 @@ fn range_cell(value: Option<i32>, combination: &Option<String>) -> String {

/// Null-allele and partial-repeat counts folded into one cell, so two columns
/// that are empty for nearly every marker don't widen the table.
fn notes(null_alleles: i64, complex: i64, t: &T) -> String {
fn notes(null_alleles: i64, complex: i64, mixed_shape: i64, t: &T) -> String {
let mut parts = Vec::new();
if null_alleles > 0 {
parts.push(format!("{null_alleles} {}", t.get("str.markers.note.null")));
}
if complex > 0 {
parts.push(format!("{complex} {}", t.get("str.markers.note.partial")));
}
// Explains why `distinct` can exceed what the shown range accounts for.
if mixed_shape > 0 {
parts.push(format!("{mixed_shape} {}", t.get("str.markers.note.mixedShape")));
}
parts.join(" · ")
}

Expand Down Expand Up @@ -168,7 +172,7 @@ fn to_row(m: du_db::ystr::MarkerStat, t: &T) -> MarkerRow {
observations: fmt_count(m.observations),
samples: fmt_count(m.samples),
distinct_values: fmt_count(m.distinct_values),
notes: notes(m.null_alleles, m.complex_count, t),
notes: notes(m.null_alleles, m.complex_count, m.mixed_shape_count, t),
rate: fmt_rate(m.mutation_rate),
rate_ci: ci,
rate_source: m.rate_source.unwrap_or_default(),
Expand Down
1 change: 1 addition & 0 deletions rust/locales/en.txt
Original file line number Diff line number Diff line change
Expand Up @@ -800,6 +800,7 @@ str.markers.multiCopy=multi-copy
str.markers.multiCopy.help=Palindromic or duplicated locus, reported as a copy vector (e.g. 11-15) and not used for age estimation.
str.markers.note.null=null
str.markers.note.partial=partial
str.markers.note.mixedShape=other shape
str.markers.status.measured=Measured rate
str.markers.status.default=Default rate
str.markers.status.excluded=Not scored
Expand Down
1 change: 1 addition & 0 deletions rust/locales/es.txt
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,7 @@ str.markers.multiCopy=multicopia
str.markers.multiCopy.help=Locus palindrómico o duplicado, notificado como vector de copias (p. ej. 11-15) y no utilizado para estimar la edad.
str.markers.note.null=nulo
str.markers.note.partial=parcial
str.markers.note.mixedShape=otra forma
str.markers.status.measured=Tasa medida
str.markers.status.default=Tasa por defecto
str.markers.status.excluded=No puntuado
Expand Down
1 change: 1 addition & 0 deletions rust/locales/fr.txt
Original file line number Diff line number Diff line change
Expand Up @@ -794,6 +794,7 @@ str.markers.multiCopy=multicopie
str.markers.multiCopy.help=Locus palindromique ou dupliqué, rapporté sous forme de vecteur de copies (p. ex. 11-15) et non utilisé pour l'estimation de l'âge.
str.markers.note.null=nul
str.markers.note.partial=partiel
str.markers.note.mixedShape=autre forme
str.markers.status.measured=Taux mesuré
str.markers.status.default=Taux par défaut
str.markers.status.excluded=Non évalué
Expand Down
23 changes: 23 additions & 0 deletions rust/migrations/0072_str_marker_stat_mixed_shape.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
-- Record observations reported in a marker's minority value shape.
--
-- Some markers are reported inconsistently by the vendor export: DYS413 has 866
-- multi-copy calls (23-23, 21-23, …) and exactly one bare `23`. The report picked
-- whichever shape was non-null with the scalar winning ties, so that single
-- observation replaced the whole copy-vector range — the page showed
-- min=modal=max=23 for a marker with 25 distinct observed values, presenting one
-- sample's value as the corpus range.
--
-- The range is now taken from whichever shape the majority of observations use
-- (which also fixes the mirror case: a mostly-simple marker with a stray vector),
-- and the minority count is kept here so the discrepancy is visible in the report
-- rather than silently discarded — the same reasoning as null_alleles and
-- complex_count.

ALTER TABLE genomics.str_marker_stat
ADD COLUMN mixed_shape_count BIGINT NOT NULL DEFAULT 0;

COMMENT ON COLUMN genomics.str_marker_stat.mixed_shape_count IS
'Observations in the marker''s minority value shape (single count vs copy vector); excluded from the reported range';

-- Existing rows carry the old shape-precedence result; the next
-- `du-jobs run-once str-marker-stats` recomputes them.
Loading