From 9e18e17bda1b59c3203dbb3d707652cef6ab98eb Mon Sep 17 00:00:00 2001 From: cecemei Date: Tue, 25 Aug 2026 12:02:01 -0700 Subject: [PATCH 1/5] pruning --- .../query/filter/FilterSegmentPruner.java | 116 ++++++++++++++---- .../query/filter/FilterSegmentPrunerTest.java | 44 +++++++ 2 files changed, 134 insertions(+), 26 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java index 6ef7565302b0..f16574b5e3f1 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java @@ -19,10 +19,14 @@ package org.apache.druid.query.filter; +import com.google.common.collect.Range; import com.google.common.collect.RangeSet; import org.apache.druid.error.InvalidInput; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.timeline.ClusterGroupTuples; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.partition.ShardSpec; @@ -73,39 +77,72 @@ public FilterSegmentPruner( public boolean include(DataSegment segment) { final ShardSpec shard = segment.getShardSpec(); - boolean include = true; if (shard != null) { final Map> filterDomain = new HashMap<>(); - final List dimensions = shard.getDomainDimensions(); - for (String dimension : dimensions) { - final VirtualColumns.Node shardNode = shard.getDomainVirtualColumns().getNode(dimension); - if (shardNode != null) { - final VirtualColumn queryEquivalent = getQueryEquivalent(shardNode); - if (queryEquivalent != null) { - if (filterFields == null || filterFields.contains(queryEquivalent.getOutputName())) { - final Optional> optFilterRangeSet = rangeCache - .computeIfAbsent( - queryEquivalent.getOutputName(), - d -> Optional.ofNullable(filter.getDimensionRangeSet(d)) - ); - optFilterRangeSet.ifPresent(stringRangeSet -> filterDomain.put( - shardNode.getVirtualColumn().getOutputName(), - stringRangeSet - )); - } - } - } else if (filterFields == null || filterFields.contains(dimension)) { - final Optional> optFilterRangeSet = - rangeCache.computeIfAbsent(dimension, d -> Optional.ofNullable(filter.getDimensionRangeSet(d))); - optFilterRangeSet.ifPresent(stringRangeSet -> filterDomain.put(dimension, stringRangeSet)); - } + for (String dimension : shard.getDomainDimensions()) { + addToFilterDomain(dimension, shard.getDomainVirtualColumns(), filterDomain); } if (!filterDomain.isEmpty() && !shard.possibleInDomain(filterDomain)) { - include = false; + return false; } } - return include; + + final ClusterGroupTuples clusterGroups = segment.getClusterGroups(); + if (clusterGroups != null && !possibleInClusterGroups(clusterGroups)) { + return false; + } + + return true; + } + + private boolean possibleInClusterGroups(ClusterGroupTuples clusterGroups) + { + final RowSignature clusteringColumns = clusterGroups.clusteringColumns(); + final int numColumns = clusteringColumns.size(); + + final Map> filterDomain = new HashMap<>(); + for (int i = 0; i < numColumns; i++) { + final String column = clusteringColumns.getColumnName(i); + if (!ColumnType.STRING.equals(clusteringColumns.getColumnType(i).orElse(null))) { + continue; + } + addToFilterDomain(column, clusterGroups.virtualColumns(), filterDomain); + } + + if (filterDomain.isEmpty()) { + // Filter doesn't constrain any string clustering column. + return true; + } + + for (final List tuple : clusterGroups.tuples()) { + if (tupleMatchesDomain(clusteringColumns, tuple, filterDomain)) { + return true; + } + } + + return false; + } + + private static boolean tupleMatchesDomain( + RowSignature clusteringColumns, + List tuple, + Map> filterDomain + ) + { + for (int i = 0; i < clusteringColumns.size(); i++) { + final RangeSet domainRangeSet = filterDomain.get(clusteringColumns.getColumnName(i)); + if (domainRangeSet == null) { + continue; + } + final Object rawValue = tuple.get(i); + // Nulls are less than empty String in segments + final Range valueRange = rawValue == null ? Range.lessThan("") : Range.singleton((String) rawValue); + if (domainRangeSet.subRangeSet(valueRange).isEmpty()) { + return false; + } + } + return true; } @Override @@ -164,6 +201,33 @@ public String toString() '}'; } + /** + * Adds the filter's {@link RangeSet} for {@code column} to {@code filterDomain}, resolving through + * {@code domainVirtualColumns} to the query's equivalent virtual column if {@code column} is virtual there. + */ + private void addToFilterDomain( + String column, + VirtualColumns domainVirtualColumns, + Map> filterDomain + ) + { + final VirtualColumns.Node domainNode = domainVirtualColumns.getNode(column); + if (domainNode != null) { + final VirtualColumn queryEquivalent = getQueryEquivalent(domainNode); + if (queryEquivalent != null && filterFields.contains(queryEquivalent.getOutputName())) { + final Optional> optFilterRangeSet = rangeCache.computeIfAbsent( + queryEquivalent.getOutputName(), + d -> Optional.ofNullable(filter.getDimensionRangeSet(d)) + ); + optFilterRangeSet.ifPresent(rangeSet -> filterDomain.put(column, rangeSet)); + } + } else if (filterFields.contains(column)) { + final Optional> optFilterRangeSet = + rangeCache.computeIfAbsent(column, d -> Optional.ofNullable(filter.getDimensionRangeSet(d))); + optFilterRangeSet.ifPresent(rangeSet -> filterDomain.put(column, rangeSet)); + } + } + @Nullable private VirtualColumn getQueryEquivalent(VirtualColumns.Node node) { diff --git a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java index d5c0d38bebaa..d8d8bc21de84 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java @@ -26,8 +26,10 @@ import org.apache.druid.query.expression.TestExprMacroTable; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.NestedFieldVirtualColumn; +import org.apache.druid.timeline.ClusterGroupTuples; import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.SegmentId; import org.apache.druid.timeline.partition.DimensionRangeShardSpec; @@ -329,6 +331,39 @@ void testPruneNumericIn() Assertions.assertTrue(pruner.include(seg)); } + @Test + void testPruneClusterGroupTuples() + { + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final RowSignature clusteringColumns = RowSignature.builder().add("dim1", ColumnType.STRING).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + List.of(List.of("abc"), List.of("xyz")) + ); + + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + final DimFilter matchingFilter = new EqualityFilter("dim1", ColumnType.STRING, "abc", null); + final DimFilter nonMatchingFilter = new EqualityFilter("dim1", ColumnType.STRING, "foo", null); + + Assertions.assertTrue(new FilterSegmentPruner(matchingFilter, null, null).include(seg)); + Assertions.assertFalse(new FilterSegmentPruner(nonMatchingFilter, null, null).include(seg)); + } + + @Test + void testClusterGroupTuplesSkipsNonStringColumns() + { + // Numeric columns are skipped for pruning (see druid issue #19408), so this must not prune. + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final RowSignature clusteringColumns = RowSignature.builder().add("id", ColumnType.LONG).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples(clusteringColumns, List.of(List.of(100L), List.of(200L))); + + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + final DimFilter filter = new EqualityFilter("id", ColumnType.LONG, 999L, null); + + Assertions.assertTrue(new FilterSegmentPruner(filter, null, null).include(seg)); + } + private ShardSpec makeRange( String column, int partitionNumber, @@ -385,4 +420,13 @@ private DataSegment makeDataSegment(String intervalString, ShardSpec shardSpec) .shardSpec(shardSpec) .build(); } + + private DataSegment makeDataSegment(String intervalString, ShardSpec shardSpec, ClusterGroupTuples clusterGroups) + { + Interval interval = Intervals.of(intervalString); + return DataSegment.builder(SegmentId.of("prune-test", interval, "0", shardSpec)) + .shardSpec(shardSpec) + .clusterGroups(clusterGroups) + .build(); + } } From 58f8bcc0b2c51c5b1943751ffdb2b1d5ee142425 Mon Sep 17 00:00:00 2001 From: cecemei Date: Tue, 25 Aug 2026 12:13:48 -0700 Subject: [PATCH 2/5] pruning2 --- .../query/filter/FilterSegmentPruner.java | 6 +-- .../query/filter/FilterSegmentPrunerTest.java | 37 ++++++++++++++++++- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java index f16574b5e3f1..2bf94385210a 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java @@ -51,7 +51,7 @@ public class FilterSegmentPruner implements SegmentPruner private final Set filterFields; private final VirtualColumns virtualColumns; private final Map>> rangeCache; - private final Map> shardEquivalenceCache; + private final Map> virtualColumnEquivalenceCache; public FilterSegmentPruner( DimFilter filter, @@ -63,7 +63,7 @@ public FilterSegmentPruner( this.filterFields = filterFields == null ? filter.getRequiredColumns() : filterFields; this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns; this.rangeCache = new HashMap<>(); - this.shardEquivalenceCache = new HashMap<>(); + this.virtualColumnEquivalenceCache = new HashMap<>(); } @@ -231,7 +231,7 @@ private void addToFilterDomain( @Nullable private VirtualColumn getQueryEquivalent(VirtualColumns.Node node) { - final Optional cached = shardEquivalenceCache.computeIfAbsent( + final Optional cached = virtualColumnEquivalenceCache.computeIfAbsent( node, n -> Optional.ofNullable(virtualColumns.findEquivalent(n)) ); diff --git a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java index d8d8bc21de84..98e04bfd8d00 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java @@ -284,7 +284,7 @@ void testEqualsAndHashcode() { EqualsVerifier.forClass(FilterSegmentPruner.class) .usingGetClass() - .withIgnoredFields("rangeCache", "shardEquivalenceCache") + .withIgnoredFields("rangeCache", "virtualColumnEquivalenceCache") .verify(); } @@ -364,6 +364,41 @@ void testClusterGroupTuplesSkipsNonStringColumns() Assertions.assertTrue(new FilterSegmentPruner(filter, null, null).include(seg)); } + @Test + void testPruneClusterGroupTuplesVirtualColumn() + { + final VirtualColumns clusterVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("vdim1", "concat(dim1, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final RowSignature clusteringColumns = RowSignature.builder().add("vdim1", ColumnType.STRING).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + clusterVirtualColumns, + List.of(List.of("abcfoo"), List.of("xyzfoo")) + ); + + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + // same expression, same name + VirtualColumns queryVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("vdim1", "concat(dim1, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final DimFilter matchingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "abcfoo", null); + final DimFilter nonMatchingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "deffoo", null); + Assertions.assertTrue(new FilterSegmentPruner(matchingFilter, null, queryVirtualColumns).include(seg)); + Assertions.assertFalse(new FilterSegmentPruner(nonMatchingFilter, null, queryVirtualColumns).include(seg)); + + // same expression, different name: still resolved via virtual column equivalence + queryVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("v0", "concat(dim1, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final DimFilter matchingFilterDifferentName = new EqualityFilter("v0", ColumnType.STRING, "abcfoo", null); + final DimFilter nonMatchingFilterDifferentName = new EqualityFilter("v0", ColumnType.STRING, "deffoo", null); + Assertions.assertTrue(new FilterSegmentPruner(matchingFilterDifferentName, null, queryVirtualColumns).include(seg)); + Assertions.assertFalse(new FilterSegmentPruner(nonMatchingFilterDifferentName, null, queryVirtualColumns).include(seg)); + } + private ShardSpec makeRange( String column, int partitionNumber, From 19c1a92f5cd799a27a88b08b5d529c69b0dcf9a9 Mon Sep 17 00:00:00 2001 From: cecemei Date: Wed, 26 Aug 2026 16:38:06 -0700 Subject: [PATCH 3/5] pruning3 --- .../query/filter/FilterSegmentPruner.java | 33 +++-- .../query/filter/FilterSegmentPrunerTest.java | 123 ++++++++++++++++++ 2 files changed, 144 insertions(+), 12 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java index 2bf94385210a..e53b16d3a0b1 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java @@ -202,8 +202,12 @@ public String toString() } /** - * Adds the filter's {@link RangeSet} for {@code column} to {@code filterDomain}, resolving through - * {@code domainVirtualColumns} to the query's equivalent virtual column if {@code column} is virtual there. + * Adds the filter's {@link RangeSet} for {@code column} to {@code filterDomain}, if the filter constrains it. + *

+ * If {@code domainVirtualColumns} considers {@code column} virtual, only a query virtual column with an + * equivalent expression can be matched against it, if none exists, nothing is added and this column is never pruned on. + *

+ * Otherwise, {@code column} is a plain physical column, it can only be used for pruning if it's a non-virtual column in the query. */ private void addToFilterDomain( String column, @@ -214,18 +218,23 @@ private void addToFilterDomain( final VirtualColumns.Node domainNode = domainVirtualColumns.getNode(column); if (domainNode != null) { final VirtualColumn queryEquivalent = getQueryEquivalent(domainNode); - if (queryEquivalent != null && filterFields.contains(queryEquivalent.getOutputName())) { - final Optional> optFilterRangeSet = rangeCache.computeIfAbsent( - queryEquivalent.getOutputName(), - d -> Optional.ofNullable(filter.getDimensionRangeSet(d)) - ); - optFilterRangeSet.ifPresent(rangeSet -> filterDomain.put(column, rangeSet)); + if (queryEquivalent != null) { + addRangeSetIfPresent(queryEquivalent.getOutputName(), column, filterDomain); } - } else if (filterFields.contains(column)) { - final Optional> optFilterRangeSet = - rangeCache.computeIfAbsent(column, d -> Optional.ofNullable(filter.getDimensionRangeSet(d))); - optFilterRangeSet.ifPresent(rangeSet -> filterDomain.put(column, rangeSet)); + } else if (virtualColumns.getNode(column) == null) { + // Query doesn't shadow the materialized column with its own virtual column of the same name. + addRangeSetIfPresent(column, column, filterDomain); + } + } + + private void addRangeSetIfPresent(String filterField, String domainColumn, Map> filterDomain) + { + if (!filterFields.contains(filterField)) { + return; } + final Optional> optFilterRangeSet = + rangeCache.computeIfAbsent(filterField, d -> Optional.ofNullable(filter.getDimensionRangeSet(d))); + optFilterRangeSet.ifPresent(rangeSet -> filterDomain.put(domainColumn, rangeSet)); } @Nullable diff --git a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java index 98e04bfd8d00..1585eef5b7df 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java @@ -350,6 +350,47 @@ void testPruneClusterGroupTuples() Assertions.assertFalse(new FilterSegmentPruner(nonMatchingFilter, null, null).include(seg)); } + @Test + void testPruneClusterGroupTuplesMultipleColumns() + { + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final RowSignature clusteringColumns = RowSignature.builder() + .add("dim1", ColumnType.STRING) + .add("dim2", ColumnType.STRING) + .build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + List.of(List.of("abc", "xyz"), List.of("def", "uvw")) + ); + + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + // matches the first tuple on both columns + final DimFilter matchingFilter = new AndDimFilter( + new EqualityFilter("dim1", ColumnType.STRING, "abc", null), + new EqualityFilter("dim2", ColumnType.STRING, "xyz", null) + ); + Assertions.assertTrue(new FilterSegmentPruner(matchingFilter, null, null).include(seg)); + + // each value individually matches a tuple, but not the same tuple, so the combination must prune + final DimFilter mismatchedCombinationFilter = new AndDimFilter( + new EqualityFilter("dim1", ColumnType.STRING, "abc", null), + new EqualityFilter("dim2", ColumnType.STRING, "uvw", null) + ); + Assertions.assertFalse(new FilterSegmentPruner(mismatchedCombinationFilter, null, null).include(seg)); + + // constraining only one of the two clustering columns still matches via the second tuple + final DimFilter singleColumnFilter = new EqualityFilter("dim2", ColumnType.STRING, "uvw", null); + Assertions.assertTrue(new FilterSegmentPruner(singleColumnFilter, null, null).include(seg)); + + // neither tuple matches + final DimFilter nonMatchingFilter = new AndDimFilter( + new EqualityFilter("dim1", ColumnType.STRING, "abc", null), + new EqualityFilter("dim2", ColumnType.STRING, "foo", null) + ); + Assertions.assertFalse(new FilterSegmentPruner(nonMatchingFilter, null, null).include(seg)); + } + @Test void testClusterGroupTuplesSkipsNonStringColumns() { @@ -399,6 +440,88 @@ void testPruneClusterGroupTuplesVirtualColumn() Assertions.assertFalse(new FilterSegmentPruner(nonMatchingFilterDifferentName, null, queryVirtualColumns).include(seg)); } + @Test + void testPruneClusterGroupTuplesVirtualColumnSameNameDifferentExpressionNeverPrunes() + { + final VirtualColumns clusterVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("vdim1", "concat(dim1, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final RowSignature clusteringColumns = RowSignature.builder().add("vdim1", ColumnType.STRING).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + clusterVirtualColumns, + List.of(List.of("abcfoo"), List.of("xyzfoo")) + ); + + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + // query's vdim1 is a different expression, so it has no equivalent on the segment side and must never prune, + // even though the filter value would not match any tuple if it were (incorrectly) compared directly + final VirtualColumns queryVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("vdim1", "concat(dim1, 'bar')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final DimFilter nonMatchingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "nomatch", null); + Assertions.assertTrue(new FilterSegmentPruner(nonMatchingFilter, null, queryVirtualColumns).include(seg)); + + final DimFilter matchingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "abcfoo", null); + Assertions.assertTrue(new FilterSegmentPruner(matchingFilter, null, queryVirtualColumns).include(seg)); + } + + @Test + void testPruneClusterGroupTuplesVirtualColumnNoQueryVirtualColumnNeverPrunes() + { + // The segment's cluster groups record "vdim1" as derived from an expression. A query with no virtual column + // of its own named "vdim1" has no way to prove it means the same expression, so pruning must not assume the + // filter value can be compared against the tuple's virtual-column-derived value: never prune. + final VirtualColumns clusterVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("vdim1", "concat(dim1, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final RowSignature clusteringColumns = RowSignature.builder().add("vdim1", ColumnType.STRING).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + clusterVirtualColumns, + List.of(List.of("abcfoo"), List.of("xyzfoo")) + ); + + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + final DimFilter matchingLookingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "abcfoo", null); + final DimFilter nonMatchingLookingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "deffoo", null); + + // no query virtual columns at all: neither filter value can be resolved against the domain's virtual column + Assertions.assertTrue(new FilterSegmentPruner(matchingLookingFilter, null, null).include(seg)); + Assertions.assertTrue(new FilterSegmentPruner(nonMatchingLookingFilter, null, null).include(seg)); + } + + @Test + void testPruneClusterGroupTuplesShadowedByQueryVirtualColumnNeverPrunes() + { + // "dim1" is a plain materialized clustering column with no virtual columns on the segment side. If the query + // defines its own virtual column named "dim1", it shadows the real column, so the query is no longer + // referring to the materialized clustering values and the segment must never prune, even for a filter value + // that looks like it wouldn't match any of the real tuple values. + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final RowSignature clusteringColumns = RowSignature.builder().add("dim1", ColumnType.STRING).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + List.of(List.of("abc"), List.of("xyz")) + ); + + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + final VirtualColumns queryVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("dim1", "concat(dim2, 'zzz')", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + + final DimFilter nonMatchingLookingFilter = new EqualityFilter("dim1", ColumnType.STRING, "nomatch", null); + Assertions.assertTrue(new FilterSegmentPruner(nonMatchingLookingFilter, null, queryVirtualColumns).include(seg)); + + final DimFilter matchingLookingFilter = new EqualityFilter("dim1", ColumnType.STRING, "abc", null); + Assertions.assertTrue(new FilterSegmentPruner(matchingLookingFilter, null, queryVirtualColumns).include(seg)); + } + private ShardSpec makeRange( String column, int partitionNumber, From badbeae36a2485b8a790bddea9ee6479a3c7d799 Mon Sep 17 00:00:00 2001 From: cecemei Date: Thu, 27 Aug 2026 11:27:19 -0700 Subject: [PATCH 4/5] transitive shadow --- .../apache/druid/segment/VirtualColumns.java | 9 ++++- .../query/filter/FilterSegmentPrunerTest.java | 38 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java index 48d1decb559a..3a5416e24770 100644 --- a/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java +++ b/processing/src/main/java/org/apache/druid/segment/VirtualColumns.java @@ -282,7 +282,14 @@ public VirtualColumn findEquivalent(Node otherNode) toCheckForEquivalence = otherVirtualColumn.rewriteRequiredColumns(equivalenceRewriteMap); } - return equivalence.get().get(toCheckForEquivalence.getEquivalanceKey()); + VirtualColumn matched = equivalence.get().get(toCheckForEquivalence.getEquivalanceKey()); + if (matched != null && + // guardrail check for expression collision when a virtual column shadows the physical column + // e.x. otherNode v0 = dim1 and VCs dim1 = dim2 plus q = dim1, q can be treated as equivalent to v0 even though it reads physical column dim2 + getNode(matched.getOutputName()).getDependencies().size() == otherNode.getDependencies().size()) { + return matched; + } + return null; } /** diff --git a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java index 1585eef5b7df..ec347373ac0c 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java @@ -522,6 +522,44 @@ void testPruneClusterGroupTuplesShadowedByQueryVirtualColumnNeverPrunes() Assertions.assertTrue(new FilterSegmentPruner(matchingLookingFilter, null, queryVirtualColumns).include(seg)); } + @Test + void testPruneClusterGroupTuplesTransitivelyShadowedByQueryVirtualColumnNeverPrunes() + { + // The segment's cluster group is keyed by virtual column "v0" = "dim1", i.e. the real physical "dim1" column. + // The query defines its own virtual column "dim1" = "dim2", which shadows the physical column, plus a second + // virtual column "q" = "dim1" that therefore transitively reads "dim2", not the real "dim1". Matching "q" + // against "v0" by comparing raw, unresolved expression text ("dim1" == "dim1") would wrongly treat them as + // equivalent, even though "q" doesn't actually read the column "v0" reads. Equivalence must be resolved + // through the query's own virtual column dependency graph, so pruning on "q" must never occur here. + final VirtualColumns clusterVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("v0", "dim1", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + final RowSignature clusteringColumns = RowSignature.builder().add("v0", ColumnType.STRING).build(); + final ClusterGroupTuples tuples = new ClusterGroupTuples( + clusteringColumns, + clusterVirtualColumns, + List.of(List.of("abc"), List.of("xyz")) + ); + + final String interval = "2026-01-01T00:00:00Z/2026-01-02T00:00:00Z"; + final DataSegment seg = makeDataSegment(interval, makeRange("dim1", 0, null, null), tuples); + + final VirtualColumns queryVirtualColumns = VirtualColumns.create( + new ExpressionVirtualColumn("dim1", "dim2", ColumnType.STRING, TestExprMacroTable.INSTANCE), + new ExpressionVirtualColumn("q", "dim1", ColumnType.STRING, TestExprMacroTable.INSTANCE) + ); + + // A filter value that matches none of the tuple's "v0" values must still never prune, since "q" doesn't + // actually read what "v0" reads. + final DimFilter nonMatchingLookingFilter = new EqualityFilter("q", ColumnType.STRING, "nomatch", null); + Assertions.assertTrue(new FilterSegmentPruner(nonMatchingLookingFilter, null, queryVirtualColumns).include(seg)); + + // Nor should a filter value that happens to match one of the tuple's "v0" values: that match is coincidental + // and says nothing about "dim2", which is what "q" actually reads. + final DimFilter matchingLookingFilter = new EqualityFilter("q", ColumnType.STRING, "abc", null); + Assertions.assertTrue(new FilterSegmentPruner(matchingLookingFilter, null, queryVirtualColumns).include(seg)); + } + private ShardSpec makeRange( String column, int partitionNumber, From 19fa4ecb18b9a69957d86dc557295e1f9ac17c63 Mon Sep 17 00:00:00 2001 From: cecemei Date: Fri, 28 Aug 2026 14:56:54 -0700 Subject: [PATCH 5/5] direct-query --- .../query/filter/FilterSegmentPruner.java | 22 +++++++++++-------- .../query/filter/FilterSegmentPrunerTest.java | 12 +++++----- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java index e53b16d3a0b1..570dcb3c5e0c 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java +++ b/processing/src/main/java/org/apache/druid/query/filter/FilterSegmentPruner.java @@ -81,7 +81,7 @@ public boolean include(DataSegment segment) if (shard != null) { final Map> filterDomain = new HashMap<>(); for (String dimension : shard.getDomainDimensions()) { - addToFilterDomain(dimension, shard.getDomainVirtualColumns(), filterDomain); + addToFilterDomain(dimension, shard.getDomainVirtualColumns(), filterDomain, false); } if (!filterDomain.isEmpty() && !shard.possibleInDomain(filterDomain)) { return false; @@ -107,7 +107,10 @@ private boolean possibleInClusterGroups(ClusterGroupTuples clusterGroups) if (!ColumnType.STRING.equals(clusteringColumns.getColumnType(i).orElse(null))) { continue; } - addToFilterDomain(column, clusterGroups.virtualColumns(), filterDomain); + // Cluster group virtual columns are wired into real query execution (they're merged into the segment's + // live VirtualColumns at cursor construction time), so "column" is directly queryable by that exact name + // as long as the query doesn't shadow it with a differently-defined virtual column of its own. + addToFilterDomain(column, clusterGroups.virtualColumns(), filterDomain, true); } if (filterDomain.isEmpty()) { @@ -202,17 +205,16 @@ public String toString() } /** - * Adds the filter's {@link RangeSet} for {@code column} to {@code filterDomain}, if the filter constrains it. - *

- * If {@code domainVirtualColumns} considers {@code column} virtual, only a query virtual column with an - * equivalent expression can be matched against it, if none exists, nothing is added and this column is never pruned on. - *

- * Otherwise, {@code column} is a plain physical column, it can only be used for pruning if it's a non-virtual column in the query. + * Adds the filter's {@link RangeSet} for {@code column} to {@code filterDomain}, if the filter constrains it and + * {@code column} can be safely matched: as a query virtual column with an equivalent expression, directly by name + * when {@code allowDirectAccess} (only valid for domain virtual columns that are themselves directly queryable, + * e.g. clustering groups), or as an unshadowed physical column. Otherwise, nothing is added. */ private void addToFilterDomain( String column, VirtualColumns domainVirtualColumns, - Map> filterDomain + Map> filterDomain, + boolean allowDirectAccess ) { final VirtualColumns.Node domainNode = domainVirtualColumns.getNode(column); @@ -220,6 +222,8 @@ private void addToFilterDomain( final VirtualColumn queryEquivalent = getQueryEquivalent(domainNode); if (queryEquivalent != null) { addRangeSetIfPresent(queryEquivalent.getOutputName(), column, filterDomain); + } else if (allowDirectAccess && virtualColumns.getNode(column) == null) { + addRangeSetIfPresent(column, column, filterDomain); } } else if (virtualColumns.getNode(column) == null) { // Query doesn't shadow the materialized column with its own virtual column of the same name. diff --git a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java index ec347373ac0c..ca85d801155c 100644 --- a/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java +++ b/processing/src/test/java/org/apache/druid/query/filter/FilterSegmentPrunerTest.java @@ -469,11 +469,11 @@ void testPruneClusterGroupTuplesVirtualColumnSameNameDifferentExpressionNeverPru } @Test - void testPruneClusterGroupTuplesVirtualColumnNoQueryVirtualColumnNeverPrunes() + void testPruneClusterGroupTuplesVirtualColumnNoQueryVirtualColumnPrunesDirectly() { - // The segment's cluster groups record "vdim1" as derived from an expression. A query with no virtual column - // of its own named "vdim1" has no way to prove it means the same expression, so pruning must not assume the - // filter value can be compared against the tuple's virtual-column-derived value: never prune. + // The segment's cluster groups record "vdim1" as derived from an expression, but cluster group virtual + // columns are directly queryable. A query with no virtual column of its own named "vdim1" doesn't shadow it, + // so the filter value can be compared directly against the tuple's virtual-column-derived value. final VirtualColumns clusterVirtualColumns = VirtualColumns.create( new ExpressionVirtualColumn("vdim1", "concat(dim1, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE) ); @@ -490,9 +490,9 @@ void testPruneClusterGroupTuplesVirtualColumnNoQueryVirtualColumnNeverPrunes() final DimFilter matchingLookingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "abcfoo", null); final DimFilter nonMatchingLookingFilter = new EqualityFilter("vdim1", ColumnType.STRING, "deffoo", null); - // no query virtual columns at all: neither filter value can be resolved against the domain's virtual column + // no query virtual columns at all, and "vdim1" isn't shadowed, so it's matched directly by name Assertions.assertTrue(new FilterSegmentPruner(matchingLookingFilter, null, null).include(seg)); - Assertions.assertTrue(new FilterSegmentPruner(nonMatchingLookingFilter, null, null).include(seg)); + Assertions.assertFalse(new FilterSegmentPruner(nonMatchingLookingFilter, null, null).include(seg)); } @Test