From 5958200f034dbdbf08c4d3d8a5067dc93e0ca46e Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Mon, 24 Aug 2026 12:25:21 +0200 Subject: [PATCH 1/3] geometries: fix the maximum of a bounding box over negative coordinates MinMaxDeriver seeded the maxima with Double.MIN_VALUE, which is the smallest positive value rather than the most negative one, so every Math.max against it won on an axis whose coordinates are all negative and the maximum stayed at 4.9E-324. Spatial extents are derived from this, so the extent of data west of Greenwich or south of the equator came out wrong. --- .../domain/transform/MinMaxDeriver.java | 4 +- .../domain/MinMaxDeriverSpec.groovy | 98 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy diff --git a/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java b/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java index f7af81672..d7c03b56c 100644 --- a/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java +++ b/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java @@ -119,7 +119,9 @@ public double[][] visit(PolyhedralSurface geometry) { private static double[][] initMinMax(int dimensions) { double[][] minMax = new double[2][dimensions]; Arrays.fill(minMax[0], Double.MAX_VALUE); - Arrays.fill(minMax[1], Double.MIN_VALUE); + // -Double.MAX_VALUE, not Double.MIN_VALUE: the latter is the smallest positive value, which + // would survive every Math.max on an axis whose coordinates are all negative + Arrays.fill(minMax[1], -Double.MAX_VALUE); return minMax; } diff --git a/xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy b/xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy new file mode 100644 index 000000000..bf797525b --- /dev/null +++ b/xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy @@ -0,0 +1,98 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.geometries.domain + +import de.ii.xtraplatform.geometries.domain.transform.MinMaxDeriver +import spock.lang.Specification + +class MinMaxDeriverSpec extends Specification { + + static Polygon polygon(double... coordinates) { + return Polygon.of(List.of(PositionList.of(Axes.XY, coordinates))) + } + + def 'positive coordinates'() { + + given: + def geometry = polygon(7.0d, 50.0d, 7.1d, 50.0d, 7.1d, 50.1d, 7.0d, 50.1d, 7.0d, 50.0d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [7.0d, 50.0d] as double[] + minMax[1] == [7.1d, 50.1d] as double[] + } + + def 'coordinates that are all negative on one axis'() { + + given: 'a polygon west of Greenwich, so every x is negative' + def geometry = polygon(-118.0d, 33.8d, -117.9d, 33.8d, -117.9d, 34.0d, -118.0d, 34.0d, -118.0d, 33.8d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: 'the maximum is the largest x, not the seed value' + minMax[0] == [-118.0d, 33.8d] as double[] + minMax[1] == [-117.9d, 34.0d] as double[] + } + + def 'coordinates that are all negative on both axes'() { + + given: + def geometry = polygon(-70.0d, -33.5d, -69.9d, -33.5d, -69.9d, -33.4d, -70.0d, -33.4d, -70.0d, -33.5d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-70.0d, -33.5d] as double[] + minMax[1] == [-69.9d, -33.4d] as double[] + } + + def 'negative coordinates across the components of a multi geometry'() { + + given: + def geometry = MultiPolygon.of(List.of( + polygon(-10.0d, -10.0d, -9.0d, -10.0d, -9.0d, -9.0d, -10.0d, -9.0d, -10.0d, -10.0d), + polygon(-8.0d, -8.0d, -7.0d, -8.0d, -7.0d, -7.0d, -8.0d, -7.0d, -8.0d, -8.0d))) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-10.0d, -10.0d] as double[] + minMax[1] == [-7.0d, -7.0d] as double[] + } + + def 'a single negative point'() { + + given: + def geometry = Point.of(-118.0d, -33.8d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-118.0d, -33.8d] as double[] + minMax[1] == [-118.0d, -33.8d] as double[] + } + + def 'a negative line string'() { + + given: + def geometry = LineString.of(PositionList.of(Axes.XY, [-5.0d, -6.0d, -3.0d, -8.0d] as double[])) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-5.0d, -8.0d] as double[] + minMax[1] == [-3.0d, -6.0d] as double[] + } +} From 3dc8ab2591a0a4651268a55fb7d9845b31ab72cb Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Mon, 24 Aug 2026 12:27:51 +0200 Subject: [PATCH 2/3] features: use the GeoPackage spatial index for spatial predicates A spatial predicate on a GPKG provider was encoded as a bare ST_Intersects(geom, ...). Unlike PostGIS, where ST_Intersects embeds the && bounding box operator and GiST applies by itself, SQLite cannot infer from a spatial operator that the R-Tree of the geometry column is relevant: unless the query names the rtree__ virtual table, the operator is evaluated for every row. The dialect now contributes a bounding box predicate over that table, added as a conjunct to the exact predicate. On a 41.6 GB GeoPackage with 24.6 million rows, a bbox request for ten features goes from 149 s to 0.3 s while returning the same features. The conjunct is only added where it is implied by the exact predicate, so that it can change neither the result nor the meaning of the predicate under negation: - only for operators whose match implies that the bounding boxes intersect, which excludes S_DISJOINT - only for a geometry column of the main table, since the semi-join form of a joined property would need the predicate inside its subquery - only for a column that an R-Tree is known for, determined once per provider on startup - never below a negation, because a NULL geometry has no entry in the R-Tree while the exact predicate is NULL for it, and negating those two is not the same The index is joined on the primary key of the feature table rather than on rowid. A GeoPackage feature table is required to have an INTEGER PRIMARY KEY, which SQLite makes an alias of rowid, but a file that does not follow that requirement would otherwise match the wrong rows. --- .../features/sql/app/FilterEncoderSql.java | 268 ++++++++++++++++-- .../sql/domain/FeatureProviderSql.java | 22 +- .../features/sql/domain/SqlClientBasic.java | 4 + .../features/sql/domain/SqlDbmsAdapter.java | 14 + .../features/sql/domain/SqlDialect.java | 41 +++ .../features/sql/domain/SqlDialectGpkg.java | 37 +++ .../sql/infra/db/SqlDbmsAdapterGpkg.java | 42 +++ .../FilterEncoderSqlSpatialIndexSpec.groovy | 189 ++++++++++++ .../sql/domain/SqlDialectGpkgSpec.groovy | 52 ++++ 9 files changed, 650 insertions(+), 19 deletions(-) create mode 100644 xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy create mode 100644 xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java index 8c595e3f7..0c896998c 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java @@ -46,6 +46,7 @@ import de.ii.xtraplatform.cql.domain.Property; import de.ii.xtraplatform.cql.domain.Scalar; import de.ii.xtraplatform.cql.domain.ScalarLiteral; +import de.ii.xtraplatform.cql.domain.SpatialFunction; import de.ii.xtraplatform.cql.domain.SpatialOperation; import de.ii.xtraplatform.cql.domain.Temporal; import de.ii.xtraplatform.cql.domain.TemporalLiteral; @@ -71,6 +72,7 @@ import de.ii.xtraplatform.geometries.domain.PositionList; import de.ii.xtraplatform.geometries.domain.transform.CoordinatesTransformer; import de.ii.xtraplatform.geometries.domain.transform.ImmutableCrsTransform; +import de.ii.xtraplatform.geometries.domain.transform.MinMaxDeriver; import java.time.Instant; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -108,6 +110,10 @@ public class FilterEncoderSql { private final String accentiCollation; private final Map customFunctions; private final java.util.function.Function> mappingResolver; + // geometry columns with a spatial index that the query has to name explicitly, keyed by + // "table.column" in lower case and mapped to the column the index is keyed on; see + // SqlDbmsAdapter.getSpatialIndexes + private final Map spatialIndexes; BiFunction, Optional, Geometry> coordinatesTransformer; public FilterEncoderSql( @@ -148,6 +154,29 @@ public FilterEncoderSql( List customFunctions, String accentiCollation, java.util.function.Function> mappingResolver) { + this( + nativeCrs, + sqlDialect, + crsTransformerFactory, + crsInfo, + cql, + customFunctions, + accentiCollation, + mappingResolver, + Map.of()); + } + + public FilterEncoderSql( + EpsgCrs nativeCrs, + SqlDialect sqlDialect, + CrsTransformerFactory crsTransformerFactory, + CrsInfo crsInfo, + Cql cql, + List customFunctions, + String accentiCollation, + java.util.function.Function> mappingResolver, + Map spatialIndexes) { + this.spatialIndexes = spatialIndexes; this.nativeCrs = nativeCrs; this.sqlDialect = sqlDialect; this.crsTransformerFactory = crsTransformerFactory; @@ -166,6 +195,123 @@ public FilterEncoderSql( this.coordinatesTransformer = this::transformCoordinatesIfNecessary; } + /** + * Adds the spatial index predicate of the dialect to an exact spatial predicate, as a conjunct. + * + *

The conjunct is only added where it is implied by the exact predicate, so that it can change + * neither the result nor the meaning of the predicate under negation: the operator has to be one + * whose match implies that the bounding boxes intersect, the geometry has to be a column of the + * main table that a spatial index is known for, and the other operand has to be a geometry + * literal whose bounding box is known. + * + * @param literalEnvelopes the bounding boxes of the geometry literals of the filter being + * encoded, keyed by the SQL they were encoded to + * @param acceleratedPredicates records the added conjunct, so that {@link + * #withoutSpatialIndexPredicates} can drop it again under a negation + * @param geometryTableAndColumn resolves the table and column of the geometry operand, evaluated + * only once everything else already matched + */ + private String withSpatialIndexPredicate( + SpatialFunction operator, + List children, + String mainExpression, + String predicate, + Map literalEnvelopes, + Map acceleratedPredicates, + Supplier>> geometryTableAndColumn) { + if (spatialIndexes.isEmpty() + || !SqlDialect.SPATIAL_OPERATORS_IMPLYING_BBOX_INTERSECTION.contains(operator) + // Only the direct conjunct form of a property reference addresses the geometry as a column + // of the main table (see visit(Property)). In the semi-join form the geometry belongs to a + // joined table, where the predicate would have to go inside the subquery instead. + || !(mainExpression.startsWith("%1$s") && mainExpression.endsWith("%2$s"))) { + return predicate; + } + + double[][] envelope = + children.stream() + .map(literalEnvelopes::get) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + + if (Objects.isNull(envelope)) { + return predicate; + } + + return geometryTableAndColumn + .get() + .flatMap( + tableColumn -> + Optional.ofNullable( + spatialIndexes.get( + String.format("%s.%s", tableColumn.first(), tableColumn.second()) + .toLowerCase(Locale.ROOT))) + .flatMap( + keyColumn -> + sqlDialect.getSpatialIndexPredicate( + tableColumn.first(), + tableColumn.second(), + "A", + keyColumn, + envelope[0], + envelope[1]))) + .map( + indexPredicate -> { + String accelerated = String.format("(%s AND %s)", indexPredicate, predicate); + // remembered so that a negation can drop it again, see + // withoutSpatialIndexPredicates + acceleratedPredicates.put(accelerated, predicate); + return accelerated; + }) + .orElse(predicate); + } + + /** + * Removes the spatial index predicates that were added to the operand of a negation. + * + *

The conjunct is implied by the exact predicate for every row that has a geometry, so adding + * it changes nothing there, negated or not. A row whose geometry is NULL is the exception: it has + * no entry in the spatial index, so the conjunct is false for it, while the exact predicate is + * NULL. Both keep the row out of a positive result, but negated the one yields true and the other + * NULL. Dropping the conjunct under a negation keeps that case as it was. + */ + private static String withoutSpatialIndexPredicates( + String expression, Map acceleratedPredicates) { + String withoutIndexPredicates = expression; + + for (Entry accelerated : acceleratedPredicates.entrySet()) { + withoutIndexPredicates = + withoutIndexPredicates.replace(accelerated.getKey(), accelerated.getValue()); + } + + return withoutIndexPredicates; + } + + /** + * The bounding box of a geometry literal, derived after the same transformation that {@link + * de.ii.xtraplatform.cql.domain.CqlToText} applies before encoding it, so that the bounding box + * is guaranteed to describe the geometry that ends up in the query. Null for an empty geometry. + */ + private double[][] envelopeOf(GeometryNode geometry) { + return coordinatesTransformer + .apply(geometry.getGeometry(), geometry.getCrs().or(() -> geometry.getGeometry().getCrs())) + .accept(new MinMaxDeriver()); + } + + /** + * The column of a plain {@code A.} reference. Anything else — a sub-decoder expression, a + * date function — is not a column that a spatial index could be looked up for. + */ + private static Optional plainColumnOfMainTable(String qualifiedColumn) { + if (!qualifiedColumn.startsWith("A.") + || qualifiedColumn.indexOf('(') >= 0 + || qualifiedColumn.indexOf(' ') >= 0) { + return Optional.empty(); + } + return Optional.of(qualifiedColumn.substring(2)); + } + private Optional renderCustomFunction( de.ii.xtraplatform.cql.domain.Function function, List children) { CustomFunction customFunction = @@ -617,6 +763,12 @@ private static Predicate getPropertyNameMatcher( private class CqlToSql extends CqlToText { private final SchemaSql rootSchema; + // bounding boxes of the geometry literals of the filter being encoded, keyed by the SQL they + // were encoded to; a visitor encodes a single filter, so this neither leaks nor is shared + private final Map literalEnvelopes = new LinkedHashMap<>(); + // spatial predicates that a spatial index predicate was added to, mapped to the predicate + // without it + private final Map acceleratedPredicates = new LinkedHashMap<>(); private CqlToSql(SchemaSql rootSchema) { super(coordinatesTransformer); @@ -1260,13 +1412,41 @@ public String visit(BinarySpatialOperation spatialOperation, List childr List expressions = processBinary(spatialOperation.getArgs(), children); - return String.format( + String predicate = + String.format( + expressions.get(0), + String.format("%s(", operator.first()), + operator + .second() + .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) + .orElse(String.format(", %s)%s", expressions.get(1), match))); + + return withSpatialIndexPredicate( + spatialOperation.getSpatialOperator(), + children, expressions.get(0), - String.format("%s(", operator.first()), - operator - .second() - .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) - .orElse(String.format(", %s)%s", expressions.get(1), match))); + predicate, + literalEnvelopes, + acceleratedPredicates, + () -> geometryTableAndColumn(spatialOperation)); + } + + /** The table and column of the geometry operand of a spatial predicate, if it is a column. */ + private Optional> geometryTableAndColumn( + BinarySpatialOperation spatialOperation) { + return spatialOperation.getArgs().stream() + .filter(Property.class::isInstance) + .map(Property.class::cast) + .findFirst() + .flatMap( + property -> { + String propertyName = property.getName().replaceAll("^\"|\"$", ""); + boolean allowColumnFallback = !propertyName.contains("."); + SchemaSql table = getTable(propertyName, false, allowColumnFallback); + return plainColumnOfMainTable( + getQualifiedColumn(table, propertyName, "A", allowColumnFallback).first()) + .map(column -> Tuple.of(rootSchema.getName(), column)); + }); } @Override @@ -1308,7 +1488,10 @@ public String visit(TemporalLiteral temporalLiteral, List children) { @Override public String visit(GeometryNode geometry, List children) { - return sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + String expression = + sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + literalEnvelopes.computeIfAbsent(expression, ignore -> envelopeOf(geometry)); + return expression; } @Override @@ -1510,7 +1693,7 @@ public String visit(LogicalOperation logicalOperation, List children) { public String visit(Not not, List children) { String operator = LOGICAL_OPERATORS.get(not.getClass()); - String operation = children.get(0); + String operation = withoutSpatialIndexPredicates(children.get(0), acceleratedPredicates); if (operation.contains("(SELECT")) { // The child predicate is (or contains) an EXISTS-style semi-join on a joined property // (A.id IN (SELECT ... WHERE )). The string surgery below would push the @@ -1560,7 +1743,9 @@ public String visit(Not not, List children) { operation.substring(0, pos), operator, operation.substring(pos + 1, length)); } - return super.visit(not, children); + // operation, not children: Not is unary, and operation is the operand with any spatial index + // predicate removed + return super.visit(not, ImmutableList.of(operation)); } @Override @@ -1628,6 +1813,13 @@ private CqlToSql2(SqlQueryMapping mapping) { this(mapping, null); } + // bounding boxes of the geometry literals of the filter being encoded, keyed by the SQL they + // were encoded to; a visitor encodes a single filter, so this neither leaks nor is shared + private final Map literalEnvelopes = new LinkedHashMap<>(); + // spatial predicates that a spatial index predicate was added to, mapped to the predicate + // without it + private final Map acceleratedPredicates = new LinkedHashMap<>(); + private CqlToSql2(SqlQueryMapping mapping, CteCollector collector) { super(coordinatesTransformer); this.mapping = mapping; @@ -2393,13 +2585,48 @@ public String visit(BinarySpatialOperation spatialOperation, List childr List expressions = processBinary(spatialOperation.getArgs(), children); - return String.format( + String predicate = + String.format( + expressions.get(0), + String.format("%s(", operator.first()), + operator + .second() + .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) + .orElse(String.format(", %s)%s", expressions.get(1), match))); + + return withSpatialIndexPredicate( + spatialOperation.getSpatialOperator(), + children, expressions.get(0), - String.format("%s(", operator.first()), - operator - .second() - .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) - .orElse(String.format(", %s)%s", expressions.get(1), match))); + predicate, + literalEnvelopes, + acceleratedPredicates, + () -> geometryTableAndColumn(spatialOperation)); + } + + /** The table and column of the geometry operand of a spatial predicate, if it is a column. */ + private Optional> geometryTableAndColumn( + BinarySpatialOperation spatialOperation) { + return spatialOperation.getArgs().stream() + .filter(Property.class::isInstance) + .map(Property.class::cast) + .findFirst() + .flatMap( + property -> { + String propertyName = property.getName().replaceAll("^\"|\"$", ""); + boolean allowColumnFallback = !propertyName.contains("."); + de.ii.xtraplatform.base.domain.util.Tuple table = + getTableColumn(propertyName, false, allowColumnFallback); + return plainColumnOfMainTable( + getQualifiedColumn( + table.first(), + table.second(), + propertyName, + "A", + allowColumnFallback) + .first()) + .map(column -> Tuple.of(mapping.getMainTable().getName(), column)); + }); } @Override @@ -2441,7 +2668,10 @@ public String visit(TemporalLiteral temporalLiteral, List children) { @Override public String visit(GeometryNode geometry, List children) { - return sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + String expression = + sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + literalEnvelopes.computeIfAbsent(expression, ignore -> envelopeOf(geometry)); + return expression; } @Override @@ -2645,7 +2875,7 @@ public String visit(LogicalOperation logicalOperation, List children) { public String visit(Not not, List children) { String operator = LOGICAL_OPERATORS.get(not.getClass()); - String operation = children.get(0); + String operation = withoutSpatialIndexPredicates(children.get(0), acceleratedPredicates); if (operation.contains("(SELECT")) { // The child predicate is (or contains) an EXISTS-style semi-join on a joined property // (A.id IN (SELECT ... WHERE )). The string surgery below would push the @@ -2695,7 +2925,9 @@ public String visit(Not not, List children) { operation.substring(0, pos), operator, operation.substring(pos + 1, length)); } - return super.visit(not, children); + // operation, not children: Not is unary, and operation is the operand with any spatial index + // predicate removed + return super.visit(not, ImmutableList.of(operation)); } @Override diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 2110d7475..147c01eb1 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -114,6 +114,7 @@ import de.ii.xtraplatform.streams.domain.Reactive.Stream; import de.ii.xtraplatform.streams.domain.Reactive.Transformer; import de.ii.xtraplatform.values.domain.ValueStore; +import java.sql.SQLException; import java.time.ZoneId; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; @@ -613,7 +614,8 @@ protected boolean onStartup() throws InterruptedException { type -> Optional.ofNullable(queryMappings.get(type)) .filter(mappings -> mappings.size() == 1) - .map(mappings -> mappings.get(0))); + .map(mappings -> mappings.get(0)), + getSpatialIndexes()); AggregateStatsQueryGenerator queryGeneratorSql = new AggregateStatsQueryGenerator(sqlDialect, filterEncoder); @@ -667,6 +669,24 @@ protected boolean onStartup() throws InterruptedException { return true; } + /** + * The geometry columns with a spatial index that the query generator has to name explicitly to + * make use of. Without it, spatial queries keep working, only without the index, so a failure to + * determine it is logged and does not keep the provider from starting. + */ + private Map getSpatialIndexes() { + try { + return getSqlClient().getSpatialIndexes(); + } catch (SQLException | RuntimeException e) { + LogContext.errorAsWarn( + LOGGER, + e, + "Could not determine the spatial indexes of the feature provider with id '{}'. Spatial queries will not use a spatial index.", + getId()); + return Map.of(); + } + } + @Override protected void onStarted() { changes() diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java index 1a9264355..5b5122a5a 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java @@ -27,4 +27,8 @@ default DbInfo getDbInfo() throws SQLException { default Map getGeoInfo() throws SQLException { return getDbmsAdapter().getGeoInfo(getConnection(), getDbInfo()); } + + default Map getSpatialIndexes() throws SQLException { + return getDbmsAdapter().getSpatialIndexes(this); + } } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java index 34a0dec60..5eabde8c6 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java @@ -40,6 +40,20 @@ public interface SqlDbmsAdapter { Map getGeoInfo(Connection connection, DbInfo dbInfo) throws SQLException; + /** + * Returns the geometry columns that have a spatial index which the query generator has to name + * explicitly to make use of, keyed by {@code table.column} in lower case (SQL identifiers are + * compared case-insensitively here, as SQLite does for ASCII). The value is the column of the + * table that the index entries are keyed on, which a query has to join the index on. + * + *

The default is an empty map, which is correct for every DBMS whose spatial operators consult + * the spatial index by themselves. It takes the client rather than a connection so that those + * adapters do not have to be handed one they will not use. + */ + default Map getSpatialIndexes(SqlClientBasic sqlClient) throws SQLException { + return Map.of(); + } + DbInfo getDbInfo(Connection connection) throws SQLException; Collator getRowSortingCollator(Optional defaultCollation); diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java index 4ba6e8d2f..2c3733e1b 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java @@ -138,6 +138,28 @@ default String getSpatialOperatorMatch(SpatialFunction spatialFunction) { return ""; } + /** + * Returns a predicate that restricts the rows to those whose geometry bounding box intersects the + * bounding box given as {@code min}/{@code max}, evaluated with a spatial index on {@code + * table}.{@code column}, or an empty optional if the dialect cannot use a spatial index that way. + * + *

The predicate is a conjunct that is added to the exact spatial predicate, never a + * replacement for it. Dialects whose spatial operators already consult the spatial index by + * themselves (such as PostGIS, where {@code ST_Intersects} includes the {@code &&} bounding box + * operator) do not implement this. + * + * @param table the table holding the geometry column + * @param column the geometry column + * @param alias the alias the table is referenced by in the query + * @param keyColumn the column of the table that the index entries are keyed on + * @param min the lower corner of the bounding box, in the native CRS + * @param max the upper corner of the bounding box, in the native CRS + */ + default Optional getSpatialIndexPredicate( + String table, String column, String alias, String keyColumn, double[] min, double[] max) { + return Optional.empty(); + } + default String getTemporalOperator(TemporalFunction temporalFunction) { // this is implementation specific return null; @@ -166,4 +188,23 @@ default String applyToExpression( Map SPATIAL_OPERATORS_3D = new ImmutableMap.Builder().build(); + + /** + * The spatial operators for which a match implies that the bounding boxes of both operands + * intersect. Only for these may a bounding box predicate from a spatial index be added as a + * conjunct: the conjunct is then implied by the exact predicate, so it changes neither the result + * nor the meaning of the predicate under negation. + * + *

{@code S_DISJOINT} is absent because two geometries can be disjoint while being arbitrarily + * far apart, so a bounding box predicate would discard matching rows. + */ + Set SPATIAL_OPERATORS_IMPLYING_BBOX_INTERSECTION = + ImmutableSet.of( + SpatialFunction.S_EQUALS, + SpatialFunction.S_TOUCHES, + SpatialFunction.S_WITHIN, + SpatialFunction.S_OVERLAPS, + SpatialFunction.S_CROSSES, + SpatialFunction.S_INTERSECTS, + SpatialFunction.S_CONTAINS); } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java index 42af4c0b7..797bb8554 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java @@ -206,6 +206,43 @@ public String escapeString(String value) { return value.replaceAll("'", "''"); } + /** + * GeoPackage keeps the bounding boxes of a geometry column in an R-Tree, a virtual table named + * {@code rtree_

_}. SQLite cannot infer from a spatial operator that such a table + * is relevant, so unless the query names it, the operator is evaluated for every row of the + * table. The predicate returned here names it, which turns the table scan into an index lookup. + * + *

The R-Tree stores the bounding boxes as 32-bit floats, rounded outwards, so the lookup never + * discards a geometry that the exact predicate would have matched. + */ + @Override + public Optional getSpatialIndexPredicate( + String table, String column, String alias, String keyColumn, double[] min, double[] max) { + if (min.length < 2 || max.length < 2) { + return Optional.empty(); + } + + return Optional.of( + String.format( + "%s.%s IN (SELECT id FROM %s WHERE maxx >= %s AND minx <= %s AND maxy >= %s AND miny <= %s)", + alias, + quoteIdentifier(keyColumn), + quoteIdentifier(String.format("rtree_%s_%s", table, column)), + toNumericLiteral(min[0]), + toNumericLiteral(max[0]), + toNumericLiteral(min[1]), + toNumericLiteral(max[1]))); + } + + private static String quoteIdentifier(String identifier) { + return String.format("\"%s\"", identifier.replace("\"", "\"\"")); + } + + private static String toNumericLiteral(double value) { + // Double.toString is independent of the default locale, unlike String.format("%f", ...) + return Double.toString(value); + } + @Override public String applyToExpression( String table, String name, Map subDecoderPaths, boolean spatial) { diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java index 96eaa17c9..735e4bce0 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java @@ -9,12 +9,14 @@ import com.github.azahnen.dagger.annotations.AutoBind; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import de.ii.xtraplatform.base.domain.AppContext; import de.ii.xtraplatform.blobs.domain.ResourceStore; import de.ii.xtraplatform.crs.domain.EpsgCrs; import de.ii.xtraplatform.crs.domain.EpsgCrs.Force; import de.ii.xtraplatform.features.sql.domain.ConnectionInfoSql; import de.ii.xtraplatform.features.sql.domain.ImmutableGeoInfo; +import de.ii.xtraplatform.features.sql.domain.SqlClientBasic; import de.ii.xtraplatform.features.sql.domain.SqlDbmsAdapter; import de.ii.xtraplatform.features.sql.domain.SqlDialect; import de.ii.xtraplatform.features.sql.domain.SqlDialectGpkg; @@ -172,6 +174,46 @@ public List getSystemTables() { "SpatialIndex"); } + /** + * Reports the geometry columns that have a GeoPackage R-Tree, mapped to the column that the + * R-Tree entries are keyed on. + * + *

The R-Tree virtual table itself is looked up instead of the {@code gpkg_rtree_index} row in + * {@code gpkg_extensions}, because it is the table that the generated query has to name, and the + * two can disagree: the extension may be registered for a column whose R-Tree was dropped. + * + *

The key column is the primary key of the feature table, which is what the R-Tree triggers + * store. It is reported instead of assuming {@code rowid}: a GeoPackage feature table is required + * to have an {@code INTEGER PRIMARY KEY}, which SQLite makes an alias of {@code rowid}, but in a + * file that does not follow that requirement the two differ and joining on {@code rowid} would + * match the wrong rows. Feature tables with a composite primary key are left out, as their rows + * cannot be addressed by the single id column of an R-Tree. + */ + @Override + public Map getSpatialIndexes(SqlClientBasic sqlClient) throws SQLException { + String query = + "SELECT gc.table_name, gc.column_name, ti.name FROM gpkg_geometry_columns gc" + + " JOIN sqlite_master m ON m.type = 'table' AND lower(m.name) = lower('rtree_' || gc.table_name || '_' || gc.column_name)" + + " JOIN pragma_table_info(gc.table_name) ti ON ti.pk = 1" + + " GROUP BY gc.table_name, gc.column_name HAVING count(*) = 1;"; + + ImmutableMap.Builder spatialIndexes = ImmutableMap.builder(); + + // the connection comes from the pool, closing it hands it back + try (Connection connection = sqlClient.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query)) { + while (resultSet.next()) { + spatialIndexes.put( + String.format("%s.%s", resultSet.getString(1), resultSet.getString(2)) + .toLowerCase(Locale.ROOT), + resultSet.getString(3)); + } + } + + return spatialIndexes.build(); + } + @Override public Map getGeoInfo(Connection connection, DbInfo dbInfo) throws SQLException { if (!(dbInfo instanceof DbInfoGpkg) diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy new file mode 100644 index 000000000..1738a9beb --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy @@ -0,0 +1,189 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import de.ii.xtraplatform.cql.app.CqlFilterExamples +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.cql.domain.And +import de.ii.xtraplatform.cql.domain.Bbox +import de.ii.xtraplatform.cql.domain.IsNull +import de.ii.xtraplatform.cql.domain.Not +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.SDisjoint +import de.ii.xtraplatform.cql.domain.SIntersects +import de.ii.xtraplatform.cql.domain.SWithin +import de.ii.xtraplatform.cql.domain.SpatialLiteral +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.FeatureSchemaFixtures +import de.ii.xtraplatform.features.domain.MappingOperationResolver +import de.ii.xtraplatform.features.domain.MappingRuleFixtures +import de.ii.xtraplatform.features.json.app.DecoderFactoryJson +import de.ii.xtraplatform.features.domain.SchemaBase +import de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings +import de.ii.xtraplatform.features.sql.domain.ImmutableSchemaSql +import de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults +import de.ii.xtraplatform.features.sql.domain.SqlDialectGpkg +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import de.ii.xtraplatform.features.sql.domain.SqlPathParser +import de.ii.xtraplatform.features.sql.domain.SqlQueryMapping +import spock.lang.Shared +import spock.lang.Specification + +import java.util.function.Function + +class FilterEncoderSqlSpatialIndexSpec extends Specification { + + static final String EXACT_WITHIN = "ST_Within(A.location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326))" + static final String RTREE_LOOKUP = "A.\"id\" IN (SELECT id FROM \"rtree_building_location\" WHERE maxx >= -118.0 AND minx <= -117.9 AND maxy >= 33.8 AND miny <= 34.0)" + + @Shared + SqlQueryMapping unfaelleMapping + + def setupSpec() { + def cql = new CqlImpl() + def pathParser = new SqlPathParser(new ImmutableSqlPathDefaults.Builder().build(), cql, + Map.of("JSON", new DecoderFactoryJson(), "EXPRESSION", new DecoderFactorySqlExpression())) + def mappingDeriver = new SqlMappingDeriver(pathParser, new ImmutableQueryGeneratorSettings.Builder().build()) + def schema = FeatureSchemaFixtures.fromYaml("strassen_unfaelle2") + def resolved = schema.accept(new MappingOperationResolver(), List.of()) + unfaelleMapping = mappingDeriver.derive(MappingRuleFixtures.fromYaml("strassen_unfaelle2"), resolved).get(0) + } + + static FilterEncoderSql encoder(dialect, Map spatialIndexes) { + return new FilterEncoderSql(OgcCrs.CRS84, dialect, null, null, new CqlImpl(), List.of(), null, + { type -> Optional.empty() } as Function, spatialIndexes) + } + + def 'gpkg, indexed geometry column on the main table: r-tree lookup is added as a conjunct'() { + + given: 'a GeoPackage provider whose geometry column has an r-tree' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: 'a bbox filter on that column is encoded' + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the r-tree is queried in addition to the exact predicate' + actual == "(${RTREE_LOOKUP} AND ${EXACT_WITHIN})" + } + + def 'gpkg, geometry column without an r-tree: predicate is unchanged'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("other_table.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == EXACT_WITHIN + } + + def 'table and column are matched case-insensitively, as sqlite compares identifiers'() { + + given: 'a source path that capitalises the column differently than the database reports it' + def schema = new ImmutableSchemaSql.Builder() + .name("Building") + .type(SchemaBase.Type.OBJECT) + .sortKey("id") + .addProperties(new ImmutableSchemaSql.Builder() + .name("Location") + .sourcePath("Location") + .type(SchemaBase.Type.GEOMETRY) + .parentPath(["Building"]) + .build()) + .build() + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: + def filter = SWithin.of(Property.of("Location"), + SpatialLiteral.of(Bbox.of(-118.0d, 33.8d, -117.9d, 34.0d, OgcCrs.CRS84))) + String actual = filterEncoder.encode(filter, schema) + + then: 'the r-tree of that column is still found' + actual == "(A.\"id\" IN (SELECT id FROM \"rtree_Building_Location\" WHERE maxx >= -118.0 AND minx <= -117.9 AND maxy >= 33.8 AND miny <= 34.0)" + + " AND ST_Within(A.Location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326)))" + } + + def 'postgis: no conjunct is added, ST_Within consults the index by itself'() { + + given: + def filterEncoder = encoder(new SqlDialectPgis(), Map.of("building.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == EXACT_WITHIN + } + + def 'S_DISJOINT is not accelerated, a disjoint geometry may lie outside the bbox'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def filter = SDisjoint.of(Property.of("location"), + SpatialLiteral.of(Bbox.of(-118.0, 33.8, -117.9, 34.0, OgcCrs.CRS84))) + + when: + String actual = filterEncoder.encode(filter, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the exact predicate is emitted on its own' + actual == "ST_Disjoint(A.location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326))" + } + + def 'under NOT the conjunct is dropped, a NULL geometry has no r-tree entry'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: + String actual = filterEncoder.encode(Not.of(CqlFilterExamples.EXAMPLE_15), QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the exact predicate is negated on its own, as it was before' + actual == "NOT (${EXACT_WITHIN})" + } + + def 'a negation elsewhere in the filter does not stop the conjunct'() { + + given: 'a filter that negates an unrelated predicate and intersects the geometry' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def filter = And.of(Not.of(IsNull.of(Property.of("location"))), CqlFilterExamples.EXAMPLE_15) + + when: + String actual = filterEncoder.encode(filter, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == "(A.location IS NOT NULL AND (${RTREE_LOOKUP} AND ${EXACT_WITHIN}))" + } + + def 'geometry reached through a join is not accelerated, the conjunct would address the wrong table'() { + + given: 'the geometry lives in a joined table, so the predicate goes inside a semi-join' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id", "geometry.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_16, QuerySchemaFixtures.JOINED_GEOMETRY) + + then: + actual == "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE ST_Intersects(AB.location, ST_GeomFromText('POLYGON((-10.0 -10.0,10.0 -10.0,10.0 10.0,-10.0 -10.0))',4326)))" + } + + def 'the mapping based encoder, which the items query uses, is accelerated too'() { + + given: 'a filter on the primary geometry of a mapping whose column has an r-tree' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("unfaelle_point.geom", "fid")) + def filter = SIntersects.of(Property.of("geometry"), + SpatialLiteral.of(Bbox.of(7.0, 50.0, 7.1, 50.1, OgcCrs.CRS84))) + + when: + String actual = filterEncoder.encode(filter, unfaelleMapping) + + then: + actual == "(A.\"fid\" IN (SELECT id FROM \"rtree_unfaelle_point_geom\" WHERE maxx >= 7.0 AND minx <= 7.1 AND maxy >= 50.0 AND miny <= 50.1) AND " + + "ST_Intersects(A.geom, ST_GeomFromText('POLYGON((7.0 50.0,7.1 50.0,7.1 50.1,7.0 50.1,7.0 50.0))',4326)))" + } +} diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy new file mode 100644 index 000000000..b694030aa --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy @@ -0,0 +1,52 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.domain + +import spock.lang.Shared +import spock.lang.Specification + +class SqlDialectGpkgSpec extends Specification { + + @Shared + SqlDialectGpkg dialect + + def setupSpec() { + dialect = new SqlDialectGpkg() + } + + def 'the spatial index predicate names the r-tree and the key column'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + "pv_pot_dach", "geom", "A", "id", [365204.0d, 5621522.0d] as double[], [365938.0d, 5622652.0d] as double[]) + + then: + actual.get() == 'A."id" IN (SELECT id FROM "rtree_pv_pot_dach_geom"' + + ' WHERE maxx >= 365204.0 AND minx <= 365938.0 AND maxy >= 5621522.0 AND miny <= 5622652.0)' + } + + def 'a bounding box without two axes yields no predicate'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + "t", "geom", "A", "id", [1.0d] as double[], [2.0d] as double[]) + + then: + actual.isEmpty() + } + + def 'identifiers are quoted, and embedded quotes escaped'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + 'we"ird', "geom", "A", 'k"ey', [1.0d, 2.0d] as double[], [3.0d, 4.0d] as double[]) + + then: + actual.get().startsWith('A."k""ey" IN (SELECT id FROM "rtree_we""ird_geom"') + } +} From b85ca02a366e707a51462f9e753ef362b3852996 Mon Sep 17 00:00:00 2001 From: Clemens Portele Date: Mon, 24 Aug 2026 12:27:51 +0200 Subject: [PATCH 3/3] features: use the GeoPackage spatial index for spatial predicates A spatial predicate on a GPKG provider was encoded as a bare ST_Intersects(geom, ...). Unlike PostGIS, where ST_Intersects embeds the && bounding box operator and GiST applies by itself, SQLite cannot infer from a spatial operator that the R-Tree of the geometry column is relevant: unless the query names the rtree_

_ virtual table, the operator is evaluated for every row. The dialect now contributes a bounding box predicate over that table, added as a conjunct to the exact predicate. On a 41.6 GB GeoPackage with 24.6 million rows, a bbox request for ten features goes from 149 s to 0.3 s while returning the same features. The conjunct is only added where it is implied by the exact predicate, so that it can change neither the result nor the meaning of the predicate under negation: - only for operators whose match implies that the bounding boxes intersect, which excludes S_DISJOINT - only for a geometry column of the main table, since the semi-join form of a joined property would need the predicate inside its subquery - only for a column that an R-Tree is known for, determined once per provider on startup - never below a negation, because a NULL geometry has no entry in the R-Tree while the exact predicate is NULL for it, and negating those two is not the same The index is joined on the primary key of the feature table rather than on rowid. A GeoPackage feature table is required to have an INTEGER PRIMARY KEY, which SQLite makes an alias of rowid, but a file that does not follow that requirement would otherwise match the wrong rows. --- .../features/sql/app/FilterEncoderSql.java | 268 ++++++++++++++++-- .../sql/domain/FeatureProviderSql.java | 22 +- .../features/sql/domain/SqlClientBasic.java | 4 + .../features/sql/domain/SqlDbmsAdapter.java | 14 + .../features/sql/domain/SqlDialect.java | 41 +++ .../features/sql/domain/SqlDialectGpkg.java | 37 +++ .../sql/infra/db/SqlDbmsAdapterGpkg.java | 42 +++ .../FilterEncoderSqlSpatialIndexSpec.groovy | 206 ++++++++++++++ .../sql/domain/SqlDialectGpkgSpec.groovy | 52 ++++ 9 files changed, 667 insertions(+), 19 deletions(-) create mode 100644 xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy create mode 100644 xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java index 8c595e3f7..0c896998c 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java @@ -46,6 +46,7 @@ import de.ii.xtraplatform.cql.domain.Property; import de.ii.xtraplatform.cql.domain.Scalar; import de.ii.xtraplatform.cql.domain.ScalarLiteral; +import de.ii.xtraplatform.cql.domain.SpatialFunction; import de.ii.xtraplatform.cql.domain.SpatialOperation; import de.ii.xtraplatform.cql.domain.Temporal; import de.ii.xtraplatform.cql.domain.TemporalLiteral; @@ -71,6 +72,7 @@ import de.ii.xtraplatform.geometries.domain.PositionList; import de.ii.xtraplatform.geometries.domain.transform.CoordinatesTransformer; import de.ii.xtraplatform.geometries.domain.transform.ImmutableCrsTransform; +import de.ii.xtraplatform.geometries.domain.transform.MinMaxDeriver; import java.time.Instant; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -108,6 +110,10 @@ public class FilterEncoderSql { private final String accentiCollation; private final Map customFunctions; private final java.util.function.Function> mappingResolver; + // geometry columns with a spatial index that the query has to name explicitly, keyed by + // "table.column" in lower case and mapped to the column the index is keyed on; see + // SqlDbmsAdapter.getSpatialIndexes + private final Map spatialIndexes; BiFunction, Optional, Geometry> coordinatesTransformer; public FilterEncoderSql( @@ -148,6 +154,29 @@ public FilterEncoderSql( List customFunctions, String accentiCollation, java.util.function.Function> mappingResolver) { + this( + nativeCrs, + sqlDialect, + crsTransformerFactory, + crsInfo, + cql, + customFunctions, + accentiCollation, + mappingResolver, + Map.of()); + } + + public FilterEncoderSql( + EpsgCrs nativeCrs, + SqlDialect sqlDialect, + CrsTransformerFactory crsTransformerFactory, + CrsInfo crsInfo, + Cql cql, + List customFunctions, + String accentiCollation, + java.util.function.Function> mappingResolver, + Map spatialIndexes) { + this.spatialIndexes = spatialIndexes; this.nativeCrs = nativeCrs; this.sqlDialect = sqlDialect; this.crsTransformerFactory = crsTransformerFactory; @@ -166,6 +195,123 @@ public FilterEncoderSql( this.coordinatesTransformer = this::transformCoordinatesIfNecessary; } + /** + * Adds the spatial index predicate of the dialect to an exact spatial predicate, as a conjunct. + * + *

The conjunct is only added where it is implied by the exact predicate, so that it can change + * neither the result nor the meaning of the predicate under negation: the operator has to be one + * whose match implies that the bounding boxes intersect, the geometry has to be a column of the + * main table that a spatial index is known for, and the other operand has to be a geometry + * literal whose bounding box is known. + * + * @param literalEnvelopes the bounding boxes of the geometry literals of the filter being + * encoded, keyed by the SQL they were encoded to + * @param acceleratedPredicates records the added conjunct, so that {@link + * #withoutSpatialIndexPredicates} can drop it again under a negation + * @param geometryTableAndColumn resolves the table and column of the geometry operand, evaluated + * only once everything else already matched + */ + private String withSpatialIndexPredicate( + SpatialFunction operator, + List children, + String mainExpression, + String predicate, + Map literalEnvelopes, + Map acceleratedPredicates, + Supplier>> geometryTableAndColumn) { + if (spatialIndexes.isEmpty() + || !SqlDialect.SPATIAL_OPERATORS_IMPLYING_BBOX_INTERSECTION.contains(operator) + // Only the direct conjunct form of a property reference addresses the geometry as a column + // of the main table (see visit(Property)). In the semi-join form the geometry belongs to a + // joined table, where the predicate would have to go inside the subquery instead. + || !(mainExpression.startsWith("%1$s") && mainExpression.endsWith("%2$s"))) { + return predicate; + } + + double[][] envelope = + children.stream() + .map(literalEnvelopes::get) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + + if (Objects.isNull(envelope)) { + return predicate; + } + + return geometryTableAndColumn + .get() + .flatMap( + tableColumn -> + Optional.ofNullable( + spatialIndexes.get( + String.format("%s.%s", tableColumn.first(), tableColumn.second()) + .toLowerCase(Locale.ROOT))) + .flatMap( + keyColumn -> + sqlDialect.getSpatialIndexPredicate( + tableColumn.first(), + tableColumn.second(), + "A", + keyColumn, + envelope[0], + envelope[1]))) + .map( + indexPredicate -> { + String accelerated = String.format("(%s AND %s)", indexPredicate, predicate); + // remembered so that a negation can drop it again, see + // withoutSpatialIndexPredicates + acceleratedPredicates.put(accelerated, predicate); + return accelerated; + }) + .orElse(predicate); + } + + /** + * Removes the spatial index predicates that were added to the operand of a negation. + * + *

The conjunct is implied by the exact predicate for every row that has a geometry, so adding + * it changes nothing there, negated or not. A row whose geometry is NULL is the exception: it has + * no entry in the spatial index, so the conjunct is false for it, while the exact predicate is + * NULL. Both keep the row out of a positive result, but negated the one yields true and the other + * NULL. Dropping the conjunct under a negation keeps that case as it was. + */ + private static String withoutSpatialIndexPredicates( + String expression, Map acceleratedPredicates) { + String withoutIndexPredicates = expression; + + for (Entry accelerated : acceleratedPredicates.entrySet()) { + withoutIndexPredicates = + withoutIndexPredicates.replace(accelerated.getKey(), accelerated.getValue()); + } + + return withoutIndexPredicates; + } + + /** + * The bounding box of a geometry literal, derived after the same transformation that {@link + * de.ii.xtraplatform.cql.domain.CqlToText} applies before encoding it, so that the bounding box + * is guaranteed to describe the geometry that ends up in the query. Null for an empty geometry. + */ + private double[][] envelopeOf(GeometryNode geometry) { + return coordinatesTransformer + .apply(geometry.getGeometry(), geometry.getCrs().or(() -> geometry.getGeometry().getCrs())) + .accept(new MinMaxDeriver()); + } + + /** + * The column of a plain {@code A.} reference. Anything else — a sub-decoder expression, a + * date function — is not a column that a spatial index could be looked up for. + */ + private static Optional plainColumnOfMainTable(String qualifiedColumn) { + if (!qualifiedColumn.startsWith("A.") + || qualifiedColumn.indexOf('(') >= 0 + || qualifiedColumn.indexOf(' ') >= 0) { + return Optional.empty(); + } + return Optional.of(qualifiedColumn.substring(2)); + } + private Optional renderCustomFunction( de.ii.xtraplatform.cql.domain.Function function, List children) { CustomFunction customFunction = @@ -617,6 +763,12 @@ private static Predicate getPropertyNameMatcher( private class CqlToSql extends CqlToText { private final SchemaSql rootSchema; + // bounding boxes of the geometry literals of the filter being encoded, keyed by the SQL they + // were encoded to; a visitor encodes a single filter, so this neither leaks nor is shared + private final Map literalEnvelopes = new LinkedHashMap<>(); + // spatial predicates that a spatial index predicate was added to, mapped to the predicate + // without it + private final Map acceleratedPredicates = new LinkedHashMap<>(); private CqlToSql(SchemaSql rootSchema) { super(coordinatesTransformer); @@ -1260,13 +1412,41 @@ public String visit(BinarySpatialOperation spatialOperation, List childr List expressions = processBinary(spatialOperation.getArgs(), children); - return String.format( + String predicate = + String.format( + expressions.get(0), + String.format("%s(", operator.first()), + operator + .second() + .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) + .orElse(String.format(", %s)%s", expressions.get(1), match))); + + return withSpatialIndexPredicate( + spatialOperation.getSpatialOperator(), + children, expressions.get(0), - String.format("%s(", operator.first()), - operator - .second() - .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) - .orElse(String.format(", %s)%s", expressions.get(1), match))); + predicate, + literalEnvelopes, + acceleratedPredicates, + () -> geometryTableAndColumn(spatialOperation)); + } + + /** The table and column of the geometry operand of a spatial predicate, if it is a column. */ + private Optional> geometryTableAndColumn( + BinarySpatialOperation spatialOperation) { + return spatialOperation.getArgs().stream() + .filter(Property.class::isInstance) + .map(Property.class::cast) + .findFirst() + .flatMap( + property -> { + String propertyName = property.getName().replaceAll("^\"|\"$", ""); + boolean allowColumnFallback = !propertyName.contains("."); + SchemaSql table = getTable(propertyName, false, allowColumnFallback); + return plainColumnOfMainTable( + getQualifiedColumn(table, propertyName, "A", allowColumnFallback).first()) + .map(column -> Tuple.of(rootSchema.getName(), column)); + }); } @Override @@ -1308,7 +1488,10 @@ public String visit(TemporalLiteral temporalLiteral, List children) { @Override public String visit(GeometryNode geometry, List children) { - return sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + String expression = + sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + literalEnvelopes.computeIfAbsent(expression, ignore -> envelopeOf(geometry)); + return expression; } @Override @@ -1510,7 +1693,7 @@ public String visit(LogicalOperation logicalOperation, List children) { public String visit(Not not, List children) { String operator = LOGICAL_OPERATORS.get(not.getClass()); - String operation = children.get(0); + String operation = withoutSpatialIndexPredicates(children.get(0), acceleratedPredicates); if (operation.contains("(SELECT")) { // The child predicate is (or contains) an EXISTS-style semi-join on a joined property // (A.id IN (SELECT ... WHERE )). The string surgery below would push the @@ -1560,7 +1743,9 @@ public String visit(Not not, List children) { operation.substring(0, pos), operator, operation.substring(pos + 1, length)); } - return super.visit(not, children); + // operation, not children: Not is unary, and operation is the operand with any spatial index + // predicate removed + return super.visit(not, ImmutableList.of(operation)); } @Override @@ -1628,6 +1813,13 @@ private CqlToSql2(SqlQueryMapping mapping) { this(mapping, null); } + // bounding boxes of the geometry literals of the filter being encoded, keyed by the SQL they + // were encoded to; a visitor encodes a single filter, so this neither leaks nor is shared + private final Map literalEnvelopes = new LinkedHashMap<>(); + // spatial predicates that a spatial index predicate was added to, mapped to the predicate + // without it + private final Map acceleratedPredicates = new LinkedHashMap<>(); + private CqlToSql2(SqlQueryMapping mapping, CteCollector collector) { super(coordinatesTransformer); this.mapping = mapping; @@ -2393,13 +2585,48 @@ public String visit(BinarySpatialOperation spatialOperation, List childr List expressions = processBinary(spatialOperation.getArgs(), children); - return String.format( + String predicate = + String.format( + expressions.get(0), + String.format("%s(", operator.first()), + operator + .second() + .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) + .orElse(String.format(", %s)%s", expressions.get(1), match))); + + return withSpatialIndexPredicate( + spatialOperation.getSpatialOperator(), + children, expressions.get(0), - String.format("%s(", operator.first()), - operator - .second() - .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) - .orElse(String.format(", %s)%s", expressions.get(1), match))); + predicate, + literalEnvelopes, + acceleratedPredicates, + () -> geometryTableAndColumn(spatialOperation)); + } + + /** The table and column of the geometry operand of a spatial predicate, if it is a column. */ + private Optional> geometryTableAndColumn( + BinarySpatialOperation spatialOperation) { + return spatialOperation.getArgs().stream() + .filter(Property.class::isInstance) + .map(Property.class::cast) + .findFirst() + .flatMap( + property -> { + String propertyName = property.getName().replaceAll("^\"|\"$", ""); + boolean allowColumnFallback = !propertyName.contains("."); + de.ii.xtraplatform.base.domain.util.Tuple table = + getTableColumn(propertyName, false, allowColumnFallback); + return plainColumnOfMainTable( + getQualifiedColumn( + table.first(), + table.second(), + propertyName, + "A", + allowColumnFallback) + .first()) + .map(column -> Tuple.of(mapping.getMainTable().getName(), column)); + }); } @Override @@ -2441,7 +2668,10 @@ public String visit(TemporalLiteral temporalLiteral, List children) { @Override public String visit(GeometryNode geometry, List children) { - return sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + String expression = + sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + literalEnvelopes.computeIfAbsent(expression, ignore -> envelopeOf(geometry)); + return expression; } @Override @@ -2645,7 +2875,7 @@ public String visit(LogicalOperation logicalOperation, List children) { public String visit(Not not, List children) { String operator = LOGICAL_OPERATORS.get(not.getClass()); - String operation = children.get(0); + String operation = withoutSpatialIndexPredicates(children.get(0), acceleratedPredicates); if (operation.contains("(SELECT")) { // The child predicate is (or contains) an EXISTS-style semi-join on a joined property // (A.id IN (SELECT ... WHERE )). The string surgery below would push the @@ -2695,7 +2925,9 @@ public String visit(Not not, List children) { operation.substring(0, pos), operator, operation.substring(pos + 1, length)); } - return super.visit(not, children); + // operation, not children: Not is unary, and operation is the operand with any spatial index + // predicate removed + return super.visit(not, ImmutableList.of(operation)); } @Override diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 2110d7475..147c01eb1 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -114,6 +114,7 @@ import de.ii.xtraplatform.streams.domain.Reactive.Stream; import de.ii.xtraplatform.streams.domain.Reactive.Transformer; import de.ii.xtraplatform.values.domain.ValueStore; +import java.sql.SQLException; import java.time.ZoneId; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; @@ -613,7 +614,8 @@ protected boolean onStartup() throws InterruptedException { type -> Optional.ofNullable(queryMappings.get(type)) .filter(mappings -> mappings.size() == 1) - .map(mappings -> mappings.get(0))); + .map(mappings -> mappings.get(0)), + getSpatialIndexes()); AggregateStatsQueryGenerator queryGeneratorSql = new AggregateStatsQueryGenerator(sqlDialect, filterEncoder); @@ -667,6 +669,24 @@ protected boolean onStartup() throws InterruptedException { return true; } + /** + * The geometry columns with a spatial index that the query generator has to name explicitly to + * make use of. Without it, spatial queries keep working, only without the index, so a failure to + * determine it is logged and does not keep the provider from starting. + */ + private Map getSpatialIndexes() { + try { + return getSqlClient().getSpatialIndexes(); + } catch (SQLException | RuntimeException e) { + LogContext.errorAsWarn( + LOGGER, + e, + "Could not determine the spatial indexes of the feature provider with id '{}'. Spatial queries will not use a spatial index.", + getId()); + return Map.of(); + } + } + @Override protected void onStarted() { changes() diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java index 1a9264355..5b5122a5a 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java @@ -27,4 +27,8 @@ default DbInfo getDbInfo() throws SQLException { default Map getGeoInfo() throws SQLException { return getDbmsAdapter().getGeoInfo(getConnection(), getDbInfo()); } + + default Map getSpatialIndexes() throws SQLException { + return getDbmsAdapter().getSpatialIndexes(this); + } } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java index 34a0dec60..5eabde8c6 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java @@ -40,6 +40,20 @@ public interface SqlDbmsAdapter { Map getGeoInfo(Connection connection, DbInfo dbInfo) throws SQLException; + /** + * Returns the geometry columns that have a spatial index which the query generator has to name + * explicitly to make use of, keyed by {@code table.column} in lower case (SQL identifiers are + * compared case-insensitively here, as SQLite does for ASCII). The value is the column of the + * table that the index entries are keyed on, which a query has to join the index on. + * + *

The default is an empty map, which is correct for every DBMS whose spatial operators consult + * the spatial index by themselves. It takes the client rather than a connection so that those + * adapters do not have to be handed one they will not use. + */ + default Map getSpatialIndexes(SqlClientBasic sqlClient) throws SQLException { + return Map.of(); + } + DbInfo getDbInfo(Connection connection) throws SQLException; Collator getRowSortingCollator(Optional defaultCollation); diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java index 4ba6e8d2f..2c3733e1b 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java @@ -138,6 +138,28 @@ default String getSpatialOperatorMatch(SpatialFunction spatialFunction) { return ""; } + /** + * Returns a predicate that restricts the rows to those whose geometry bounding box intersects the + * bounding box given as {@code min}/{@code max}, evaluated with a spatial index on {@code + * table}.{@code column}, or an empty optional if the dialect cannot use a spatial index that way. + * + *

The predicate is a conjunct that is added to the exact spatial predicate, never a + * replacement for it. Dialects whose spatial operators already consult the spatial index by + * themselves (such as PostGIS, where {@code ST_Intersects} includes the {@code &&} bounding box + * operator) do not implement this. + * + * @param table the table holding the geometry column + * @param column the geometry column + * @param alias the alias the table is referenced by in the query + * @param keyColumn the column of the table that the index entries are keyed on + * @param min the lower corner of the bounding box, in the native CRS + * @param max the upper corner of the bounding box, in the native CRS + */ + default Optional getSpatialIndexPredicate( + String table, String column, String alias, String keyColumn, double[] min, double[] max) { + return Optional.empty(); + } + default String getTemporalOperator(TemporalFunction temporalFunction) { // this is implementation specific return null; @@ -166,4 +188,23 @@ default String applyToExpression( Map SPATIAL_OPERATORS_3D = new ImmutableMap.Builder().build(); + + /** + * The spatial operators for which a match implies that the bounding boxes of both operands + * intersect. Only for these may a bounding box predicate from a spatial index be added as a + * conjunct: the conjunct is then implied by the exact predicate, so it changes neither the result + * nor the meaning of the predicate under negation. + * + *

{@code S_DISJOINT} is absent because two geometries can be disjoint while being arbitrarily + * far apart, so a bounding box predicate would discard matching rows. + */ + Set SPATIAL_OPERATORS_IMPLYING_BBOX_INTERSECTION = + ImmutableSet.of( + SpatialFunction.S_EQUALS, + SpatialFunction.S_TOUCHES, + SpatialFunction.S_WITHIN, + SpatialFunction.S_OVERLAPS, + SpatialFunction.S_CROSSES, + SpatialFunction.S_INTERSECTS, + SpatialFunction.S_CONTAINS); } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java index 42af4c0b7..797bb8554 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java @@ -206,6 +206,43 @@ public String escapeString(String value) { return value.replaceAll("'", "''"); } + /** + * GeoPackage keeps the bounding boxes of a geometry column in an R-Tree, a virtual table named + * {@code rtree_

_}. SQLite cannot infer from a spatial operator that such a table + * is relevant, so unless the query names it, the operator is evaluated for every row of the + * table. The predicate returned here names it, which turns the table scan into an index lookup. + * + *

The R-Tree stores the bounding boxes as 32-bit floats, rounded outwards, so the lookup never + * discards a geometry that the exact predicate would have matched. + */ + @Override + public Optional getSpatialIndexPredicate( + String table, String column, String alias, String keyColumn, double[] min, double[] max) { + if (min.length < 2 || max.length < 2) { + return Optional.empty(); + } + + return Optional.of( + String.format( + "%s.%s IN (SELECT id FROM %s WHERE maxx >= %s AND minx <= %s AND maxy >= %s AND miny <= %s)", + alias, + quoteIdentifier(keyColumn), + quoteIdentifier(String.format("rtree_%s_%s", table, column)), + toNumericLiteral(min[0]), + toNumericLiteral(max[0]), + toNumericLiteral(min[1]), + toNumericLiteral(max[1]))); + } + + private static String quoteIdentifier(String identifier) { + return String.format("\"%s\"", identifier.replace("\"", "\"\"")); + } + + private static String toNumericLiteral(double value) { + // Double.toString is independent of the default locale, unlike String.format("%f", ...) + return Double.toString(value); + } + @Override public String applyToExpression( String table, String name, Map subDecoderPaths, boolean spatial) { diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java index 96eaa17c9..735e4bce0 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java @@ -9,12 +9,14 @@ import com.github.azahnen.dagger.annotations.AutoBind; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import de.ii.xtraplatform.base.domain.AppContext; import de.ii.xtraplatform.blobs.domain.ResourceStore; import de.ii.xtraplatform.crs.domain.EpsgCrs; import de.ii.xtraplatform.crs.domain.EpsgCrs.Force; import de.ii.xtraplatform.features.sql.domain.ConnectionInfoSql; import de.ii.xtraplatform.features.sql.domain.ImmutableGeoInfo; +import de.ii.xtraplatform.features.sql.domain.SqlClientBasic; import de.ii.xtraplatform.features.sql.domain.SqlDbmsAdapter; import de.ii.xtraplatform.features.sql.domain.SqlDialect; import de.ii.xtraplatform.features.sql.domain.SqlDialectGpkg; @@ -172,6 +174,46 @@ public List getSystemTables() { "SpatialIndex"); } + /** + * Reports the geometry columns that have a GeoPackage R-Tree, mapped to the column that the + * R-Tree entries are keyed on. + * + *

The R-Tree virtual table itself is looked up instead of the {@code gpkg_rtree_index} row in + * {@code gpkg_extensions}, because it is the table that the generated query has to name, and the + * two can disagree: the extension may be registered for a column whose R-Tree was dropped. + * + *

The key column is the primary key of the feature table, which is what the R-Tree triggers + * store. It is reported instead of assuming {@code rowid}: a GeoPackage feature table is required + * to have an {@code INTEGER PRIMARY KEY}, which SQLite makes an alias of {@code rowid}, but in a + * file that does not follow that requirement the two differ and joining on {@code rowid} would + * match the wrong rows. Feature tables with a composite primary key are left out, as their rows + * cannot be addressed by the single id column of an R-Tree. + */ + @Override + public Map getSpatialIndexes(SqlClientBasic sqlClient) throws SQLException { + String query = + "SELECT gc.table_name, gc.column_name, ti.name FROM gpkg_geometry_columns gc" + + " JOIN sqlite_master m ON m.type = 'table' AND lower(m.name) = lower('rtree_' || gc.table_name || '_' || gc.column_name)" + + " JOIN pragma_table_info(gc.table_name) ti ON ti.pk = 1" + + " GROUP BY gc.table_name, gc.column_name HAVING count(*) = 1;"; + + ImmutableMap.Builder spatialIndexes = ImmutableMap.builder(); + + // the connection comes from the pool, closing it hands it back + try (Connection connection = sqlClient.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query)) { + while (resultSet.next()) { + spatialIndexes.put( + String.format("%s.%s", resultSet.getString(1), resultSet.getString(2)) + .toLowerCase(Locale.ROOT), + resultSet.getString(3)); + } + } + + return spatialIndexes.build(); + } + @Override public Map getGeoInfo(Connection connection, DbInfo dbInfo) throws SQLException { if (!(dbInfo instanceof DbInfoGpkg) diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy new file mode 100644 index 000000000..fd554ebe6 --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy @@ -0,0 +1,206 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import de.ii.xtraplatform.cql.app.CqlFilterExamples +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.cql.domain.And +import de.ii.xtraplatform.cql.domain.Bbox +import de.ii.xtraplatform.cql.domain.IsNull +import de.ii.xtraplatform.cql.domain.Not +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.SDisjoint +import de.ii.xtraplatform.cql.domain.SIntersects +import de.ii.xtraplatform.cql.domain.SWithin +import de.ii.xtraplatform.cql.domain.SpatialLiteral +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.FeatureSchemaFixtures +import de.ii.xtraplatform.features.domain.MappingOperationResolver +import de.ii.xtraplatform.features.domain.MappingRuleFixtures +import de.ii.xtraplatform.features.json.app.DecoderFactoryJson +import de.ii.xtraplatform.features.domain.SchemaBase +import de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings +import de.ii.xtraplatform.features.sql.domain.ImmutableSchemaSql +import de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults +import de.ii.xtraplatform.features.sql.domain.SqlDialectGpkg +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import de.ii.xtraplatform.features.sql.domain.SqlPathParser +import de.ii.xtraplatform.features.sql.domain.SqlQueryMapping +import spock.lang.Shared +import spock.lang.Specification + +import java.util.function.Function + +class FilterEncoderSqlSpatialIndexSpec extends Specification { + + static final String EXACT_WITHIN = "ST_Within(A.location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326))" + static final String RTREE_LOOKUP = "A.\"id\" IN (SELECT id FROM \"rtree_building_location\" WHERE maxx >= -118.0 AND minx <= -117.9 AND maxy >= 33.8 AND miny <= 34.0)" + + @Shared + SqlQueryMapping unfaelleMapping + + def setupSpec() { + def cql = new CqlImpl() + def pathParser = new SqlPathParser(new ImmutableSqlPathDefaults.Builder().build(), cql, + Map.of("JSON", new DecoderFactoryJson(), "EXPRESSION", new DecoderFactorySqlExpression())) + def mappingDeriver = new SqlMappingDeriver(pathParser, new ImmutableQueryGeneratorSettings.Builder().build()) + def schema = FeatureSchemaFixtures.fromYaml("strassen_unfaelle2") + def resolved = schema.accept(new MappingOperationResolver(), List.of()) + unfaelleMapping = mappingDeriver.derive(MappingRuleFixtures.fromYaml("strassen_unfaelle2"), resolved).get(0) + } + + static FilterEncoderSql encoder(dialect, Map spatialIndexes) { + return new FilterEncoderSql(OgcCrs.CRS84, dialect, null, null, new CqlImpl(), List.of(), null, + { type -> Optional.empty() } as Function, spatialIndexes) + } + + def 'gpkg, indexed geometry column on the main table: r-tree lookup is added as a conjunct'() { + + given: 'a GeoPackage provider whose geometry column has an r-tree' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: 'a bbox filter on that column is encoded' + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the r-tree is queried in addition to the exact predicate' + actual == "(${RTREE_LOOKUP} AND ${EXACT_WITHIN})" + } + + def 'gpkg, geometry column without an r-tree: predicate is unchanged'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("other_table.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == EXACT_WITHIN + } + + def 'table and column are matched case-insensitively, as sqlite compares identifiers'() { + + given: 'a source path that capitalises the column differently than the database reports it' + def schema = new ImmutableSchemaSql.Builder() + .name("Building") + .type(SchemaBase.Type.OBJECT) + .sortKey("id") + .addProperties(new ImmutableSchemaSql.Builder() + .name("Location") + .sourcePath("Location") + .type(SchemaBase.Type.GEOMETRY) + .parentPath(["Building"]) + .build()) + .build() + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: + def filter = SWithin.of(Property.of("Location"), + SpatialLiteral.of(Bbox.of(-118.0d, 33.8d, -117.9d, 34.0d, OgcCrs.CRS84))) + String actual = filterEncoder.encode(filter, schema) + + then: 'the r-tree of that column is still found' + actual == "(A.\"id\" IN (SELECT id FROM \"rtree_Building_Location\" WHERE maxx >= -118.0 AND minx <= -117.9 AND maxy >= 33.8 AND miny <= 34.0)" + + " AND ST_Within(A.Location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326)))" + } + + def 'postgis: no conjunct is added, ST_Within consults the index by itself'() { + + given: + def filterEncoder = encoder(new SqlDialectPgis(), Map.of("building.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == EXACT_WITHIN + } + + def 'S_DISJOINT is not accelerated, a disjoint geometry may lie outside the bbox'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def filter = SDisjoint.of(Property.of("location"), + SpatialLiteral.of(Bbox.of(-118.0, 33.8, -117.9, 34.0, OgcCrs.CRS84))) + + when: + String actual = filterEncoder.encode(filter, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the exact predicate is emitted on its own' + actual == "ST_Disjoint(A.location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326))" + } + + def 'S_DISJOINT and NOT S_INTERSECTS mean the same and are encoded the same way'() { + + given: 'the same predicate, spelled both ways' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def bbox = SpatialLiteral.of(Bbox.of(-118.0d, 33.8d, -117.9d, 34.0d, OgcCrs.CRS84)) + + when: + String disjoint = filterEncoder.encode( + SDisjoint.of(Property.of("location"), bbox), QuerySchemaFixtures.SIMPLE_GEOMETRY) + String notIntersects = filterEncoder.encode( + Not.of(SIntersects.of(Property.of("location"), bbox)), QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'neither consults the r-tree, so the two spellings cannot disagree' + !disjoint.contains("rtree_") + !notIntersects.contains("rtree_") + } + + def 'under NOT the conjunct is dropped, a NULL geometry has no r-tree entry'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: + String actual = filterEncoder.encode(Not.of(CqlFilterExamples.EXAMPLE_15), QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the exact predicate is negated on its own, as it was before' + actual == "NOT (${EXACT_WITHIN})" + } + + def 'a negation elsewhere in the filter does not stop the conjunct'() { + + given: 'a filter that negates an unrelated predicate and intersects the geometry' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def filter = And.of(Not.of(IsNull.of(Property.of("location"))), CqlFilterExamples.EXAMPLE_15) + + when: + String actual = filterEncoder.encode(filter, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == "(A.location IS NOT NULL AND (${RTREE_LOOKUP} AND ${EXACT_WITHIN}))" + } + + def 'geometry reached through a join is not accelerated, the conjunct would address the wrong table'() { + + given: 'the geometry lives in a joined table, so the predicate goes inside a semi-join' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id", "geometry.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_16, QuerySchemaFixtures.JOINED_GEOMETRY) + + then: + actual == "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE ST_Intersects(AB.location, ST_GeomFromText('POLYGON((-10.0 -10.0,10.0 -10.0,10.0 10.0,-10.0 -10.0))',4326)))" + } + + def 'the mapping based encoder, which the items query uses, is accelerated too'() { + + given: 'a filter on the primary geometry of a mapping whose column has an r-tree' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("unfaelle_point.geom", "fid")) + def filter = SIntersects.of(Property.of("geometry"), + SpatialLiteral.of(Bbox.of(7.0, 50.0, 7.1, 50.1, OgcCrs.CRS84))) + + when: + String actual = filterEncoder.encode(filter, unfaelleMapping) + + then: + actual == "(A.\"fid\" IN (SELECT id FROM \"rtree_unfaelle_point_geom\" WHERE maxx >= 7.0 AND minx <= 7.1 AND maxy >= 50.0 AND miny <= 50.1) AND " + + "ST_Intersects(A.geom, ST_GeomFromText('POLYGON((7.0 50.0,7.1 50.0,7.1 50.1,7.0 50.1,7.0 50.0))',4326)))" + } +} diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy new file mode 100644 index 000000000..b694030aa --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy @@ -0,0 +1,52 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.domain + +import spock.lang.Shared +import spock.lang.Specification + +class SqlDialectGpkgSpec extends Specification { + + @Shared + SqlDialectGpkg dialect + + def setupSpec() { + dialect = new SqlDialectGpkg() + } + + def 'the spatial index predicate names the r-tree and the key column'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + "pv_pot_dach", "geom", "A", "id", [365204.0d, 5621522.0d] as double[], [365938.0d, 5622652.0d] as double[]) + + then: + actual.get() == 'A."id" IN (SELECT id FROM "rtree_pv_pot_dach_geom"' + + ' WHERE maxx >= 365204.0 AND minx <= 365938.0 AND maxy >= 5621522.0 AND miny <= 5622652.0)' + } + + def 'a bounding box without two axes yields no predicate'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + "t", "geom", "A", "id", [1.0d] as double[], [2.0d] as double[]) + + then: + actual.isEmpty() + } + + def 'identifiers are quoted, and embedded quotes escaped'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + 'we"ird', "geom", "A", 'k"ey', [1.0d, 2.0d] as double[], [3.0d, 4.0d] as double[]) + + then: + actual.get().startsWith('A."k""ey" IN (SELECT id FROM "rtree_we""ird_geom"') + } +}