diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java index 8c595e3f7..0c896998c 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/app/FilterEncoderSql.java @@ -46,6 +46,7 @@ import de.ii.xtraplatform.cql.domain.Property; import de.ii.xtraplatform.cql.domain.Scalar; import de.ii.xtraplatform.cql.domain.ScalarLiteral; +import de.ii.xtraplatform.cql.domain.SpatialFunction; import de.ii.xtraplatform.cql.domain.SpatialOperation; import de.ii.xtraplatform.cql.domain.Temporal; import de.ii.xtraplatform.cql.domain.TemporalLiteral; @@ -71,6 +72,7 @@ import de.ii.xtraplatform.geometries.domain.PositionList; import de.ii.xtraplatform.geometries.domain.transform.CoordinatesTransformer; import de.ii.xtraplatform.geometries.domain.transform.ImmutableCrsTransform; +import de.ii.xtraplatform.geometries.domain.transform.MinMaxDeriver; import java.time.Instant; import java.time.LocalDate; import java.time.format.DateTimeFormatter; @@ -108,6 +110,10 @@ public class FilterEncoderSql { private final String accentiCollation; private final Map customFunctions; private final java.util.function.Function> mappingResolver; + // geometry columns with a spatial index that the query has to name explicitly, keyed by + // "table.column" in lower case and mapped to the column the index is keyed on; see + // SqlDbmsAdapter.getSpatialIndexes + private final Map spatialIndexes; BiFunction, Optional, Geometry> coordinatesTransformer; public FilterEncoderSql( @@ -148,6 +154,29 @@ public FilterEncoderSql( List customFunctions, String accentiCollation, java.util.function.Function> mappingResolver) { + this( + nativeCrs, + sqlDialect, + crsTransformerFactory, + crsInfo, + cql, + customFunctions, + accentiCollation, + mappingResolver, + Map.of()); + } + + public FilterEncoderSql( + EpsgCrs nativeCrs, + SqlDialect sqlDialect, + CrsTransformerFactory crsTransformerFactory, + CrsInfo crsInfo, + Cql cql, + List customFunctions, + String accentiCollation, + java.util.function.Function> mappingResolver, + Map spatialIndexes) { + this.spatialIndexes = spatialIndexes; this.nativeCrs = nativeCrs; this.sqlDialect = sqlDialect; this.crsTransformerFactory = crsTransformerFactory; @@ -166,6 +195,123 @@ public FilterEncoderSql( this.coordinatesTransformer = this::transformCoordinatesIfNecessary; } + /** + * Adds the spatial index predicate of the dialect to an exact spatial predicate, as a conjunct. + * + *

The conjunct is only added where it is implied by the exact predicate, so that it can change + * neither the result nor the meaning of the predicate under negation: the operator has to be one + * whose match implies that the bounding boxes intersect, the geometry has to be a column of the + * main table that a spatial index is known for, and the other operand has to be a geometry + * literal whose bounding box is known. + * + * @param literalEnvelopes the bounding boxes of the geometry literals of the filter being + * encoded, keyed by the SQL they were encoded to + * @param acceleratedPredicates records the added conjunct, so that {@link + * #withoutSpatialIndexPredicates} can drop it again under a negation + * @param geometryTableAndColumn resolves the table and column of the geometry operand, evaluated + * only once everything else already matched + */ + private String withSpatialIndexPredicate( + SpatialFunction operator, + List children, + String mainExpression, + String predicate, + Map literalEnvelopes, + Map acceleratedPredicates, + Supplier>> geometryTableAndColumn) { + if (spatialIndexes.isEmpty() + || !SqlDialect.SPATIAL_OPERATORS_IMPLYING_BBOX_INTERSECTION.contains(operator) + // Only the direct conjunct form of a property reference addresses the geometry as a column + // of the main table (see visit(Property)). In the semi-join form the geometry belongs to a + // joined table, where the predicate would have to go inside the subquery instead. + || !(mainExpression.startsWith("%1$s") && mainExpression.endsWith("%2$s"))) { + return predicate; + } + + double[][] envelope = + children.stream() + .map(literalEnvelopes::get) + .filter(Objects::nonNull) + .findFirst() + .orElse(null); + + if (Objects.isNull(envelope)) { + return predicate; + } + + return geometryTableAndColumn + .get() + .flatMap( + tableColumn -> + Optional.ofNullable( + spatialIndexes.get( + String.format("%s.%s", tableColumn.first(), tableColumn.second()) + .toLowerCase(Locale.ROOT))) + .flatMap( + keyColumn -> + sqlDialect.getSpatialIndexPredicate( + tableColumn.first(), + tableColumn.second(), + "A", + keyColumn, + envelope[0], + envelope[1]))) + .map( + indexPredicate -> { + String accelerated = String.format("(%s AND %s)", indexPredicate, predicate); + // remembered so that a negation can drop it again, see + // withoutSpatialIndexPredicates + acceleratedPredicates.put(accelerated, predicate); + return accelerated; + }) + .orElse(predicate); + } + + /** + * Removes the spatial index predicates that were added to the operand of a negation. + * + *

The conjunct is implied by the exact predicate for every row that has a geometry, so adding + * it changes nothing there, negated or not. A row whose geometry is NULL is the exception: it has + * no entry in the spatial index, so the conjunct is false for it, while the exact predicate is + * NULL. Both keep the row out of a positive result, but negated the one yields true and the other + * NULL. Dropping the conjunct under a negation keeps that case as it was. + */ + private static String withoutSpatialIndexPredicates( + String expression, Map acceleratedPredicates) { + String withoutIndexPredicates = expression; + + for (Entry accelerated : acceleratedPredicates.entrySet()) { + withoutIndexPredicates = + withoutIndexPredicates.replace(accelerated.getKey(), accelerated.getValue()); + } + + return withoutIndexPredicates; + } + + /** + * The bounding box of a geometry literal, derived after the same transformation that {@link + * de.ii.xtraplatform.cql.domain.CqlToText} applies before encoding it, so that the bounding box + * is guaranteed to describe the geometry that ends up in the query. Null for an empty geometry. + */ + private double[][] envelopeOf(GeometryNode geometry) { + return coordinatesTransformer + .apply(geometry.getGeometry(), geometry.getCrs().or(() -> geometry.getGeometry().getCrs())) + .accept(new MinMaxDeriver()); + } + + /** + * The column of a plain {@code A.} reference. Anything else — a sub-decoder expression, a + * date function — is not a column that a spatial index could be looked up for. + */ + private static Optional plainColumnOfMainTable(String qualifiedColumn) { + if (!qualifiedColumn.startsWith("A.") + || qualifiedColumn.indexOf('(') >= 0 + || qualifiedColumn.indexOf(' ') >= 0) { + return Optional.empty(); + } + return Optional.of(qualifiedColumn.substring(2)); + } + private Optional renderCustomFunction( de.ii.xtraplatform.cql.domain.Function function, List children) { CustomFunction customFunction = @@ -617,6 +763,12 @@ private static Predicate getPropertyNameMatcher( private class CqlToSql extends CqlToText { private final SchemaSql rootSchema; + // bounding boxes of the geometry literals of the filter being encoded, keyed by the SQL they + // were encoded to; a visitor encodes a single filter, so this neither leaks nor is shared + private final Map literalEnvelopes = new LinkedHashMap<>(); + // spatial predicates that a spatial index predicate was added to, mapped to the predicate + // without it + private final Map acceleratedPredicates = new LinkedHashMap<>(); private CqlToSql(SchemaSql rootSchema) { super(coordinatesTransformer); @@ -1260,13 +1412,41 @@ public String visit(BinarySpatialOperation spatialOperation, List childr List expressions = processBinary(spatialOperation.getArgs(), children); - return String.format( + String predicate = + String.format( + expressions.get(0), + String.format("%s(", operator.first()), + operator + .second() + .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) + .orElse(String.format(", %s)%s", expressions.get(1), match))); + + return withSpatialIndexPredicate( + spatialOperation.getSpatialOperator(), + children, expressions.get(0), - String.format("%s(", operator.first()), - operator - .second() - .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) - .orElse(String.format(", %s)%s", expressions.get(1), match))); + predicate, + literalEnvelopes, + acceleratedPredicates, + () -> geometryTableAndColumn(spatialOperation)); + } + + /** The table and column of the geometry operand of a spatial predicate, if it is a column. */ + private Optional> geometryTableAndColumn( + BinarySpatialOperation spatialOperation) { + return spatialOperation.getArgs().stream() + .filter(Property.class::isInstance) + .map(Property.class::cast) + .findFirst() + .flatMap( + property -> { + String propertyName = property.getName().replaceAll("^\"|\"$", ""); + boolean allowColumnFallback = !propertyName.contains("."); + SchemaSql table = getTable(propertyName, false, allowColumnFallback); + return plainColumnOfMainTable( + getQualifiedColumn(table, propertyName, "A", allowColumnFallback).first()) + .map(column -> Tuple.of(rootSchema.getName(), column)); + }); } @Override @@ -1308,7 +1488,10 @@ public String visit(TemporalLiteral temporalLiteral, List children) { @Override public String visit(GeometryNode geometry, List children) { - return sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + String expression = + sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + literalEnvelopes.computeIfAbsent(expression, ignore -> envelopeOf(geometry)); + return expression; } @Override @@ -1510,7 +1693,7 @@ public String visit(LogicalOperation logicalOperation, List children) { public String visit(Not not, List children) { String operator = LOGICAL_OPERATORS.get(not.getClass()); - String operation = children.get(0); + String operation = withoutSpatialIndexPredicates(children.get(0), acceleratedPredicates); if (operation.contains("(SELECT")) { // The child predicate is (or contains) an EXISTS-style semi-join on a joined property // (A.id IN (SELECT ... WHERE )). The string surgery below would push the @@ -1560,7 +1743,9 @@ public String visit(Not not, List children) { operation.substring(0, pos), operator, operation.substring(pos + 1, length)); } - return super.visit(not, children); + // operation, not children: Not is unary, and operation is the operand with any spatial index + // predicate removed + return super.visit(not, ImmutableList.of(operation)); } @Override @@ -1628,6 +1813,13 @@ private CqlToSql2(SqlQueryMapping mapping) { this(mapping, null); } + // bounding boxes of the geometry literals of the filter being encoded, keyed by the SQL they + // were encoded to; a visitor encodes a single filter, so this neither leaks nor is shared + private final Map literalEnvelopes = new LinkedHashMap<>(); + // spatial predicates that a spatial index predicate was added to, mapped to the predicate + // without it + private final Map acceleratedPredicates = new LinkedHashMap<>(); + private CqlToSql2(SqlQueryMapping mapping, CteCollector collector) { super(coordinatesTransformer); this.mapping = mapping; @@ -2393,13 +2585,48 @@ public String visit(BinarySpatialOperation spatialOperation, List childr List expressions = processBinary(spatialOperation.getArgs(), children); - return String.format( + String predicate = + String.format( + expressions.get(0), + String.format("%s(", operator.first()), + operator + .second() + .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) + .orElse(String.format(", %s)%s", expressions.get(1), match))); + + return withSpatialIndexPredicate( + spatialOperation.getSpatialOperator(), + children, expressions.get(0), - String.format("%s(", operator.first()), - operator - .second() - .map(mask -> String.format(", %s, 'mask=%s')%s", expressions.get(1), mask, match)) - .orElse(String.format(", %s)%s", expressions.get(1), match))); + predicate, + literalEnvelopes, + acceleratedPredicates, + () -> geometryTableAndColumn(spatialOperation)); + } + + /** The table and column of the geometry operand of a spatial predicate, if it is a column. */ + private Optional> geometryTableAndColumn( + BinarySpatialOperation spatialOperation) { + return spatialOperation.getArgs().stream() + .filter(Property.class::isInstance) + .map(Property.class::cast) + .findFirst() + .flatMap( + property -> { + String propertyName = property.getName().replaceAll("^\"|\"$", ""); + boolean allowColumnFallback = !propertyName.contains("."); + de.ii.xtraplatform.base.domain.util.Tuple table = + getTableColumn(propertyName, false, allowColumnFallback); + return plainColumnOfMainTable( + getQualifiedColumn( + table.first(), + table.second(), + propertyName, + "A", + allowColumnFallback) + .first()) + .map(column -> Tuple.of(mapping.getMainTable().getName(), column)); + }); } @Override @@ -2441,7 +2668,10 @@ public String visit(TemporalLiteral temporalLiteral, List children) { @Override public String visit(GeometryNode geometry, List children) { - return sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + String expression = + sqlDialect.applyToWkt(super.visit(geometry, children), nativeCrs.getCode()); + literalEnvelopes.computeIfAbsent(expression, ignore -> envelopeOf(geometry)); + return expression; } @Override @@ -2645,7 +2875,7 @@ public String visit(LogicalOperation logicalOperation, List children) { public String visit(Not not, List children) { String operator = LOGICAL_OPERATORS.get(not.getClass()); - String operation = children.get(0); + String operation = withoutSpatialIndexPredicates(children.get(0), acceleratedPredicates); if (operation.contains("(SELECT")) { // The child predicate is (or contains) an EXISTS-style semi-join on a joined property // (A.id IN (SELECT ... WHERE )). The string surgery below would push the @@ -2695,7 +2925,9 @@ public String visit(Not not, List children) { operation.substring(0, pos), operator, operation.substring(pos + 1, length)); } - return super.visit(not, children); + // operation, not children: Not is unary, and operation is the operand with any spatial index + // predicate removed + return super.visit(not, ImmutableList.of(operation)); } @Override diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java index 2110d7475..147c01eb1 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/FeatureProviderSql.java @@ -114,6 +114,7 @@ import de.ii.xtraplatform.streams.domain.Reactive.Stream; import de.ii.xtraplatform.streams.domain.Reactive.Transformer; import de.ii.xtraplatform.values.domain.ValueStore; +import java.sql.SQLException; import java.time.ZoneId; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.ArrayList; @@ -613,7 +614,8 @@ protected boolean onStartup() throws InterruptedException { type -> Optional.ofNullable(queryMappings.get(type)) .filter(mappings -> mappings.size() == 1) - .map(mappings -> mappings.get(0))); + .map(mappings -> mappings.get(0)), + getSpatialIndexes()); AggregateStatsQueryGenerator queryGeneratorSql = new AggregateStatsQueryGenerator(sqlDialect, filterEncoder); @@ -667,6 +669,24 @@ protected boolean onStartup() throws InterruptedException { return true; } + /** + * The geometry columns with a spatial index that the query generator has to name explicitly to + * make use of. Without it, spatial queries keep working, only without the index, so a failure to + * determine it is logged and does not keep the provider from starting. + */ + private Map getSpatialIndexes() { + try { + return getSqlClient().getSpatialIndexes(); + } catch (SQLException | RuntimeException e) { + LogContext.errorAsWarn( + LOGGER, + e, + "Could not determine the spatial indexes of the feature provider with id '{}'. Spatial queries will not use a spatial index.", + getId()); + return Map.of(); + } + } + @Override protected void onStarted() { changes() diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java index 1a9264355..5b5122a5a 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlClientBasic.java @@ -27,4 +27,8 @@ default DbInfo getDbInfo() throws SQLException { default Map getGeoInfo() throws SQLException { return getDbmsAdapter().getGeoInfo(getConnection(), getDbInfo()); } + + default Map getSpatialIndexes() throws SQLException { + return getDbmsAdapter().getSpatialIndexes(this); + } } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java index 34a0dec60..5eabde8c6 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDbmsAdapter.java @@ -40,6 +40,20 @@ public interface SqlDbmsAdapter { Map getGeoInfo(Connection connection, DbInfo dbInfo) throws SQLException; + /** + * Returns the geometry columns that have a spatial index which the query generator has to name + * explicitly to make use of, keyed by {@code table.column} in lower case (SQL identifiers are + * compared case-insensitively here, as SQLite does for ASCII). The value is the column of the + * table that the index entries are keyed on, which a query has to join the index on. + * + *

The default is an empty map, which is correct for every DBMS whose spatial operators consult + * the spatial index by themselves. It takes the client rather than a connection so that those + * adapters do not have to be handed one they will not use. + */ + default Map getSpatialIndexes(SqlClientBasic sqlClient) throws SQLException { + return Map.of(); + } + DbInfo getDbInfo(Connection connection) throws SQLException; Collator getRowSortingCollator(Optional defaultCollation); diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java index 4ba6e8d2f..2c3733e1b 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialect.java @@ -138,6 +138,28 @@ default String getSpatialOperatorMatch(SpatialFunction spatialFunction) { return ""; } + /** + * Returns a predicate that restricts the rows to those whose geometry bounding box intersects the + * bounding box given as {@code min}/{@code max}, evaluated with a spatial index on {@code + * table}.{@code column}, or an empty optional if the dialect cannot use a spatial index that way. + * + *

The predicate is a conjunct that is added to the exact spatial predicate, never a + * replacement for it. Dialects whose spatial operators already consult the spatial index by + * themselves (such as PostGIS, where {@code ST_Intersects} includes the {@code &&} bounding box + * operator) do not implement this. + * + * @param table the table holding the geometry column + * @param column the geometry column + * @param alias the alias the table is referenced by in the query + * @param keyColumn the column of the table that the index entries are keyed on + * @param min the lower corner of the bounding box, in the native CRS + * @param max the upper corner of the bounding box, in the native CRS + */ + default Optional getSpatialIndexPredicate( + String table, String column, String alias, String keyColumn, double[] min, double[] max) { + return Optional.empty(); + } + default String getTemporalOperator(TemporalFunction temporalFunction) { // this is implementation specific return null; @@ -166,4 +188,23 @@ default String applyToExpression( Map SPATIAL_OPERATORS_3D = new ImmutableMap.Builder().build(); + + /** + * The spatial operators for which a match implies that the bounding boxes of both operands + * intersect. Only for these may a bounding box predicate from a spatial index be added as a + * conjunct: the conjunct is then implied by the exact predicate, so it changes neither the result + * nor the meaning of the predicate under negation. + * + *

{@code S_DISJOINT} is absent because two geometries can be disjoint while being arbitrarily + * far apart, so a bounding box predicate would discard matching rows. + */ + Set SPATIAL_OPERATORS_IMPLYING_BBOX_INTERSECTION = + ImmutableSet.of( + SpatialFunction.S_EQUALS, + SpatialFunction.S_TOUCHES, + SpatialFunction.S_WITHIN, + SpatialFunction.S_OVERLAPS, + SpatialFunction.S_CROSSES, + SpatialFunction.S_INTERSECTS, + SpatialFunction.S_CONTAINS); } diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java index 42af4c0b7..797bb8554 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkg.java @@ -206,6 +206,43 @@ public String escapeString(String value) { return value.replaceAll("'", "''"); } + /** + * GeoPackage keeps the bounding boxes of a geometry column in an R-Tree, a virtual table named + * {@code rtree__}. SQLite cannot infer from a spatial operator that such a table + * is relevant, so unless the query names it, the operator is evaluated for every row of the + * table. The predicate returned here names it, which turns the table scan into an index lookup. + * + *

The R-Tree stores the bounding boxes as 32-bit floats, rounded outwards, so the lookup never + * discards a geometry that the exact predicate would have matched. + */ + @Override + public Optional getSpatialIndexPredicate( + String table, String column, String alias, String keyColumn, double[] min, double[] max) { + if (min.length < 2 || max.length < 2) { + return Optional.empty(); + } + + return Optional.of( + String.format( + "%s.%s IN (SELECT id FROM %s WHERE maxx >= %s AND minx <= %s AND maxy >= %s AND miny <= %s)", + alias, + quoteIdentifier(keyColumn), + quoteIdentifier(String.format("rtree_%s_%s", table, column)), + toNumericLiteral(min[0]), + toNumericLiteral(max[0]), + toNumericLiteral(min[1]), + toNumericLiteral(max[1]))); + } + + private static String quoteIdentifier(String identifier) { + return String.format("\"%s\"", identifier.replace("\"", "\"\"")); + } + + private static String toNumericLiteral(double value) { + // Double.toString is independent of the default locale, unlike String.format("%f", ...) + return Double.toString(value); + } + @Override public String applyToExpression( String table, String name, Map subDecoderPaths, boolean spatial) { diff --git a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java index 96eaa17c9..735e4bce0 100644 --- a/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java +++ b/xtraplatform-features-sql/src/main/java/de/ii/xtraplatform/features/sql/infra/db/SqlDbmsAdapterGpkg.java @@ -9,12 +9,14 @@ import com.github.azahnen.dagger.annotations.AutoBind; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import de.ii.xtraplatform.base.domain.AppContext; import de.ii.xtraplatform.blobs.domain.ResourceStore; import de.ii.xtraplatform.crs.domain.EpsgCrs; import de.ii.xtraplatform.crs.domain.EpsgCrs.Force; import de.ii.xtraplatform.features.sql.domain.ConnectionInfoSql; import de.ii.xtraplatform.features.sql.domain.ImmutableGeoInfo; +import de.ii.xtraplatform.features.sql.domain.SqlClientBasic; import de.ii.xtraplatform.features.sql.domain.SqlDbmsAdapter; import de.ii.xtraplatform.features.sql.domain.SqlDialect; import de.ii.xtraplatform.features.sql.domain.SqlDialectGpkg; @@ -172,6 +174,46 @@ public List getSystemTables() { "SpatialIndex"); } + /** + * Reports the geometry columns that have a GeoPackage R-Tree, mapped to the column that the + * R-Tree entries are keyed on. + * + *

The R-Tree virtual table itself is looked up instead of the {@code gpkg_rtree_index} row in + * {@code gpkg_extensions}, because it is the table that the generated query has to name, and the + * two can disagree: the extension may be registered for a column whose R-Tree was dropped. + * + *

The key column is the primary key of the feature table, which is what the R-Tree triggers + * store. It is reported instead of assuming {@code rowid}: a GeoPackage feature table is required + * to have an {@code INTEGER PRIMARY KEY}, which SQLite makes an alias of {@code rowid}, but in a + * file that does not follow that requirement the two differ and joining on {@code rowid} would + * match the wrong rows. Feature tables with a composite primary key are left out, as their rows + * cannot be addressed by the single id column of an R-Tree. + */ + @Override + public Map getSpatialIndexes(SqlClientBasic sqlClient) throws SQLException { + String query = + "SELECT gc.table_name, gc.column_name, ti.name FROM gpkg_geometry_columns gc" + + " JOIN sqlite_master m ON m.type = 'table' AND lower(m.name) = lower('rtree_' || gc.table_name || '_' || gc.column_name)" + + " JOIN pragma_table_info(gc.table_name) ti ON ti.pk = 1" + + " GROUP BY gc.table_name, gc.column_name HAVING count(*) = 1;"; + + ImmutableMap.Builder spatialIndexes = ImmutableMap.builder(); + + // the connection comes from the pool, closing it hands it back + try (Connection connection = sqlClient.getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(query)) { + while (resultSet.next()) { + spatialIndexes.put( + String.format("%s.%s", resultSet.getString(1), resultSet.getString(2)) + .toLowerCase(Locale.ROOT), + resultSet.getString(3)); + } + } + + return spatialIndexes.build(); + } + @Override public Map getGeoInfo(Connection connection, DbInfo dbInfo) throws SQLException { if (!(dbInfo instanceof DbInfoGpkg) diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy new file mode 100644 index 000000000..fd554ebe6 --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/app/FilterEncoderSqlSpatialIndexSpec.groovy @@ -0,0 +1,206 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.app + +import de.ii.xtraplatform.cql.app.CqlFilterExamples +import de.ii.xtraplatform.cql.app.CqlImpl +import de.ii.xtraplatform.cql.domain.And +import de.ii.xtraplatform.cql.domain.Bbox +import de.ii.xtraplatform.cql.domain.IsNull +import de.ii.xtraplatform.cql.domain.Not +import de.ii.xtraplatform.cql.domain.Property +import de.ii.xtraplatform.cql.domain.SDisjoint +import de.ii.xtraplatform.cql.domain.SIntersects +import de.ii.xtraplatform.cql.domain.SWithin +import de.ii.xtraplatform.cql.domain.SpatialLiteral +import de.ii.xtraplatform.crs.domain.OgcCrs +import de.ii.xtraplatform.features.domain.FeatureSchemaFixtures +import de.ii.xtraplatform.features.domain.MappingOperationResolver +import de.ii.xtraplatform.features.domain.MappingRuleFixtures +import de.ii.xtraplatform.features.json.app.DecoderFactoryJson +import de.ii.xtraplatform.features.domain.SchemaBase +import de.ii.xtraplatform.features.sql.domain.ImmutableQueryGeneratorSettings +import de.ii.xtraplatform.features.sql.domain.ImmutableSchemaSql +import de.ii.xtraplatform.features.sql.domain.ImmutableSqlPathDefaults +import de.ii.xtraplatform.features.sql.domain.SqlDialectGpkg +import de.ii.xtraplatform.features.sql.domain.SqlDialectPgis +import de.ii.xtraplatform.features.sql.domain.SqlPathParser +import de.ii.xtraplatform.features.sql.domain.SqlQueryMapping +import spock.lang.Shared +import spock.lang.Specification + +import java.util.function.Function + +class FilterEncoderSqlSpatialIndexSpec extends Specification { + + static final String EXACT_WITHIN = "ST_Within(A.location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326))" + static final String RTREE_LOOKUP = "A.\"id\" IN (SELECT id FROM \"rtree_building_location\" WHERE maxx >= -118.0 AND minx <= -117.9 AND maxy >= 33.8 AND miny <= 34.0)" + + @Shared + SqlQueryMapping unfaelleMapping + + def setupSpec() { + def cql = new CqlImpl() + def pathParser = new SqlPathParser(new ImmutableSqlPathDefaults.Builder().build(), cql, + Map.of("JSON", new DecoderFactoryJson(), "EXPRESSION", new DecoderFactorySqlExpression())) + def mappingDeriver = new SqlMappingDeriver(pathParser, new ImmutableQueryGeneratorSettings.Builder().build()) + def schema = FeatureSchemaFixtures.fromYaml("strassen_unfaelle2") + def resolved = schema.accept(new MappingOperationResolver(), List.of()) + unfaelleMapping = mappingDeriver.derive(MappingRuleFixtures.fromYaml("strassen_unfaelle2"), resolved).get(0) + } + + static FilterEncoderSql encoder(dialect, Map spatialIndexes) { + return new FilterEncoderSql(OgcCrs.CRS84, dialect, null, null, new CqlImpl(), List.of(), null, + { type -> Optional.empty() } as Function, spatialIndexes) + } + + def 'gpkg, indexed geometry column on the main table: r-tree lookup is added as a conjunct'() { + + given: 'a GeoPackage provider whose geometry column has an r-tree' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: 'a bbox filter on that column is encoded' + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the r-tree is queried in addition to the exact predicate' + actual == "(${RTREE_LOOKUP} AND ${EXACT_WITHIN})" + } + + def 'gpkg, geometry column without an r-tree: predicate is unchanged'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("other_table.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == EXACT_WITHIN + } + + def 'table and column are matched case-insensitively, as sqlite compares identifiers'() { + + given: 'a source path that capitalises the column differently than the database reports it' + def schema = new ImmutableSchemaSql.Builder() + .name("Building") + .type(SchemaBase.Type.OBJECT) + .sortKey("id") + .addProperties(new ImmutableSchemaSql.Builder() + .name("Location") + .sourcePath("Location") + .type(SchemaBase.Type.GEOMETRY) + .parentPath(["Building"]) + .build()) + .build() + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: + def filter = SWithin.of(Property.of("Location"), + SpatialLiteral.of(Bbox.of(-118.0d, 33.8d, -117.9d, 34.0d, OgcCrs.CRS84))) + String actual = filterEncoder.encode(filter, schema) + + then: 'the r-tree of that column is still found' + actual == "(A.\"id\" IN (SELECT id FROM \"rtree_Building_Location\" WHERE maxx >= -118.0 AND minx <= -117.9 AND maxy >= 33.8 AND miny <= 34.0)" + + " AND ST_Within(A.Location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326)))" + } + + def 'postgis: no conjunct is added, ST_Within consults the index by itself'() { + + given: + def filterEncoder = encoder(new SqlDialectPgis(), Map.of("building.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_15, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == EXACT_WITHIN + } + + def 'S_DISJOINT is not accelerated, a disjoint geometry may lie outside the bbox'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def filter = SDisjoint.of(Property.of("location"), + SpatialLiteral.of(Bbox.of(-118.0, 33.8, -117.9, 34.0, OgcCrs.CRS84))) + + when: + String actual = filterEncoder.encode(filter, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the exact predicate is emitted on its own' + actual == "ST_Disjoint(A.location, ST_GeomFromText('POLYGON((-118.0 33.8,-117.9 33.8,-117.9 34.0,-118.0 34.0,-118.0 33.8))',4326))" + } + + def 'S_DISJOINT and NOT S_INTERSECTS mean the same and are encoded the same way'() { + + given: 'the same predicate, spelled both ways' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def bbox = SpatialLiteral.of(Bbox.of(-118.0d, 33.8d, -117.9d, 34.0d, OgcCrs.CRS84)) + + when: + String disjoint = filterEncoder.encode( + SDisjoint.of(Property.of("location"), bbox), QuerySchemaFixtures.SIMPLE_GEOMETRY) + String notIntersects = filterEncoder.encode( + Not.of(SIntersects.of(Property.of("location"), bbox)), QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'neither consults the r-tree, so the two spellings cannot disagree' + !disjoint.contains("rtree_") + !notIntersects.contains("rtree_") + } + + def 'under NOT the conjunct is dropped, a NULL geometry has no r-tree entry'() { + + given: + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + + when: + String actual = filterEncoder.encode(Not.of(CqlFilterExamples.EXAMPLE_15), QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: 'the exact predicate is negated on its own, as it was before' + actual == "NOT (${EXACT_WITHIN})" + } + + def 'a negation elsewhere in the filter does not stop the conjunct'() { + + given: 'a filter that negates an unrelated predicate and intersects the geometry' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id")) + def filter = And.of(Not.of(IsNull.of(Property.of("location"))), CqlFilterExamples.EXAMPLE_15) + + when: + String actual = filterEncoder.encode(filter, QuerySchemaFixtures.SIMPLE_GEOMETRY) + + then: + actual == "(A.location IS NOT NULL AND (${RTREE_LOOKUP} AND ${EXACT_WITHIN}))" + } + + def 'geometry reached through a join is not accelerated, the conjunct would address the wrong table'() { + + given: 'the geometry lives in a joined table, so the predicate goes inside a semi-join' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("building.location", "id", "geometry.location", "id")) + + when: + String actual = filterEncoder.encode(CqlFilterExamples.EXAMPLE_16, QuerySchemaFixtures.JOINED_GEOMETRY) + + then: + actual == "A.id IN (SELECT AA.id FROM building AA JOIN geometry AB ON (AA.id=AB.id) WHERE ST_Intersects(AB.location, ST_GeomFromText('POLYGON((-10.0 -10.0,10.0 -10.0,10.0 10.0,-10.0 -10.0))',4326)))" + } + + def 'the mapping based encoder, which the items query uses, is accelerated too'() { + + given: 'a filter on the primary geometry of a mapping whose column has an r-tree' + def filterEncoder = encoder(new SqlDialectGpkg(), Map.of("unfaelle_point.geom", "fid")) + def filter = SIntersects.of(Property.of("geometry"), + SpatialLiteral.of(Bbox.of(7.0, 50.0, 7.1, 50.1, OgcCrs.CRS84))) + + when: + String actual = filterEncoder.encode(filter, unfaelleMapping) + + then: + actual == "(A.\"fid\" IN (SELECT id FROM \"rtree_unfaelle_point_geom\" WHERE maxx >= 7.0 AND minx <= 7.1 AND maxy >= 50.0 AND miny <= 50.1) AND " + + "ST_Intersects(A.geom, ST_GeomFromText('POLYGON((7.0 50.0,7.1 50.0,7.1 50.1,7.0 50.1,7.0 50.0))',4326)))" + } +} diff --git a/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy new file mode 100644 index 000000000..b694030aa --- /dev/null +++ b/xtraplatform-features-sql/src/test/groovy/de/ii/xtraplatform/features/sql/domain/SqlDialectGpkgSpec.groovy @@ -0,0 +1,52 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.features.sql.domain + +import spock.lang.Shared +import spock.lang.Specification + +class SqlDialectGpkgSpec extends Specification { + + @Shared + SqlDialectGpkg dialect + + def setupSpec() { + dialect = new SqlDialectGpkg() + } + + def 'the spatial index predicate names the r-tree and the key column'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + "pv_pot_dach", "geom", "A", "id", [365204.0d, 5621522.0d] as double[], [365938.0d, 5622652.0d] as double[]) + + then: + actual.get() == 'A."id" IN (SELECT id FROM "rtree_pv_pot_dach_geom"' + + ' WHERE maxx >= 365204.0 AND minx <= 365938.0 AND maxy >= 5621522.0 AND miny <= 5622652.0)' + } + + def 'a bounding box without two axes yields no predicate'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + "t", "geom", "A", "id", [1.0d] as double[], [2.0d] as double[]) + + then: + actual.isEmpty() + } + + def 'identifiers are quoted, and embedded quotes escaped'() { + + when: + Optional actual = dialect.getSpatialIndexPredicate( + 'we"ird', "geom", "A", 'k"ey', [1.0d, 2.0d] as double[], [3.0d, 4.0d] as double[]) + + then: + actual.get().startsWith('A."k""ey" IN (SELECT id FROM "rtree_we""ird_geom"') + } +} diff --git a/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java b/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java index f7af81672..d7c03b56c 100644 --- a/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java +++ b/xtraplatform-geometries/src/main/java/de/ii/xtraplatform/geometries/domain/transform/MinMaxDeriver.java @@ -119,7 +119,9 @@ public double[][] visit(PolyhedralSurface geometry) { private static double[][] initMinMax(int dimensions) { double[][] minMax = new double[2][dimensions]; Arrays.fill(minMax[0], Double.MAX_VALUE); - Arrays.fill(minMax[1], Double.MIN_VALUE); + // -Double.MAX_VALUE, not Double.MIN_VALUE: the latter is the smallest positive value, which + // would survive every Math.max on an axis whose coordinates are all negative + Arrays.fill(minMax[1], -Double.MAX_VALUE); return minMax; } diff --git a/xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy b/xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy new file mode 100644 index 000000000..bf797525b --- /dev/null +++ b/xtraplatform-geometries/src/test/groovy/de/ii/xtraplatform/geometries/domain/MinMaxDeriverSpec.groovy @@ -0,0 +1,98 @@ +/* + * Copyright 2026 interactive instruments GmbH + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ +package de.ii.xtraplatform.geometries.domain + +import de.ii.xtraplatform.geometries.domain.transform.MinMaxDeriver +import spock.lang.Specification + +class MinMaxDeriverSpec extends Specification { + + static Polygon polygon(double... coordinates) { + return Polygon.of(List.of(PositionList.of(Axes.XY, coordinates))) + } + + def 'positive coordinates'() { + + given: + def geometry = polygon(7.0d, 50.0d, 7.1d, 50.0d, 7.1d, 50.1d, 7.0d, 50.1d, 7.0d, 50.0d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [7.0d, 50.0d] as double[] + minMax[1] == [7.1d, 50.1d] as double[] + } + + def 'coordinates that are all negative on one axis'() { + + given: 'a polygon west of Greenwich, so every x is negative' + def geometry = polygon(-118.0d, 33.8d, -117.9d, 33.8d, -117.9d, 34.0d, -118.0d, 34.0d, -118.0d, 33.8d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: 'the maximum is the largest x, not the seed value' + minMax[0] == [-118.0d, 33.8d] as double[] + minMax[1] == [-117.9d, 34.0d] as double[] + } + + def 'coordinates that are all negative on both axes'() { + + given: + def geometry = polygon(-70.0d, -33.5d, -69.9d, -33.5d, -69.9d, -33.4d, -70.0d, -33.4d, -70.0d, -33.5d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-70.0d, -33.5d] as double[] + minMax[1] == [-69.9d, -33.4d] as double[] + } + + def 'negative coordinates across the components of a multi geometry'() { + + given: + def geometry = MultiPolygon.of(List.of( + polygon(-10.0d, -10.0d, -9.0d, -10.0d, -9.0d, -9.0d, -10.0d, -9.0d, -10.0d, -10.0d), + polygon(-8.0d, -8.0d, -7.0d, -8.0d, -7.0d, -7.0d, -8.0d, -7.0d, -8.0d, -8.0d))) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-10.0d, -10.0d] as double[] + minMax[1] == [-7.0d, -7.0d] as double[] + } + + def 'a single negative point'() { + + given: + def geometry = Point.of(-118.0d, -33.8d) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-118.0d, -33.8d] as double[] + minMax[1] == [-118.0d, -33.8d] as double[] + } + + def 'a negative line string'() { + + given: + def geometry = LineString.of(PositionList.of(Axes.XY, [-5.0d, -6.0d, -3.0d, -8.0d] as double[])) + + when: + double[][] minMax = geometry.accept(new MinMaxDeriver()) + + then: + minMax[0] == [-5.0d, -8.0d] as double[] + minMax[1] == [-3.0d, -6.0d] as double[] + } +}