Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,8 @@ public FeatureTokenDecoderGml(
Map<String, EpsgCrs> srsNameMappings = new LinkedHashMap<>(inputProfile.getSrsNameMappings());
Set<String> verticalSrsNames = new HashSet<>();
collectVariantReferenceSystems(featureSchema, srsNameMappings, verticalSrsNames);
this.geometryDecoder = new GeometryDecoderGml(srsNameMappings, verticalSrsNames);
this.geometryDecoder =
new GeometryDecoderGml(srsNameMappings, verticalSrsNames, inputProfile.getSupportedCrs());
this.buffer = new StringBuilder();

List<String> wrappers = new ArrayList<>(2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,13 @@ default String getFeatureMemberElementName() {
*/
Set<String> getObjectTypeSuffixedProperties();

/**
* The coordinate reference systems that a {@code srsName} attribute in the document may declare.
* An empty list does not restrict them; a value that resolves to another coordinate reference
* system is rejected.
*/
List<EpsgCrs> getSupportedCrs();

static FeatureTokenDecoderGmlInputProfile empty() {
return ImmutableFeatureTokenDecoderGmlInputProfile.builder().build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,8 @@ static class Frame {

private final Deque<Frame> stack = new ArrayDeque<>();
private final Map<String, EpsgCrs> srsNameMappings;
// the coordinate reference systems that a srsName may resolve to; empty does not restrict them
private final List<EpsgCrs> supportedCrs;
private final Set<String> verticalSrsNames;
private boolean waitingForInput = false;
private Geometry<?> result;
Expand Down Expand Up @@ -164,7 +166,7 @@ public GeometryDecoderGml() {
* built-in parsers cannot handle.
*/
public GeometryDecoderGml(Map<String, EpsgCrs> srsNameMappings) {
this(srsNameMappings, Set.of());
this(srsNameMappings, Set.of(), List.of());
}

/**
Expand All @@ -173,8 +175,20 @@ public GeometryDecoderGml(Map<String, EpsgCrs> srsNameMappings) {
* {@link #getVerticalValue()}) instead of being decoded into a {@link Geometry}.
*/
public GeometryDecoderGml(Map<String, EpsgCrs> srsNameMappings, Set<String> verticalSrsNames) {
this(srsNameMappings, verticalSrsNames, List.of());
}

/**
* @param supportedCrs the coordinate reference systems that a {@code srsName} attribute may
* resolve to; an empty list does not restrict them
*/
public GeometryDecoderGml(
Map<String, EpsgCrs> srsNameMappings,
Set<String> verticalSrsNames,
List<EpsgCrs> supportedCrs) {
this.srsNameMappings = srsNameMappings == null ? Map.of() : srsNameMappings;
this.verticalSrsNames = verticalSrsNames == null ? Set.of() : verticalSrsNames;
this.supportedCrs = supportedCrs == null ? List.of() : supportedCrs;
}

/** The verbatim srsName of the outermost geometry element of the last completed decode. */
Expand Down Expand Up @@ -353,7 +367,8 @@ private boolean handleStart(
this.rawSrsName = parser.getAttributeValue(null, "srsName");
this.verticalMode = rawSrsName != null && verticalSrsNames.contains(rawSrsName);
}
Optional<EpsgCrs> explicitCrs = parseSrsName(parser, srsNameMappings);
Optional<EpsgCrs> explicitCrs =
parseSrsName(parser, srsNameMappings).map(this::checkSupported);
f.crs = explicitCrs.or(() -> defaultCrs).or(this::inheritedCrs);
OptionalInt dim = parseSrsDimension(parser);
if (dim.isEmpty()) {
Expand Down Expand Up @@ -540,6 +555,22 @@ private OptionalInt inheritedSrsDimension() {
return OptionalInt.empty();
}

/**
* A coordinate reference system declared on a geometry must be one of the supported ones. Only
* the code is compared, the axis order is a property of the encoding, not of the coordinate
* reference system.
*/
private EpsgCrs checkSupported(EpsgCrs crs) {
if (!supportedCrs.isEmpty()
&& supportedCrs.stream().noneMatch(supported -> supported.getCode() == crs.getCode())) {
throw new IllegalArgumentException(
String.format(
"The coordinate reference system '%s' in a 'srsName' attribute is not supported here. Supported coordinate reference systems: %s.",
crs.toUriString(), supportedCrs.stream().map(EpsgCrs::toUriString).toList()));
}
return crs;
}

private static Optional<EpsgCrs> parseSrsName(
AsyncXMLStreamReader<AsyncByteArrayFeeder> parser, Map<String, EpsgCrs> srsNameMappings) {
String srsName = parser.getAttributeValue(null, "srsName");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import de.ii.xtraplatform.geometries.domain.transcode.json.GeometryDecoderJson;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Objects;
import java.util.Optional;

Expand All @@ -44,6 +45,7 @@ public class FeatureTokenDecoderGeoJson
private final Optional<String> nullValue;
private final EpsgCrs crs;
private final Axes axes;
private final GeometryDecoderJson geometryDecoder;

private boolean started;
private int depth = -1;
Expand All @@ -68,6 +70,15 @@ public class FeatureTokenDecoderGeoJson
downstream;

public FeatureTokenDecoderGeoJson(Optional<String> nullValue, EpsgCrs crs, Axes axes) {
this(nullValue, crs, axes, List.of());
}

/**
* @param supportedCrs the coordinate reference systems that a {@code coordRefSys} member in the
* document may declare; an empty list does not restrict them
*/
public FeatureTokenDecoderGeoJson(
Optional<String> nullValue, EpsgCrs crs, Axes axes, List<EpsgCrs> supportedCrs) {
super();
try {
this.parser = JSON_FACTORY.createNonBlockingByteArrayParser();
Expand All @@ -78,6 +89,7 @@ public FeatureTokenDecoderGeoJson(Optional<String> nullValue, EpsgCrs crs, Axes
this.nullValue = nullValue;
this.crs = crs;
this.axes = axes;
this.geometryDecoder = new GeometryDecoderJson(false, supportedCrs);
}

@Override
Expand Down Expand Up @@ -146,7 +158,7 @@ public boolean advanceParser() {
if (geometryDepth == 0) {
JsonNode geomNode = OBJECT_MAPPER.readTree(geometryBuffer.asParser());
Geometry<?> geometry =
new GeometryDecoderJson().decode(geomNode, Optional.of(crs), Optional.of(axes));
geometryDecoder.decode(geomNode, Optional.of(crs), Optional.of(axes));
context.setGeometry(geometry);
context.pathTracker().track(geometryFieldName, 0);
downstream.onGeometry(context);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -582,7 +582,8 @@ protected boolean onStartup() throws InterruptedException {
? dbmsAdapters.get(getData().getConnectionInfo().getDialect()).getDefaultSchemas()
: getData().getConnectionInfo().getSchemas();
this.sourceSchemaValidator =
new SourceSchemaValidatorSql(validationSchemas, this::getSqlClient);
new SourceSchemaValidatorSql(
validationSchemas, getData().getNativeTimeZone(), this::getSqlClient);

this.pathParser3 = createPathParser3(getData().getSourcePathDefaults(), cql, subdecoders);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
import de.ii.xtraplatform.features.domain.SchemaBase.Type;
import de.ii.xtraplatform.features.domain.Tuple;
import de.ii.xtraplatform.features.sql.domain.ValueTypeMapping;
import java.sql.JDBCType;
import java.util.Collection;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import org.slf4j.Logger;
Expand Down Expand Up @@ -106,6 +108,34 @@ public boolean isColumnTemporal(String table, String name) {
.isPresent();
}

/**
* Whether the column has a time zone, e.g. PostgreSQL {@code timestamptz}. Drivers differ in the
* JDBC type they report for such columns, so the database-specific type name is checked, too.
*/
public boolean isColumnTemporalWithTimeZone(String table, String name) {
return getColumn(table, name)
.map(Column::getColumnDataType)
.filter(
t ->
Objects.equals(t.getJavaSqlType(), JDBCType.TIMESTAMP_WITH_TIMEZONE)
|| Objects.equals(t.getJavaSqlType(), JDBCType.TIME_WITH_TIMEZONE)
|| hasTimeZoneInName(t.getDatabaseSpecificTypeName())
|| hasTimeZoneInName(t.getName()))
.isPresent();
}

private static boolean hasTimeZoneInName(String typeName) {
if (Objects.isNull(typeName)) {
return false;
}
// "timestamptz"/"timetz" (PostgreSQL), "TIMESTAMP WITH TIME ZONE" and Oracle's
// "TIMESTAMP WITH LOCAL TIME ZONE"; "timestamp without time zone" must not match
String name = typeName.toLowerCase(Locale.ROOT);
return name.endsWith("tz")
|| name.contains("with time zone")
|| name.contains("with local time zone");
}

public Optional<Column> getColumn(String tableName, String columnName) {
return getColumn(tableName, columnName, false, false);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@
import de.ii.xtraplatform.features.sql.domain.SqlClient;
import de.ii.xtraplatform.features.sql.domain.SqlRelation;
import java.io.IOException;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.slf4j.Logger;
Expand All @@ -35,12 +39,17 @@ public class SourceSchemaValidatorSql implements SourceSchemaValidator<SchemaSql
"%s: column '%s' in table '%s' should not be used as %s, neither a primary key nor a unique constraint can be found";
public static final String COLUMN_CANNOT_BE_USED_AS =
"%s: column '%s' in table '%s' cannot be used as %s";
public static final String COLUMN_WITH_TIME_ZONE =
"%s: column '%s' in table '%s' has a time zone, but 'nativeTimeZone' is set to '%s'. Values of such a column are read in UTC and would then be interpreted as '%s', which shifts them. 'nativeTimeZone' is only applied to values without a time zone; either set it to UTC or use a column without a time zone.";

private final List<String> schemas;
private final Optional<ZoneId> nativeTimeZone;
private Supplier<SqlClient> sqlClient;

public SourceSchemaValidatorSql(List<String> schemas, Supplier<SqlClient> sqlClient) {
public SourceSchemaValidatorSql(
List<String> schemas, Optional<ZoneId> nativeTimeZone, Supplier<SqlClient> sqlClient) {
this.schemas = schemas;
this.nativeTimeZone = nativeTimeZone;
this.sqlClient = sqlClient;
}

Expand Down Expand Up @@ -213,13 +222,37 @@ private ValidationResult validate(
tableSchema.getName(),
"datetime"));
}

if (attribute.isTemporal()
&& hasNonUtcTimeZone()
&& schemaInfo.isColumnTemporalWithTimeZone(
tableSchema.getName(), attribute.getName())) {
result.addWarnings(
String.format(
COLUMN_WITH_TIME_ZONE,
String.format(
"Potentially incorrect datetime values for property '%s' in type '%s'",
attribute.getSourcePath().orElse("???"), typeName),
attribute.getName(),
tableSchema.getName(),
nativeTimeZone.get(),
nativeTimeZone.get()));
}
}
}
});

return result.build();
}

// The values of a column with a time zone are read in UTC (the time zone of every connection),
// so only a native time zone other than UTC changes their meaning.
private boolean hasNonUtcTimeZone() {
return nativeTimeZone
.filter(zone -> !Objects.equals(zone.getRules(), ZoneOffset.UTC.getRules()))
.isPresent();
}

private List<String> getAllUsedTables(List<SchemaSql> schemaSql) {
return schemaSql.stream()
.flatMap(schemaSql1 -> schemaSql1.getAllObjects().stream())
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright 2026 interactive instruments GmbH
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package de.ii.xtraplatform.features.sql.infra.db

import schemacrawler.crawl.MutableColumn
import schemacrawler.crawl.MutableColumnDataType
import schemacrawler.crawl.MutableTable
import schemacrawler.schema.Column
import schemacrawler.schema.ColumnDataType
import schemacrawler.schema.DataTypeType
import schemacrawler.schema.JavaSqlType
import schemacrawler.schema.JavaSqlTypeGroup
import schemacrawler.schema.Table
import schemacrawler.schemacrawler.SchemaReference
import spock.lang.Specification

import java.sql.JDBCType
import java.sql.Timestamp

/**
* Detection of columns with a time zone. Only the values of such a column carry an offset in the
* data source, so the provider option "nativeTimeZone" must not be applied to them; the source
* schema validator reports the combination as a warning. Drivers disagree about the JDBC type they
* report for these columns, hence the fallback to the type name.
*/
class SchemaInfoSpec extends Specification {

def 'temporal columns with and without a time zone'() {

given: 'a table with the temporal column types of the supported dialects'
SchemaReference schema = new SchemaReference("catalog", "schema")
Table table = new MutableTable(schema, "table1")
addColumn(schema, table, "tstz", "timestamptz", JDBCType.TIMESTAMP)
addColumn(schema, table, "tstz_jdbc", "timestamptz", JDBCType.TIMESTAMP_WITH_TIMEZONE)
addColumn(schema, table, "tstz_name", "TIMESTAMP WITH TIME ZONE", JDBCType.TIMESTAMP)
addColumn(schema, table, "tstz_local", "TIMESTAMP(6) WITH LOCAL TIME ZONE", JDBCType.TIMESTAMP)
addColumn(schema, table, "ttz", "timetz", JDBCType.TIME)
addColumn(schema, table, "ts", "timestamp", JDBCType.TIMESTAMP)
addColumn(schema, table, "ts_name", "timestamp without time zone", JDBCType.TIMESTAMP)
addColumn(schema, table, "d", "date", JDBCType.DATE)
SchemaInfo schemaInfo = new SchemaInfo([table])

expect: 'the columns that carry an offset are detected, independently of the JDBC type'
schemaInfo.isColumnTemporalWithTimeZone("table1", "tstz")
schemaInfo.isColumnTemporalWithTimeZone("table1", "tstz_jdbc")
schemaInfo.isColumnTemporalWithTimeZone("table1", "tstz_name")
schemaInfo.isColumnTemporalWithTimeZone("table1", "tstz_local")
schemaInfo.isColumnTemporalWithTimeZone("table1", "ttz")

and: 'columns without a time zone are not, in particular not "without time zone"'
!schemaInfo.isColumnTemporalWithTimeZone("table1", "ts")
!schemaInfo.isColumnTemporalWithTimeZone("table1", "ts_name")
!schemaInfo.isColumnTemporalWithTimeZone("table1", "d")

and: 'an unknown column is not reported'
!schemaInfo.isColumnTemporalWithTimeZone("table1", "unknown")
!schemaInfo.isColumnTemporalWithTimeZone("unknown", "ts")

and: 'both are temporal columns'
schemaInfo.isColumnTemporal("table1", "tstz")
schemaInfo.isColumnTemporal("table1", "ts")
}

def 'a column that is not temporal has no time zone'() {

given: 'a table with a string and a geometry column'
SchemaReference schema = new SchemaReference("catalog", "schema")
Table table = new MutableTable(schema, "table1")
addColumn(schema, table, "s", "varchar", JDBCType.VARCHAR)
addColumn(schema, table, "geom", "geometry", JDBCType.OTHER)
SchemaInfo schemaInfo = new SchemaInfo([table])

expect:
!schemaInfo.isColumnTemporalWithTimeZone("table1", "s")
!schemaInfo.isColumnTemporalWithTimeZone("table1", "geom")
}

// the schemacrawler.crawl classes are not public, so every member access goes through Groovy's
// dynamic dispatch (as in SchemaGeneratorSqlSpec) and the variables are typed as the interfaces
private static void addColumn(
SchemaReference schema, Table table, String name, String typeName, JDBCType jdbcType) {
Column column = new MutableColumn(table, name)
ColumnDataType columnDataType = new MutableColumnDataType(schema, typeName, DataTypeType.system)
columnDataType.setJavaSqlType(new JavaSqlType(jdbcType, Timestamp.class, JavaSqlTypeGroup.temporal))
column.setColumnDataType(columnDataType)
table.addColumn(column)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,10 @@ public CompletionStage<Result> runWith(
}

if (stepMetadata) {
source = source.via(new FeatureTokenTransformerMetadata(resultBuilder));
source =
source.via(
new FeatureTokenTransformerMetadata(
resultBuilder, data.getNativeTimeZone().orElse(ZoneId.of("UTC"))));
}

FeatureTokenTransformerAudit auditTransformer = null;
Expand Down Expand Up @@ -288,7 +291,10 @@ public <X> CompletionStage<ResultReduced<X>> runWith(
source = source.via(new FeatureTokenTransformerWeakETag(resultBuilder));
}
if (stepMetadata) {
source = source.via(new FeatureTokenTransformerMetadata(resultBuilder));
source =
source.via(
new FeatureTokenTransformerMetadata(
resultBuilder, data.getNativeTimeZone().orElse(ZoneId.of("UTC"))));
}

FeatureTokenTransformerAudit auditTransformer = null;
Expand Down
Loading
Loading