diff --git a/build.gradle b/build.gradle index 159075f112..939b8f01f8 100644 --- a/build.gradle +++ b/build.gradle @@ -604,6 +604,140 @@ project(':sbe-samples') { } } +project(':sbe-jackson') { + apply plugin: 'maven-publish' + apply plugin: 'signing' + + dependencies { + api project(':sbe-tool') + api libs.jackson.databind + } + + def generatedDir = "${layout.buildDirectory.get()}/generated-src" + def generatedClassesDir = "${layout.buildDirectory.get()}/classes/java/generated" + sourceSets { + generated { + java.srcDir generatedDir + compileClasspath += project(':sbe-tool').sourceSets.main.runtimeClasspath + } + test { + resources.srcDir project(':sbe-tool').file('src/test/resources') + } + } + + compileGeneratedJava.dependsOn 'generateTestCodecs' + compileTestJava.dependsOn compileGeneratedJava + + testing { + suites { + test { + dependencies { + implementation files(generatedClassesDir) + implementation libs.hamcrest + implementation platform(libs.junit.bom) + implementation "org.junit.jupiter:junit-jupiter-params" + implementation(libs.jqwik) { + // Exclude JUnit 5 dependencies that are already provided due to useJUnitJupiter + exclude group: 'org.junit.platform', module: 'junit-platform-commons' + exclude group: 'org.junit.platform', module: 'junit-platform-engine' + } + } + } + } + } + + tasks.register('generateTestCodecs', JavaExec) { + mainClass.set('uk.co.real_logic.sbe.SbeTool') + classpath = project(':sbe-tool').sourceSets.main.runtimeClasspath + jvmArgs('--add-opens', 'java.base/jdk.internal.misc=ALL-UNNAMED') + systemProperties( + 'sbe.output.dir': generatedDir.toString(), + 'sbe.target.language': 'Java', + 'sbe.validation.stop.on.error': 'true', + 'sbe.validation.xsd': validationXsdPath.toString(), + 'sbe.generate.precedence.checks': 'false') + def schemaDir = project(':sbe-tool').file('src/test/resources') + def schemas = [new File(schemaDir, 'json-printer-test-schema.xml'), + new File(schemaDir, 'example-extension-schema.xml'), + new File(schemaDir, 'composite-elements-schema.xml'), + new File(schemaDir, 'group-with-data-schema.xml')] + args = schemas.collect { it.path } + inputs.files(schemas) + inputs.file(validationXsdPath) + inputs.files(project(':sbe-tool').sourceSets.main.runtimeClasspath) + outputs.dir(generatedDir) + doFirst { + delete generatedDir + } + } + + // Runs the test suite a second time against the latest Jackson 2.21.x release. + configurations { + latestJacksonTestRuntimeClasspath { + extendsFrom testRuntimeClasspath + resolutionStrategy.force "com.fasterxml.jackson.core:jackson-databind:${libs.versions.jackson.latest.get()}" + } + } + + tasks.register('testLatestJackson', Test) { + description = 'Runs the sbe-jackson tests against the latest Jackson 2.21.x release.' + group = 'verification' + dependsOn 'compileTestJava' + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.output + sourceSets.main.output + + files(generatedClassesDir) + configurations.latestJacksonTestRuntimeClasspath + useJUnitPlatform() + jvmArgs('--add-opens', 'java.base/jdk.internal.misc=ALL-UNNAMED') + javaLauncher.set(toolchainLauncher) + testLogging { + exceptionFormat = 'full' + events = ["FAILED", "STANDARD_OUT", "STANDARD_ERROR"] + } + } + + jar { + manifest.attributes( + 'Specification-Title': 'Simple Binary Encoding', + 'Specification-Version': '1.0', + 'Implementation-Title': 'SBE', + 'Implementation-Version': sbeVersion, + 'Implementation-Vendor': 'Adaptive Financial Consulting Limited', + 'Automatic-Module-Name': 'uk.co.real_logic.sbe.jackson' + ) + } + + java { + withSourcesJar() + withJavadocJar() + } + + publishing { + publications { + sbeJackson(MavenPublication) { + from components.java + pom(projectPom) + } + } + + repositories { + maven { + url = !isReleaseVersion ? sonatypeCentralPortalSnapshotsRepoUrl : sonatypeCentralPortalReleasesRepoUrl + credentials { + username = sonatypeCentralPortalUsername + password = sonatypeCentralPortalPassword + } + } + } + } + + signing { + if (signingKey != null) { + useInMemoryPgpKeys(signingKey, signingPassword) + } + sign publishing.publications.sbeJackson + } +} + project(':sbe-benchmarks') { apply plugin: 'com.gradleup.shadow' @@ -611,6 +745,8 @@ project(':sbe-benchmarks') { implementation libs.jmh.core annotationProcessor libs.jmh.generator.annprocess implementation project(':sbe-tool') + implementation project(':sbe-jackson') + implementation libs.jackson.databind implementation files("${layout.buildDirectory.get()}/classes/java/generated") } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dc7d6adc01..312ec60941 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -5,6 +5,8 @@ commons-codec = "1.15" commons-lang3 = "3.8.1" hamcrest = "3.0" httpcore = "4.4.14" +jackson = "2.16.1" +jackson-latest = "2.21.6" jqwik = "1.10.1" jmh = "1.37" json = "20260814" @@ -20,6 +22,7 @@ commons-codec = { group = "commons-codec", name = "commons-codec", version.ref = commons-lang3 = { group = "org.apache.commons", name = "commons-lang3", version.ref = "commons-lang3" } hamcrest = { group = "org.hamcrest", name = "hamcrest", version.ref = "hamcrest" } httpcore = { group = "org.apache.httpcomponents", name = "httpcore", version.ref = "httpcore" } +jackson-databind = { group = "com.fasterxml.jackson.core", name = "jackson-databind", version.ref = "jackson" } jqwik = { group = "net.jqwik", name = "jqwik", version.ref = "jqwik" } json = { group = "org.json", name = "json", version.ref = "json" } jmh-core = { group = "org.openjdk.jmh", name = "jmh-core", version.ref = "jmh" } diff --git a/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/NaiveCarTokenListener.java b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/NaiveCarTokenListener.java new file mode 100644 index 0000000000..aaa48a188b --- /dev/null +++ b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/NaiveCarTokenListener.java @@ -0,0 +1,203 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.benchmarks; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.DirectBuffer; +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.ir.Encoding; +import uk.co.real_logic.sbe.ir.Token; +import uk.co.real_logic.sbe.otf.AbstractTokenListener; +import uk.co.real_logic.sbe.otf.Types; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * Deliberately simple stock-node baseline for this car.xml corpus, not a general SBE adapter. + * Re-walks tokens and enum values on every decode; no compiled field plan or cached value nodes. + * Optional fields, version transitions, binary data and uint64 are outside this corpus. + */ +final class NaiveCarTokenListener extends AbstractTokenListener +{ + private static final JsonNodeFactory FACTORY = JsonNodeFactory.instance; + private final ObjectNode[] objects = new ObjectNode[8]; + private final ArrayNode[] groups = new ArrayNode[8]; + private int depth; + private int compositeDepth; + + ObjectNode root() + { + return objects[0]; + } + + @Override + public void onBeginMessage(final Token token) + { + depth = 0; + compositeDepth = 0; + objects[0] = FACTORY.objectNode(); + } + + @Override + public void onEncoding( + final Token fieldToken, final DirectBuffer buffer, final int bufferIndex, + final Token typeToken, final int actingVersion) + { + final Encoding encoding = typeToken.encoding(); + final String name = compositeDepth > 0 ? typeToken.name() : fieldToken.name(); + final PrimitiveType type = encoding.primitiveType(); + final JsonNode value; + if (typeToken.isConstantEncoding()) + { + value = PrimitiveType.CHAR == type ? FACTORY.textNode(encoding.constValue().toString()) : + integerNode(type, encoding.constValue().longValue()); + } + else if (PrimitiveType.CHAR == type) + { + final byte[] bytes = new byte[typeToken.arrayLength()]; + buffer.getBytes(bufferIndex, bytes); + int length = 0; + while (length < bytes.length && bytes[length] != 0) + { + length++; + } + value = FACTORY.textNode(new String(bytes, 0, length, StandardCharsets.US_ASCII)); + } + else if (typeToken.arrayLength() > 1) + { + final ArrayNode array = FACTORY.arrayNode(); + for (int i = 0; i < typeToken.arrayLength(); i++) + { + array.add(primitive(buffer, bufferIndex + i * type.size(), encoding)); + } + value = array; + } + else + { + value = primitive(buffer, bufferIndex, encoding); + } + objects[depth].set(name, value); + } + + @Override + public void onEnum( + final Token fieldToken, final DirectBuffer buffer, final int bufferIndex, + final List tokens, final int fromIndex, final int toIndex, final int actingVersion) + { + final long raw = Types.getLong(buffer, bufferIndex, tokens.get(fromIndex + 1).encoding()); + for (int i = fromIndex + 1; i < toIndex; i++) + { + final Token value = tokens.get(i); + if (raw == value.encoding().constValue().longValue()) + { + objects[depth].put(fieldToken.name(), value.name()); + return; + } + } + objects[depth].set(fieldToken.name(), integerNode(tokens.get(fromIndex + 1).encoding().primitiveType(), raw)); + } + + @Override + public void onBitSet( + final Token fieldToken, final DirectBuffer buffer, final int bufferIndex, + final List tokens, final int fromIndex, final int toIndex, final int actingVersion) + { + final Encoding encoding = tokens.get(fromIndex + 1).encoding(); + objects[depth].set(fieldToken.name(), integerNode(encoding.primitiveType(), + Types.getLong(buffer, bufferIndex, encoding))); + } + + @Override + public void onBeginComposite( + final Token fieldToken, final List tokens, final int fromIndex, final int toIndex) + { + final ObjectNode child = FACTORY.objectNode(); + objects[depth].set(fieldToken.name(), child); + objects[++depth] = child; + compositeDepth++; + } + + @Override + public void onEndComposite( + final Token fieldToken, final List tokens, final int fromIndex, final int toIndex) + { + objects[depth--] = null; + compositeDepth--; + } + + @Override + public void onGroupHeader(final Token token, final int numInGroup) + { + final ArrayNode group = FACTORY.arrayNode(); + objects[depth].set(token.name(), group); + groups[depth] = numInGroup == 0 ? null : group; + } + + @Override + public void onBeginGroup(final Token token, final int groupIndex, final int numInGroup) + { + final ObjectNode entry = FACTORY.objectNode(); + groups[depth].add(entry); + objects[++depth] = entry; + } + + @Override + public void onEndGroup(final Token token, final int groupIndex, final int numInGroup) + { + objects[depth--] = null; + if (groupIndex + 1 == numInGroup) + { + groups[depth] = null; + } + } + + @Override + public void onVarData( + final Token fieldToken, final DirectBuffer buffer, final int bufferIndex, + final int length, final Token typeToken) + { + final byte[] bytes = new byte[length]; + buffer.getBytes(bufferIndex, bytes); + objects[depth].put(fieldToken.name(), + new String(bytes, Charset.forName(typeToken.encoding().characterEncoding()))); + } + + private static JsonNode primitive(final DirectBuffer buffer, final int index, final Encoding encoding) + { + switch (encoding.primitiveType()) + { + case FLOAT: + return FACTORY.numberNode(buffer.getFloat(index, encoding.byteOrder())); + + case DOUBLE: + return FACTORY.numberNode(buffer.getDouble(index, encoding.byteOrder())); + + default: + return integerNode(encoding.primitiveType(), Types.getLong(buffer, index, encoding)); + } + } + + private static JsonNode integerNode(final PrimitiveType type, final long value) + { + return PrimitiveType.UINT32 == type || PrimitiveType.INT64 == type ? + FACTORY.numberNode(value) : FACTORY.numberNode((int)value); + } +} diff --git a/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonCarState.java b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonCarState.java new file mode 100644 index 0000000000..44992df688 --- /dev/null +++ b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonCarState.java @@ -0,0 +1,213 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.benchmarks; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.openjdk.jmh.annotations.*; +import uk.co.real_logic.sbe.CarBenchmark; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.jackson.SbeJson; +import uk.co.real_logic.sbe.jackson.SbeJsonDecoder; +import uk.co.real_logic.sbe.jackson.SbeJsonEncoder; +import uk.co.real_logic.sbe.json.JsonPrinter; +import uk.co.real_logic.sbe.otf.OtfHeaderDecoder; +import uk.co.real_logic.sbe.otf.OtfMessageDecoder; +import uk.co.real_logic.sbe.xml.IrGenerator; +import uk.co.real_logic.sbe.xml.ParserOptions; +import uk.co.real_logic.sbe.xml.XmlSchemaParser; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * Shared trial data for all three axes. Setup also checks the benchmark adapters against the generated corpus. + * All mutable codecs, output buffers and future borrowed documents belong to one benchmark thread. + */ +@State(Scope.Thread) +public class SbeJsonCarState +{ + /** Corpus size; HEAVY is bounded deliberately, not the schema's nested group maximum. */ + @Param({ "BASELINE", "HEAVY" }) + public String corpus; + + final UnsafeBuffer input = new UnsafeBuffer(ByteBuffer.allocateDirect(64 * 1024)); + final UnsafeBuffer output = new UnsafeBuffer(ByteBuffer.allocateDirect(64 * 1024)); + final ObjectMapper mapper = new ObjectMapper(); + final MessageHeaderEncoder generatedHeader = new MessageHeaderEncoder(); + final CarEncoder generatedEncoder = new CarEncoder(); + final NaiveCarTokenListener listener = new NaiveCarTokenListener(); + final StringBuilder printed = new StringBuilder(64 * 1024); + final ByteArrayOutputStream jsonOutput = new ByteArrayOutputStream(64 * 1024); + Ir ir; + SbeJson sbeJson; + SbeJsonDecoder decoder; + SbeJsonEncoder encoder; + OtfHeaderDecoder header; + JsonPrinter printer; + ObjectNode tree; + int length; + + @Setup(Level.Trial) + public void setup() throws Exception + { + try (InputStream stream = Objects.requireNonNull( + SbeJsonCarState.class.getResourceAsStream("/car.xml"), "car.xml")) + { + ir = new IrGenerator().generate(XmlSchemaParser.parse(stream, ParserOptions.DEFAULT)); + } + sbeJson = SbeJson.builder(ir).build(); + decoder = sbeJson.newDecoder(); + encoder = sbeJson.newEncoder(CarEncoder.TEMPLATE_ID); + header = new OtfHeaderDecoder(ir.headerStructure()); + printer = new JsonPrinter(ir); + + CarBenchmark.encode(generatedHeader, generatedEncoder, input, 0); + length = MessageHeaderEncoder.ENCODED_LENGTH + generatedEncoder.encodedLength(); + tree = decoder.decodeCopy(input, 0, length); + if ("HEAVY".equals(corpus)) + { + expandCorpus(); + length = encodeGenerated(input); + tree = decoder.decodeCopy(input, 0, length); + } + + require(tree.equals(decodeOtf()), "OTF tree differs from decodeCopy"); + require(length == encodeGenerated(output), "generated length differs"); + requireBytesEqual(); + require(length == encoder.encodedLength(tree), "sizing pass differs"); + require(length == encoder.encode(tree, output, 0, output.capacity()), "plan length differs"); + requireBytesEqual(); + + printer.print(printed, input, 0); + final ObjectNode printerTree = (ObjectNode)mapper.readTree(printed.toString()); + // JsonPrinter uses a choice object and readTree uses DoubleNode for floats. Re-encoding compares + // the actual wire values without charging normalization to either measured decoder. + printerTree.set("extras", tree.get("extras")); + require(length == encoder.encode(printerTree, output, 0, output.capacity()), "printer length differs"); + requireBytesEqual(); + require(mapper.readTree(mapper.writeValueAsBytes(tree)).equals(mapper.readTree(tree.toString())), + "byte serialization differs"); + mapper.writeValue(jsonOutput, tree); + require(mapper.readTree(jsonOutput.toByteArray()).equals(mapper.readTree(tree.toString())), + "stream serialization differs"); + jsonOutput.reset(); + } + + ObjectNode decodeOtf() + { + final int templateId = header.getTemplateId(input, 0); + OtfMessageDecoder.decode(input, header.encodedLength(), header.getSchemaVersion(input, 0), + header.getBlockLength(input, 0), ir.getMessage(templateId), listener); + return listener.root(); + } + + int encodeGenerated(final UnsafeBuffer destination) + { + final CarEncoder car = generatedEncoder.wrapAndApplyHeader(destination, 0, generatedHeader); + car.serialNumber(tree.get("serialNumber").longValue()) + .modelYear(tree.get("modelYear").intValue()) + .available(BooleanType.valueOf(tree.get("available").textValue())) + .code(Model.valueOf(tree.get("code").textValue())) + .vehicleCode(tree.get("vehicleCode").textValue()); + final JsonNode numbers = tree.get("someNumbers"); + for (int i = 0; i < numbers.size(); i++) + { + car.someNumbers(i, numbers.get(i).intValue()); + } + final int extras = tree.get("extras").intValue(); + car.extras().clear().sunRoof((extras & 1) != 0).sportsPack((extras & 2) != 0) + .cruiseControl((extras & 4) != 0); + final JsonNode engine = tree.get("engine"); + car.engine().capacity(engine.get("capacity").intValue()) + .numCylinders((short)engine.get("numCylinders").intValue()) + .manufacturerCode(engine.get("manufacturerCode").textValue()); + + final JsonNode fuel = tree.get("fuelFigures"); + final CarEncoder.FuelFiguresEncoder fuelEncoder = car.fuelFiguresCount(fuel.size()); + for (int i = 0; i < fuel.size(); i++) + { + final JsonNode entry = fuel.get(i); + fuelEncoder.next().speed(entry.get("speed").intValue()).mpg(entry.get("mpg").floatValue()); + } + final JsonNode performance = tree.get("performanceFigures"); + final CarEncoder.PerformanceFiguresEncoder performanceEncoder = car.performanceFiguresCount(performance.size()); + for (int i = 0; i < performance.size(); i++) + { + final JsonNode entry = performance.get(i); + performanceEncoder.next().octaneRating((short)entry.get("octaneRating").intValue()); + final JsonNode acceleration = entry.get("acceleration"); + final CarEncoder.PerformanceFiguresEncoder.AccelerationEncoder accelerationEncoder = + performanceEncoder.accelerationCount(acceleration.size()); + for (int j = 0; j < acceleration.size(); j++) + { + final JsonNode value = acceleration.get(j); + accelerationEncoder.next().mph(value.get("mph").intValue()).seconds(value.get("seconds").floatValue()); + } + } + car.manufacturer(tree.get("manufacturer").textValue()); + car.model(tree.get("model").textValue()); + return MessageHeaderEncoder.ENCODED_LENGTH + car.encodedLength(); + } + + private void expandCorpus() + { + final ArrayNode fuel = (ArrayNode)tree.get("fuelFigures"); + final ObjectNode fuelEntry = (ObjectNode)fuel.get(0); + fuel.removeAll(); + for (int i = 0; i < 32; i++) + { + fuel.add(fuelEntry.deepCopy().put("speed", 256 + i)); + } + final ArrayNode performance = (ArrayNode)tree.get("performanceFigures"); + final ObjectNode performanceEntry = (ObjectNode)performance.get(0); + final ArrayNode acceleration = (ArrayNode)performanceEntry.get("acceleration"); + final ObjectNode accelerationEntry = (ObjectNode)acceleration.get(0); + acceleration.removeAll(); + for (int i = 0; i < 16; i++) + { + acceleration.add(accelerationEntry.deepCopy().put("mph", 256 + i)); + } + performance.removeAll(); + for (int i = 0; i < 8; i++) + { + performance.add(performanceEntry.deepCopy()); + } + tree.put("manufacturer", "Café".repeat(1024)); + tree.put("model", "Modèle".repeat(683)); + } + + private void requireBytesEqual() + { + for (int i = 0; i < length; i++) + { + require(input.getByte(i) == output.getByte(i), "wire bytes differ at " + i); + } + } + + private static void require(final boolean condition, final String message) + { + if (!condition) + { + throw new IllegalStateException(message); + } + } +} diff --git a/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonDecodeBenchmark.java b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonDecodeBenchmark.java new file mode 100644 index 0000000000..6dc106d091 --- /dev/null +++ b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonDecodeBenchmark.java @@ -0,0 +1,50 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.benchmarks; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class SbeJsonDecodeBenchmark +{ + @Benchmark + public void decodeCopy(final SbeJsonCarState state, final Blackhole blackhole) + { + blackhole.consume(state.decoder.decodeCopy(state.input, 0, state.length)); + } + + @Benchmark + public void jsonPrinterReadTree(final SbeJsonCarState state, final Blackhole blackhole) throws IOException + { + state.printed.setLength(0); + state.printer.print(state.printed, state.input, 0); + blackhole.consume(state.mapper.readTree(state.printed.toString())); + } + + @Benchmark + public void naiveOtfTree(final SbeJsonCarState state, final Blackhole blackhole) + { + blackhole.consume(state.decodeOtf()); + } + + // Add decodeInto(BorrowedDocument) and writeJson(JsonGenerator) benchmarks here when those APIs land. + // Keep the document/generator in SbeJsonCarState and provision/warm them in its trial setup. +} diff --git a/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonEncodeBenchmark.java b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonEncodeBenchmark.java new file mode 100644 index 0000000000..7d1d3b5693 --- /dev/null +++ b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonEncodeBenchmark.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.benchmarks; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class SbeJsonEncodeBenchmark +{ + @Benchmark + public void encode(final SbeJsonCarState state, final Blackhole blackhole) + { + blackhole.consume(state.encoder.encode(state.tree, state.output, 0, state.output.capacity())); + blackhole.consume(state.output); + } + + @Benchmark + public void encodedLengthAndEncode(final SbeJsonCarState state, final Blackhole blackhole) + { + final int length = state.encoder.encodedLength(state.tree); + blackhole.consume(state.encoder.encode(state.tree, state.output, 0, length)); + blackhole.consume(state.output); + } + + @Benchmark + public void generatedFromTree(final SbeJsonCarState state, final Blackhole blackhole) + { + blackhole.consume(state.encodeGenerated(state.output)); + blackhole.consume(state.output); + } +} diff --git a/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonSerializeBenchmark.java b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonSerializeBenchmark.java new file mode 100644 index 0000000000..77798fb8b0 --- /dev/null +++ b/sbe-benchmarks/src/main/java/uk/co/real_logic/sbe/benchmarks/SbeJsonSerializeBenchmark.java @@ -0,0 +1,44 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.benchmarks; + +import org.openjdk.jmh.annotations.*; +import org.openjdk.jmh.infra.Blackhole; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; + +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +public class SbeJsonSerializeBenchmark +{ + @Benchmark + public void writeValueAsBytes(final SbeJsonCarState state, final Blackhole blackhole) throws IOException + { + blackhole.consume(state.mapper.writeValueAsBytes(state.tree)); + } + + @Benchmark + public void writeValueToReusableStream(final SbeJsonCarState state, final Blackhole blackhole) throws IOException + { + state.jsonOutput.reset(); + state.mapper.writeValue(state.jsonOutput, state.tree); + blackhole.consume(state.jsonOutput.size()); + blackhole.consume(state.jsonOutput); + } + + // The tree is decoded once at setup. Add the direct writeJson path alongside these baselines later. +} diff --git a/sbe-jackson/BENCHMARKS.md b/sbe-jackson/BENCHMARKS.md new file mode 100644 index 0000000000..d28610d469 --- /dev/null +++ b/sbe-jackson/BENCHMARKS.md @@ -0,0 +1,89 @@ +# SbeJson benchmark harness + +These JMH benchmarks implement the harness portion of DESIGN.md §13 step 7. They make no +performance or zero-allocation claims. Generated codecs still come from the existing +`sbe-benchmarks/src/main/resources/car.xml` Gradle generation task. + +Build and list from the repository root: + +```sh +./gradlew :sbe-benchmarks:build :sbe-jackson:check +java -jar sbe-benchmarks/build/libs/sbe-benchmarks.jar -l +``` + +Run each axis separately (JMH filters match the fully qualified class names): + +```sh +java --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -jar sbe-benchmarks/build/libs/sbe-benchmarks.jar '.*SbeJsonDecodeBenchmark.*' -f 3 -wi 5 -i 10 -prof gc +java --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -jar sbe-benchmarks/build/libs/sbe-benchmarks.jar '.*SbeJsonEncodeBenchmark.*' -f 3 -wi 5 -i 10 -prof gc +java --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -jar sbe-benchmarks/build/libs/sbe-benchmarks.jar '.*SbeJsonSerializeBenchmark.*' -f 3 -wi 5 -i 10 -prof gc +``` + +For a short wiring/correctness smoke run of every method and corpus: + +```sh +java --add-opens=java.base/jdk.internal.misc=ALL-UNNAMED -jar sbe-benchmarks/build/libs/sbe-benchmarks.jar '.*SbeJson.*Benchmark.*' -f 1 -wi 1 -i 1 -w 1s -r 1s -t 1 -prof gc -jvmArgsAppend '-Xms256m -Xmx256m' -foe true +``` + +Use `-p corpus=BASELINE` or `-p corpus=HEAVY` to select data; otherwise both run. Full runs should +use an idle machine and record JVM, CPU, heap, GC and flags. The fixed smoke heap keeps the smoke +run modest; choose and record an appropriate heap for real runs. Do not infer rankings from smoke +results or compare measurements collected under different conditions. + +| Axis | Methods | Measured work | +| --- | --- | --- | +| Decode | `decodeCopy`, `jsonPrinterReadTree`, `naiveOtfTree` | Wire to a fresh stock Jackson tree; printer includes text production and parsing. | +| Encode | `encode`, `encodedLengthAndEncode`, `generatedFromTree` | The same predecoded tree to a reused SBE destination, including header. Sizing variant measures both passes. | +| Serialize | `writeValueAsBytes`, `writeValueToReusableStream` | The predecoded stock tree to JSON; decode is excluded. | + +Every method uses `SbeJsonCarState` (`Scope.Thread`), average nanoseconds per operation, and a +`Blackhole`. Trial setup parses IR, builds SbeJson/decoder/encoder/ObjectMapper, prepares direct +input/output buffers, warms printer/serializer buffers, and verifies the adapters. The state holds +the generated CarEncoder and header, the reusable OTF listener, a StringBuilder, and a 64 KiB +ByteArrayOutputStream. There is no invocation-level setup or harness buffer allocation. + +BASELINE is exactly `CarBenchmark.encode`: 3 fuel entries, 2 performance entries, 3 acceleration +entries per performance entry, and the original short strings. HEAVY uses the same schema with +32 fuel entries, 8 performance entries, 16 acceleration entries each, and Latin-1 manufacturer/model +strings of 4096/4098 bytes. Speeds exceed the small integer cache. This is a bounded stress corpus, +not an extension-version schema or a schema-maximum corpus: uint16 maximum nested counts would +exceed the default total group budget and dominate routine runs. Empty groups, version transitions, +uint64 high-bit values, binary data and hostile inputs remain future corpus work. + +Trial setup checks exact equality of the naive OTF and decodeCopy trees, and byte equality of +both encoders against the generated corpus. JsonPrinter output is re-encoded and compared after +normalizing its extras object outside the measured method. Both serialization outputs are parsed +and checked outside measurement. A failed check aborts the trial. + +Interpretation and limitations: + +- `gc.alloc.rate.norm` is allocated bytes per operation (B/op), not live/retained memory and not + allocation throughput (`gc.alloc.rate`, MB/s). Stock-tree decoding and byte-array serialization + allocate by design. Tiny nonzero normalized values can include measurement overhead. +- C2 escape analysis can eliminate allocations whose results do not escape. Blackholes help keep + results observable but do not establish a zero-allocation guarantee. Cross-check real runs with + `-prof jfr` allocation events and the `sbe-jackson` ThreadMXBean allocation guard tests described + in DESIGN.md §12 when the borrowed-path branch supplies them (they are absent on this harness's + base). JFR events are sampled, so an absence of events is not proof. A separate run with + `-jvmArgsAppend '-XX:-DoEscapeAnalysis'` is diagnostic, not a replacement production result. +- JsonPrinter emits extras as named booleans; decodeCopy and the naive listener emit a numeric + mask. readTree also uses different numeric node types (notably DoubleNode versus FloatNode). + Outside this corpus, unsigned uint64, unknown enums, absent fields, binary and NaN differ too; + see DESIGN.md §9. Printer results include pretty text, a String copy, and a second parse. +- The naive listener is deliberately limited to this Car corpus. It allocates fresh stock nodes + and text buffers, walks enum tokens on each call, and does not implement SbeJson validation, + bounds/limit checks or general optional/version semantics. +- The generated encoder walks the same tree on every invocation, including enum name resolution, + string access, nested groups and var-data. It uses generated String setters (including their + charset conversion allocations), skips wire-absent constants, and does not perform SbeJson's + full validation. It is a schema-specific baseline, not an equivalent validation engine. +- `writeValueAsBytes` always returns a new byte array and has no OutputStream overload. + `writeValueToReusableStream` instead calls `ObjectMapper.writeValue` after resetting the warmed + stream; it consumes the stream and size without `toByteArray()`. Both include Jackson generator + creation/closing and float formatting; neither includes tree construction. + +When borrowed APIs land, add a document/generator to the shared state and provision them in trial +setup, then add `decodeInto(BorrowedDocument)` and `writeJson(JsonGenerator)` methods at the marked +slots. Specify whether generator lifecycle/flush is measured, make bytes observable, and report +`retainedBytes()` alongside allocation. The existing state and corpus need no restructuring. +No implementation tuning is part of this scaffold. diff --git a/sbe-jackson/DESIGN.md b/sbe-jackson/DESIGN.md new file mode 100644 index 0000000000..5e710f69e8 --- /dev/null +++ b/sbe-jackson/DESIGN.md @@ -0,0 +1,574 @@ +# sbe-jackson — SBE ⇄ Jackson `JsonNode` codec + +Revision 2 (2026-09-16). Supersedes revision 1 in full. + +## Revision 2 — what changed and why + +Revision 1 was a single-model design. Revision 2 is the result of a three-round cross-critique +between two independent reviewers plus one owner decision. The architecture (IR compiled once +into a flat plan, mutable custom nodes in a borrowed skeleton tree, encoder walks the plan not +the JSON) survived. The following revision-1 claims were wrong or under-specified and are replaced: + +| Revision 1 | Revision 2 | Reason | +|---|---|---| +| `NodeMode.REUSE / ALLOCATE` flag, both returning a bare `ObjectNode` | Ownership in the type system: `decodeCopy()` returns a stock tree, `decodeInto(BorrowedDocument)` returns a borrowed one | A flag makes borrowed lifetime invisible at call sites | +| Bounds checked against `buffer.capacity()` | Explicit `length`; every read checked against `offset + length` | SBE header carries no total length; a receive buffer holds several messages | +| Group entries: "skip `blockLength − plan.blockLength`" | `cursor = entryBase + actingBlockLength`; each present field checked to fit inside the acting block | Negative skip when acting block is smaller than the plan's | +| Absent-by-version fields emit `null` / nullValue / const | Property omitted; skeleton per effective layout | Collapses "not in this version" with "optional sentinel"; nonsense for required fields | +| Newer `actingVersion` clamped to schema version | Rejected (`NewerVersions.REJECT`); a `PROJECT_ROOT` projection that reports partial consumption is phase 2 | Block length locates the fixed block only; unknown groups / var-data make later sections unlocatable | +| Skeleton keyed by observed `(templateId, actingVersion)` | Keyed by effective-layout index (distinct `sinceVersion` thresholds); bounded by `Limits.maxRetainedBytes` | Observed version numbers are unbounded | +| Bit set → object of named booleans | Numeric mask by default; `BitSetStyle.OBJECT` documented lossy | Named-only loses unnamed bits; object-with-fallback made one field bimodal | +| Group pools capped by length-type max (255 / 65535) | `Limits`: total group entries, var-data bytes, depth, retained bytes | Nested groups multiply; 65535 × 65535 is not a bound | +| `REUSE_CHECKED`: generation stamps on leaves | `newCheckedDecoder()`: fresh skeleton per decode, previous tree poisoned; coverage list documented | Old and new references are the same object; a stamp cannot tell them apart | +| `Sbe*Node.equals` accepts stock counterpart | Symmetric within own class only; `SbeJsonNodes.semanticEquals` for cross-family | `TextNode.equals(SbeStringNode)` is false; asymmetric `equals` breaks `ObjectNode.equals` | +| "Streaming serialize is zero-alloc" | Integer / string / binary writes zero-alloc on `UTF8JsonGenerator`; `writeNumber(double/float)` allocates until a custom formatter | `NumberOutput.toString` path in jackson-core 2.16–2.21 | +| Constants ignored on encode | Supplied constants always validated; omitted is fine | Silently accepting a contradictory value | +| `strictEncoding(boolean)` | `UnknownProperties.ERROR` (default) / `IGNORE` | One flag hid two unrelated policies | +| Encoder stateless and shared | Thread-confined encoder bound to a template (`newEncoder(templateId)`) | Leaves room for scratch state and generated backends without changing the contract | +| No streaming path (non-goal) | `writeJson(JsonGenerator)` ships; `encode(JsonParser)` ships with schema-order rule for variable sections | Both reuse the plan unchanged; the parser path needs no replay buffer under that rule | +| Codegen "rejected" | Deferred behind a `MessageCodec` SPI; built only when benchmarks justify | Interpreter is needed anyway for runtime schemas and as the oracle | +| "1.5–3× slower reads than flyweights", "switch beats itable" | Withdrawn as numbers; hypotheses for JMH | Never measured | + +## 1. Goal + +Convert SBE messages (any schema, loaded at runtime from IR) to and from Jackson 2.x `JsonNode` trees +with the smallest possible steady-state allocation and the highest throughput a generic +(non-generated) implementation can reach. Provide a direct `JsonGenerator` sink for consumers who +want bytes rather than a tree. + +### Allocation honesty statement + +A tree of stock Jackson nodes allocates at least one object per value; that path +(`decodeCopy`) is offered and is never zero-allocation. Zero steady-state allocation is reached only +through a **borrowed document**: a per-decoder skeleton tree whose leaves are custom mutable +`NumericNode` / `ValueNode` subclasses overwritten in place, valid until the next decode into the +same document. + +The allocation claim is scoped, never library-wide: + +> 0 B/op measured for a successful `decodeInto` on an adequately provisioned `BorrowedDocument`, +> for this template and corpus, using the supported access pattern (`longValue`, `doubleValue`, +> `booleanValue`, `get`, `path`, `size`), after warm-up, with retained bytes reported alongside. +> `writeJson` on `UTF8JsonGenerator` is 0 B/op for integer, string and binary content. +> `encode` from a tree is 0 B/op except through `binaryValue()`. + +Operations that allocate by construction: + +- `textValue()` on a string leaf (one `String`, cached until the leaf is next written), `asText()` on numbers. +- `numberValue()` (boxes outside the `Long` cache), `bigIntegerValue()`, `decimalValue()`. +- `binaryValue()` (returns an exact-size copy — `BinaryNode` contract). Zero-alloc access is + `SbeBinaryNode.byteArray()` + `length()`. +- Iterators: `fields()`, `fieldNames()`, `elements()`. +- `snapshot()` / `deepCopy()`, `ObjectMapper.writeValueAsBytes` result array, `toString()`. +- `writeNumber(double)` / `writeNumber(float)` on any stock generator (formats through a `String`). + A buffer-based Schubfach/Ryu formatter into `char[24]` + `writeNumber(char[],int,int)` is phase 2. +- `writeNumber(BigInteger)`; uint64 above `Long.MAX_VALUE` in `decodeCopy` trees (`BigIntegerNode`). +- Warm-up: pools and scratch arrays grow to their high-water mark within `Limits`. +- `decodeCopy` entirely; the caller's `readTree` on the encode side; one exception object per error. + +Compatibility hazard (not an allocation): downstream code doing `instanceof TextNode` / +`LongNode` on a borrowed tree fails. Use `decodeCopy` or `snapshot()` for such consumers. + +## 2. Non-goals + +- No changes to the SBE code generator or to `sbe-all`. +- No generated Jackson adapters in the first release. The `MessageCodec` SPI exists so they can be added + later if benchmarks justify them (§15). +- No arbitrary-order variable sections on the `JsonParser` → SBE path. Groups and var-data must arrive in + schema order; otherwise use the tree path. +- No schema-agnostic "guess the type" mode. Everything is driven by IR. +- No timestamp / `semanticType` formatting, no decimal-composite → `double` conversion. Numbers stay numbers. +- No byte-identical round-trip guarantee. Padding, NaN payloads and unknown extension bytes are not preserved; + the promise is semantic equality. + +## 3. Assumptions (decided, not blocking) + +| Topic | Decision | +|---|---| +| Placement | New Gradle module `sbe-jackson` in this repo (`settings.gradle`). Depends only on public `sbe-tool` API (`uk.co.real_logic.sbe.ir.*`, `uk.co.real_logic.sbe.otf.*`) so the same code can move to a standalone repo. | +| Java | 17, matching `sourceCompatibility` in `build.gradle`. | +| Jackson | 2.x. Compile against 2.16.1 (lowest in local cache with all APIs used), run the test suite against 2.21.4 too. See §11 for Jackson 3. | +| Agrona | 2.6.0 already provided via `sbe-tool` `api` dependency. | +| Runtime strategy | IR compiled once into a flat plan; no per-message `List` walking (§5). | +| Dispatch | Single `FieldPlan` class with a `byte kind`, `switch` in the decode/encode loops (mirrors `OtfMessageDecoder`). A struct-of-arrays variant (`byte[] kind`, `int[] offset`, …) is built as a JMH alternative; whichever measures faster ships. | +| Frame | Decode takes `(buffer, offset, length)`. Every read is bounded by `offset + length`, never by `buffer.capacity()`. | +| Header | Not part of the body tree. `OtfHeaderDecoder` is reused for reading; `templateId`, `actingVersion`, `blockLength` are exposed on `BorrowedDocument` (borrowed path) and on `SbeJsonDecoder.lastHeader()` — a thread-confined `HeaderView` overwritten by every decode, and the only way to read the header after `decodeCopy` or `writeJson`. Encode writes the header from IR: the supplied IR's `blockLength` and `version`. | +| Version targeting | One `SbeJson` encodes exactly its IR's version. Encoding an older layout means building a second `SbeJson` from that version's IR; there is no header-version override. | +| Thread-safety | Compiled plans (`SbeJson`) shared and immutable. `SbeJsonDecoder` and `SbeJsonEncoder` are thread-confined, non-reentrant instances. The interpreter encoder happens to be stateless internally; the contract does not promise it. | +| Error model | One unchecked `SbeJsonException`. `IOException` only from methods that write to a caller-supplied `JsonGenerator`. | + +## 4. Architecture + +```mermaid +flowchart LR + IR[Ir + HeaderStructure] --> PC[PlanCompiler] + PC --> MP["MessagePlan[] (by templateId / name)"] + MP --> D[SbeJsonDecoder] + MP --> E[SbeJsonEncoder] + BUF[(DirectBuffer)] --> D + D --> C["ObjectNode (decodeCopy: fresh stock tree)"] + D --> B["BorrowedDocument (decodeInto: skeleton + Sbe*Node leaves)"] + D --> G[(JsonGenerator)] + U["JsonNode (caller built)"] --> E + P[(JsonParser)] --> E + E --> OUT[(MutableDirectBuffer)] + D -. via .-> SPI[MessageCodec SPI: PlanMessageCodec today, generated adapters later] +``` + +Public API: + +```java +final Ir ir = new IrDecoder("car.sbeir").decode(); // or XmlSchemaParser + IrGenerator + +final SbeJson sbeJson = SbeJson.builder(ir) + .enumStyle(EnumStyle.NAME) // unknown raw value -> number + .bitSetStyle(BitSetStyle.MASK) // OBJECT is a presentation policy, lossy for unnamed bits + .charArrayStyle(CharArrayStyle.NUL_TERMINATED) // EXACT keeps all N chars including NULs + .unknownProperties(UnknownProperties.ERROR) // default; IGNORE opt-in + .newerVersions(NewerVersions.REJECT) // only option in release 1; PROJECT_ROOT is phase 2 (§13) + .exceptionStackTraces(true) // false for hostile-input gateways + .limits(Limits.builder() + .maxGroupEntries(10_000) // total across all groups and nesting + .maxVarDataBytes(1 << 20) + .maxDepth(8) + .maxRetainedBytes(16 << 20).build()) + .build(); // compiles plans once; immutable; shared + +// one per thread +final SbeJsonDecoder decoder = sbeJson.newDecoder(); + +// fresh stock tree, retainable, allocates +final ObjectNode tree = decoder.decodeCopy(buffer, offset, length); +final HeaderView header = decoder.lastHeader(); // templateId, actingVersion, blockLength; overwritten by the next decode + +// borrowed, zero-allocation steady state +final BorrowedDocument doc = decoder.newDocument(); +final int consumed = decoder.decodeInto(buffer, offset, length, doc); +doc.root(); // JsonNode, valid until the next decodeInto(doc) +doc.snapshot(); // independent stock tree (allocates) +doc.templateId(); doc.actingVersion(); doc.blockLength(); doc.valid(); + +// no tree +decoder.writeJson(buffer, offset, length, jsonGenerator); + +// one per thread, bound to a template and the IR's version +final SbeJsonEncoder encoder = sbeJson.newEncoder("Car"); // or newEncoder(templateId) +final int written = encoder.encode(body, dst, dstOffset, dstAvailable); // header + body, single pass +final int needed = encoder.encodedLength(body); // optional sizing pass +encoder.encode(jsonParser, dst, dstOffset, dstAvailable); // schema-order variable sections + +// debug: allocates per decode, poisons the previous tree +final SbeJsonDecoder checked = sbeJson.newCheckedDecoder(); +``` + +Backend scope: `MessageCodec` covers `decodeInto` and `encode(JsonNode)`. `decodeCopy` is implemented as +`decodeInto` on a decoder-private `BorrowedDocument` followed by `snapshot()`, so it rides on the same backend. +`writeJson` and `encode(JsonParser)` are interpreter-only in release 1; a phase-2 generated adapter that wants +them extends the SPI then. + +Classes (all in `uk.co.real_logic.sbe.jackson`): + +| Class | Role | +|---|---| +| `SbeJson` | Builder + facade. `Int2ObjectHashMap` by templateId, `Map` by name, `OtfHeaderDecoder`, header layout for encoding, policies, `Limits`. | +| `EnumStyle`, `BitSetStyle`, `CharArrayStyle`, `UnknownProperties`, `NewerVersions` | Policy enums. | +| `Limits` | Total group entries, var-data bytes, nesting depth, retained bytes per document. | +| `PlanCompiler` | `Ir` → `MessagePlan`. Mirrors the token walk of `OtfMessageDecoder` (fields, then groups, then var-data) but records instead of dispatching. No Jackson import. | +| `MessagePlan` | `templateId`, `name`, `blockLength`, `schemaVersion`, flat `FieldPlan[]`, root child range, sorted distinct `sinceVersion` thresholds (effective-layout table). No Jackson import. | +| `FieldPlan` | One flat record per field / composite member / group / var-data: `kind`, `name` (interned), `offset` (scope-relative), `primitiveType`, `byteOrder`, `arrayLength`, `encodedLength`, `sinceVersion`, `presence`, `nullValueLong`, `nullValueDouble`, `minValue`, `maxValue`, const value, `childStart`, `childEnd`, `enumValues` (sorted `long[]`) + `enumNames`, `choiceNames` + `choiceBits` + `knownMask`, `Object2IntHashMap` name lookup, `characterEncoding` tag, group dimension layout (`blockLengthType/offset`, `numInGroupType/offset`, `dimensionSize`), var-data length type / offset. No Jackson import. | +| `MessageCodec` | SPI: `int decodeInto(DirectBuffer, int, int, BorrowedDocument)`, `int encode(JsonNode, MutableDirectBuffer, int, int)`. One per template. | +| `PlanMessageCodec` | The interpreter. `switch (kind)` loops over `FieldPlan[]`. Only production backend in release 1. | +| `SbeJsonDecoder` | Thread-confined. `decodeCopy`, `decodeInto`, `writeJson`, `newDocument`, `lastHeader`. Routes by header templateId. | +| `HeaderView` | Thread-confined per decoder; `templateId`, `schemaId`, `actingVersion`, `blockLength` of the last decode on that decoder. Overwritten by every decode. | +| `SbeJsonEncoder` | Thread-confined, bound to one template. `encode(JsonNode…)`, `encode(JsonParser…)`, `encodedLength`. | +| `BorrowedDocument` | Per decoder: skeleton registry keyed by `(templateId, layoutIndex)`, leaf references by `FieldPlan` index, per-group `ArrayNode` + pooled entry skeletons (recursive), scratch `char[]` / `byte[]`, `valid`, header ints, `retainedBytes`. | +| `JacksonCaches` | Per `SbeJson`: prebuilt immutable constant nodes, enum name `TextNode`s, `SerializedString` field names. The only place plan metadata meets Jackson objects. | +| `SbeLongNode` | `NumericNode` subclass, mutable `long`, `NumberType` tag INT/LONG, `unsigned64` flag. | +| `SbeDoubleNode` | `NumericNode` subclass, mutable `double`, tag FLOAT/DOUBLE. | +| `SbeStringNode` | `ValueNode` subclass, `char[]` + length, lazily cached `String`. | +| `SbeBinaryNode` | `ValueNode` subclass, `byte[]` + length, node type BINARY. | +| `SbeJsonNodes` | `semanticEquals(JsonNode, JsonNode)` — nodeType + value walk across custom and stock families. | +| `Utf8` | Hand-rolled UTF-8 decode (buffer → `char[]`, U+FFFD on invalid wire bytes), encode (`CharSequence` → buffer; an unpaired surrogate is reported by `encodedLength`, never replaced), validity scan, surrogate aware. | +| `SbeJsonException` | Unchecked; all validation failures. Error code, templateId, byte offset, copied path indices; message formatted at throw. | +| `ErrorCode` | Enum: `UNKNOWN_TEMPLATE`, `UNSUPPORTED_VERSION`, `FRAME_OVERFLOW`, `FIELD_OUTSIDE_BLOCK`, `LIMIT_EXCEEDED`, `MISSING_REQUIRED`, `TYPE_MISMATCH`, `OUT_OF_RANGE`, `UNKNOWN_ENUM`, `UNKNOWN_CHOICE`, `UNKNOWN_PROPERTY`, `CONSTANT_MISMATCH`, `SECTION_OUT_OF_ORDER`, `DESTINATION_OVERFLOW`. | + +## 5. Plan compilation (once per `SbeJson`) + +`PlanCompiler` walks the IR tokens for each message exactly as `OtfMessageDecoder` would and emits one +`FieldPlan` per leaf or container, in decode order (block fields, then groups, then var-data, recursively). + +- Offsets are **scope-relative**: relative to the message block start for root fields, relative to the entry + start for group fields. Composite members carry the composite's offset added in, so one read per leaf. +- `encodedLength` is stored per leaf so the decoder can check `offset + encodedLength <= actingBlockLength`. +- Enum tables: `long[] enumValues` sorted for binary search, parallel `String[] enumNames`; reverse + `Object2IntHashMap` for encoding. Bit sets: `choiceBits`, `choiceNames`, `knownMask`. +- Group dimension layout, var-data length type and offset, `characterEncoding` tag, presence, null / min / + max values, constant value, `sinceVersion`, `deprecated` are all captured; nothing is re-derived per message. +- Effective-layout table per message: sorted distinct `sinceVersion` values across the whole message. + `actingVersion` → layout index by binary search. Bound is `thresholds + 1` per template regardless of which + version numbers appear on the wire. +- Field names are interned once (`String.intern()`) so `ObjectNode.get(name)` hits a cached hash. +- Jackson-specific caches (`JacksonCaches`) are built alongside but live in a separate object so the plan + classes stay free of Jackson imports (§11). + +## 6. Decode algorithm + +Common prologue for all three entry points: + +``` +frameEnd = offset + length // checked: >= offset, <= capacity +header = OtfHeaderDecoder at offset (checked to fit) +plan = plans[templateId] // unknown -> UNKNOWN_TEMPLATE +if actingVersion > plan.schemaVersion: UNSUPPORTED_VERSION // release 1; PROJECT_ROOT projection is phase 2 (§13) +layout = plan.layoutIndex(actingVersion) +``` + +Entry walk (root and every group entry share it): + +``` +decodeEntry(plan, entryFields, buf, entryBase, actingBlockLength, actingVersion, frameEnd, target): + checked(entryBase + actingBlockLength <= frameEnd) + for f in entryFields: + if f.sinceVersion > actingVersion: continue // property omitted; not in this layout's skeleton + if f.isConstant: emit constant (no bytes read); continue + checked(f.offset + f.encodedLength <= actingBlockLength) // FIELD_OUTSIDE_BLOCK + read at entryBase + f.offset ... + cursor = entryBase + actingBlockLength + for g in groups: + if g.sinceVersion > actingVersion: continue // omitted; consumes no bytes + read dims at cursor (checked); numInGroup validated against g.numInGroupType min..max from IR + count against Limits.maxGroupEntries (running total for the document), maxDepth + cursor += dimensionSize + array.removeAll() // ArrayList.clear(), no allocation + for i in 0..numInGroup: + entry = pool.entryAt(i) // grow to high-water within Limits + cursor = decodeEntry(g, entry fields, buf, cursor, dimBlockLength, actingVersion, frameEnd, entry) + array.add(entry) + for v in varData: + if v.sinceVersion > actingVersion: continue + read length at cursor (checked); against Limits.maxVarDataBytes + copy bytes into leaf scratch; cursor += lengthSize + len + return cursor +``` + +Per-construct behaviour in the borrowed path: + +| Construct | Decode | +|---|---| +| int8/16/32, uint8/16/32, int64 | `SbeLongNode.set(Types.getLong(...))`. Optional and equal to nullValue → `NullNode`. `numberType()` INT for ≤32-bit signed and uint8/16, LONG for int64/uint32. | +| uint64 | `setUnsigned(raw)`; `numberType()` LONG when `raw >= 0`, BIG_INTEGER otherwise; `serialize()` writes unsigned digits via `char[20]` + `writeNumber(char[],int,int)`. | +| float/double | `SbeDoubleNode.set`. Optional null test is `Double.isNaN(v)` when the sentinel is NaN, value compare otherwise. | +| `char` | one-char string. | +| `char[N]` | `NUL_TERMINATED`: copy up to first NUL or N. `EXACT`: all N chars. | +| numeric `[N]` | skeleton `ArrayNode` of N leaves set in place; never cleared. | +| enum | binary search → prebuilt immutable `TextNode` (shared). Unknown raw → number leaf. `ORDINAL` style → number always. | +| bit set | `MASK`: `SbeLongNode` with the raw value. `OBJECT`: skeleton `ObjectNode` of named booleans (`BooleanNode` singletons), documented lossy for bits outside `knownMask`. | +| composite | recurse into child skeleton `ObjectNode`. | +| constant | prebuilt stock node from `JacksonCaches`; consumes no bytes. | +| group | as above. Empty → `[]`. | +| var-data text | `Utf8.decode` (or ASCII copy) into `SbeStringNode` scratch `char[]`. | +| var-data binary (`characterEncoding` null / `binary`) | copy into `SbeBinaryNode` scratch `byte[]`; serializes base64. | +| `sinceVersion > actingVersion` | property absent from this layout's skeleton. | +| `deprecated` | decoded normally. | + +Presence toggling: required leaves are fixed in the skeleton and never replaced. Optional leaves keep a +per-slot `boolean present`; `ObjectNode.replace(name, leaf | NullNode)` runs only on a transition, so the +steady per-field cost is one buffer read and one field store. `replace` on an existing key is a +`LinkedHashMap.put` on an existing entry: no allocation, insertion order preserved. + +Skeleton registry: keyed `(templateId, layoutIndex)`, built lazily, retained bytes counted against +`Limits.maxRetainedBytes`. Exceeding the budget is a `SbeJsonException`, not a silent eviction. + +Failure leaves `doc.valid() == false` until the next successful `decodeInto(doc)`. `valid()` describes the +document, not whether some reference the caller extracted earlier belongs to the current decode. + +`decodeCopy` runs the same walk with a stock-node `JsonNodeFactory`; uint64 above `Long.MAX_VALUE` becomes +`BigIntegerNode`. + +`writeJson` runs the same walk against a `JsonGenerator`: `writeFieldName(SerializedString)`, +`writeNumber(long)`, `writeString(char[],0,n)`, `writeBinary(b64, byte[],0,n)`. Var-data text on an on-heap +buffer with UTF-8 / ASCII encoding is scanned once for validity and then written with +`writeUTF8String(src.byteArray(), src.wrapAdjustment() + payloadOffset, len)`; invalid bytes or other +charsets go through `Utf8.decode` to `char[]` and `writeString`. `writeNumber(double)` allocates on stock +generators until phase 2 (§13). A persistent generator per thread is the measured configuration. + +## 7. Encode algorithm (`JsonNode` → SBE) + +Walk the **plan**, never the JSON. `obj.get(name)` is a `HashMap.get` with a cached hash; `fields()` and +`fieldNames()` allocate iterators and are never called on the happy path. + +``` +encode(body, plan, dst, off, available): + write header (plan.blockLength, templateId, schemaId, schemaVersion) + pos = off + headerLen + encodeBlock(plan.rootFields, body, dst, pos) // fixed offsets: JSON key order irrelevant + pos += plan.blockLength // padding bytes: caller zero-fills if comparing bytes + pos = encodeTail(plan.rootGroups, plan.rootVarData, body, dst, pos) // schema order + // per ObjectNode (root, each composite, each group entry), not root only: + if unknownProperties == ERROR and recognizedCount(obj) != obj.size(): slow path names the first unknown key + return pos - off +``` + +Groups: `ArrayNode.size()` gives the count directly; no counting pre-pass. Validate against the dimension +type's min..max and `Limits`, write the dimension header, then an index loop (`array.get(i)`, no iterator). +Var-data: bytes are written at `pos + lengthSize` straight from `textValue()` via `Utf8.encode` (no +`String.getBytes`), then the length is back-filled. + +Coercion policy: + +| Field | Accepts | Rejects | +|---|---|---| +| ints | `isIntegralNumber()` and `canConvertToLong()` within schema min..max; uint64 also `BigIntegerNode` or decimal string | floating node (`5.0` → error, no silent truncation), out of range (including an optional scalar's own null sentinel: `cupHolderCount: 255` is `OUT_OF_RANGE`; omit or `null` to select it), negative into unsigned | +| float/double | `isNumber()` within schema min..max; strings `"NaN"`, `"Infinity"`, `"-Infinity"` | other strings, finite value out of range (an optional scalar's finite sentinel included) | +| optional numeric `[N]` | each element as above, **plus** the null sentinel per element (compared at wire precision: a float field compares as `float`) so decoded arrays, which expose sentinels as numbers, re-encode byte-identically; missing / `null` fills every element with the sentinel | element out of range, wrong element count | +| `char[N]` | `textValue()` length ≤ N, `charAt` loop, NUL pad | char > 0x7F into an ASCII field, too long, unpaired surrogate or character unmappable in the field charset (`TYPE_MISMATCH`) | +| enum | name via `Object2IntHashMap`, or integral | unknown name | +| bit set | integral mask, or `ObjectNode` of booleans | unknown choice name | +| composite | `ObjectNode` | other | +| group | `ArrayNode` of `ObjectNode`; missing → dimensions with `numInGroup = 0` | size outside dimension type range or `Limits` | +| var-data text | `textValue()` → `Utf8.encode` into `dst` (other charsets through a bounded `CharsetEncoder`) | length > length type max, unpaired surrogate or unmappable character (`TYPE_MISMATCH`) | +| var-data binary | `binaryValue()` (allocates; inherent) | | +| constant | omitted, or present and equal to the schema constant | present and different | +| missing / `null` | optional → nullValue; group → empty; var-data → length 0 | required → error | + +Text policy (fixed `char[N]` and var-data alike, every charset): malformed input is **rejected** with +`TYPE_MISMATCH`, never replaced. An unpaired UTF-16 surrogate in a Java string has no encoding in any charset; the +strict coercion rule above (`5.0` into an int is an error) applies to text too, so no U+FFFD or `?` ever reaches the +wire from the encoder. The decoder still maps invalid UTF-8 *wire bytes* to U+FFFD, because rejecting a received +message for one bad byte in a string is not the decoder's call. The encoder retains nothing owned by the caller +after a call returns or throws: the destination reference is cleared in a `finally`. + +Unknown properties: `ERROR` by default. The happy path counts recognized properties during the plan walk and +compares with `size()` **per `ObjectNode`** — root, every composite, every group entry — otherwise unknown keys +inside groups would pass silently; the slow path that names the offending key runs only on mismatch, so the check +is free on valid input. `IGNORE` skips the count. + +Single pass: on overflow or validation failure the destination region is invalid and the exception carries the +field path. `encodedLength(body)` is an optional second traversal (counts UTF-8 bytes) for callers who want +capacity certainty first; the input must not change between the two calls. Neither call is transactional. + +`encode(JsonParser …)`: block fields in any order (fixed offsets); groups and var-data must arrive in schema +order (`SECTION_OUT_OF_ORDER` otherwise). Unknown counts and lengths are handled by reserving the dimension +or length prefix and back-filling after `]` / the string end. No replay buffer; callers with arbitrary-order +input use the tree path. + +## 8. Custom node semantics (the part that silently breaks) + +| Method | Rule | +|---|---| +| `deepCopy()` | `ValueNode` default returns `this` — an aliasing bug for mutable nodes. Return the stock equivalent (`LongNode` / `IntNode` / `BigIntegerNode` / `DoubleNode` / `FloatNode` / `TextNode` / `BinaryNode`). | +| `asToken()`, `getNodeType()` | Exactly as stock: `VALUE_NUMBER_INT` / NUMBER, `VALUE_NUMBER_FLOAT` / NUMBER, `VALUE_STRING` / STRING, `VALUE_EMBEDDED_OBJECT` / BINARY. | +| `numberType()`, `isInt` / `isLong` / `isBigInteger` / `isFloat` / `isDouble` | `JsonNode` defaults are `false`; override consistently with the SBE type. | +| `isIntegralNumber` / `isFloatingPointNumber` / `isNaN` | Override; the encoder gates on these when round-tripping borrowed trees. | +| `canConvertToInt` / `canConvertToLong` | Range based; uint64 with the high bit set → `canConvertToLong() == false`. | +| `intValue` / `longValue` / `doubleValue` | Field reads, no allocation. `numberValue()` and friends box on demand. | +| `serialize()` | `writeNumber(long)`, `writeNumber(double)` (allocates on stock generators — phase 2), `writeString(char[],0,len)`, `writeBinary(b64, byte[],0,len)`. uint64 above `Long.MAX_VALUE`: unsigned digits into a per-node `char[20]`, `writeNumber(char[],int,int)` — string-free on `UTF8JsonGenerator` and `WriterBasedJsonGenerator`; the base `JsonGenerator` implementation builds a `String`. | +| `equals` / `hashCode` | Symmetric **within the custom class only**. `TextNode.equals` requires `instanceof TextNode`, so accepting a stock counterpart one-way would break the contract and make `ObjectNode.equals` order dependent. Cross-family comparison is `SbeJsonNodes.semanticEquals(a, b)`; tests compare via `snapshot()`. | +| `textValue()` | `SbeStringNode` builds a `String` once and caches it until the next `set`. | + +Borrowed-document contract (Javadoc on `BorrowedDocument`): + +- Traversal-only from the application's side. Mutating containers (`put`, `remove`, `add`) invalidates the skeleton. +- Every value simultaneously visible in the tree has its own storage; no leaf is shared between two positions. +- `root()` and anything reached from it are valid until the next `decodeInto(doc)`. Retaining a reference past + that is a contract violation that release builds do not detect. +- `newCheckedDecoder()` detects it in debug: each decode builds a fresh skeleton and poisons the previous one + (`stale = true`; every accessor throws `IllegalStateException("borrowed node used after next decode")`). + Coverage: `Sbe*Node` value accessors and `serialize`, `ObjectNode` `get` / `path` / `has` / `size`, `ArrayNode` + `get` / `size`. Not covered: iterators or `Map.Entry` objects already obtained before the next decode, and + shared immutable leaves (`NullNode`, `BooleanNode`, enum name `TextNode`s, constants), which are valid by + construction. + +Jackson 2.x minors add abstract methods to `NumericNode`. Compile against 2.16.1; run the suite against 2.21.4. + +## 9. JSON conventions (round-trip table) + +| SBE | JSON (decode) | Encode accepts | +|---|---|---| +| signed ints, uint8/16/32 | number | integral node in range | +| uint64 | unsigned decimal number (never through `double`) | integral node, `BigIntegerNode`, decimal string | +| float / double | number; NaN / ±Infinity per Jackson `WRITE_NAN_AS_STRINGS` | number or `"NaN"` / `"Infinity"` / `"-Infinity"` | +| optional field at null sentinel | `null` | `null` or absent | +| required field | value | value (absent → error) | +| absent in acting version | property omitted | ignored if present | +| `char` | 1-char string | 1-char string | +| `char[N]` | string, `NUL_TERMINATED` (default) or `EXACT` | well-formed string ≤ N chars, NUL padded | +| numeric `[N]` | array of N numbers (optional: sentinels exposed as numbers, never `null` elements) | array of N numbers; optional arrays accept the sentinel per element | +| enum | name (`NAME`, default); unknown raw → number; `ORDINAL` → number | name or number | +| bit set | number mask (`MASK`, default); `OBJECT` → `{choice: bool}` (lossy for unnamed bits) | number or object | +| composite | nested object | nested object | +| constant | value from IR (consumes no bytes) | omitted, or equal to the constant | +| group | array of objects; empty → `[]` | array; missing → `numInGroup = 0` | +| var-data text | string in schema charset | well-formed string (unpaired surrogate → `TYPE_MISMATCH`) | +| var-data binary | base64 string (`BinaryNode` semantics) | base64 string | +| header | not in body; `templateId` / `actingVersion` / `blockLength` on `BorrowedDocument` or `decoder.lastHeader()` | derived from IR | + +Documented divergences from `JsonPrinter`: uint64 printed unsigned (printer prints signed), bit sets as +mask (printer prints choice names), binary as base64 (printer emits `PrintBufferUtil.hexDump` when `characterEncoding` is null), absent-by-version omitted +(printer emits nullValue / constant), unknown enum as number (printer prints `"null"`), NaN as string. + +## 10. Module layout and build changes + +``` +sbe-jackson/ + DESIGN.md + src/main/java/uk/co/real_logic/sbe/jackson/{SbeJson, EnumStyle, BitSetStyle, CharArrayStyle, + UnknownProperties, NewerVersions, Limits, PlanCompiler, MessagePlan, FieldPlan, MessageCodec, + PlanMessageCodec, SbeJsonDecoder, SbeJsonEncoder, BorrowedDocument, HeaderView, JacksonCaches, + SbeLongNode, SbeDoubleNode, SbeStringNode, SbeBinaryNode, SbeJsonNodes, Utf8, + SbeJsonException, ErrorCode, package-info}.java + src/test/java/uk/co/real_logic/sbe/jackson/... + src/test/resources/ (reuse sbe-tool test schemas via a resources dependency, plus small edge-case schemas) +``` + +- `settings.gradle`: `include 'sbe-jackson'`. +- `gradle/libs.versions.toml`: `jackson = "2.16.1"` compile baseline; `jackson-databind` library entry. +- `build.gradle`: new `project(':sbe-jackson')` block: `api project(':sbe-tool')`, `api libs.jackson.databind`, + test deps `junit`, `hamcrest`, `jqwik`; a `testLatestJackson` task that forces `2.21.4`. Checkstyle and + the repo's `-Werror` javac flags apply unchanged (final parameters, brace style). +- `sbe-benchmarks`: `implementation project(':sbe-jackson')`; Jackson enters only the benchmark shadow jar. +- Nothing added to `sbe-all`. + +## 11. Jackson 3 versus Jackson 2 + +Short answer: not better for speed, only better if the consumers are already on Jackson 3. + +| Aspect | Jackson 2.x (2.21.4) | Jackson 3.x (3.2.2) | +|---|---|---| +| Tree performance | `ObjectNode` over `LinkedHashMap`, same mutable API | Same internals, no measurable difference for this workload | +| Node subclassing surface | `NumericNode`: 10 abstract methods; `TextNode`; `serialize(JsonGenerator, SerializerProvider)`; checked `IOException` | `NumericNode` adds `isNaN()` and `_asString()`; `StringNode` replaces `TextNode`; `serialize(JsonGenerator, SerializationContext)`; unchecked `JacksonException` (cleaner hot path) | +| Generator APIs needed | `writeNumber(char[],int,int)`, `writeString(char[],int,int)`, `writeBinary(Base64Variant,byte[],int,int)`, `writeUTF8String` all present | All present with the same shapes | +| Ecosystem (Sept 2026) | Spring Boot 3 line, most libraries | Spring Boot 4 line, `tools.jackson.*` packages, cannot share one artifact with 2.x | +| API stability for our custom nodes | Mature, slow churn | Younger, abstract sets still moving in minors | + +Decision: target Jackson 2 as asked. `PlanCompiler`, `MessagePlan`, `FieldPlan`, `MessageCodec`, `Limits`, +`Utf8` and the buffer read/write helpers carry no Jackson import, so a `sbe-jackson3` adapter module is only +`JacksonCaches`, the node classes, `BorrowedDocument`, decoder and encoder. Do not attempt a single artifact +that supports both; package names differ and the `serialize` signatures conflict. + +## 12. Verification plan + +Correctness: + +- `PlanCompilerTest`: offsets, sizes, dimension layouts, layout tables for `car.xml` and + `fix-message-samples.xml` equal what `OtfMessageDecoder` visits (record via a `TokenListener` spy). +- `DecodeConformanceTest`: for every sample message, `readTree(JsonPrinter.print(...))` semantically equals + `decodeCopy(...)` and `decodeInto(...).snapshot()` modulo the divergences listed in §9. +- `EncoderOracleTest`: bytes from `encode` equal bytes from the generated `CarEncoder` for the same tree + (both buffers zero-filled first; padding is unspecified). This is the oracle for the from-scratch plan + encoder, since `sbe-tool` has no OTF encoder. +- `RoundTripTest`: generated `Car` encoder bytes → decode → encode → decode; compare via `semanticEquals`. +- `EncodeValidationTest`: every reject row of §7 (missing required, unsigned negative, float into int, char + overflow, var-data length overflow, unknown enum name, unknown choice, wrong container type, unknown property + under `ERROR`, contradictory constant, group count over `Limits`, frame overflow). +- `NodeCompatibilityTest`: on a borrowed tree, `writeTree`, `writeValueAsBytes`, `treeToValue(Map.class)`, + `convertValue`, `toString`, `at("/ptr")`; `snapshot()` independence (next decode does not change the copy); + `equals` symmetric within custom classes; `semanticEquals` against stock trees both directions. +- `VersioningTest`: v1 message with v2 schema (absent fields omitted; trailing block bytes skipped; group + entries with a smaller acting block); message whose header `blockLength` exceeds the schema's; newer + `actingVersion` rejected with `UNSUPPORTED_VERSION`; `lastHeader()` populated after `decodeCopy`; two + acting versions interleaved on one document (forces skeleton switch, bounded registry). +- `CheckedDecoderTest`: stale reference through each covered accessor throws; covered list in §8 is exact. +- `LimitsTest`: hostile `numInGroup` at nested depth stops at `maxGroupEntries`; var-data over `maxVarDataBytes`; + `maxDepth`; `maxRetainedBytes` on skeleton growth. Malformed length within `capacity()` but past `offset + + length` is rejected. +- Property tests with `jqwik`: random values within primitive ranges, random group counts (0 through the type's + IR `maxValue`), random UTF-8 strings including surrogate pairs and invalid sequences, embedded NULs in + var-data, random property order in encode input, round trip through encode/decode. +- Edge list: empty message body, nested groups (2 levels) with empty inner groups, optional composite members, + uint64 with high bit set, `float` NaN in optional vs required field, ASCII field receiving non-ASCII, + var-data of length 0, `numInGroup` of `uint8` at 254 (255 is the null sentinel and must be rejected), message + with multiple var-data fields, off-heap buffer on the `writeJson` UTF-8 fast path (`wrapAdjustment`). + +Allocation and speed (JMH in `sbe-benchmarks`, reusing `car.xml` and `fix-message-samples.xml` next to +`CarBenchmark` and `MarketDataBenchmark`): + +- Vary three axes independently: engine (`PlanMessageCodec`, later a generated adapter), ownership + (`decodeCopy`, `decodeInto`), output mode (tree, `writeJson`). Encode with and without `encodedLength`. +- Baselines in the same class: `JsonPrinter` → `ObjectMapper.readTree`; naive `TokenListener` building stock + nodes per message. +- Corpus: values outside the small-int cache, uint64 high bit, non-ASCII text, empty / typical / max groups, + nested groups, presence transitions on every message (forces the `replace` path), two acting versions + interleaved, float fields (to show the phase-2 gap). +- `-prof gc`: acceptance is `gc.alloc.rate.norm == 0 B/op` for `decodeInto`, `writeJson` on integer/string/binary + corpora, and `encode` (non-binary) after warm-up. Report retained bytes (`doc.retainedBytes()`) next to it. +- Cross-checks: `-prof jfr` allocation events; opt-in unit guard (`@Tag("allocation")`) using + `ThreadMXBean.getThreadAllocatedBytes` around 1k decodes after 20k warm-up, asserting 0; a run with + `-XX:-DoEscapeAnalysis` as a diagnostic only (it changes the workload; the production JVM configuration is the + primary result). +- `-prof perfasm` once on the `switch (kind)` loop versus the struct-of-arrays variant; keep whichever wins. + +## 13. Implementation steps + +1. Scaffold module, build wiring, version catalog entries, empty test task matrix. Gate: `./gradlew :sbe-jackson:test` green. +2. `PlanCompiler` + `MessagePlan` + `FieldPlan` + layout table, with `PlanCompilerTest` against the spy listener. +3. `decodeCopy` with stock nodes over `PlanMessageCodec`. `DecodeConformanceTest`, `VersioningTest` (older + message, newer rejected), `LimitsTest`. +4. Encoder from stock trees. `EncoderOracleTest`, `RoundTripTest`, `EncodeValidationTest`, jqwik properties. +5. Borrowed path: `Sbe*Node`, `BorrowedDocument`, skeleton registry, pools, presence transitions, + `SbeJsonNodes.semanticEquals`, `newCheckedDecoder`. `NodeCompatibilityTest`, `CheckedDecoderTest`, + allocation guard. +6. `writeJson` (generator sink) and `encode(JsonParser)` (schema-order rule, back-fill). Allocation guard + on integer/string/binary corpora. +7. JMH benchmarks on all three axes with both baselines; tune (kind ordering, struct-of-arrays trial, unsigned + digit writer, `Utf8` loops). +8. Javadoc (borrowed contract, checked-decoder coverage list, allocation claim wording), README usage section, + changelog line, second Jackson version test task. + +Phase 2 (after release 1, each gated on benchmark evidence): + +- Buffer-based double/float formatter into `char[24]` + `writeNumber(char[],int,int)` so `writeJson` and + `SbeDoubleNode.serialize()` stop allocating on `UTF8JsonGenerator`. +- Generated `MessageCodec` adapters (per template, calling generated flyweights or reading the buffer directly) + behind the SPI, benchmarked against `PlanMessageCodec`; promoted only if the gain justifies regeneration cost. +- `NewerVersions.PROJECT_ROOT` for newer versions if a consumer needs it: decode root fields and root-level + known sections in schema order, then stop; add `complete()` to `BorrowedDocument` (false here) and return + bytes consumed up to the last known section. +- `tryDecodeInto(..., ErrorState)` status-return API for gateways where malformed input is routine. + +Each step is red → green → refactor; no step starts before the previous gate passes. + +## 14. Risks + +- Downstream code doing `instanceof TextNode` / `LongNode` on a borrowed tree misbehaves. Mitigation: + documented as a compatibility hazard; `decodeCopy`; `snapshot()`. +- Borrowed-tree misuse (retaining across decodes). Mitigation: Javadoc contract; `newCheckedDecoder()` with an + exact coverage list — it does not claim to catch iterators obtained before the next decode. +- The plan encoder implements SBE layout from scratch (`sbe-tool` has no OTF encoder). Mitigation: + `EncoderOracleTest` against generated encoder bytes on every schema in the test set. +- Jackson minor releases adding abstract methods to `NumericNode`. Mitigation: two-version test matrix. +- `writeNumber(char[],int,int)` allocating on a generator other than `UTF8JsonGenerator` / + `WriterBasedJsonGenerator`. Mitigation: allocation claim names the generator; `-prof gc` on `treeSerialize`. +- `writeNumber(double)` allocates on all stock generators. Mitigation: documented gap; phase-2 formatter. +- Hostile messages forcing large retained arenas. Mitigation: `Limits` on total entries, bytes, depth and + retained memory, checked before growth; `retainedBytes()` exposed. +- Skeleton registry growth from many acting versions. Mitigation: keyed by effective layout, bounded by the + threshold count, counted in `maxRetainedBytes`. +- Newer-version messages silently misdecoded. Mitigation: `REJECT` is the only release-1 behaviour; the phase-2 + projection must report `complete() == false` and partial consumption. +- Performance assumptions (`switch` vs calls, flat array vs struct-of-arrays, interpreter vs generated) are + unmeasured. Mitigation: all are JMH variants, none are promises. + +## 15. Alternatives considered + +- **Generated Jackson adapters as the core** (emit `XxxJsonCodec` per message, over generated flyweights or the + raw buffer): straight-line code and constant offsets, likely the fastest field reads. Deferred behind the + `MessageCodec` SPI: the interpreter is needed regardless for runtime-loaded schemas and as the cross-engine + oracle; two generator outputs on day one is not a small library; the var-data path through generated string + setters costs an extra copy; and the speed advantage is unmeasured. Revisit with JMH data (§13 phase 2). +- **Per-message `TokenListener` over `List`** (what `JsonPrinter` does): no plan compilation, but every + message re-derives offsets, names and enum tables. Kept only as a benchmark baseline. +- **`decodeInto(ObjectNode target)` reusing an ordinary stock tree**: saves container allocation but still + allocates every `LongNode` outside the small-int cache and every `TextNode`, and mutates a caller-owned tree. + A third ownership model that reaches zero for nothing. Rejected. +- **Custom `JsonNodeFactory` as an arena**: hands out preallocated leaves by bump index, but `ObjectNode.put` of + a new key still allocates a `LinkedHashMap.Entry`, and a shared mutable leaf would alias fields. Kept only for + immutable caches (constants, enum names) inside `JacksonCaches`. +- **Generation stamps on reused leaves for stale-reference detection**: old and new references are the same + object, so no stamp value distinguishes them. Replaced by the checked decoder (fresh skeleton + poison). +- **`TokenBuffer` for parser replay or as an intermediate**: allocates `Segment` chains and boxes numerics; an + extra copy before `readTree`. Rejected for the core; the parser path restricts variable-section order instead. +- **Version clamping for newer messages**: block lengths skip extra fixed bytes but cannot locate past unknown + groups or var-data. Replaced by `REJECT`; a root projection with partial-consumption reporting is phase 2. +- **Lazy buffer-backed tree** (decode fields on first access): defers work for uninspected fields but borrows + the source buffer and needs group-offset scanning; more lifetime complexity than the skeleton. Not pursued. diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/BitSetStyle.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/BitSetStyle.java new file mode 100644 index 0000000000..9b60ac0fcf --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/BitSetStyle.java @@ -0,0 +1,32 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * How bit sets are represented in JSON. + */ +public enum BitSetStyle +{ + /** + * The raw mask as a number. Lossless. + */ + MASK, + + /** + * An object of {@code {choiceName: boolean}}. Bits without a declared choice are lost. + */ + OBJECT +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/CharArrayStyle.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/CharArrayStyle.java new file mode 100644 index 0000000000..8a876a5974 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/CharArrayStyle.java @@ -0,0 +1,32 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * How fixed-length {@code char[N]} fields are decoded to strings. + */ +public enum CharArrayStyle +{ + /** + * Characters up to the first NUL, or all N when no NUL is present. + */ + NUL_TERMINATED, + + /** + * All N characters, including any NULs. + */ + EXACT +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/EnumStyle.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/EnumStyle.java new file mode 100644 index 0000000000..ff5aaea240 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/EnumStyle.java @@ -0,0 +1,32 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * How enum values are represented in JSON. + */ +public enum EnumStyle +{ + /** + * The valid value name as a string; a raw value not declared by the enum decodes as a number. + */ + NAME, + + /** + * The raw encoded value as a number. + */ + ORDINAL +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/ErrorCode.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/ErrorCode.java new file mode 100644 index 0000000000..3d4ebb1ac6 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/ErrorCode.java @@ -0,0 +1,95 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * Classification of a failure raised as an {@link SbeJsonException}. + */ +public enum ErrorCode +{ + /** + * The header template id (or a name passed to {@link SbeJson#newEncoder(String)}) does not identify a message + * in the IR, or the header schema id differs from the IR's schema id. + */ + UNKNOWN_TEMPLATE, + + /** + * The message acting version is newer than the schema version the codec was built from. + */ + UNSUPPORTED_VERSION, + + /** + * A read would extend past {@code offset + length} of the supplied frame. + */ + FRAME_OVERFLOW, + + /** + * A field present in the acting version does not fit inside the acting block length of its scope. + */ + FIELD_OUTSIDE_BLOCK, + + /** + * A {@link Limits} budget was exceeded. + */ + LIMIT_EXCEEDED, + + /** + * A required property is missing or {@code null} on encode. + */ + MISSING_REQUIRED, + + /** + * A JSON node has the wrong shape for the SBE field, e.g. a floating point number into an integer field. + */ + TYPE_MISMATCH, + + /** + * A value is outside the range permitted by the schema: numeric min / max, dimension type range, string or + * array length, or a bit set mask wider than its encoding type. + */ + OUT_OF_RANGE, + + /** + * An enum name or raw value is not a valid value of the enum. + */ + UNKNOWN_ENUM, + + /** + * A bit set choice name is not declared by the set. + */ + UNKNOWN_CHOICE, + + /** + * A JSON property does not correspond to any field in the acting layout and + * {@link UnknownProperties#ERROR} is in force. + */ + UNKNOWN_PROPERTY, + + /** + * A constant field was supplied with a value different from the schema constant. + */ + CONSTANT_MISMATCH, + + /** + * On the streaming encode path a group or var-data section arrived out of schema order. + */ + SECTION_OUT_OF_ORDER, + + /** + * The destination buffer region is too small for the encoded message. + */ + DESTINATION_OVERFLOW +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/FieldPlan.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/FieldPlan.java new file mode 100644 index 0000000000..bde6d12042 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/FieldPlan.java @@ -0,0 +1,342 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.collections.Object2IntHashMap; +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.ir.Encoding; + +import java.nio.ByteOrder; +import java.nio.charset.Charset; + +/** + * One flat record per field, composite member, group or var-data section of a message. Produced once by + * {@link PlanCompiler}; read by the codecs in a {@code switch (kind)} loop. Carries no Jackson types. + *

+ * Offsets are scope-relative: relative to the block start for root fields and to the entry start for group + * fields. Composite members carry their composite's offset added in, so one read per leaf. + */ +final class FieldPlan +{ + /** + * int8 / int16 / int32 / int64 / uint8 / uint16 / uint32 scalar read as a {@code long}. + */ + static final byte KIND_INT = 0; + + /** + * uint64 scalar; the raw {@code long} is interpreted unsigned. + */ + static final byte KIND_UINT64 = 1; + + /** + * 32-bit float scalar. + */ + static final byte KIND_FLOAT = 2; + + /** + * 64-bit double scalar. + */ + static final byte KIND_DOUBLE = 3; + + /** + * Single {@code char}. + */ + static final byte KIND_CHAR = 4; + + /** + * {@code char[N]} decoded as a string. + */ + static final byte KIND_CHAR_ARRAY = 5; + + /** + * Fixed length array of a numeric primitive. + */ + static final byte KIND_NUMERIC_ARRAY = 6; + + /** + * Enum; {@link #enumValues} / {@link #enumNames} hold the table. + */ + static final byte KIND_ENUM = 7; + + /** + * Bit set; {@link #choiceNames} / {@link #choiceBits} hold the table. + */ + static final byte KIND_BIT_SET = 8; + + /** + * Composite; children in {@code [childStart, childEnd)}. + */ + static final byte KIND_COMPOSITE = 9; + + /** + * Repeating group; children in {@code [childStart, childEnd)}: fields, then groups, then var-data. + */ + static final byte KIND_GROUP = 10; + + /** + * Variable length data section. + */ + static final byte KIND_VAR_DATA = 11; + + /** + * Character encoding tag: ASCII or unspecified on a {@code char} field. + */ + static final byte ENC_ASCII = 0; + + /** + * Character encoding tag: UTF-8. + */ + static final byte ENC_UTF8 = 1; + + /** + * Character encoding tag: binary var-data (no character encoding). + */ + static final byte ENC_BINARY = 2; + + /** + * Character encoding tag: any other charset, resolved into {@link #charset}. + */ + static final byte ENC_OTHER = 3; + + final byte kind; + final String name; + final int index; + final int offset; + final PrimitiveType primitiveType; + final ByteOrder byteOrder; + final int arrayLength; + final int encodedLength; + final int sinceVersion; + final int deprecated; + final Encoding.Presence presence; + final boolean constant; + final boolean optional; + final long nullValueLong; + final double nullValueDouble; + final long minValueLong; + final long maxValueLong; + final double minValueDouble; + final double maxValueDouble; + final long constLong; + final double constDouble; + final String constString; + final int childStart; + final int childEnd; + final long[] enumValues; + final String[] enumNames; + final Object2IntHashMap enumNameToIndex; + final String[] choiceNames; + final int[] choiceBits; + final long knownMask; + final Object2IntHashMap choiceNameToBit; + final byte characterEncodingTag; + final String characterEncoding; + final Charset charset; + final int blockLength; + final PrimitiveType blockLengthType; + final int blockLengthOffset; + final PrimitiveType numInGroupType; + final int numInGroupOffset; + final ByteOrder numInGroupByteOrder; + final long numInGroupMin; + final long numInGroupMax; + final int dimensionSize; + final PrimitiveType lengthType; + final int lengthOffset; + final long lengthMax; + final int dataOffset; + + FieldPlan(final Builder b) + { + kind = b.kind; + name = b.name.intern(); + index = b.index; + offset = b.offset; + primitiveType = b.primitiveType; + byteOrder = b.byteOrder; + arrayLength = b.arrayLength; + encodedLength = b.encodedLength; + sinceVersion = b.sinceVersion; + deprecated = b.deprecated; + presence = b.presence; + constant = b.presence == Encoding.Presence.CONSTANT; + optional = b.presence == Encoding.Presence.OPTIONAL; + nullValueLong = b.nullValueLong; + nullValueDouble = b.nullValueDouble; + minValueLong = b.minValueLong; + maxValueLong = b.maxValueLong; + minValueDouble = b.minValueDouble; + maxValueDouble = b.maxValueDouble; + constLong = b.constLong; + constDouble = b.constDouble; + constString = b.constString; + childStart = b.childStart; + childEnd = b.childEnd; + enumValues = b.enumValues; + enumNames = b.enumNames; + enumNameToIndex = b.enumNameToIndex; + choiceNames = b.choiceNames; + choiceBits = b.choiceBits; + knownMask = b.knownMask; + choiceNameToBit = b.choiceNameToBit; + characterEncodingTag = b.characterEncodingTag; + characterEncoding = b.characterEncoding; + charset = b.charset; + blockLength = b.blockLength; + blockLengthType = b.blockLengthType; + blockLengthOffset = b.blockLengthOffset; + numInGroupType = b.numInGroupType; + numInGroupOffset = b.numInGroupOffset; + numInGroupByteOrder = b.numInGroupByteOrder; + numInGroupMin = b.numInGroupMin; + numInGroupMax = b.numInGroupMax; + dimensionSize = b.dimensionSize; + lengthType = b.lengthType; + lengthOffset = b.lengthOffset; + lengthMax = b.lengthMax; + dataOffset = b.dataOffset; + } + + /** + * Whether this plan describes a container whose children are in {@code [childStart, childEnd)}. + * + * @return true for composites and groups. + */ + boolean isContainer() + { + return kind == KIND_COMPOSITE || kind == KIND_GROUP; + } + + /** + * Whether this plan describes a fixed-length field in the block (as opposed to a group or var-data). + * + * @return true for everything except groups and var-data. + */ + boolean isBlockField() + { + return kind < KIND_GROUP; + } + + /** + * Index of the enum value equal to {@code raw}, or -1. + * + * @param raw encoded value. + * @return index into {@link #enumValues} / {@link #enumNames}, or -1 when unknown. + */ + int enumIndexOf(final long raw) + { + final long[] values = enumValues; + int low = 0; + int high = values.length - 1; + while (low <= high) + { + final int mid = (low + high) >>> 1; + final long midVal = values[mid]; + if (midVal < raw) + { + low = mid + 1; + } + else if (midVal > raw) + { + high = mid - 1; + } + else + { + return mid; + } + } + + return -1; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() + { + return "FieldPlan{" + + "kind=" + kind + + ", name='" + name + '\'' + + ", index=" + index + + ", offset=" + offset + + ", primitiveType=" + primitiveType + + ", arrayLength=" + arrayLength + + ", encodedLength=" + encodedLength + + ", sinceVersion=" + sinceVersion + + ", presence=" + presence + + ", childStart=" + childStart + + ", childEnd=" + childEnd + + '}'; + } + + /** + * Mutable holder used only during compilation. + */ + static final class Builder + { + byte kind; + String name; + int index; + int offset; + PrimitiveType primitiveType; + ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; + int arrayLength = 1; + int encodedLength; + int sinceVersion; + int deprecated; + Encoding.Presence presence = Encoding.Presence.REQUIRED; + long nullValueLong; + double nullValueDouble = Double.NaN; + long minValueLong = Long.MIN_VALUE; + long maxValueLong = Long.MAX_VALUE; + double minValueDouble = -Double.MAX_VALUE; + double maxValueDouble = Double.MAX_VALUE; + long constLong; + double constDouble; + String constString; + int childStart; + int childEnd; + long[] enumValues; + String[] enumNames; + Object2IntHashMap enumNameToIndex; + String[] choiceNames; + int[] choiceBits; + long knownMask; + Object2IntHashMap choiceNameToBit; + byte characterEncodingTag = ENC_ASCII; + String characterEncoding; + Charset charset; + int blockLength; + PrimitiveType blockLengthType; + int blockLengthOffset; + PrimitiveType numInGroupType; + int numInGroupOffset; + ByteOrder numInGroupByteOrder = ByteOrder.LITTLE_ENDIAN; + long numInGroupMin; + long numInGroupMax; + int dimensionSize; + PrimitiveType lengthType; + int lengthOffset; + long lengthMax; + int dataOffset; + + FieldPlan build() + { + return new FieldPlan(this); + } + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/HeaderLayout.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/HeaderLayout.java new file mode 100644 index 0000000000..36da81086e --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/HeaderLayout.java @@ -0,0 +1,139 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.DirectBuffer; +import org.agrona.MutableDirectBuffer; +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.ir.HeaderStructure; +import uk.co.real_logic.sbe.ir.Token; + +import java.nio.ByteOrder; + +/** + * Layout of the message header taken from the IR {@link HeaderStructure}: offset, primitive type and byte order + * of each of the four standard members, consulted individually on both read and write (a header may mix byte + * orders per member). Reads return {@code long} so that uint32 members at or above 2^31 never surface as an + * {@code IllegalStateException}; the decoder range checks them. Header members other than the four standard + * ones are written as zero bytes. + */ +final class HeaderLayout +{ + /** + * Index of the block length member in the per-member arrays. + */ + static final int BLOCK_LENGTH = 0; + + /** + * Index of the template id member in the per-member arrays. + */ + static final int TEMPLATE_ID = 1; + + /** + * Index of the schema id member in the per-member arrays. + */ + static final int SCHEMA_ID = 2; + + /** + * Index of the schema version member in the per-member arrays. + */ + static final int SCHEMA_VERSION = 3; + + private static final String[] MEMBER_NAMES = { + HeaderStructure.BLOCK_LENGTH, HeaderStructure.TEMPLATE_ID, HeaderStructure.SCHEMA_ID, + HeaderStructure.SCHEMA_VERSION + }; + + private final int encodedLength; + private final int[] offsets = new int[MEMBER_NAMES.length]; + private final PrimitiveType[] types = new PrimitiveType[MEMBER_NAMES.length]; + private final ByteOrder[] byteOrders = new ByteOrder[MEMBER_NAMES.length]; + + HeaderLayout(final HeaderStructure headerStructure) + { + encodedLength = headerStructure.tokens().get(0).encodedLength(); + + for (final Token token : headerStructure.tokens()) + { + for (int member = 0; member < MEMBER_NAMES.length; member++) + { + if (MEMBER_NAMES[member].equals(token.name()) && null != token.encoding().primitiveType()) + { + offsets[member] = token.offset(); + types[member] = token.encoding().primitiveType(); + byteOrders[member] = token.encoding().byteOrder(); + } + } + } + + for (int member = 0; member < MEMBER_NAMES.length; member++) + { + if (null == types[member]) + { + throw new IllegalArgumentException("header is missing member " + MEMBER_NAMES[member]); + } + } + } + + int encodedLength() + { + return encodedLength; + } + + /** + * Read one header member with its own type and byte order. + * + * @param buffer buffer holding the header. + * @param offset offset of the header. + * @param member one of {@link #BLOCK_LENGTH}, {@link #TEMPLATE_ID}, {@link #SCHEMA_ID}, + * {@link #SCHEMA_VERSION}. + * @return the value widened to a long (unsigned for unsigned types). + */ + long read(final DirectBuffer buffer, final int offset, final int member) + { + return WireTypes.getLong(buffer, offset + offsets[member], types[member], byteOrders[member]); + } + + int memberOffset(final int member) + { + return offsets[member]; + } + + String memberName(final int member) + { + return MEMBER_NAMES[member]; + } + + void write( + final MutableDirectBuffer buffer, + final int offset, + final int blockLength, + final int templateId, + final int schemaId, + final int version) + { + buffer.setMemory(offset, encodedLength, (byte)0); + put(buffer, offset, BLOCK_LENGTH, blockLength); + put(buffer, offset, TEMPLATE_ID, templateId); + put(buffer, offset, SCHEMA_ID, schemaId); + put(buffer, offset, SCHEMA_VERSION, version); + } + + private void put(final MutableDirectBuffer buffer, final int offset, final int member, final long value) + { + WireTypes.putLong(buffer, offset + offsets[member], types[member], byteOrders[member], value); + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/HeaderView.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/HeaderView.java new file mode 100644 index 0000000000..b7755b3ab8 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/HeaderView.java @@ -0,0 +1,113 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * Header fields of the last message decoded on a {@link SbeJsonDecoder}. One instance per decoder, overwritten + * by every decode (including decodes that fail after the header was read), so it is thread-confined like the + * decoder and must be read before the next decode. + */ +public final class HeaderView +{ + private int templateId; + private int schemaId; + private int actingVersion; + private int blockLength; + private boolean populated; + + HeaderView() + { + } + + void set(final int templateId, final int schemaId, final int actingVersion, final int blockLength) + { + this.templateId = templateId; + this.schemaId = schemaId; + this.actingVersion = actingVersion; + this.blockLength = blockLength; + this.populated = true; + } + + void clear() + { + populated = false; + } + + /** + * Whether a header has been read on this decoder since it was created. + * + * @return true once the first decode has read a header. + */ + public boolean populated() + { + return populated; + } + + /** + * Template id from the header of the last decode. + * + * @return the template id. + */ + public int templateId() + { + return templateId; + } + + /** + * Schema id from the header of the last decode. + * + * @return the schema id. + */ + public int schemaId() + { + return schemaId; + } + + /** + * Acting (schema) version from the header of the last decode. + * + * @return the acting version. + */ + public int actingVersion() + { + return actingVersion; + } + + /** + * Root block length from the header of the last decode. + * + * @return the acting block length of the root. + */ + public int blockLength() + { + return blockLength; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() + { + return "HeaderView{" + + "templateId=" + templateId + + ", schemaId=" + schemaId + + ", actingVersion=" + actingVersion + + ", blockLength=" + blockLength + + ", populated=" + populated + + '}'; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/JacksonCaches.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/JacksonCaches.java new file mode 100644 index 0000000000..7c66bdc3f3 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/JacksonCaches.java @@ -0,0 +1,129 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.core.io.SerializedString; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.TextNode; + +import java.math.BigInteger; + +/** + * Prebuilt immutable Jackson objects for one {@link MessagePlan}: constant value nodes, enum name nodes and + * serialized field names. The only place plan metadata meets Jackson objects; built once per {@link SbeJson}. + */ +final class JacksonCaches +{ + private static final BigInteger TWO_POW_64 = BigInteger.ONE.shiftLeft(64); + + private final JsonNode[] constants; + private final TextNode[][] enumNames; + private final SerializedString[] fieldNames; + + JacksonCaches(final MessagePlan plan, final EnumStyle enumStyle) + { + final JsonNodeFactory factory = JsonNodeFactory.instance; + final FieldPlan[] fields = plan.fields; + constants = new JsonNode[fields.length]; + enumNames = new TextNode[fields.length][]; + fieldNames = new SerializedString[fields.length]; + + for (int i = 0; i < fields.length; i++) + { + final FieldPlan f = fields[i]; + fieldNames[i] = new SerializedString(f.name); + + if (FieldPlan.KIND_ENUM == f.kind) + { + final TextNode[] names = new TextNode[f.enumNames.length]; + for (int n = 0; n < names.length; n++) + { + names[n] = factory.textNode(f.enumNames[n]); + } + enumNames[i] = names; + } + + if (f.constant) + { + constants[i] = constantNode(f, enumStyle, factory); + } + } + } + + JsonNode constant(final int fieldIndex) + { + return constants[fieldIndex]; + } + + TextNode enumName(final int fieldIndex, final int valueIndex) + { + return enumNames[fieldIndex][valueIndex]; + } + + SerializedString fieldName(final int fieldIndex) + { + return fieldNames[fieldIndex]; + } + + /** + * Stock node for a uint64 raw value interpreted unsigned. + * + * @param factory node factory. + * @param raw raw 64 bits. + * @return {@code LongNode} when non-negative, otherwise a {@code BigIntegerNode}. + */ + static JsonNode unsignedLongNode(final JsonNodeFactory factory, final long raw) + { + if (raw >= 0) + { + return factory.numberNode(raw); + } + + return factory.numberNode(BigInteger.valueOf(raw).add(TWO_POW_64)); + } + + private static JsonNode constantNode(final FieldPlan f, final EnumStyle enumStyle, final JsonNodeFactory factory) + { + switch (f.kind) + { + case FieldPlan.KIND_INT: + return WireTypes.fitsInt(f.primitiveType) ? + factory.numberNode((int)f.constLong) : factory.numberNode(f.constLong); + + case FieldPlan.KIND_UINT64: + return unsignedLongNode(factory, f.constLong); + + case FieldPlan.KIND_FLOAT: + return factory.numberNode((float)f.constDouble); + + case FieldPlan.KIND_DOUBLE: + return factory.numberNode(f.constDouble); + + case FieldPlan.KIND_CHAR: + case FieldPlan.KIND_CHAR_ARRAY: + return factory.textNode(f.constString); + + case FieldPlan.KIND_ENUM: + return EnumStyle.NAME == enumStyle ? + factory.textNode(f.constString) : + PlanMessageCodec.numericNode(f.primitiveType, f.constLong, factory); + + default: + throw new IllegalStateException("constant not supported for kind " + f.kind + ": " + f); + } + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/Limits.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/Limits.java new file mode 100644 index 0000000000..15fc9824b6 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/Limits.java @@ -0,0 +1,223 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * Resource budgets enforced while decoding or encoding a single message. Exceeding any of them raises + * {@link SbeJsonException} with {@link ErrorCode#LIMIT_EXCEEDED}. + *

+ * Group counts and var-data lengths on the wire are attacker controlled; the schema's dimension and length + * types bound each individually (65535 entries per group, 4 GiB per var-data field) but nested groups multiply, + * so the budgets here are totals per message. + */ +public final class Limits +{ + /** + * Default total number of group entries across all groups and nesting levels of one message. + */ + public static final int DEFAULT_MAX_GROUP_ENTRIES = 100_000; + + /** + * Default total number of var-data payload bytes across all var-data fields of one message. + */ + public static final int DEFAULT_MAX_VAR_DATA_BYTES = 16 << 20; + + /** + * Default maximum group nesting depth. The root block is depth 0; a top level group is depth 1. + */ + public static final int DEFAULT_MAX_DEPTH = 16; + + /** + * Default budget for bytes retained per borrowed document (skeleton trees, pools, scratch arrays). + */ + public static final long DEFAULT_MAX_RETAINED_BYTES = 64L << 20; + + private static final Limits DEFAULTS = builder().build(); + + private final int maxGroupEntries; + private final int maxVarDataBytes; + private final int maxDepth; + private final long maxRetainedBytes; + + private Limits(final Builder builder) + { + maxGroupEntries = builder.maxGroupEntries; + maxVarDataBytes = builder.maxVarDataBytes; + maxDepth = builder.maxDepth; + maxRetainedBytes = builder.maxRetainedBytes; + } + + /** + * Limits with all default values. + * + * @return the default limits. + */ + public static Limits defaults() + { + return DEFAULTS; + } + + /** + * Create a new builder initialised with the default values. + * + * @return a new builder. + */ + public static Builder builder() + { + return new Builder(); + } + + /** + * Total number of group entries permitted across all groups and nesting levels of one message. + * + * @return total number of group entries permitted per message. + */ + public int maxGroupEntries() + { + return maxGroupEntries; + } + + /** + * Total number of var-data payload bytes permitted across all var-data fields of one message. + * + * @return total var-data bytes permitted per message. + */ + public int maxVarDataBytes() + { + return maxVarDataBytes; + } + + /** + * Maximum group nesting depth. The root block is depth 0. + * + * @return maximum group nesting depth. + */ + public int maxDepth() + { + return maxDepth; + } + + /** + * Budget for bytes retained per borrowed document. + * + * @return retained byte budget per document. + */ + public long maxRetainedBytes() + { + return maxRetainedBytes; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() + { + return "Limits{" + + "maxGroupEntries=" + maxGroupEntries + + ", maxVarDataBytes=" + maxVarDataBytes + + ", maxDepth=" + maxDepth + + ", maxRetainedBytes=" + maxRetainedBytes + + '}'; + } + + /** + * Builder for {@link Limits}. + */ + public static final class Builder + { + private int maxGroupEntries = DEFAULT_MAX_GROUP_ENTRIES; + private int maxVarDataBytes = DEFAULT_MAX_VAR_DATA_BYTES; + private int maxDepth = DEFAULT_MAX_DEPTH; + private long maxRetainedBytes = DEFAULT_MAX_RETAINED_BYTES; + + Builder() + { + } + + /** + * Set the total number of group entries permitted per message. + * + * @param maxGroupEntries total across all groups and nesting levels; must be non-negative. + * @return this for a fluent API. + */ + public Builder maxGroupEntries(final int maxGroupEntries) + { + this.maxGroupEntries = requireNonNegative(maxGroupEntries, "maxGroupEntries"); + return this; + } + + /** + * Set the total number of var-data payload bytes permitted per message. + * + * @param maxVarDataBytes total across all var-data fields; must be non-negative. + * @return this for a fluent API. + */ + public Builder maxVarDataBytes(final int maxVarDataBytes) + { + this.maxVarDataBytes = requireNonNegative(maxVarDataBytes, "maxVarDataBytes"); + return this; + } + + /** + * Set the maximum group nesting depth. + * + * @param maxDepth maximum depth; the root block is depth 0; must be non-negative. + * @return this for a fluent API. + */ + public Builder maxDepth(final int maxDepth) + { + this.maxDepth = requireNonNegative(maxDepth, "maxDepth"); + return this; + } + + /** + * Set the budget for bytes retained per borrowed document. + * + * @param maxRetainedBytes retained byte budget; must be non-negative. + * @return this for a fluent API. + */ + public Builder maxRetainedBytes(final long maxRetainedBytes) + { + if (maxRetainedBytes < 0) + { + throw new IllegalArgumentException("maxRetainedBytes must be non-negative: " + maxRetainedBytes); + } + this.maxRetainedBytes = maxRetainedBytes; + return this; + } + + /** + * Build the immutable {@link Limits}. + * + * @return the limits. + */ + public Limits build() + { + return new Limits(this); + } + + private static int requireNonNegative(final int value, final String name) + { + if (value < 0) + { + throw new IllegalArgumentException(name + " must be non-negative: " + value); + } + + return value; + } + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/MessageCodec.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/MessageCodec.java new file mode 100644 index 0000000000..6b1c192f33 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/MessageCodec.java @@ -0,0 +1,82 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.DirectBuffer; +import org.agrona.MutableDirectBuffer; + +/** + * Backend for one message template. {@link PlanMessageCodec} is the interpreter over a {@link MessagePlan}; + * generated adapters may implement this later without changing {@link SbeJsonDecoder} / {@link SbeJsonEncoder}. + *

+ * The decoder performs the frame and header prologue (frame bounds, template routing, version check) and hands + * the body to the codec. The encoder writes the header itself so that a single {@link #encode} call produces a + * complete message. + *

+ * {@code decodeInto(DirectBuffer, int, int, BorrowedDocument)} joins this interface with the borrowed-document + * path (DESIGN.md section 13 step 5). + */ +interface MessageCodec +{ + /** + * The plan this codec was built from. + * + * @return the compiled plan. + */ + MessagePlan plan(); + + /** + * Decode a message body into a fresh tree of stock Jackson nodes. + * + * @param buffer source buffer. + * @param bodyOffset offset of the root block (just after the header). + * @param frameEnd exclusive end of the frame; no read may reach it. + * @param actingBlockLength root block length from the header. + * @param actingVersion acting version from the header, already checked against the schema version. + * @param context scratch state owned by the calling decoder. + * @return the root object. + */ + ObjectNode decodeCopy( + DirectBuffer buffer, + int bodyOffset, + int frameEnd, + int actingBlockLength, + int actingVersion, + WalkContext context); + + /** + * Encode header and body in one pass. + * + * @param body root object of the message. + * @param dst destination buffer. + * @param offset where the header starts. + * @param available bytes available from {@code offset}. + * @param context scratch state owned by the calling encoder. + * @return bytes written including the header. + */ + int encode(JsonNode body, MutableDirectBuffer dst, int offset, int available, WalkContext context); + + /** + * Compute the encoded length of header and body without writing. + * + * @param body root object of the message. + * @param context scratch state owned by the calling encoder. + * @return bytes that {@link #encode} would write. + */ + int encodedLength(JsonNode body, WalkContext context); +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/MessagePlan.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/MessagePlan.java new file mode 100644 index 0000000000..e87883dce9 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/MessagePlan.java @@ -0,0 +1,119 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import java.util.Arrays; + +/** + * Compiled layout of one message template: a flat {@link FieldPlan} array in decode order with the root child + * range, plus the effective-layout table (sorted distinct {@code sinceVersion} thresholds). Immutable and shared + * between threads. Carries no Jackson types. + */ +final class MessagePlan +{ + final int templateId; + final String name; + final int blockLength; + final int schemaId; + final int schemaVersion; + final FieldPlan[] fields; + final int rootStart; + final int rootEnd; + final int[] versionThresholds; + final int maxGroupDepth; + + MessagePlan( + final int templateId, + final String name, + final int blockLength, + final int schemaId, + final int schemaVersion, + final FieldPlan[] fields, + final int rootStart, + final int rootEnd, + final int[] versionThresholds, + final int maxGroupDepth) + { + this.templateId = templateId; + this.name = name.intern(); + this.blockLength = blockLength; + this.schemaId = schemaId; + this.schemaVersion = schemaVersion; + this.fields = fields; + this.rootStart = rootStart; + this.rootEnd = rootEnd; + this.versionThresholds = versionThresholds; + this.maxGroupDepth = maxGroupDepth; + } + + /** + * Index of the effective layout for an acting version: the number of thresholds less than or equal to the + * version. Two acting versions that admit the same set of fields share a layout index. + * + * @param actingVersion version from the message header. + * @return layout index in {@code [0, layoutCount())}. + */ + int layoutIndex(final int actingVersion) + { + final int[] thresholds = versionThresholds; + int low = 0; + int high = thresholds.length; + while (low < high) + { + final int mid = (low + high) >>> 1; + if (thresholds[mid] <= actingVersion) + { + low = mid + 1; + } + else + { + high = mid; + } + } + + return low; + } + + /** + * Number of distinct effective layouts: one more than the number of thresholds. + * + * @return the bound on layout indices. + */ + int layoutCount() + { + return versionThresholds.length + 1; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() + { + return "MessagePlan{" + + "templateId=" + templateId + + ", name='" + name + '\'' + + ", blockLength=" + blockLength + + ", schemaId=" + schemaId + + ", schemaVersion=" + schemaVersion + + ", fields=" + fields.length + + ", rootStart=" + rootStart + + ", rootEnd=" + rootEnd + + ", versionThresholds=" + Arrays.toString(versionThresholds) + + ", maxGroupDepth=" + maxGroupDepth + + '}'; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/NewerVersions.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/NewerVersions.java new file mode 100644 index 0000000000..7fc5f7d21b --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/NewerVersions.java @@ -0,0 +1,27 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * Treatment of messages whose acting version is newer than the schema version of the IR. + */ +public enum NewerVersions +{ + /** + * Raise {@link ErrorCode#UNSUPPORTED_VERSION}. The only option in release 1. + */ + REJECT +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanCompiler.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanCompiler.java new file mode 100644 index 0000000000..4755b71103 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanCompiler.java @@ -0,0 +1,570 @@ +/** + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.collections.Object2IntHashMap; +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.PrimitiveValue; +import uk.co.real_logic.sbe.ir.Encoding; +import uk.co.real_logic.sbe.ir.HeaderStructure; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Signal; +import uk.co.real_logic.sbe.ir.Token; + +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.TreeSet; + +/** + * Compiles the IR token list of a message into a {@link MessagePlan}. Mirrors the token walk of + * {@link uk.co.real_logic.sbe.otf.OtfMessageDecoder} (block fields, then groups, then var-data, recursively) but + * records offsets, sizes and tables instead of dispatching. Carries no Jackson types. + */ +final class PlanCompiler +{ + private static final int MISSING = -1; + + private final Ir ir; + private final List out = new ArrayList<>(); + private final TreeSet versions = new TreeSet<>(); + private int maxGroupDepth; + + private PlanCompiler(final Ir ir) + { + this.ir = ir; + } + + /** + * Compile the plan for one message. + * + * @param ir the IR the message belongs to. + * @param msgTokens tokens for the message as returned by {@link Ir#getMessage(long)}. + * @return the compiled plan. + */ + static MessagePlan compile(final Ir ir, final List msgTokens) + { + return new PlanCompiler(ir).compileMessage(msgTokens); + } + + private MessagePlan compileMessage(final List tokens) + { + final Token msgToken = tokens.get(0); + if (Signal.BEGIN_MESSAGE != msgToken.signal()) + { + throw new IllegalArgumentException("expected BEGIN_MESSAGE but found " + msgToken.signal()); + } + + final long range = compileScope(tokens, 1, tokens.size() - 1, 0); + + final FieldPlan[] fields = new FieldPlan[out.size()]; + for (int i = 0; i < fields.length; i++) + { + final FieldPlan.Builder b = out.get(i); + b.index = i; + fields[i] = b.build(); + } + + final int[] thresholds = new int[versions.size()]; + int t = 0; + for (final Integer version : versions) + { + thresholds[t++] = version; + } + + return new MessagePlan( + msgToken.id(), + msgToken.name(), + msgToken.encodedLength(), + ir.id(), + ir.version(), + fields, + rangeStart(range), + rangeEnd(range), + thresholds, + maxGroupDepth); + } + + /* + * Compile the direct children of a block scope (message root or group body) in {@code [from, to)}, then the + * descendants of every container child, so that each scope's children are contiguous. + */ + private long compileScope(final List tokens, final int from, final int to, final int depth) + { + maxGroupDepth = Math.max(maxGroupDepth, depth); + final int start = out.size(); + final List containers = new ArrayList<>(); + + int i = from; + while (i < to) + { + final Token token = tokens.get(i); + switch (token.signal()) + { + case BEGIN_FIELD: + compileField(tokens, i, containers); + break; + + case BEGIN_GROUP: + compileGroup(tokens, i, containers); + break; + + case BEGIN_VAR_DATA: + compileVarData(tokens, i); + break; + + default: + throw new IllegalStateException("unexpected token in block scope: " + token); + } + + i += token.componentTokenCount(); + } + + final int end = out.size(); + + for (final int[] container : containers) + { + final FieldPlan.Builder b = out.get(container[0]); + final long range = b.kind == FieldPlan.KIND_GROUP ? + compileScope(tokens, container[1], container[2], depth + 1) : + compileCompositeMembers(tokens, container[1], container[2], b.offset, b.sinceVersion); + b.childStart = rangeStart(range); + b.childEnd = rangeEnd(range); + } + + return pack(start, end); + } + + private void compileField(final List tokens, final int fieldIdx, final List containers) + { + final Token fieldToken = tokens.get(fieldIdx); + final Token typeToken = tokens.get(fieldIdx + 1); + final FieldPlan.Builder b = newBuilder(fieldToken.name(), fieldToken.version(), fieldToken.deprecated()); + final int typeIdx = fieldIdx + 1; + + switch (typeToken.signal()) + { + case ENCODING: + fillEncoding(b, typeToken, 0); + break; + + case BEGIN_COMPOSITE: + fillComposite(b, typeToken, 0); + containers.add(new int[]{ out.size(), typeIdx + 1, typeIdx + typeToken.componentTokenCount() - 1 }); + break; + + case BEGIN_ENUM: + fillEnum(b, tokens, typeIdx, 0, fieldToken); + break; + + case BEGIN_SET: + fillSet(b, tokens, typeIdx, 0); + break; + + default: + throw new IllegalStateException("unexpected field type token: " + typeToken); + } + + out.add(b); + } + + /* + * Compile the members of a composite in {@code [from, to)}; offsets are relative to the enclosing scope so the + * composite's own scope-relative offset is added to every member. + */ + private long compileCompositeMembers( + final List tokens, final int from, final int to, final int compositeOffset, final int parentVersion) + { + final int start = out.size(); + final List containers = new ArrayList<>(); + + int i = from; + while (i < to) + { + final Token token = tokens.get(i); + final FieldPlan.Builder b = newBuilder( + token.name(), Math.max(parentVersion, token.version()), token.deprecated()); + + switch (token.signal()) + { + case ENCODING: + fillEncoding(b, token, compositeOffset); + break; + + case BEGIN_COMPOSITE: + fillComposite(b, token, compositeOffset); + containers.add(new int[]{ out.size(), i + 1, i + token.componentTokenCount() - 1 }); + break; + + case BEGIN_ENUM: + fillEnum(b, tokens, i, compositeOffset, null); + break; + + case BEGIN_SET: + fillSet(b, tokens, i, compositeOffset); + break; + + default: + throw new IllegalStateException("unexpected composite member token: " + token); + } + + out.add(b); + i += token.componentTokenCount(); + } + + final int end = out.size(); + + for (final int[] container : containers) + { + final FieldPlan.Builder b = out.get(container[0]); + final long range = compileCompositeMembers( + tokens, container[1], container[2], b.offset, b.sinceVersion); + b.childStart = rangeStart(range); + b.childEnd = rangeEnd(range); + } + + return pack(start, end); + } + + private void compileGroup(final List tokens, final int groupIdx, final List containers) + { + final Token groupToken = tokens.get(groupIdx); + final Token dimensionToken = tokens.get(groupIdx + 1); + final FieldPlan.Builder b = newBuilder(groupToken.name(), groupToken.version(), groupToken.deprecated()); + b.kind = FieldPlan.KIND_GROUP; + b.blockLength = groupToken.encodedLength(); + b.dimensionSize = dimensionToken.encodedLength(); + b.encodedLength = dimensionToken.encodedLength(); + + final int dimensionEnd = groupIdx + 1 + dimensionToken.componentTokenCount(); + final Token blockLengthToken = findMember( + tokens, groupIdx + 2, dimensionEnd, HeaderStructure.BLOCK_LENGTH, groupIdx + 2); + final Token numInGroupToken = findMember(tokens, groupIdx + 2, dimensionEnd, "numInGroup", groupIdx + 3); + + b.blockLengthType = blockLengthToken.encoding().primitiveType(); + b.blockLengthOffset = blockLengthToken.offset(); + b.byteOrder = blockLengthToken.encoding().byteOrder(); + b.numInGroupType = numInGroupToken.encoding().primitiveType(); + b.numInGroupOffset = numInGroupToken.offset(); + b.numInGroupByteOrder = numInGroupToken.encoding().byteOrder(); + b.numInGroupMin = numInGroupToken.encoding().applicableMinValue().longValue(); + b.numInGroupMax = numInGroupToken.encoding().applicableMaxValue().longValue(); + + containers.add(new int[]{ out.size(), dimensionEnd, groupIdx + groupToken.componentTokenCount() - 1 }); + out.add(b); + } + + private void compileVarData(final List tokens, final int varDataIdx) + { + final Token varDataToken = tokens.get(varDataIdx); + final Token compositeToken = tokens.get(varDataIdx + 1); + final int compositeEnd = varDataIdx + 1 + compositeToken.componentTokenCount(); + final Token lengthToken = findMember(tokens, varDataIdx + 2, compositeEnd, "length", varDataIdx + 2); + final Token dataToken = findMember(tokens, varDataIdx + 2, compositeEnd, "varData", varDataIdx + 3); + + final FieldPlan.Builder b = newBuilder( + varDataToken.name(), varDataToken.version(), varDataToken.deprecated()); + b.kind = FieldPlan.KIND_VAR_DATA; + b.lengthType = lengthToken.encoding().primitiveType(); + b.lengthOffset = lengthToken.offset(); + b.lengthMax = lengthToken.encoding().applicableMaxValue().longValue(); + b.byteOrder = lengthToken.encoding().byteOrder(); + b.dataOffset = dataToken.offset(); + b.primitiveType = dataToken.encoding().primitiveType(); + b.presence = varDataToken.encoding().presence(); + + final String characterEncoding = dataToken.encoding().characterEncoding(); + b.characterEncoding = characterEncoding; + if (null == characterEncoding) + { + b.characterEncodingTag = FieldPlan.ENC_BINARY; + } + else + { + applyCharacterEncoding(b, characterEncoding); + } + + out.add(b); + } + + private void fillEncoding(final FieldPlan.Builder b, final Token typeToken, final int baseOffset) + { + final Encoding encoding = typeToken.encoding(); + final PrimitiveType primitiveType = encoding.primitiveType(); + b.primitiveType = primitiveType; + b.byteOrder = encoding.byteOrder(); + b.offset = baseOffset + typeToken.offset(); + b.presence = encoding.presence(); + b.encodedLength = typeToken.encodedLength(); + b.arrayLength = Math.max(1, typeToken.arrayLength()); + b.characterEncoding = encoding.characterEncoding(); + if (null != encoding.characterEncoding()) + { + applyCharacterEncoding(b, encoding.characterEncoding()); + } + + if (PrimitiveType.FLOAT == primitiveType || PrimitiveType.DOUBLE == primitiveType) + { + b.minValueDouble = encoding.applicableMinValue().doubleValue(); + b.maxValueDouble = encoding.applicableMaxValue().doubleValue(); + b.nullValueDouble = encoding.applicableNullValue().doubleValue(); + } + else + { + b.minValueLong = encoding.applicableMinValue().longValue(); + b.maxValueLong = encoding.applicableMaxValue().longValue(); + b.nullValueLong = encoding.applicableNullValue().longValue(); + } + + if (Encoding.Presence.CONSTANT == encoding.presence()) + { + fillConstant(b, encoding.constValue(), primitiveType); + } + else + { + b.kind = scalarKind(primitiveType, b.arrayLength); + } + } + + private static void fillConstant( + final FieldPlan.Builder b, final PrimitiveValue constValue, final PrimitiveType primitiveType) + { + switch (constValue.representation()) + { + case LONG: + if (PrimitiveType.CHAR == primitiveType) + { + b.kind = FieldPlan.KIND_CHAR; + b.constString = String.valueOf((char)constValue.longValue()); + } + else + { + b.kind = scalarKind(primitiveType, 1); + b.constLong = constValue.longValue(); + b.constDouble = constValue.longValue(); + } + break; + + case DOUBLE: + b.kind = scalarKind(primitiveType, 1); + b.constDouble = constValue.doubleValue(); + break; + + case BYTE_ARRAY: + b.kind = FieldPlan.KIND_CHAR_ARRAY; + b.constString = new String( + constValue.byteArrayValue(), charsetFor(constValue.characterEncoding())); + b.arrayLength = b.constString.length(); + break; + + default: + throw new IllegalStateException("unsupported constant representation: " + constValue); + } + } + + private static byte scalarKind(final PrimitiveType primitiveType, final int arrayLength) + { + switch (primitiveType) + { + case CHAR: + return arrayLength > 1 ? FieldPlan.KIND_CHAR_ARRAY : FieldPlan.KIND_CHAR; + + case FLOAT: + return arrayLength > 1 ? FieldPlan.KIND_NUMERIC_ARRAY : FieldPlan.KIND_FLOAT; + + case DOUBLE: + return arrayLength > 1 ? FieldPlan.KIND_NUMERIC_ARRAY : FieldPlan.KIND_DOUBLE; + + case UINT64: + return arrayLength > 1 ? FieldPlan.KIND_NUMERIC_ARRAY : FieldPlan.KIND_UINT64; + + default: + return arrayLength > 1 ? FieldPlan.KIND_NUMERIC_ARRAY : FieldPlan.KIND_INT; + } + } + + private static void fillComposite(final FieldPlan.Builder b, final Token compositeToken, final int baseOffset) + { + b.kind = FieldPlan.KIND_COMPOSITE; + b.offset = baseOffset + compositeToken.offset(); + b.encodedLength = compositeToken.encodedLength(); + b.presence = Encoding.Presence.REQUIRED; + } + + private void fillEnum( + final FieldPlan.Builder b, + final List tokens, + final int enumIdx, + final int baseOffset, + final Token fieldToken) + { + final Token enumToken = tokens.get(enumIdx); + final Encoding encoding = enumToken.encoding(); + b.kind = FieldPlan.KIND_ENUM; + b.primitiveType = encoding.primitiveType(); + b.byteOrder = encoding.byteOrder(); + b.offset = baseOffset + enumToken.offset(); + b.encodedLength = enumToken.encodedLength(); + b.nullValueLong = encoding.applicableNullValue().longValue(); + b.presence = null != fieldToken ? fieldToken.encoding().presence() : Encoding.Presence.REQUIRED; + + final int count = enumToken.componentTokenCount() - 2; + final long[] values = new long[count]; + final String[] names = new String[count]; + final Integer[] order = new Integer[count]; + for (int i = 0; i < count; i++) + { + final Token valueToken = tokens.get(enumIdx + 1 + i); + values[i] = valueToken.encoding().constValue().longValue(); + names[i] = valueToken.name().intern(); + order[i] = i; + } + Arrays.sort(order, (x, y) -> Long.compare(values[x], values[y])); + + b.enumValues = new long[count]; + b.enumNames = new String[count]; + b.enumNameToIndex = new Object2IntHashMap<>(MISSING); + for (int i = 0; i < count; i++) + { + b.enumValues[i] = values[order[i]]; + b.enumNames[i] = names[order[i]]; + b.enumNameToIndex.put(b.enumNames[i], i); + } + + if (Encoding.Presence.CONSTANT == b.presence) + { + final PrimitiveValue constValue = fieldToken.encoding().constValue(); + final String valueRef = new String(constValue.byteArrayValue(), StandardCharsets.UTF_8); + final int dot = valueRef.indexOf('.'); + final String valueName = -1 == dot ? valueRef : valueRef.substring(dot + 1); + final int index = b.enumNameToIndex.getValue(valueName); + if (MISSING == index) + { + throw new IllegalStateException("constant enum value not found: " + valueRef); + } + b.constString = b.enumNames[index]; + b.constLong = b.enumValues[index]; + } + } + + private static void fillSet( + final FieldPlan.Builder b, final List tokens, final int setIdx, final int baseOffset) + { + final Token setToken = tokens.get(setIdx); + final Encoding encoding = setToken.encoding(); + b.kind = FieldPlan.KIND_BIT_SET; + b.primitiveType = encoding.primitiveType(); + b.byteOrder = encoding.byteOrder(); + b.offset = baseOffset + setToken.offset(); + b.encodedLength = setToken.encodedLength(); + b.presence = Encoding.Presence.REQUIRED; + b.minValueLong = 0; + b.maxValueLong = 8 == encoding.primitiveType().size() ? -1L : (1L << (8 * encoding.primitiveType().size())) - 1; + + final int count = setToken.componentTokenCount() - 2; + b.choiceNames = new String[count]; + b.choiceBits = new int[count]; + b.choiceNameToBit = new Object2IntHashMap<>(MISSING); + long knownMask = 0; + for (int i = 0; i < count; i++) + { + final Token choiceToken = tokens.get(setIdx + 1 + i); + final int bit = (int)choiceToken.encoding().constValue().longValue(); + b.choiceNames[i] = choiceToken.name().intern(); + b.choiceBits[i] = bit; + b.choiceNameToBit.put(b.choiceNames[i], bit); + knownMask |= 1L << bit; + } + b.knownMask = knownMask; + } + + private static void applyCharacterEncoding(final FieldPlan.Builder b, final String characterEncoding) + { + final Charset charset = charsetFor(characterEncoding); + b.charset = charset; + if (StandardCharsets.US_ASCII.equals(charset)) + { + b.characterEncodingTag = FieldPlan.ENC_ASCII; + } + else if (StandardCharsets.UTF_8.equals(charset)) + { + b.characterEncodingTag = FieldPlan.ENC_UTF8; + } + else + { + b.characterEncodingTag = FieldPlan.ENC_OTHER; + } + } + + /** + * Resolve a schema {@code characterEncoding} to a charset; {@code null} means ASCII for character fields. + * + * @param characterEncoding schema value or {@code null}. + * @return the charset. + */ + static Charset charsetFor(final String characterEncoding) + { + if (null == characterEncoding || "ASCII".equalsIgnoreCase(characterEncoding)) + { + return StandardCharsets.US_ASCII; + } + + return Charset.forName(characterEncoding); + } + + private FieldPlan.Builder newBuilder(final String name, final int sinceVersion, final int deprecated) + { + final FieldPlan.Builder b = new FieldPlan.Builder(); + b.name = name; + b.sinceVersion = sinceVersion; + b.deprecated = deprecated; + versions.add(sinceVersion); + + return b; + } + + private static Token findMember( + final List tokens, final int from, final int to, final String name, final int fallbackIdx) + { + for (int i = from; i < to; i++) + { + final Token token = tokens.get(i); + if (Signal.ENCODING == token.signal() && name.equals(token.name())) + { + return token; + } + } + + return tokens.get(fallbackIdx); + } + + private static long pack(final int start, final int end) + { + return ((long)start << 32) | end; + } + + private static int rangeStart(final long range) + { + return (int)(range >>> 32); + } + + private static int rangeEnd(final long range) + { + return (int)range; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanMessageCodec.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanMessageCodec.java new file mode 100644 index 0000000000..09034d9c2e --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanMessageCodec.java @@ -0,0 +1,580 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.DirectBuffer; +import org.agrona.MutableDirectBuffer; +import uk.co.real_logic.sbe.PrimitiveType; + +/** + * Interpreter backend: {@code switch (kind)} loops over the {@link FieldPlan} array of one template. The only + * production backend in release 1. Immutable and shared; all per-walk state lives in the {@link WalkContext}. + */ +final class PlanMessageCodec implements MessageCodec +{ + private final MessagePlan plan; + private final JacksonCaches caches; + private final SbeJson config; + private final Limits limits; + + PlanMessageCodec(final MessagePlan plan, final JacksonCaches caches, final SbeJson config) + { + this.plan = plan; + this.caches = caches; + this.config = config; + this.limits = config.limits(); + } + + /** + * {@inheritDoc} + */ + @Override + public MessagePlan plan() + { + return plan; + } + + /** + * {@inheritDoc} + */ + @Override + public ObjectNode decodeCopy( + final DirectBuffer buffer, + final int bodyOffset, + final int frameEnd, + final int actingBlockLength, + final int actingVersion, + final WalkContext ctx) + { + ctx.reset(); + final ObjectNode root = ctx.factory.objectNode(); + decodeEntry( + plan.rootStart, plan.rootEnd, buffer, bodyOffset, actingBlockLength, actingVersion, frameEnd, root, ctx); + + return root; + } + + /** + * {@inheritDoc} + */ + @Override + public int encode( + final JsonNode body, + final MutableDirectBuffer dst, + final int offset, + final int available, + final WalkContext ctx) + { + return treeEncoder(ctx).encode(body, dst, offset, available); + } + + /** + * {@inheritDoc} + */ + @Override + public int encodedLength(final JsonNode body, final WalkContext ctx) + { + return treeEncoder(ctx).encodedLength(body); + } + + /* + * The tree encoder is retained on the thread-confined context so that one instance serves every call of + * the owning SbeJsonEncoder. + */ + private PlanTreeEncoder treeEncoder(final WalkContext ctx) + { + PlanTreeEncoder encoder = ctx.treeEncoder; + if (null == encoder || encoder.plan() != plan) + { + encoder = new PlanTreeEncoder(plan, config, ctx); + ctx.treeEncoder = encoder; + } + + return encoder; + } + + private int decodeEntry( + final int childStart, + final int childEnd, + final DirectBuffer buffer, + final int entryBase, + final int actingBlockLength, + final int actingVersion, + final int frameEnd, + final ObjectNode target, + final WalkContext ctx) + { + if (actingBlockLength > frameEnd - entryBase) + { + throw ctx.error(plan, ErrorCode.FRAME_OVERFLOW, null, entryBase, + "block length " + actingBlockLength + " exceeds frame by " + + (actingBlockLength - (frameEnd - entryBase)) + " bytes"); + } + + final FieldPlan[] fields = plan.fields; + int cursor = entryBase + actingBlockLength; + + for (int i = childStart; i < childEnd; i++) + { + final FieldPlan f = fields[i]; + if (f.sinceVersion > actingVersion) + { + continue; + } + + switch (f.kind) + { + case FieldPlan.KIND_GROUP: + cursor = decodeGroup(f, buffer, cursor, actingVersion, frameEnd, target, ctx); + break; + + case FieldPlan.KIND_VAR_DATA: + cursor = decodeVarData(f, buffer, cursor, frameEnd, target, ctx); + break; + + case FieldPlan.KIND_COMPOSITE: + target.set(f.name, decodeComposite(f, buffer, entryBase, actingBlockLength, actingVersion, ctx)); + break; + + default: + if (f.constant) + { + target.set(f.name, caches.constant(f.index)); + } + else + { + checkInsideBlock(f, entryBase, actingBlockLength, ctx); + target.set(f.name, decodeLeaf(f, buffer, entryBase + f.offset, ctx.factory)); + } + break; + } + } + + return cursor; + } + + /* + * Every present, non-constant leaf must fit inside the acting block of its scope. Composites are not checked + * as a whole: a composite whose newer members are absent in the acting version is legitimately shorter. + */ + private void checkInsideBlock( + final FieldPlan f, final int entryBase, final int actingBlockLength, final WalkContext ctx) + { + if (f.offset + f.encodedLength > actingBlockLength) + { + throw ctx.error(plan, ErrorCode.FIELD_OUTSIDE_BLOCK, f, entryBase + f.offset, + "field ends at " + (f.offset + f.encodedLength) + " but acting block length is " + + actingBlockLength); + } + } + + private ObjectNode decodeComposite( + final FieldPlan composite, + final DirectBuffer buffer, + final int entryBase, + final int actingBlockLength, + final int actingVersion, + final WalkContext ctx) + { + final ObjectNode node = ctx.factory.objectNode(); + final FieldPlan[] fields = plan.fields; + ctx.push(composite.index); + for (int i = composite.childStart; i < composite.childEnd; i++) + { + final FieldPlan member = fields[i]; + if (member.sinceVersion > actingVersion) + { + continue; + } + if (member.constant) + { + node.set(member.name, caches.constant(member.index)); + } + else if (FieldPlan.KIND_COMPOSITE == member.kind) + { + node.set( + member.name, decodeComposite(member, buffer, entryBase, actingBlockLength, actingVersion, ctx)); + } + else + { + checkInsideBlock(member, entryBase, actingBlockLength, ctx); + node.set(member.name, decodeLeaf(member, buffer, entryBase + member.offset, ctx.factory)); + } + } + ctx.pop(); + + return node; + } + + private JsonNode decodeLeaf( + final FieldPlan f, final DirectBuffer buffer, final int index, final JsonNodeFactory factory) + { + switch (f.kind) + { + case FieldPlan.KIND_INT: + { + final long value = WireTypes.getLong(buffer, index, f.primitiveType, f.byteOrder); + if (f.optional && value == f.nullValueLong) + { + return factory.nullNode(); + } + return WireTypes.fitsInt(f.primitiveType) ? factory.numberNode((int)value) : factory.numberNode(value); + } + + case FieldPlan.KIND_UINT64: + { + final long raw = buffer.getLong(index, f.byteOrder); + if (f.optional && raw == f.nullValueLong) + { + return factory.nullNode(); + } + return JacksonCaches.unsignedLongNode(factory, raw); + } + + case FieldPlan.KIND_FLOAT: + { + final float value = buffer.getFloat(index, f.byteOrder); + if (f.optional && isNull(value, f.nullValueDouble)) + { + return factory.nullNode(); + } + return factory.numberNode(value); + } + + case FieldPlan.KIND_DOUBLE: + { + final double value = buffer.getDouble(index, f.byteOrder); + if (f.optional && isNull(value, f.nullValueDouble)) + { + return factory.nullNode(); + } + return factory.numberNode(value); + } + + case FieldPlan.KIND_CHAR: + { + final byte value = buffer.getByte(index); + if (f.optional && value == f.nullValueLong) + { + return factory.nullNode(); + } + return factory.textNode(String.valueOf((char)(value & 0xFF))); + } + + case FieldPlan.KIND_CHAR_ARRAY: + return factory.textNode(decodeCharArray(f, buffer, index)); + + case FieldPlan.KIND_NUMERIC_ARRAY: + return decodeNumericArray(f, buffer, index, factory); + + case FieldPlan.KIND_ENUM: + return decodeEnum(f, buffer, index, factory); + + case FieldPlan.KIND_BIT_SET: + return decodeBitSet(f, buffer, index, factory); + + default: + throw new IllegalStateException("unexpected kind " + f.kind + " for " + f); + } + } + + /* + * ASCII and UTF-8 arrays are terminated at the first zero byte (NUL is a single byte in both). Other charsets + * are decoded in full and terminated at the first NUL character, since their encoded forms may contain zero + * bytes inside ordinary characters. + */ + private String decodeCharArray(final FieldPlan f, final DirectBuffer buffer, final int index) + { + final boolean nulTerminated = CharArrayStyle.NUL_TERMINATED == config.charArrayStyle(); + + if (FieldPlan.ENC_ASCII == f.characterEncodingTag) + { + int length = f.arrayLength; + final char[] chars = new char[length]; + for (int i = 0; i < length; i++) + { + final byte b = buffer.getByte(index + i); + if (nulTerminated && 0 == b) + { + length = i; + break; + } + chars[i] = (char)(b & 0xFF); + } + return new String(chars, 0, length); + } + + int length = f.arrayLength; + if (nulTerminated && FieldPlan.ENC_UTF8 == f.characterEncodingTag) + { + for (int i = 0; i < f.arrayLength; i++) + { + if (0 == buffer.getByte(index + i)) + { + length = i; + break; + } + } + } + + final byte[] bytes = new byte[length]; + buffer.getBytes(index, bytes, 0, length); + final String decoded = new String(bytes, f.charset); + if (nulTerminated && FieldPlan.ENC_UTF8 != f.characterEncodingTag) + { + final int nul = decoded.indexOf('\0'); + return nul < 0 ? decoded : decoded.substring(0, nul); + } + + return decoded; + } + + private static ArrayNode decodeNumericArray( + final FieldPlan f, final DirectBuffer buffer, final int index, final JsonNodeFactory factory) + { + final ArrayNode array = factory.arrayNode(f.arrayLength); + final PrimitiveType type = f.primitiveType; + final int size = type.size(); + for (int i = 0; i < f.arrayLength; i++) + { + final int elementIndex = index + i * size; + switch (type) + { + case FLOAT: + array.add(buffer.getFloat(elementIndex, f.byteOrder)); + break; + + case DOUBLE: + array.add(buffer.getDouble(elementIndex, f.byteOrder)); + break; + + case UINT64: + array.add(JacksonCaches.unsignedLongNode(factory, buffer.getLong(elementIndex, f.byteOrder))); + break; + + default: + final long value = WireTypes.getLong(buffer, elementIndex, type, f.byteOrder); + if (WireTypes.fitsInt(type)) + { + array.add((int)value); + } + else + { + array.add(value); + } + break; + } + } + + return array; + } + + private JsonNode decodeEnum( + final FieldPlan f, final DirectBuffer buffer, final int index, final JsonNodeFactory factory) + { + final long raw = WireTypes.getLong(buffer, index, f.primitiveType, f.byteOrder); + if (f.optional && raw == f.nullValueLong) + { + return factory.nullNode(); + } + + if (EnumStyle.NAME == config.enumStyle()) + { + final int valueIndex = f.enumIndexOf(raw); + if (valueIndex >= 0) + { + return caches.enumName(f.index, valueIndex); + } + } + + return numericNode(f.primitiveType, raw, factory); + } + + private JsonNode decodeBitSet( + final FieldPlan f, final DirectBuffer buffer, final int index, final JsonNodeFactory factory) + { + final long raw = WireTypes.getLong(buffer, index, f.primitiveType, f.byteOrder); + if (BitSetStyle.MASK == config.bitSetStyle()) + { + return numericNode(f.primitiveType, raw, factory); + } + + final ObjectNode node = factory.objectNode(); + for (int i = 0; i < f.choiceNames.length; i++) + { + node.set(f.choiceNames[i], factory.booleanNode(0 != ((raw >>> f.choiceBits[i]) & 1L))); + } + + return node; + } + + /** + * Stock number node for a raw value of an integer encoding type: unsigned ({@code BigIntegerNode} when the + * high bit is set) for uint64, {@code IntNode} for types that fit an int (including the byte of a + * {@code char} encoded enum), {@code LongNode} otherwise. + * + * @param type encoding type of the value. + * @param raw raw value as read from the wire (or the constant). + * @param factory node factory. + * @return the number node. + */ + static JsonNode numericNode(final PrimitiveType type, final long raw, final JsonNodeFactory factory) + { + if (PrimitiveType.UINT64 == type) + { + return JacksonCaches.unsignedLongNode(factory, raw); + } + + return WireTypes.fitsInt(type) || PrimitiveType.CHAR == type ? + factory.numberNode((int)raw) : factory.numberNode(raw); + } + + private int decodeGroup( + final FieldPlan g, + final DirectBuffer buffer, + final int dimensionOffset, + final int actingVersion, + final int frameEnd, + final ObjectNode target, + final WalkContext ctx) + { + if (ctx.depth() + 1 > limits.maxDepth()) + { + throw ctx.error(plan, ErrorCode.LIMIT_EXCEEDED, g, dimensionOffset, + "group nesting depth " + (ctx.depth() + 1) + " exceeds maxDepth " + limits.maxDepth()); + } + if (g.dimensionSize > frameEnd - dimensionOffset) + { + throw ctx.error(plan, ErrorCode.FRAME_OVERFLOW, g, dimensionOffset, + "group dimensions of " + g.dimensionSize + " bytes do not fit in the frame"); + } + + final long blockLength = WireTypes.getLong( + buffer, dimensionOffset + g.blockLengthOffset, g.blockLengthType, g.byteOrder); + final long numInGroup = WireTypes.getLong( + buffer, dimensionOffset + g.numInGroupOffset, g.numInGroupType, g.numInGroupByteOrder); + + if (numInGroup < g.numInGroupMin || numInGroup > g.numInGroupMax) + { + throw ctx.error(plan, ErrorCode.OUT_OF_RANGE, g, dimensionOffset + g.numInGroupOffset, + "numInGroup " + numInGroup + " outside [" + g.numInGroupMin + ", " + g.numInGroupMax + "]"); + } + if (ctx.addGroupEntries(numInGroup) > limits.maxGroupEntries()) + { + throw ctx.error(plan, ErrorCode.LIMIT_EXCEEDED, g, dimensionOffset + g.numInGroupOffset, + "total group entries exceed maxGroupEntries " + limits.maxGroupEntries()); + } + + int cursor = dimensionOffset + g.dimensionSize; + final int count = (int)numInGroup; + // An empty group consumes only its dimensions; each present entry is checked against the frame below. + if (count > 0 && blockLength > frameEnd - cursor) + { + throw ctx.error(plan, ErrorCode.FRAME_OVERFLOW, g, dimensionOffset + g.blockLengthOffset, + "group block length " + blockLength + " exceeds the frame"); + } + + final ArrayNode array = ctx.factory.arrayNode(count); + ctx.push(g.index); + for (int i = 0; i < count; i++) + { + ctx.element(i); + final ObjectNode entry = ctx.factory.objectNode(); + cursor = decodeEntry( + g.childStart, g.childEnd, buffer, cursor, (int)blockLength, actingVersion, frameEnd, entry, ctx); + array.add(entry); + } + ctx.pop(); + target.set(g.name, array); + + return cursor; + } + + private int decodeVarData( + final FieldPlan v, + final DirectBuffer buffer, + final int lengthOffset, + final int frameEnd, + final ObjectNode target, + final WalkContext ctx) + { + if (v.dataOffset > frameEnd - lengthOffset) + { + throw ctx.error(plan, ErrorCode.FRAME_OVERFLOW, v, lengthOffset, + "var-data length prefix does not fit in the frame"); + } + + final long length = WireTypes.getLong(buffer, lengthOffset + v.lengthOffset, v.lengthType, v.byteOrder); + if (length > v.lengthMax) + { + throw ctx.error(plan, ErrorCode.OUT_OF_RANGE, v, lengthOffset + v.lengthOffset, + "var-data length " + length + " exceeds the length type maximum " + v.lengthMax); + } + if (ctx.addVarDataBytes(length) > limits.maxVarDataBytes()) + { + throw ctx.error(plan, ErrorCode.LIMIT_EXCEEDED, v, lengthOffset + v.lengthOffset, + "total var-data bytes exceed maxVarDataBytes " + limits.maxVarDataBytes()); + } + + final int dataIndex = lengthOffset + v.dataOffset; + if (length > frameEnd - dataIndex) + { + throw ctx.error(plan, ErrorCode.FRAME_OVERFLOW, v, dataIndex, + "var-data of " + length + " bytes exceeds the frame by " + (length - (frameEnd - dataIndex))); + } + + final int len = (int)length; + final JsonNode node; + switch (v.characterEncodingTag) + { + case FieldPlan.ENC_BINARY: + { + final byte[] bytes = new byte[len]; + buffer.getBytes(dataIndex, bytes, 0, len); + node = ctx.factory.binaryNode(bytes); + break; + } + + case FieldPlan.ENC_UTF8: + node = ctx.factory.textNode(buffer.getStringWithoutLengthUtf8(dataIndex, len)); + break; + + case FieldPlan.ENC_ASCII: + node = ctx.factory.textNode(buffer.getStringWithoutLengthAscii(dataIndex, len)); + break; + + default: + { + final byte[] bytes = new byte[len]; + buffer.getBytes(dataIndex, bytes, 0, len); + node = ctx.factory.textNode(new String(bytes, v.charset)); + break; + } + } + target.set(v.name, node); + + return dataIndex + len; + } + + private static boolean isNull(final double value, final double nullValue) + { + return Double.isNaN(nullValue) ? Double.isNaN(value) : value == nullValue; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanTreeEncoder.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanTreeEncoder.java new file mode 100644 index 0000000000..80e2f1f5ed --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/PlanTreeEncoder.java @@ -0,0 +1,1051 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.MutableDirectBuffer; +import uk.co.real_logic.sbe.PrimitiveType; + +import java.io.IOException; +import java.math.BigInteger; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.util.Iterator; +import java.util.Map; + +/** + * One encode (or sizing) pass of a {@link JsonNode} tree over a {@link MessagePlan}. Walks the plan, never the + * JSON: {@code obj.get(name)} per field, {@code ArrayNode.size()} for counts, no iterators on the happy path. + * Unknown properties are detected per {@code ObjectNode} by comparing the recognised count with + * {@code size()}; the slow path that names the offending key runs only on mismatch. + *

+ * The same walk runs in sizing mode (no destination, limit {@code Integer.MAX_VALUE}) for + * {@link SbeJsonEncoder#encodedLength}; every position is still bounds checked so a message that would exceed + * the supported {@code int} size is rejected with {@link ErrorCode#DESTINATION_OVERFLOW} instead of wrapping. + * Var-data lengths are validated before any payload is materialised. + *

+ * Text is coerced strictly: an unpaired UTF-16 surrogate, or a character the field's charset cannot represent, + * is {@link ErrorCode#TYPE_MISMATCH} for fixed {@code char[N]} and var-data alike; nothing is silently replaced. + * Null sentinels: an optional scalar takes its sentinel from omission or JSON {@code null} only, so a numeric + * value equal to the sentinel fails the schema range like any other number. Elements of an optional numeric + * array additionally accept the sentinel (compared at wire precision) so that decoded arrays re-encode. + *

+ * One instance is retained per thread-confined {@link SbeJsonEncoder} (via {@link WalkContext#treeEncoder}). + * The destination is released when a call returns or throws; nothing owned by the caller is retained. + */ +final class PlanTreeEncoder +{ + private static final BigInteger TWO_POW_64 = BigInteger.ONE.shiftLeft(64); + private static final BigInteger MAX_UINT64 = TWO_POW_64.subtract(BigInteger.ONE); + + private final MessagePlan plan; + private final SbeJson config; + private final Limits limits; + private final WalkContext ctx; + private MutableDirectBuffer dst; + private int limit; + private boolean sizing; + + PlanTreeEncoder(final MessagePlan plan, final SbeJson config, final WalkContext ctx) + { + this.plan = plan; + this.config = config; + this.limits = config.limits(); + this.ctx = ctx; + } + + MessagePlan plan() + { + return plan; + } + + int encode(final JsonNode body, final MutableDirectBuffer dst, final int offset, final int available) + { + if (offset < 0 || available < 0 || available > dst.capacity() - offset) + { + throw error(ErrorCode.DESTINATION_OVERFLOW, null, offset, + "destination [" + offset + ", " + offset + " + " + available + ") outside buffer capacity " + + dst.capacity()); + } + this.dst = dst; + this.limit = offset + available; + this.sizing = false; + try + { + return walk(body, offset) - offset; + } + finally + { + this.dst = null; + } + } + + int encodedLength(final JsonNode body) + { + this.dst = null; + this.limit = Integer.MAX_VALUE; + this.sizing = true; + + return walk(body, 0); + } + + private int walk(final JsonNode body, final int offset) + { + ctx.reset(); + final ObjectNode root = requireObject(body, null, offset); + final HeaderLayout header = config.headerLayout(); + final int headerLength = header.encodedLength(); + ensure(offset, headerLength, null); + if (!sizing) + { + header.write(dst, offset, plan.blockLength, plan.templateId, plan.schemaId, plan.schemaVersion); + } + + return encodeEntry(plan.rootStart, plan.rootEnd, root, offset + headerLength, plan.blockLength); + } + + private int encodeEntry( + final int childStart, final int childEnd, final ObjectNode obj, final int entryBase, final int blockLength) + { + ensure(entryBase, blockLength, null); + if (!sizing) + { + dst.setMemory(entryBase, blockLength, (byte)0); + } + + final FieldPlan[] fields = plan.fields; + int recognised = 0; + int cursor = entryBase + blockLength; + + for (int i = childStart; i < childEnd; i++) + { + final FieldPlan f = fields[i]; + final JsonNode node = obj.get(f.name); + if (null != node) + { + recognised++; + } + + switch (f.kind) + { + case FieldPlan.KIND_GROUP: + cursor = encodeGroup(f, node, cursor); + break; + + case FieldPlan.KIND_VAR_DATA: + cursor = encodeVarData(f, node, cursor); + break; + + default: + encodeBlockField(f, node, entryBase); + break; + } + } + + checkUnknownProperties(obj, recognised, childStart, childEnd, entryBase); + + return cursor; + } + + private void checkUnknownProperties( + final ObjectNode obj, final int recognised, final int childStart, final int childEnd, final int offset) + { + if (UnknownProperties.ERROR != config.unknownProperties() || recognised == obj.size()) + { + return; + } + + final FieldPlan[] fields = plan.fields; + final Iterator names = obj.fieldNames(); + while (names.hasNext()) + { + final String name = names.next(); + boolean known = false; + for (int i = childStart; i < childEnd; i++) + { + if (fields[i].name.equals(name)) + { + known = true; + break; + } + } + if (!known) + { + throw error(ErrorCode.UNKNOWN_PROPERTY, null, offset, "unknown property '" + name + "'"); + } + } + } + + private void encodeBlockField(final FieldPlan f, final JsonNode node, final int entryBase) + { + final int index = entryBase + f.offset; + if (f.constant) + { + // Any supplied value, including an explicit null, must match; only omission is exempt. + if (null != node) + { + validateConstant(f, node, index); + } + return; + } + + if (null == node || node.isNull()) + { + if (!f.optional) + { + throw error(ErrorCode.MISSING_REQUIRED, f, index, "required field is missing or null"); + } + writeNull(f, index); + return; + } + + switch (f.kind) + { + case FieldPlan.KIND_INT: + putLong(f, index, signedValue(f, node, index, false)); + break; + + case FieldPlan.KIND_UINT64: + putLong(f, index, uint64Value(f, node, index, false)); + break; + + case FieldPlan.KIND_FLOAT: + case FieldPlan.KIND_DOUBLE: + { + final double value = floatingValue(f, node, index, false); + if (!sizing) + { + putNumeric(f, index, 0, value); + } + break; + } + + case FieldPlan.KIND_CHAR: + encodeChar(f, node, index); + break; + + case FieldPlan.KIND_CHAR_ARRAY: + encodeCharArray(f, node, index); + break; + + case FieldPlan.KIND_NUMERIC_ARRAY: + encodeNumericArray(f, node, index); + break; + + case FieldPlan.KIND_ENUM: + putLong(f, index, enumValue(f, node, index)); + break; + + case FieldPlan.KIND_BIT_SET: + putLong(f, index, bitSetValue(f, node, index)); + break; + + case FieldPlan.KIND_COMPOSITE: + encodeComposite(f, node, entryBase); + break; + + default: + throw new IllegalStateException("unexpected kind " + f.kind + " for " + f); + } + } + + private void encodeComposite(final FieldPlan composite, final JsonNode node, final int entryBase) + { + final ObjectNode obj = requireObject(node, composite, entryBase + composite.offset); + final FieldPlan[] fields = plan.fields; + int recognised = 0; + ctx.push(composite.index); + for (int i = composite.childStart; i < composite.childEnd; i++) + { + final FieldPlan member = fields[i]; + final JsonNode memberNode = obj.get(member.name); + if (null != memberNode) + { + recognised++; + } + encodeBlockField(member, memberNode, entryBase); + } + checkUnknownProperties( + obj, recognised, composite.childStart, composite.childEnd, entryBase + composite.offset); + ctx.pop(); + } + + private void writeNull(final FieldPlan f, final int index) + { + if (sizing) + { + return; + } + + switch (f.kind) + { + case FieldPlan.KIND_INT: + case FieldPlan.KIND_UINT64: + case FieldPlan.KIND_ENUM: + case FieldPlan.KIND_CHAR: + WireTypes.putLong(dst, index, f.primitiveType, f.byteOrder, f.nullValueLong); + break; + + case FieldPlan.KIND_FLOAT: + dst.putFloat(index, (float)f.nullValueDouble, f.byteOrder); + break; + + case FieldPlan.KIND_DOUBLE: + dst.putDouble(index, f.nullValueDouble, f.byteOrder); + break; + + case FieldPlan.KIND_NUMERIC_ARRAY: + for (int i = 0; i < f.arrayLength; i++) + { + putNumeric(f, index + i * f.primitiveType.size(), f.nullValueLong, f.nullValueDouble); + } + break; + + default: + // char arrays and bit sets: the block was zero-filled already. + break; + } + } + + private void putNumeric(final FieldPlan f, final int index, final long longValue, final double doubleValue) + { + switch (f.primitiveType) + { + case FLOAT: + dst.putFloat(index, (float)doubleValue, f.byteOrder); + break; + + case DOUBLE: + dst.putDouble(index, doubleValue, f.byteOrder); + break; + + default: + WireTypes.putLong(dst, index, f.primitiveType, f.byteOrder, longValue); + break; + } + } + + private void putLong(final FieldPlan f, final int index, final long value) + { + if (!sizing) + { + WireTypes.putLong(dst, index, f.primitiveType, f.byteOrder, value); + } + } + + /* + * Signed integer value for an int / uint8..uint32 field: an integral node within the schema range. Inside an + * optional numeric array (arrayElement) the null sentinel is also accepted so that decoded arrays re-encode; + * a scalar sentinel is selected by omission or JSON null only, never by its numeric value. + */ + private long signedValue(final FieldPlan f, final JsonNode node, final int index, final boolean arrayElement) + { + if (!node.isIntegralNumber()) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, "expected an integral number but found " + describe(node)); + } + if (!node.canConvertToLong()) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "value " + node + " does not fit a 64-bit integer"); + } + + final long value = node.longValue(); + if (arrayElement && f.optional && value == f.nullValueLong) + { + return value; + } + if (value < f.minValueLong || value > f.maxValueLong) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, + "value " + value + " outside [" + f.minValueLong + ", " + f.maxValueLong + "]"); + } + + return value; + } + + /* + * Raw value for a uint64 field: unsigned within the schema range, or (inside an optional numeric array only) + * the null sentinel; see signedValue. + */ + private long uint64Value(final FieldPlan f, final JsonNode node, final int index, final boolean arrayElement) + { + final long raw = unsignedRaw(f, node, index); + if (arrayElement && f.optional && raw == f.nullValueLong) + { + return raw; + } + if (Long.compareUnsigned(raw, f.minValueLong) < 0 || Long.compareUnsigned(raw, f.maxValueLong) > 0) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, + "value " + Long.toUnsignedString(raw) + " outside [" + Long.toUnsignedString(f.minValueLong) + + ", " + Long.toUnsignedString(f.maxValueLong) + "]"); + } + + return raw; + } + + /* + * Parse an unsigned 64-bit value from an integral node (including BigIntegerNode) or a decimal string. + * Negative values and values above 2^64 - 1 are OUT_OF_RANGE. + */ + private long unsignedRaw(final FieldPlan f, final JsonNode node, final int index) + { + final BigInteger value; + if (node.isIntegralNumber()) + { + if (node.canConvertToLong()) + { + final long v = node.longValue(); + if (v < 0) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "negative value " + v + " into an unsigned field"); + } + return v; + } + value = node.bigIntegerValue(); + } + else if (node.isTextual()) + { + try + { + value = new BigInteger(node.textValue().trim()); + } + catch (final NumberFormatException ex) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "expected an unsigned decimal string but found '" + node.textValue() + "'"); + } + } + else + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "expected an integral number or decimal string but found " + describe(node)); + } + + if (value.signum() < 0 || value.compareTo(MAX_UINT64) > 0) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "value " + value + " outside [0, " + MAX_UINT64 + "]"); + } + + return value.longValue(); + } + + /* + * Floating value within the schema range, or (inside an optional numeric array only) the null sentinel + * compared at wire precision; see signedValue. + */ + private double floatingValue(final FieldPlan f, final JsonNode node, final int index, final boolean arrayElement) + { + final double value; + if (node.isNumber()) + { + value = node.doubleValue(); + } + else if (node.isTextual()) + { + switch (node.textValue()) + { + case "NaN": + value = Double.NaN; + break; + + case "Infinity": + case "+Infinity": + value = Double.POSITIVE_INFINITY; + break; + + case "-Infinity": + value = Double.NEGATIVE_INFINITY; + break; + + default: + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "expected a number or \"NaN\" / \"Infinity\" / \"-Infinity\" but found '" + + node.textValue() + "'"); + } + } + else + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, "expected a number but found " + describe(node)); + } + + if (arrayElement && f.optional && isWireSentinel(f, value)) + { + return value; + } + if (Double.isFinite(value) && (value < f.minValueDouble || value > f.maxValueDouble)) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, + "value " + value + " outside [" + f.minValueDouble + ", " + f.maxValueDouble + "]"); + } + + return value; + } + + /* + * Whether a value is the field's null sentinel once written: a float field compares as float, so a double + * node such as -1.1 matches a float sentinel of -1.1f, and a NaN sentinel matches every NaN. + */ + private static boolean isWireSentinel(final FieldPlan f, final double value) + { + final double nullValue = f.nullValueDouble; + if (Double.isNaN(nullValue)) + { + return Double.isNaN(value); + } + + return PrimitiveType.FLOAT == f.primitiveType ? (float)value == (float)nullValue : value == nullValue; + } + + private void encodeChar(final FieldPlan f, final JsonNode node, final int index) + { + final String text = requireText(f, node, index); + if (1 != text.length()) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "expected exactly one character but found " + text.length()); + } + final char c = text.charAt(0); + if (c > 0xFF || (FieldPlan.ENC_ASCII == f.characterEncodingTag && c > 0x7F)) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, "character U+" + Integer.toHexString(c) + + " cannot be encoded in a single byte char field"); + } + putLong(f, index, c); + } + + private void encodeCharArray(final FieldPlan f, final JsonNode node, final int index) + { + final String text = requireText(f, node, index); + if (FieldPlan.ENC_ASCII == f.characterEncodingTag) + { + final int length = text.length(); + if (length > f.arrayLength) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, + "string of " + length + " characters exceeds char[" + f.arrayLength + "]"); + } + for (int i = 0; i < length; i++) + { + final char c = text.charAt(i); + if (c > 0x7F) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "character U+" + Integer.toHexString(c) + " is not ASCII"); + } + if (!sizing) + { + dst.putByte(index + i, (byte)c); + } + } + } + else + { + final byte[] bytes = boundedBytes(f, text, index, f.arrayLength, ErrorCode.OUT_OF_RANGE); + if (!sizing) + { + dst.putBytes(index, bytes); + } + } + } + + private void encodeNumericArray(final FieldPlan f, final JsonNode node, final int index) + { + if (!node.isArray()) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, "expected an array but found " + describe(node)); + } + if (node.size() != f.arrayLength) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, + "expected " + f.arrayLength + " elements but found " + node.size()); + } + + final int size = f.primitiveType.size(); + for (int i = 0; i < f.arrayLength; i++) + { + final JsonNode element = node.get(i); + final int elementIndex = index + i * size; + switch (f.primitiveType) + { + case FLOAT: + case DOUBLE: + { + final double value = floatingValue(f, element, elementIndex, true); + if (!sizing) + { + putNumeric(f, elementIndex, 0, value); + } + break; + } + + case UINT64: + putLong(f, elementIndex, uint64Value(f, element, elementIndex, true)); + break; + + default: + putLong(f, elementIndex, signedValue(f, element, elementIndex, true)); + break; + } + } + } + + private long enumValue(final FieldPlan f, final JsonNode node, final int index) + { + if (node.isTextual()) + { + final int valueIndex = f.enumNameToIndex.getValue(node.textValue()); + if (valueIndex < 0) + { + throw error(ErrorCode.UNKNOWN_ENUM, f, index, "unknown enum name '" + node.textValue() + "'"); + } + return f.enumValues[valueIndex]; + } + if (node.isIntegralNumber()) + { + final PrimitiveType type = f.primitiveType; + if (PrimitiveType.UINT64 == type) + { + return unsignedRaw(f, node, index); + } + if (!node.canConvertToLong()) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "enum value " + node + " does not fit " + type); + } + final long raw = node.longValue(); + final long min = PrimitiveType.CHAR == type ? 0 : type.minValue().longValue(); + final long max = PrimitiveType.CHAR == type ? 0xFF : type.maxValue().longValue(); + if (raw < min || raw > max) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "enum value " + raw + " does not fit " + type); + } + return raw; + } + + throw error(ErrorCode.TYPE_MISMATCH, f, index, "expected an enum name or number but found " + describe(node)); + } + + private long bitSetValue(final FieldPlan f, final JsonNode node, final int index) + { + if (node.isIntegralNumber()) + { + if (PrimitiveType.UINT64 == f.primitiveType) + { + return unsignedRaw(f, node, index); + } + if (!node.canConvertToLong()) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, "mask " + node + " does not fit " + f.primitiveType); + } + final long mask = node.longValue(); + if (mask < 0 || mask > f.maxValueLong) + { + throw error(ErrorCode.OUT_OF_RANGE, f, index, + "mask " + mask + " does not fit " + f.primitiveType); + } + return mask; + } + if (node.isObject()) + { + long mask = 0; + final Iterator> entries = node.fields(); + while (entries.hasNext()) + { + final Map.Entry entry = entries.next(); + final int bit = f.choiceNameToBit.getValue(entry.getKey()); + if (bit < 0) + { + throw error(ErrorCode.UNKNOWN_CHOICE, f, index, "unknown choice '" + entry.getKey() + "'"); + } + if (!entry.getValue().isBoolean()) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "choice '" + entry.getKey() + "' expected a boolean but found " + describe(entry.getValue())); + } + if (entry.getValue().booleanValue()) + { + mask |= 1L << bit; + } + } + return mask; + } + + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "expected an integer mask or an object of booleans but found " + describe(node)); + } + + private void validateConstant(final FieldPlan f, final JsonNode node, final int index) + { + final boolean matches; + switch (f.kind) + { + case FieldPlan.KIND_INT: + matches = node.isIntegralNumber() && node.canConvertToLong() && node.longValue() == f.constLong; + break; + + case FieldPlan.KIND_UINT64: + matches = node.isIntegralNumber() && node.bigIntegerValue().equals(unsigned(f.constLong)); + break; + + case FieldPlan.KIND_FLOAT: + matches = node.isNumber() && (float)node.doubleValue() == (float)f.constDouble; + break; + + case FieldPlan.KIND_DOUBLE: + matches = node.isNumber() && node.doubleValue() == f.constDouble; + break; + + case FieldPlan.KIND_ENUM: + if (node.isTextual()) + { + matches = node.textValue().equals(f.constString); + } + else if (PrimitiveType.UINT64 == f.primitiveType) + { + matches = node.isIntegralNumber() && node.bigIntegerValue().equals(unsigned(f.constLong)); + } + else + { + matches = node.isIntegralNumber() && node.canConvertToLong() && node.longValue() == f.constLong; + } + break; + + default: + matches = node.isTextual() && node.textValue().equals(f.constString); + break; + } + + if (!matches) + { + throw error(ErrorCode.CONSTANT_MISMATCH, f, index, + "supplied " + node + " but the schema constant is " + + (null != f.constString ? "'" + f.constString + "'" : constantDescription(f))); + } + } + + private static String constantDescription(final FieldPlan f) + { + switch (f.kind) + { + case FieldPlan.KIND_UINT64: + return Long.toUnsignedString(f.constLong); + + case FieldPlan.KIND_FLOAT: + case FieldPlan.KIND_DOUBLE: + return String.valueOf(f.constDouble); + + default: + return String.valueOf(f.constLong); + } + } + + private int encodeGroup(final FieldPlan g, final JsonNode node, final int dimensionOffset) + { + final int count; + if (null == node || node.isNull()) + { + count = 0; + } + else if (node.isArray()) + { + count = node.size(); + } + else + { + throw error(ErrorCode.TYPE_MISMATCH, g, dimensionOffset, "expected an array but found " + describe(node)); + } + + if (count < g.numInGroupMin || count > g.numInGroupMax) + { + throw error(ErrorCode.OUT_OF_RANGE, g, dimensionOffset, + "numInGroup " + count + " outside [" + g.numInGroupMin + ", " + g.numInGroupMax + "]"); + } + if (ctx.addGroupEntries(count) > limits.maxGroupEntries()) + { + throw error(ErrorCode.LIMIT_EXCEEDED, g, dimensionOffset, + "total group entries exceed maxGroupEntries " + limits.maxGroupEntries()); + } + if (ctx.depth() + 1 > limits.maxDepth()) + { + throw error(ErrorCode.LIMIT_EXCEEDED, g, dimensionOffset, + "group nesting depth " + (ctx.depth() + 1) + " exceeds maxDepth " + limits.maxDepth()); + } + + ensure(dimensionOffset, g.dimensionSize, g); + if (!sizing) + { + dst.setMemory(dimensionOffset, g.dimensionSize, (byte)0); + WireTypes.putLong( + dst, dimensionOffset + g.blockLengthOffset, g.blockLengthType, g.byteOrder, g.blockLength); + WireTypes.putLong( + dst, dimensionOffset + g.numInGroupOffset, g.numInGroupType, g.numInGroupByteOrder, count); + } + + int cursor = dimensionOffset + g.dimensionSize; + ctx.push(g.index); + for (int i = 0; i < count; i++) + { + ctx.element(i); + final ObjectNode entry = requireObject(node.get(i), null, cursor); + cursor = encodeEntry(g.childStart, g.childEnd, entry, cursor, g.blockLength); + } + ctx.pop(); + + return cursor; + } + + /* + * Var-data: the payload length is validated against the length type maximum, the remaining maxVarDataBytes + * budget and the destination before any payload bytes are materialised. + */ + private int encodeVarData(final FieldPlan v, final JsonNode node, final int lengthOffset) + { + ensure(lengthOffset, v.dataOffset, v); + final int dataIndex = lengthOffset + v.dataOffset; + + final int length; + if (null == node || node.isNull()) + { + length = 0; + } + else if (FieldPlan.ENC_BINARY == v.characterEncodingTag) + { + length = encodeBinaryVarData(v, node, lengthOffset, dataIndex); + } + else if (FieldPlan.ENC_UTF8 == v.characterEncodingTag) + { + final String text = requireText(v, node, dataIndex); + final long utf8Length = Utf8.encodedLength(text); + if (utf8Length < 0) + { + throw error(ErrorCode.TYPE_MISMATCH, v, dataIndex, + "string contains an unpaired surrogate and cannot be encoded as UTF-8"); + } + length = checkVarDataLength(v, utf8Length, lengthOffset); + if (!sizing) + { + Utf8.encode(text, dst, dataIndex); + } + } + else if (FieldPlan.ENC_ASCII == v.characterEncodingTag) + { + final String text = requireText(v, node, dataIndex); + if (!Utf8.isAscii(text)) + { + throw error(ErrorCode.TYPE_MISMATCH, v, dataIndex, "string contains non-ASCII characters"); + } + length = checkVarDataLength(v, text.length(), lengthOffset); + if (!sizing) + { + for (int i = 0; i < length; i++) + { + dst.putByte(dataIndex + i, (byte)text.charAt(i)); + } + } + } + else + { + final String text = requireText(v, node, dataIndex); + final long budget = Math.min( + v.lengthMax, Math.min(limits.maxVarDataBytes() - ctx.varDataBytes(), limit - dataIndex)); + final byte[] bytes = boundedBytes(v, text, lengthOffset, budget, null); + length = checkVarDataLength(v, bytes.length, lengthOffset); + if (!sizing) + { + dst.putBytes(dataIndex, bytes); + } + } + + ctx.addVarDataBytes(length); + if (!sizing) + { + dst.setMemory(lengthOffset, v.dataOffset, (byte)0); + WireTypes.putLong(dst, lengthOffset + v.lengthOffset, v.lengthType, v.byteOrder, length); + } + + return dataIndex + length; + } + + private int encodeBinaryVarData( + final FieldPlan v, final JsonNode node, final int lengthOffset, final int dataIndex) + { + final byte[] bytes; + if (node.isBinary()) + { + bytes = binaryValue(v, node, dataIndex); + } + else if (node.isTextual()) + { + // Count payload bytes without padding or whitespace; binaryValue validates the base64 syntax. + final String text = node.textValue(); + long decodedLength = 0; + int unitIndex = 0; + for (int i = 0; i < text.length(); i++) + { + final char c = text.charAt(i); + if (c > ' ') + { + if (unitIndex > 0 && '=' != c) + { + decodedLength++; + } + unitIndex = (unitIndex + 1) & 3; + } + } + checkVarDataLength(v, decodedLength, lengthOffset); + bytes = binaryValue(v, node, dataIndex); + } + else + { + throw error(ErrorCode.TYPE_MISMATCH, v, dataIndex, "expected base64 binary but found " + describe(node)); + } + + final int length = checkVarDataLength(v, bytes.length, lengthOffset); + if (!sizing) + { + dst.putBytes(dataIndex, bytes); + } + + return length; + } + + /* + * Encode text in a non ASCII / UTF-8 charset without materialising more than budget + 1 bytes. When the + * result would exceed the budget, overflowCode is raised if given, otherwise the var-data budget checks + * decide the code. The charset encoder reports (never replaces) malformed and unmappable input, matching + * the UTF-8 and ASCII paths. + */ + private byte[] boundedBytes( + final FieldPlan f, final String text, final int index, final long budget, final ErrorCode overflowCode) + { + final CharsetEncoder encoder = f.charset.newEncoder(); + final long upperBound = (long)Math.ceil(text.length() * (double)encoder.maxBytesPerChar()); + final long scratch = Math.min(upperBound, Math.min(budget, Integer.MAX_VALUE - 1) + 1); + final ByteBuffer out = ByteBuffer.allocate((int)Math.max(0, scratch)); + CoderResult result = encoder.encode(CharBuffer.wrap(text), out, true); + if (result.isUnderflow()) + { + result = encoder.flush(out); + } + if (result.isOverflow() || out.position() > budget) + { + if (null != overflowCode) + { + throw error(overflowCode, f, index, "string encodes to more than " + budget + " bytes"); + } + checkVarDataLength(f, budget + 1, index); + } + if (result.isError()) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, + "string contains characters that cannot be encoded in " + f.charset); + } + + final byte[] bytes = new byte[out.position()]; + out.flip(); + out.get(bytes); + + return bytes; + } + + /* + * Check a var-data payload length against the length type maximum, the remaining var-data budget and the + * destination, in that order, without consuming the budget. + */ + private int checkVarDataLength(final FieldPlan v, final long length, final int lengthOffset) + { + if (length > v.lengthMax) + { + throw error(ErrorCode.OUT_OF_RANGE, v, lengthOffset, + "var-data of " + length + " bytes exceeds the length type maximum " + v.lengthMax); + } + if (length > limits.maxVarDataBytes() - ctx.varDataBytes()) + { + throw error(ErrorCode.LIMIT_EXCEEDED, v, lengthOffset, + "total var-data bytes exceed maxVarDataBytes " + limits.maxVarDataBytes()); + } + ensure(lengthOffset + v.dataOffset, length, v); + + return (int)length; + } + + private byte[] binaryValue(final FieldPlan v, final JsonNode node, final int index) + { + try + { + return node.binaryValue(); + } + catch (final IOException ex) + { + throw error(ErrorCode.TYPE_MISMATCH, v, index, "invalid base64: " + ex.getMessage()); + } + } + + private ObjectNode requireObject(final JsonNode node, final FieldPlan f, final int index) + { + if (null == node || node.isNull()) + { + throw error(ErrorCode.MISSING_REQUIRED, f, index, "required object is missing or null"); + } + if (!node.isObject()) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, "expected an object but found " + describe(node)); + } + + return (ObjectNode)node; + } + + private String requireText(final FieldPlan f, final JsonNode node, final int index) + { + if (!node.isTextual()) + { + throw error(ErrorCode.TYPE_MISMATCH, f, index, "expected a string but found " + describe(node)); + } + + return node.textValue(); + } + + /* + * Check that length bytes at index fit before the limit. In sizing mode the limit is Integer.MAX_VALUE, so + * this also rejects messages beyond the supported size instead of overflowing int arithmetic. + */ + private void ensure(final int index, final long length, final FieldPlan f) + { + if (length > limit - index) + { + throw error(ErrorCode.DESTINATION_OVERFLOW, f, index, + "need " + length + " bytes at " + index + " but only " + Math.max(0, limit - index) + + (sizing ? " remain below the supported message size" : " available")); + } + } + + private SbeJsonException error(final ErrorCode code, final FieldPlan f, final int index, final String detail) + { + return ctx.error(plan, code, f, index, detail); + } + + private static BigInteger unsigned(final long raw) + { + return raw >= 0 ? BigInteger.valueOf(raw) : BigInteger.valueOf(raw).add(TWO_POW_64); + } + + private static String describe(final JsonNode node) + { + return node.getNodeType().name().toLowerCase() + " " + node; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJson.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJson.java new file mode 100644 index 0000000000..ddb0fd9366 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJson.java @@ -0,0 +1,364 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.collections.Int2ObjectHashMap; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Token; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Compiled, immutable, shareable codec for one schema: the {@link Ir} compiled once into a {@link MessagePlan} + * per message plus the JSON conventions chosen at build time. Hands out thread-confined + * {@link SbeJsonDecoder}s and {@link SbeJsonEncoder}s. + * + * 

{@code
+ * final SbeJson sbeJson = SbeJson.builder(ir).build();
+ * final SbeJsonDecoder decoder = sbeJson.newDecoder();          // one per thread
+ * final ObjectNode tree = decoder.decodeCopy(buffer, offset, length);
+ * final SbeJsonEncoder encoder = sbeJson.newEncoder("Car");     // one per thread, bound to a template
+ * final int written = encoder.encode(tree, dst, 0, dst.capacity());
+ * }
+ */ +public final class SbeJson +{ + private final Ir ir; + private final EnumStyle enumStyle; + private final BitSetStyle bitSetStyle; + private final CharArrayStyle charArrayStyle; + private final UnknownProperties unknownProperties; + private final NewerVersions newerVersions; + private final boolean exceptionStackTraces; + private final Limits limits; + private final HeaderLayout headerLayout; + private final Int2ObjectHashMap codecsById = new Int2ObjectHashMap<>(); + private final Map codecsByName = new HashMap<>(); + + private SbeJson(final Builder builder) + { + ir = builder.ir; + enumStyle = builder.enumStyle; + bitSetStyle = builder.bitSetStyle; + charArrayStyle = builder.charArrayStyle; + unknownProperties = builder.unknownProperties; + newerVersions = builder.newerVersions; + exceptionStackTraces = builder.exceptionStackTraces; + limits = builder.limits; + headerLayout = new HeaderLayout(ir.headerStructure()); + + for (final List tokens : ir.messages()) + { + final MessagePlan plan = PlanCompiler.compile(ir, tokens); + if (plan.maxGroupDepth > limits.maxDepth()) + { + throw new SbeJsonException( + ErrorCode.LIMIT_EXCEEDED, plan.templateId, SbeJsonException.NO_OFFSET, plan.name, + "schema group nesting depth " + plan.maxGroupDepth + " exceeds maxDepth " + limits.maxDepth(), + exceptionStackTraces); + } + final MessageCodec codec = new PlanMessageCodec(plan, new JacksonCaches(plan, enumStyle), this); + codecsById.put(plan.templateId, codec); + codecsByName.put(plan.name, codec); + } + } + + /** + * Start building a codec for a schema. + * + * @param ir the IR of the schema, from {@code IrDecoder} or {@code IrGenerator}. + * @return a builder with default policies. + */ + public static Builder builder(final Ir ir) + { + return new Builder(Objects.requireNonNull(ir, "ir")); + } + + /** + * Create a thread-confined decoder for any message of the schema. + * + * @return a new decoder. + */ + public SbeJsonDecoder newDecoder() + { + return new SbeJsonDecoder(this); + } + + /** + * Debug decoder whose borrowed documents detect use after the next decode. Arrives with the borrowed-document + * path (DESIGN.md section 13 step 5). + * + * @return never returns in this release. + * @throws UnsupportedOperationException always, until step 5 lands. + */ + public SbeJsonDecoder newCheckedDecoder() + { + // TODO(DESIGN.md section 13 step 5): fresh skeleton per decode with poisoning of the previous tree. + throw new UnsupportedOperationException("newCheckedDecoder arrives with the borrowed-document path"); + } + + /** + * Create a thread-confined encoder bound to a message template by name. + * + * @param messageName message name from the schema. + * @return a new encoder. + * @throws SbeJsonException with {@link ErrorCode#UNKNOWN_TEMPLATE} when the name is not a message. + */ + public SbeJsonEncoder newEncoder(final String messageName) + { + final MessageCodec codec = codecsByName.get(messageName); + if (null == codec) + { + throw new SbeJsonException( + ErrorCode.UNKNOWN_TEMPLATE, SbeJsonException.NO_TEMPLATE_ID, SbeJsonException.NO_OFFSET, null, + "no message named '" + messageName + "' in schema id " + ir.id(), exceptionStackTraces); + } + + return new SbeJsonEncoder(this, codec); + } + + /** + * Create a thread-confined encoder bound to a message template by id. + * + * @param templateId template id from the schema. + * @return a new encoder. + * @throws SbeJsonException with {@link ErrorCode#UNKNOWN_TEMPLATE} when the id is not a message. + */ + public SbeJsonEncoder newEncoder(final int templateId) + { + return new SbeJsonEncoder(this, codecForTemplate(templateId)); + } + + /** + * The IR this codec was compiled from. + * + * @return the IR. + */ + public Ir ir() + { + return ir; + } + + /** + * Enum representation policy. + * + * @return the enum style. + */ + public EnumStyle enumStyle() + { + return enumStyle; + } + + /** + * Bit set representation policy. + * + * @return the bit set style. + */ + public BitSetStyle bitSetStyle() + { + return bitSetStyle; + } + + /** + * Fixed-length char array decoding policy. + * + * @return the char array style. + */ + public CharArrayStyle charArrayStyle() + { + return charArrayStyle; + } + + /** + * Unknown JSON property policy on encode. + * + * @return the unknown property policy. + */ + public UnknownProperties unknownProperties() + { + return unknownProperties; + } + + /** + * Policy for messages newer than the schema. + * + * @return the newer version policy. + */ + public NewerVersions newerVersions() + { + return newerVersions; + } + + /** + * Whether thrown {@link SbeJsonException}s capture a stack trace. + * + * @return true when stack traces are writable. + */ + public boolean exceptionStackTraces() + { + return exceptionStackTraces; + } + + /** + * Resource budgets enforced per message. + * + * @return the limits. + */ + public Limits limits() + { + return limits; + } + + HeaderLayout headerLayout() + { + return headerLayout; + } + + MessageCodec codecForTemplate(final int templateId) + { + final MessageCodec codec = codecsById.get(templateId); + if (null == codec) + { + throw new SbeJsonException( + ErrorCode.UNKNOWN_TEMPLATE, templateId, SbeJsonException.NO_OFFSET, null, + "no message with template id " + templateId + " in schema id " + ir.id(), exceptionStackTraces); + } + + return codec; + } + + /** + * Builder for {@link SbeJson}. Defaults: {@link EnumStyle#NAME}, {@link BitSetStyle#MASK}, + * {@link CharArrayStyle#NUL_TERMINATED}, {@link UnknownProperties#ERROR}, {@link NewerVersions#REJECT}, + * writable stack traces, {@link Limits#defaults()}. + */ + public static final class Builder + { + private final Ir ir; + private EnumStyle enumStyle = EnumStyle.NAME; + private BitSetStyle bitSetStyle = BitSetStyle.MASK; + private CharArrayStyle charArrayStyle = CharArrayStyle.NUL_TERMINATED; + private UnknownProperties unknownProperties = UnknownProperties.ERROR; + private NewerVersions newerVersions = NewerVersions.REJECT; + private boolean exceptionStackTraces = true; + private Limits limits = Limits.defaults(); + + Builder(final Ir ir) + { + this.ir = ir; + } + + /** + * Set the enum representation. + * + * @param enumStyle policy. + * @return this for a fluent API. + */ + public Builder enumStyle(final EnumStyle enumStyle) + { + this.enumStyle = Objects.requireNonNull(enumStyle); + return this; + } + + /** + * Set the bit set representation. + * + * @param bitSetStyle policy. + * @return this for a fluent API. + */ + public Builder bitSetStyle(final BitSetStyle bitSetStyle) + { + this.bitSetStyle = Objects.requireNonNull(bitSetStyle); + return this; + } + + /** + * Set the fixed-length char array decoding. + * + * @param charArrayStyle policy. + * @return this for a fluent API. + */ + public Builder charArrayStyle(final CharArrayStyle charArrayStyle) + { + this.charArrayStyle = Objects.requireNonNull(charArrayStyle); + return this; + } + + /** + * Set the treatment of unknown JSON properties on encode. + * + * @param unknownProperties policy. + * @return this for a fluent API. + */ + public Builder unknownProperties(final UnknownProperties unknownProperties) + { + this.unknownProperties = Objects.requireNonNull(unknownProperties); + return this; + } + + /** + * Set the treatment of messages newer than the schema. + * + * @param newerVersions policy. + * @return this for a fluent API. + */ + public Builder newerVersions(final NewerVersions newerVersions) + { + this.newerVersions = Objects.requireNonNull(newerVersions); + return this; + } + + /** + * Whether thrown {@link SbeJsonException}s capture a stack trace. Set false for gateways that reject + * hostile input routinely. + * + * @param exceptionStackTraces true to capture stack traces. + * @return this for a fluent API. + */ + public Builder exceptionStackTraces(final boolean exceptionStackTraces) + { + this.exceptionStackTraces = exceptionStackTraces; + return this; + } + + /** + * Set the resource budgets. + * + * @param limits budgets. + * @return this for a fluent API. + */ + public Builder limits(final Limits limits) + { + this.limits = Objects.requireNonNull(limits); + return this; + } + + /** + * Compile the plans and build the immutable codec. + * + * @return the codec. + * @throws SbeJsonException with {@link ErrorCode#LIMIT_EXCEEDED} when a message nests groups deeper than + * {@link Limits#maxDepth()}. + */ + public SbeJson build() + { + return new SbeJson(this); + } + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonDecoder.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonDecoder.java new file mode 100644 index 0000000000..48c03390ff --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonDecoder.java @@ -0,0 +1,192 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.DirectBuffer; + +import java.io.IOException; + +/** + * Thread-confined, non-reentrant decoder for any message of one {@link SbeJson}. Routes by the header template + * id. Every decode takes an explicit frame {@code (buffer, offset, length)} and no read goes past + * {@code offset + length}: an SBE header carries no total length and a receive buffer may hold several messages. + *

+ * The header is read with the IR's per-member encodings (type and byte order); a header member whose value does + * not fit an {@code int} is rejected with {@link ErrorCode#OUT_OF_RANGE} rather than escaping as another exception. + */ +public final class SbeJsonDecoder +{ + private final SbeJson sbeJson; + private final HeaderLayout headerLayout; + private final HeaderView header = new HeaderView(); + private final WalkContext context; + + SbeJsonDecoder(final SbeJson sbeJson) + { + this.sbeJson = sbeJson; + this.headerLayout = sbeJson.headerLayout(); + this.context = new WalkContext(JsonNodeFactory.instance, sbeJson.exceptionStackTraces()); + } + + /** + * Decode one message (header plus body) into a fresh tree of stock Jackson nodes. The tree is independent of + * the buffer and of this decoder and may be retained. Allocates one node per value. + * + * @param buffer buffer holding the message. + * @param offset offset of the message header. + * @param length bytes available from {@code offset}; no read goes past {@code offset + length}. + * @return the body as an object; header fields are available from {@link #lastHeader()}. + * @throws SbeJsonException on framing or validation failure. + */ + public ObjectNode decodeCopy(final DirectBuffer buffer, final int offset, final int length) + { + final int frameEnd = prologue(buffer, offset, length); + final MessageCodec codec = route(buffer, offset); + + return codec.decodeCopy( + buffer, + offset + headerLayout.encodedLength(), + frameEnd, + header.blockLength(), + header.actingVersion(), + context); + } + + /** + * Header of the last message decoded on this decoder. Overwritten by every decode, including decodes that + * fail after the header was read. + * + * @return the header view; {@link HeaderView#populated()} is false before the first decode. + */ + public HeaderView lastHeader() + { + return header; + } + + /** + * Write one message straight to a generator without building a tree. Arrives with DESIGN.md section 13 + * step 6. + * + * @param buffer buffer holding the message. + * @param offset offset of the message header. + * @param length bytes available from {@code offset}. + * @param generator destination. + * @throws IOException from the generator. + * @throws UnsupportedOperationException always, until step 6 lands. + */ + public void writeJson(final DirectBuffer buffer, final int offset, final int length, final JsonGenerator generator) + throws IOException + { + // TODO(DESIGN.md section 13 step 6): generator sink over the same plan walk. + throw new UnsupportedOperationException("writeJson arrives with the generator sink"); + } + + /** + * Decode into a borrowed document for zero steady-state allocation. Arrives with DESIGN.md section 13 + * step 5. + * + * @param buffer buffer holding the message. + * @param offset offset of the message header. + * @param length bytes available from {@code offset}. + * @param document document created by {@link #newDocument()}. + * @return bytes consumed. + * @throws UnsupportedOperationException always, until step 5 lands. + */ + public int decodeInto(final DirectBuffer buffer, final int offset, final int length, final Object document) + { + // TODO(DESIGN.md section 13 step 5): BorrowedDocument with skeleton registry and Sbe*Node leaves. + throw new UnsupportedOperationException("decodeInto arrives with the borrowed-document path"); + } + + /** + * Create a borrowed document for {@link #decodeInto}. Arrives with DESIGN.md section 13 step 5. + * + * @return never returns in this release. + * @throws UnsupportedOperationException always, until step 5 lands. + */ + public Object newDocument() + { + // TODO(DESIGN.md section 13 step 5): BorrowedDocument. + throw new UnsupportedOperationException("newDocument arrives with the borrowed-document path"); + } + + private int prologue(final DirectBuffer buffer, final int offset, final int length) + { + if (offset < 0 || length < 0 || length > buffer.capacity() - offset) + { + throw new SbeJsonException( + ErrorCode.FRAME_OVERFLOW, SbeJsonException.NO_TEMPLATE_ID, offset, null, + "frame [" + offset + ", " + offset + " + " + length + ") outside buffer capacity " + + buffer.capacity(), sbeJson.exceptionStackTraces()); + } + + final int headerLength = headerLayout.encodedLength(); + if (headerLength > length) + { + throw new SbeJsonException( + ErrorCode.FRAME_OVERFLOW, SbeJsonException.NO_TEMPLATE_ID, offset, null, + "frame of " + length + " bytes is shorter than the " + headerLength + " byte header", + sbeJson.exceptionStackTraces()); + } + + return offset + length; + } + + private MessageCodec route(final DirectBuffer buffer, final int offset) + { + final int templateId = headerMember(buffer, offset, HeaderLayout.TEMPLATE_ID); + final int schemaId = headerMember(buffer, offset, HeaderLayout.SCHEMA_ID); + final int actingVersion = headerMember(buffer, offset, HeaderLayout.SCHEMA_VERSION); + final int blockLength = headerMember(buffer, offset, HeaderLayout.BLOCK_LENGTH); + header.set(templateId, schemaId, actingVersion, blockLength); + + if (schemaId != sbeJson.ir().id()) + { + throw new SbeJsonException( + ErrorCode.UNKNOWN_TEMPLATE, templateId, offset, null, + "header schema id " + schemaId + " does not match schema id " + sbeJson.ir().id(), + sbeJson.exceptionStackTraces()); + } + + final MessageCodec codec = sbeJson.codecForTemplate(templateId); + if (actingVersion > codec.plan().schemaVersion) + { + throw new SbeJsonException( + ErrorCode.UNSUPPORTED_VERSION, templateId, offset, codec.plan().name, + "acting version " + actingVersion + " is newer than schema version " + codec.plan().schemaVersion, + sbeJson.exceptionStackTraces()); + } + + return codec; + } + + private int headerMember(final DirectBuffer buffer, final int offset, final int member) + { + final long value = headerLayout.read(buffer, offset, member); + if (value < 0 || value > Integer.MAX_VALUE) + { + throw new SbeJsonException( + ErrorCode.OUT_OF_RANGE, SbeJsonException.NO_TEMPLATE_ID, offset + headerLayout.memberOffset(member), + null, "header " + headerLayout.memberName(member) + " value " + Long.toUnsignedString(value) + + " does not fit a 32-bit signed integer", sbeJson.exceptionStackTraces()); + } + + return (int)value; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonEncoder.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonEncoder.java new file mode 100644 index 0000000000..91c3a3c9ad --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonEncoder.java @@ -0,0 +1,113 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import org.agrona.MutableDirectBuffer; + +import java.io.IOException; + +/** + * Thread-confined, non-reentrant encoder bound to one message template and to the schema version of the + * {@link SbeJson} it came from. Walks the plan, not the JSON: block fields are written at fixed offsets so JSON + * key order is irrelevant; groups and var-data are written in schema order. + */ +public final class SbeJsonEncoder +{ + private final MessageCodec codec; + private final WalkContext context; + + SbeJsonEncoder(final SbeJson sbeJson, final MessageCodec codec) + { + this.codec = codec; + this.context = new WalkContext(JsonNodeFactory.instance, sbeJson.exceptionStackTraces()); + } + + /** + * Template id this encoder writes. + * + * @return the template id. + */ + public int templateId() + { + return codec.plan().templateId; + } + + /** + * Message name this encoder writes. + * + * @return the message name. + */ + public String messageName() + { + return codec.plan().name; + } + + /** + * Encode header and body in a single pass. On failure the destination region is undefined and the exception + * carries the field path. The destination is not retained after the call returns or throws. + *

+ * Text is coerced strictly: an unpaired UTF-16 surrogate, or a character the field's charset cannot + * represent, is rejected with {@link ErrorCode#TYPE_MISMATCH} for fixed {@code char[N]} and var-data alike; + * nothing is replaced. An optional scalar takes its null sentinel from omission or JSON {@code null} only; + * elements of an optional numeric array may also carry the sentinel value so decoded arrays re-encode. + * + * @param body root object; see DESIGN.md section 9 for the accepted shapes per field type. + * @param dst destination buffer. + * @param offset where the header starts. + * @param available bytes available from {@code offset}. + * @return bytes written including the header. + * @throws SbeJsonException on validation failure or {@link ErrorCode#DESTINATION_OVERFLOW}. + */ + public int encode(final JsonNode body, final MutableDirectBuffer dst, final int offset, final int available) + { + return codec.encode(body, dst, offset, available, context); + } + + /** + * Compute the encoded length of header and body without writing. Optional sizing pass; the input must not + * change before the following {@link #encode}. + * + * @param body root object. + * @return bytes that {@link #encode} would write. + * @throws SbeJsonException on validation failure. + */ + public int encodedLength(final JsonNode body) + { + return codec.encodedLength(body, context); + } + + /** + * Encode straight from a parser. Groups and var-data must arrive in schema order. Arrives with DESIGN.md + * section 13 step 6. + * + * @param parser source positioned before the root object. + * @param dst destination buffer. + * @param offset where the header starts. + * @param available bytes available from {@code offset}. + * @return bytes written including the header. + * @throws IOException from the parser. + * @throws UnsupportedOperationException always, until step 6 lands. + */ + public int encode(final JsonParser parser, final MutableDirectBuffer dst, final int offset, final int available) + throws IOException + { + // TODO(DESIGN.md section 13 step 6): streaming encode with schema-order rule and back-filled prefixes. + throw new UnsupportedOperationException("encode(JsonParser) arrives with the streaming path"); + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonException.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonException.java new file mode 100644 index 0000000000..9fdd53c342 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/SbeJsonException.java @@ -0,0 +1,160 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * Unchecked exception raised for every validation or framing failure in this module. + *

+ * Carries the {@link ErrorCode}, the template id of the message being processed (or {@link #NO_TEMPLATE_ID} + * when the header could not be read), the byte offset into the buffer at which the failure was detected (or + * {@link #NO_OFFSET} when not applicable, e.g. a JSON shape error) and the dotted field path, e.g. + * {@code Car.performanceFigures[1].acceleration[2].seconds}. The message is formatted at throw time. + *

+ * When {@code SbeJson.Builder#exceptionStackTraces(false)} is set the stack trace is not writable, which makes + * the exception cheap enough for gateways that reject hostile input routinely. + */ +public final class SbeJsonException extends RuntimeException +{ + /** + * Template id value when the header could not be read. + */ + public static final int NO_TEMPLATE_ID = -1; + + /** + * Byte offset value when no buffer position applies. + */ + public static final int NO_OFFSET = -1; + + private static final long serialVersionUID = 4130712906152366229L; + + private final ErrorCode code; + private final int templateId; + private final int byteOffset; + private final String path; + + /** + * Create an exception with a writable stack trace. + * + * @param code error classification. + * @param templateId template id of the message, or {@link #NO_TEMPLATE_ID}. + * @param byteOffset buffer offset of the failure, or {@link #NO_OFFSET}. + * @param path dotted field path, or {@code null} when not applicable. + * @param detail human readable detail appended to the message. + */ + public SbeJsonException( + final ErrorCode code, + final int templateId, + final int byteOffset, + final String path, + final String detail) + { + this(code, templateId, byteOffset, path, detail, true); + } + + /** + * Create an exception, optionally without a writable stack trace. + * + * @param code error classification. + * @param templateId template id of the message, or {@link #NO_TEMPLATE_ID}. + * @param byteOffset buffer offset of the failure, or {@link #NO_OFFSET}. + * @param path dotted field path, or {@code null} when not applicable. + * @param detail human readable detail appended to the message. + * @param writableStackTrace whether the stack trace should be captured. + */ + public SbeJsonException( + final ErrorCode code, + final int templateId, + final int byteOffset, + final String path, + final String detail, + final boolean writableStackTrace) + { + super(formatMessage(code, templateId, byteOffset, path, detail), null, false, writableStackTrace); + this.code = code; + this.templateId = templateId; + this.byteOffset = byteOffset; + this.path = path; + } + + /** + * Error classification. + * + * @return the error code. + */ + public ErrorCode code() + { + return code; + } + + /** + * Template id of the message being processed. + * + * @return the template id, or {@link #NO_TEMPLATE_ID}. + */ + public int templateId() + { + return templateId; + } + + /** + * Byte offset into the buffer at which the failure was detected. + * + * @return the byte offset, or {@link #NO_OFFSET}. + */ + public int byteOffset() + { + return byteOffset; + } + + /** + * Dotted field path of the failing field. + * + * @return the path, or {@code null} when not applicable. + */ + public String path() + { + return path; + } + + private static String formatMessage( + final ErrorCode code, + final int templateId, + final int byteOffset, + final String path, + final String detail) + { + final StringBuilder sb = new StringBuilder(96); + sb.append(code); + if (NO_TEMPLATE_ID != templateId) + { + sb.append(" templateId=").append(templateId); + } + if (null != path) + { + sb.append(" path=").append(path); + } + if (NO_OFFSET != byteOffset) + { + sb.append(" offset=").append(byteOffset); + } + if (null != detail) + { + sb.append(": ").append(detail); + } + + return sb.toString(); + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/UnknownProperties.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/UnknownProperties.java new file mode 100644 index 0000000000..ae4a9d0c00 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/UnknownProperties.java @@ -0,0 +1,32 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +/** + * Treatment of JSON properties that do not correspond to a field of the message on encode. + */ +public enum UnknownProperties +{ + /** + * Raise {@link ErrorCode#UNKNOWN_PROPERTY}. Checked per object: root, every composite, every group entry. + */ + ERROR, + + /** + * Skip unknown properties silently. + */ + IGNORE +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/Utf8.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/Utf8.java new file mode 100644 index 0000000000..aaad0eccaa --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/Utf8.java @@ -0,0 +1,142 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.MutableDirectBuffer; + +/** + * Hand-rolled UTF-8 encoding of a {@link CharSequence} straight into a buffer, without an intermediate + * {@code byte[]}. Surrogate pairs become four-byte sequences; an unpaired surrogate is malformed input, reported + * by {@link #encodedLength} (-1) and refused by {@link #encode}, never replaced. {@link #encodedLength} and + * {@link #encode} agree byte for byte. No Jackson import. + */ +final class Utf8 +{ + private Utf8() + { + } + + /** + * Number of bytes {@link #encode} will write for a sequence, or -1 when the sequence contains an unpaired + * surrogate and so cannot be encoded. + * + * Returned as a {@code long} so that a sequence whose encoding exceeds {@code Integer.MAX_VALUE} bytes is + * reported rather than wrapped. + * + * @param chars source characters. + * @return encoded byte count, or -1 for malformed input. + */ + static long encodedLength(final CharSequence chars) + { + final int length = chars.length(); + long bytes = 0; + for (int i = 0; i < length; i++) + { + final char c = chars.charAt(i); + if (c < 0x80) + { + bytes++; + } + else if (c < 0x800) + { + bytes += 2; + } + else if (Character.isSurrogate(c)) + { + if (!Character.isHighSurrogate(c) || i + 1 >= length || !Character.isLowSurrogate(chars.charAt(i + 1))) + { + return -1; + } + bytes += 4; + i++; + } + else + { + bytes += 3; + } + } + + return bytes; + } + + /** + * Encode a sequence into a buffer. The caller has already checked that {@link #encodedLength} bytes fit, + * which also establishes that the sequence is well-formed. + * + * @param chars source characters. + * @param buffer destination. + * @param index where the first byte goes. + * @return bytes written. + * @throws IllegalArgumentException on an unpaired surrogate; {@link #encodedLength} reports those first. + */ + static int encode(final CharSequence chars, final MutableDirectBuffer buffer, final int index) + { + final int length = chars.length(); + int pos = index; + for (int i = 0; i < length; i++) + { + final char c = chars.charAt(i); + if (c < 0x80) + { + buffer.putByte(pos++, (byte)c); + } + else if (c < 0x800) + { + buffer.putByte(pos++, (byte)(0xC0 | (c >> 6))); + buffer.putByte(pos++, (byte)(0x80 | (c & 0x3F))); + } + else if (Character.isHighSurrogate(c) && i + 1 < length && Character.isLowSurrogate(chars.charAt(i + 1))) + { + final int codePoint = Character.toCodePoint(c, chars.charAt(++i)); + buffer.putByte(pos++, (byte)(0xF0 | (codePoint >> 18))); + buffer.putByte(pos++, (byte)(0x80 | ((codePoint >> 12) & 0x3F))); + buffer.putByte(pos++, (byte)(0x80 | ((codePoint >> 6) & 0x3F))); + buffer.putByte(pos++, (byte)(0x80 | (codePoint & 0x3F))); + } + else if (Character.isSurrogate(c)) + { + throw new IllegalArgumentException("unpaired surrogate U+" + Integer.toHexString(c) + " at " + i); + } + else + { + buffer.putByte(pos++, (byte)(0xE0 | (c >> 12))); + buffer.putByte(pos++, (byte)(0x80 | ((c >> 6) & 0x3F))); + buffer.putByte(pos++, (byte)(0x80 | (c & 0x3F))); + } + } + + return pos - index; + } + + /** + * Whether every character is 7-bit ASCII. + * + * @param chars source characters. + * @return true when all characters are below 0x80. + */ + static boolean isAscii(final CharSequence chars) + { + for (int i = 0, length = chars.length(); i < length; i++) + { + if (chars.charAt(i) >= 0x80) + { + return false; + } + } + + return true; + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/WalkContext.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/WalkContext.java new file mode 100644 index 0000000000..7c5cf8879b --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/WalkContext.java @@ -0,0 +1,134 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.node.JsonNodeFactory; + +import java.util.Arrays; + +/** + * Per-decoder / per-encoder scratch state for one plan walk: the field path (for error messages), running + * {@link Limits} counters and the node factory. Thread-confined with its owner; reset at the start of every walk. + */ +final class WalkContext +{ + private static final int INITIAL_DEPTH = 16; + + final JsonNodeFactory factory; + final boolean writableStackTraces; + + /** + * Tree encoder retained across calls by the owning thread-confined encoder; created lazily by the codec. + */ + PlanTreeEncoder treeEncoder; + + private int[] pathFields = new int[INITIAL_DEPTH]; + private int[] pathElements = new int[INITIAL_DEPTH]; + private int depth; + private long groupEntries; + private long varDataBytes; + + WalkContext(final JsonNodeFactory factory, final boolean writableStackTraces) + { + this.factory = factory; + this.writableStackTraces = writableStackTraces; + } + + void reset() + { + depth = 0; + groupEntries = 0; + varDataBytes = 0; + } + + int depth() + { + return depth; + } + + void push(final int fieldIndex) + { + if (depth == pathFields.length) + { + pathFields = Arrays.copyOf(pathFields, depth * 2); + pathElements = Arrays.copyOf(pathElements, depth * 2); + } + pathFields[depth] = fieldIndex; + pathElements[depth] = -1; + depth++; + } + + void element(final int index) + { + pathElements[depth - 1] = index; + } + + void pop() + { + depth--; + } + + long addGroupEntries(final long count) + { + groupEntries += count; + return groupEntries; + } + + long addVarDataBytes(final long count) + { + varDataBytes += count; + return varDataBytes; + } + + long varDataBytes() + { + return varDataBytes; + } + + /** + * Format the current path plus an optional leaf as {@code Message.group[3].composite.field}. + * + * @param plan message being walked. + * @param leaf the failing field, or {@code null} when the failure is at the current container. + * @return the dotted path. + */ + String path(final MessagePlan plan, final FieldPlan leaf) + { + final StringBuilder sb = new StringBuilder(64); + sb.append(plan.name); + for (int i = 0; i < depth; i++) + { + sb.append('.').append(plan.fields[pathFields[i]].name); + if (pathElements[i] >= 0) + { + sb.append('[').append(pathElements[i]).append(']'); + } + } + if (null != leaf) + { + sb.append('.').append(leaf.name); + } + + return sb.toString(); + } + + SbeJsonException error( + final MessagePlan plan, final ErrorCode code, final FieldPlan leaf, final int byteOffset, final String detail) + { + return new SbeJsonException( + code, plan.templateId, byteOffset, path(plan, leaf), detail, writableStackTraces); + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/WireTypes.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/WireTypes.java new file mode 100644 index 0000000000..b6f36dd82f --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/WireTypes.java @@ -0,0 +1,123 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.DirectBuffer; +import org.agrona.MutableDirectBuffer; +import uk.co.real_logic.sbe.PrimitiveType; + +import java.nio.ByteOrder; + +/** + * Integer reads and writes for every SBE primitive type, widened to {@code long}. Unsigned 8/16/32-bit values + * are zero-extended; uint64 is returned raw and must be interpreted unsigned by the caller. No bounds checks; + * callers check against the frame first. + */ +final class WireTypes +{ + private WireTypes() + { + } + + static long getLong(final DirectBuffer buffer, final int index, final PrimitiveType type, final ByteOrder order) + { + switch (type) + { + case CHAR: + case INT8: + return buffer.getByte(index); + + case UINT8: + return buffer.getByte(index) & 0xFF; + + case INT16: + return buffer.getShort(index, order); + + case UINT16: + return buffer.getShort(index, order) & 0xFFFF; + + case INT32: + return buffer.getInt(index, order); + + case UINT32: + return buffer.getInt(index, order) & 0xFFFF_FFFFL; + + case INT64: + case UINT64: + return buffer.getLong(index, order); + + default: + throw new IllegalArgumentException("not an integer type: " + type); + } + } + + static void putLong( + final MutableDirectBuffer buffer, + final int index, + final PrimitiveType type, + final ByteOrder order, + final long value) + { + switch (type) + { + case CHAR: + case INT8: + case UINT8: + buffer.putByte(index, (byte)value); + break; + + case INT16: + case UINT16: + buffer.putShort(index, (short)value, order); + break; + + case INT32: + case UINT32: + buffer.putInt(index, (int)value, order); + break; + + case INT64: + case UINT64: + buffer.putLong(index, value, order); + break; + + default: + throw new IllegalArgumentException("not an integer type: " + type); + } + } + + /** + * Whether a primitive type is represented with {@code int} precision in JSON trees (fits in a Java int). + * + * @param type primitive type. + * @return true for int8 / int16 / int32 / uint8 / uint16. + */ + static boolean fitsInt(final PrimitiveType type) + { + switch (type) + { + case INT8: + case INT16: + case INT32: + case UINT8: + case UINT16: + return true; + + default: + return false; + } + } +} diff --git a/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/package-info.java b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/package-info.java new file mode 100644 index 0000000000..696fc147f9 --- /dev/null +++ b/sbe-jackson/src/main/java/uk/co/real_logic/sbe/jackson/package-info.java @@ -0,0 +1,24 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Conversion of SBE messages to and from Jackson 2.x {@code JsonNode} trees driven by + * {@link uk.co.real_logic.sbe.ir.Ir} loaded at runtime. + *

+ * Entry point is {@link uk.co.real_logic.sbe.jackson.SbeJson}, which compiles the IR once into flat plans and + * hands out thread-confined {@link uk.co.real_logic.sbe.jackson.SbeJsonDecoder} and + * {@link uk.co.real_logic.sbe.jackson.SbeJsonEncoder} instances. + */ +package uk.co.real_logic.sbe.jackson; diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/DecodeConformanceTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/DecodeConformanceTest.java new file mode 100644 index 0000000000..e39813d272 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/DecodeConformanceTest.java @@ -0,0 +1,222 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.json.JsonPrinter; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.function.ToIntBiFunction; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * {@code readTree(JsonPrinter.print(...))} must equal {@code decodeCopy(...)} semantically. The printer emits bit + * sets as objects of booleans, so the comparison uses {@link BitSetStyle#OBJECT}; the remaining documented + * divergences (uint64 above {@code Long.MAX_VALUE}, binary var-data, unknown enum values, NaN) are covered by + * dedicated tests below. + */ +class DecodeConformanceTest +{ + private static final int CAPACITY = 4096; + + @Test + void baselineCarMatchesJsonPrinter() throws Exception + { + assertMatchesPrinter(TestMessages.BASELINE_SCHEMA, TestMessages::encodeBaselineCar); + } + + @Test + void extensionCarMatchesJsonPrinter() throws Exception + { + assertMatchesPrinter(TestMessages.EXTENSION_SCHEMA, TestMessages::encodeExtensionCar); + } + + @Test + void compositeElementsMatchJsonPrinter() throws Exception + { + assertMatchesPrinter(TestMessages.COMPOSITE_ELEMENTS_SCHEMA, TestMessages::encodeCompositeElements); + } + + @Test + void groupWithDataMatchesJsonPrinter() throws Exception + { + assertMatchesPrinter(TestMessages.GROUP_WITH_DATA_SCHEMA, TestMessages::encodeGroupWithData); + } + + @Test + void decodeCopyAtNonZeroOffsetProducesSameTree() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + final ObjectNode atZero = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + + final UnsafeBuffer shifted = TestMessages.newBuffer(CAPACITY); + shifted.putBytes(101, buffer, 0, length); + final SbeJsonDecoder decoder = sbeJson.newDecoder(); + final ObjectNode atOffset = decoder.decodeCopy(shifted, 101, length); + + JsonNodes.assertSemanticEquals(atZero, atOffset); + assertEquals(1, decoder.lastHeader().templateId()); + assertEquals(1, decoder.lastHeader().schemaId()); + assertEquals(2, decoder.lastHeader().actingVersion()); + assertEquals(ir.getMessage(1).get(0).encodedLength(), decoder.lastHeader().blockLength()); + } + + @Test + void bitSetDecodesAsMaskByDefault() + { + final ObjectNode car = decodeBaselineCar(SbeJson.builder(TestMessages.ir(TestMessages.BASELINE_SCHEMA))); + assertEquals(6, car.get("extras").intValue()); + assertTrue(car.get("extras").isIntegralNumber()); + } + + @Test + void enumDecodesAsNameByDefaultAndOrdinalOnRequest() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final ObjectNode named = decodeBaselineCar(SbeJson.builder(ir)); + assertEquals("T", named.get("available").textValue()); + assertEquals("A", named.get("code").textValue()); + assertEquals("Petrol", named.get("engine").get("fuel").textValue()); + assertEquals(9000, named.get("engine").get("maxRpm").intValue()); + + final ObjectNode ordinal = decodeBaselineCar(SbeJson.builder(ir).enumStyle(EnumStyle.ORDINAL)); + assertEquals(1, ordinal.get("available").intValue()); + assertEquals('A', ordinal.get("code").intValue()); + } + + @Test + void unknownEnumValueDecodesAsNumber() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + buffer.putByte(8 + 10, (byte)7); + + final ObjectNode car = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + assertEquals(7, car.get("available").intValue()); + } + + @Test + void uint64AboveLongMaxDecodesUnsigned() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + buffer.putLong(8, -2L, java.nio.ByteOrder.LITTLE_ENDIAN); + + final ObjectNode car = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + final JsonNode serialNumber = car.get("serialNumber"); + assertTrue(serialNumber.isBigInteger()); + assertEquals(new BigInteger("18446744073709551614"), serialNumber.bigIntegerValue()); + } + + @Test + void binaryVarDataDecodesAsBinaryNode() throws Exception + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final baseline.MessageHeaderEncoder header = new baseline.MessageHeaderEncoder(); + final baseline.CredentialsEncoder credentials = new baseline.CredentialsEncoder(); + final byte[] password = { 0, 1, 2, (byte)0xFE, (byte)0xFF, 'x' }; + credentials.wrapAndApplyHeader(buffer, 0, header) + .login("bob") + .putEncryptedPassword(password, 0, password.length); + final int length = header.encodedLength() + credentials.encodedLength(); + + final ObjectNode tree = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + assertEquals("bob", tree.get("login").textValue()); + assertTrue(tree.get("encryptedPassword").isBinary()); + assertArrayEquals(password, tree.get("encryptedPassword").binaryValue()); + assertEquals("\"AAEC/v94\"", JsonNodes.MAPPER.writeValueAsString(tree.get("encryptedPassword"))); + } + + @Test + void nanFloatDecodesAsNanNumberWhenRequiredAndNullWhenOptional() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + final ObjectNode reference = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + + // fuelFigures[0].mpg is required: NaN stays a number. + final int fuelFiguresDims = 8 + ir.getMessage(1).get(0).encodedLength(); + buffer.putFloat(fuelFiguresDims + 3 + 2, Float.NaN, java.nio.ByteOrder.LITTLE_ENDIAN); + final ObjectNode car = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + final JsonNode mpg = car.get("fuelFigures").get(0).get("mpg"); + assertTrue(mpg.isNumber()); + assertTrue(Double.isNaN(mpg.doubleValue())); + assertEquals(reference.get("fuelFigures").get(1), car.get("fuelFigures").get(1)); + + // cupHolderCount is optional uint8: 255 decodes as null. + buffer.putByte(8 + 61, (byte)0xFF); + final ObjectNode withNull = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + assertTrue(withNull.get("cupHolderCount").isNull()); + assertTrue(withNull.has("cupHolderCount")); + } + + @ParameterizedTest + @ValueSource(strings = { "NUL_TERMINATED", "EXACT" }) + void charArrayStyleControlsTrailingNuls(final String style) + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + buffer.putBytes(8 + 32, "ab\0\0ef".getBytes(StandardCharsets.US_ASCII)); + + final ObjectNode car = SbeJson.builder(ir).charArrayStyle(CharArrayStyle.valueOf(style)).build() + .newDecoder().decodeCopy(buffer, 0, length); + final String expected = "EXACT".equals(style) ? "ab\0\0ef" : "ab"; + assertEquals(expected, car.get("vehicleCode").textValue()); + } + + private static ObjectNode decodeBaselineCar(final SbeJson.Builder builder) + { + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + + return builder.build().newDecoder().decodeCopy(buffer, 0, length); + } + + private static void assertMatchesPrinter( + final String schema, final ToIntBiFunction encoder) throws Exception + { + final Ir ir = TestMessages.ir(schema); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = encoder.applyAsInt(buffer, 0); + + final StringBuilder printed = new StringBuilder(); + new JsonPrinter(ir).print(printed, buffer, 0); + final JsonNode expected = JsonNodes.MAPPER.readTree(printed.toString()); + + final SbeJson sbeJson = SbeJson.builder(ir).bitSetStyle(BitSetStyle.OBJECT).build(); + final ObjectNode actual = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + + JsonNodes.assertSemanticEquals(expected, actual); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EdgeCaseTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EdgeCaseTest.java new file mode 100644 index 0000000000..7301791f7c --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EdgeCaseTest.java @@ -0,0 +1,518 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.core.Base64Variant; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import uk.co.real_logic.sbe.ir.Ir; + +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.function.Consumer; + +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Edge cases from {@code edge-cases-schema.xml}: uint64 enums and sets around 2^63 and 2^64, optional + * float/double sentinels, non UTF-8 text, var-data limits enforced before allocation, messages beyond the int + * range and trailing empty groups. + */ +class EdgeCaseTest +{ + private static final int CAPACITY = 1 << 17; + private static final int HEADER = 8; + private static final int BLOCK_LENGTH = 44; + private static final JsonNodeFactory F = JsonNodeFactory.instance; + private static final Ir IR = ProgrammaticIrs.edgeCases(); + private static final BigInteger TWO_POW_63 = BigInteger.ONE.shiftLeft(63); + private static final BigInteger TWO_POW_64 = BigInteger.ONE.shiftLeft(64); + private static final BigInteger MAX_UINT64 = TWO_POW_64.subtract(BigInteger.ONE); + private static final BigInteger TOP = MAX_UINT64.subtract(BigInteger.ONE); + + @Test + void uint64EnumRoundTripsAt2Pow63AndAboveInBothStyles() + { + final SbeJson names = SbeJson.builder(IR).build(); + final SbeJson ordinals = SbeJson.builder(IR).enumStyle(EnumStyle.ORDINAL).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + int length = encode(names, "Edge", edge().put("bigEnum", "HIGH"), buffer); + assertEquals(Long.MIN_VALUE, buffer.getLong(HEADER, LITTLE_ENDIAN)); + assertEquals("HIGH", names.newDecoder().decodeCopy(buffer, 0, length).get("bigEnum").textValue()); + final JsonNode high = ordinals.newDecoder().decodeCopy(buffer, 0, length).get("bigEnum"); + assertTrue(high.isBigInteger(), high.toString()); + assertEquals(TWO_POW_63, high.bigIntegerValue()); + + length = encode(names, "Edge", edge().set("bigEnum", F.numberNode(TWO_POW_63)), buffer); + assertEquals(Long.MIN_VALUE, buffer.getLong(HEADER, LITTLE_ENDIAN)); + + length = encode(names, "Edge", edge().put("bigEnum", "TOP"), buffer); + assertEquals(-2L, buffer.getLong(HEADER, LITTLE_ENDIAN)); + assertEquals(TOP, ordinals.newDecoder().decodeCopy(buffer, 0, length).get("bigEnum").bigIntegerValue()); + + // An unknown but representable value: 2^64 - 1 encodes and decodes as an unsigned number in both styles. + length = encode(names, "Edge", edge().set("bigEnum", F.numberNode(MAX_UINT64)), buffer); + assertEquals(-1L, buffer.getLong(HEADER, LITTLE_ENDIAN)); + assertEquals(MAX_UINT64, names.newDecoder().decodeCopy(buffer, 0, length).get("bigEnum").bigIntegerValue()); + assertEquals( + MAX_UINT64, ordinals.newDecoder().decodeCopy(buffer, 0, length).get("bigEnum").bigIntegerValue()); + + // Constants: the uint64 constant and the ORDINAL style constant enum are unsigned nodes. + final ObjectNode decoded = ordinals.newDecoder().decodeCopy(buffer, 0, length); + assertEquals(TOP, decoded.get("bigConst").bigIntegerValue()); + assertEquals(TOP, decoded.get("constEnum").bigIntegerValue()); + assertEquals("TOP", names.newDecoder().decodeCopy(buffer, 0, length).get("constEnum").textValue()); + encode(ordinals, "Edge", edge().set("constEnum", F.numberNode(TOP)), buffer); + } + + @Test + void uint64EnumAndSetRejectNegativeAndAbove2Pow64() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.bigEnum", e -> e.put("bigEnum", -1)); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.bigEnum", e -> e.put("bigEnum", Long.MIN_VALUE)); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.bigEnum", e -> e.set("bigEnum", F.numberNode(TWO_POW_64))); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.bigSet", e -> e.put("bigSet", -1)); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.bigSet", e -> e.set("bigSet", F.numberNode(TWO_POW_64))); + assertRejects(sbeJson, ErrorCode.CONSTANT_MISMATCH, "Edge.constEnum", e -> e.put("constEnum", -2)); + assertRejects(sbeJson, ErrorCode.CONSTANT_MISMATCH, "Edge.constEnum", e -> e.putNull("constEnum")); + } + + @Test + void uint64SetBit63RoundTripsAsMaskAndAsObject() + { + final SbeJson masks = SbeJson.builder(IR).build(); + final SbeJson objects = SbeJson.builder(IR).bitSetStyle(BitSetStyle.OBJECT).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + int length = encode(masks, "Edge", edge().set("bigSet", F.numberNode(TWO_POW_63)), buffer); + assertEquals(Long.MIN_VALUE, buffer.getLong(HEADER + 8, LITTLE_ENDIAN)); + assertEquals(TWO_POW_63, masks.newDecoder().decodeCopy(buffer, 0, length).get("bigSet").bigIntegerValue()); + final JsonNode asObject = objects.newDecoder().decodeCopy(buffer, 0, length).get("bigSet"); + assertEquals(false, asObject.get("low").booleanValue()); + assertEquals(true, asObject.get("high").booleanValue()); + + final ObjectNode viaObject = edge(); + viaObject.putObject("bigSet").put("high", true).put("low", true); + length = encode(objects, "Edge", viaObject, buffer); + assertEquals(Long.MIN_VALUE | 1L, buffer.getLong(HEADER + 8, LITTLE_ENDIAN)); + assertEquals( + TWO_POW_63.add(BigInteger.ONE), + masks.newDecoder().decodeCopy(buffer, 0, length).get("bigSet").bigIntegerValue()); + } + + @Test + void utf16CharArrayTerminatesAtTheNulCharacterNotTheFirstZeroByte() + { + final SbeJson nulTerminated = SbeJson.builder(IR).build(); + final SbeJson exact = SbeJson.builder(IR).charArrayStyle(CharArrayStyle.EXACT).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + final int length = encode(nulTerminated, "Edge", edge().put("name16", "AB"), buffer); + final byte[] name = new byte[8]; + buffer.getBytes(HEADER + 36, name); + assertArrayEquals(new byte[]{ 0, 0x41, 0, 0x42, 0, 0, 0, 0 }, name); + + assertEquals("AB", nulTerminated.newDecoder().decodeCopy(buffer, 0, length).get("name16").textValue()); + assertEquals("AB\0\0", exact.newDecoder().decodeCopy(buffer, 0, length).get("name16").textValue()); + + assertRejects(nulTerminated, ErrorCode.OUT_OF_RANGE, "Edge.name16", e -> e.put("name16", "ABCDE")); + } + + @Test + void optionalFloatAndDoubleSentinelsDecodeAsNullAndEncodeFromNull() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + final ObjectNode omitted = edge(); + int length = encode(sbeJson, "Edge", omitted, buffer); + assertTrue(Float.isNaN(buffer.getFloat(HEADER + 16, LITTLE_ENDIAN))); + assertTrue(Double.isNaN(buffer.getDouble(HEADER + 20, LITTLE_ENDIAN))); + assertEquals(-1L, buffer.getLong(HEADER + 28, LITTLE_ENDIAN)); + ObjectNode decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertTrue(decoded.get("optFloat").isNull()); + assertTrue(decoded.get("optDouble").isNull()); + assertTrue(decoded.get("optU64").isNull()); + + final ObjectNode explicitNull = edge(); + explicitNull.putNull("optFloat").putNull("optDouble").putNull("optU64"); + final UnsafeBuffer again = TestMessages.newBuffer(CAPACITY); + assertEquals(length, encode(sbeJson, "Edge", explicitNull, again)); + assertArrayEquals(Arrays.copyOf(buffer.byteArray(), length), Arrays.copyOf(again.byteArray(), length)); + + final ObjectNode present = edge().put("optFloat", 1.5f).put("optDouble", 2.5d); + present.set("optU64", F.numberNode(MAX_UINT64.subtract(BigInteger.ONE))); + length = encode(sbeJson, "Edge", present, buffer); + decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertEquals(1.5f, decoded.get("optFloat").floatValue()); + assertEquals(2.5d, decoded.get("optDouble").doubleValue()); + assertEquals(TOP, decoded.get("optU64").bigIntegerValue()); + + // A wire NaN in an optional slot is the sentinel. + buffer.putFloat(HEADER + 16, Float.NaN, LITTLE_ENDIAN); + buffer.putDouble(HEADER + 20, Double.NaN, LITTLE_ENDIAN); + decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertTrue(decoded.get("optFloat").isNull()); + assertTrue(decoded.get("optDouble").isNull()); + } + + @Test + void varDataLimitsAreEnforcedBeforeThePayloadIsMaterialised() + { + final SbeJson small = SbeJson.builder(IR).limits(Limits.builder().maxVarDataBytes(10).build()).build(); + final String longText = "x".repeat(100); + final byte[] thirtyBytes = new byte[30]; + final String base64 = Base64.getEncoder().encodeToString(thirtyBytes); + + assertRejects(small, ErrorCode.LIMIT_EXCEEDED, "Edge.text16", e -> e.put("text16", longText)); + assertRejects(small, ErrorCode.LIMIT_EXCEEDED, "Edge.blob", e -> e.put("blob", base64)); + assertRejects(small, ErrorCode.LIMIT_EXCEEDED, "Edge.blob", e -> e.set("blob", F.binaryNode(thirtyBytes))); + + // The base64 text is rejected on its counted payload size: binaryValue() fails the test if reached. + assertRejects(small, ErrorCode.LIMIT_EXCEEDED, "Edge.blob", e -> e.set("blob", new UndecodableText(base64))); + + // Within budget in the other charset path: exactly 10 bytes of UTF-16 (BOM + 4 chars). + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = encode(small, "Edge", edge().put("text16", "abcd"), buffer); + assertEquals("abcd", small.newDecoder().decodeCopy(buffer, 0, length).get("text16").textValue()); + + // Length type maximum (uint16) beats a generous var-data budget and destination. + final SbeJson defaults = SbeJson.builder(IR).build(); + assertRejects(defaults, ErrorCode.OUT_OF_RANGE, "Edge.text16", e -> e.put("text16", "y".repeat(40000))); + assertRejects(defaults, ErrorCode.OUT_OF_RANGE, "Edge.blob", + e -> e.put("blob", Base64.getEncoder().encodeToString(new byte[70000]))); + assertRejects(defaults, ErrorCode.OUT_OF_RANGE, "Edge.blob", + e -> e.set("blob", new UndecodableText(Base64.getEncoder().encodeToString(new byte[70000])))); + + // Destination smaller than the payload: rejected before decoding the base64 or encoding the text. + final UnsafeBuffer tiny = TestMessages.newBuffer(HEADER + BLOCK_LENGTH + 4 + 2 + 2 + 4); + final SbeJsonEncoder encoder = defaults.newEncoder("Edge"); + final SbeJsonException text = assertThrows(SbeJsonException.class, + () -> encoder.encode(edge().put("text16", "toolong"), tiny, 0, tiny.capacity())); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, text.code()); + assertEquals("Edge.text16", text.path()); + final SbeJsonException blob = assertThrows(SbeJsonException.class, + () -> encoder.encode(edge().set("blob", new UndecodableText(base64)), tiny, 0, tiny.capacity())); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, blob.code()); + assertEquals("Edge.blob", blob.path()); + + // Ordinary UTF-16 text and binary round-trip. + final ObjectNode full = edge().put("text16", "héllo 🚗"); + full.set("blob", F.binaryNode(new byte[]{ 1, 2, 3 })); + final int fullLength = encode(defaults, "Edge", full, buffer); + final ObjectNode decoded = defaults.newDecoder().decodeCopy(buffer, 0, fullLength); + assertEquals("héllo 🚗", decoded.get("text16").textValue()); + assertEquals(F.binaryNode(new byte[]{ 1, 2, 3 }), decoded.get("blob")); + assertEquals( + "héllo 🚗".getBytes(StandardCharsets.UTF_16).length, + buffer.getShort(fullLength - 2 - 3 - 2 - "héllo 🚗".getBytes(StandardCharsets.UTF_16).length, + LITTLE_ENDIAN)); + } + + @Test + void otherCharsetScratchIsSizedFromTheBudgetNotTheText() + { + final Ir spyIr = ProgrammaticIrs.spyCharset(); + final SbeJson small = SbeJson.builder(spyIr).limits(Limits.builder().maxVarDataBytes(10).build()).build(); + final SbeJsonEncoder encoder = small.newEncoder("Spy"); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + // Var-data: 100 chars would encode to 200 bytes; the encoder is handed budget + 1 = 11 bytes. + final ObjectNode longText = F.objectNode().put("name", "ab").put("text", "x".repeat(100)); + SpyCharset.reset(); + SbeJsonException ex = assertThrows(SbeJsonException.class, () -> encoder.encode(longText, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.LIMIT_EXCEEDED, ex.code()); + assertEquals("Spy.text", ex.path()); + assertEquals(11, SpyCharset.maxOutputCapacity()); + + SpyCharset.reset(); + ex = assertThrows(SbeJsonException.class, () -> encoder.encodedLength(longText)); + assertEquals(ErrorCode.LIMIT_EXCEEDED, ex.code()); + assertEquals(11, SpyCharset.maxOutputCapacity()); + + // Fixed char[8]: the budget is the array length, so the scratch is 9 bytes. + final ObjectNode longName = F.objectNode().put("name", "y".repeat(100)).put("text", ""); + SpyCharset.reset(); + ex = assertThrows(SbeJsonException.class, () -> encoder.encode(longName, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.OUT_OF_RANGE, ex.code()); + assertEquals("Spy.name", ex.path()); + assertEquals(9, SpyCharset.maxOutputCapacity()); + + // Within budget the text round-trips through the spy charset. + final ObjectNode fits = F.objectNode().put("name", "abc").put("text", "hello"); + SpyCharset.reset(); + final int length = encoder.encode(fits, buffer, 0, CAPACITY); + assertEquals(length, encoder.encodedLength(fits)); + assertTrue(SpyCharset.encodeCalls() > 0); + final ObjectNode decoded = small.newDecoder().decodeCopy(buffer, 0, length); + assertEquals("abc", decoded.get("name").textValue()); + assertEquals("hello", decoded.get("text").textValue()); + } + + @Test + void optionalUint64ScalarRejectsItsNumericSentinel() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.optU64", e -> e.set("optU64", F.numberNode(MAX_UINT64))); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.optU64", e -> e.put("optU64", MAX_UINT64.toString())); + assertRejects(sbeJson, ErrorCode.OUT_OF_RANGE, "Edge.optU64", e -> e.put("optU64", -1L)); + } + + @Test + void optionalFloatingArraysWithFiniteSentinelsRoundTripFromOmission() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Arrays"); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + final ObjectNode omitted = arrays(); + final int length = encoder.encode(omitted, buffer, 0, CAPACITY); + assertEquals(length, encoder.encodedLength(omitted)); + assertEquals(HEADER + 12 + 16 + 8, length); + for (int i = 0; i < 3; i++) + { + assertEquals(-1.1f, buffer.getFloat(HEADER + i * 4, LITTLE_ENDIAN)); + } + assertEquals(-1d, buffer.getDouble(HEADER + 12, LITTLE_ENDIAN)); + assertEquals(-1d, buffer.getDouble(HEADER + 20, LITTLE_ENDIAN)); + + // Decode exposes the sentinels as numbers; the decoded tree must re-encode byte-identically. + final ObjectNode decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertEquals(-1.1f, decoded.get("floats").get(2).floatValue()); + assertEquals(-1d, decoded.get("doubles").get(1).doubleValue()); + final UnsafeBuffer again = TestMessages.newBuffer(CAPACITY); + assertEquals(length, encoder.encode(decoded, again, 0, CAPACITY)); + assertEquals(length, encoder.encodedLength(decoded)); + assertArrayEquals(Arrays.copyOf(buffer.byteArray(), length), Arrays.copyOf(again.byteArray(), length)); + + // A double-precision -1.1 element is the sentinel at float (wire) precision, not a range violation. + final ObjectNode viaDouble = arrays(); + viaDouble.putArray("floats").add(-1.1d).add(-1.1d).add(-1.1d); + viaDouble.putArray("doubles").add(-1d).add(-1d); + assertEquals(length, encoder.encode(viaDouble, again, 0, CAPACITY)); + assertEquals(length, encoder.encodedLength(viaDouble)); + assertArrayEquals(Arrays.copyOf(buffer.byteArray(), length), Arrays.copyOf(again.byteArray(), length)); + + // Sentinel elements mixed with in-range values are fine; out-of-range non-sentinels still fail. + final ObjectNode mixed = arrays(); + mixed.putArray("floats").add(-1.1f).add(50f).add(100f); + mixed.putArray("doubles").add(0d).add(-1d); + final int mixedLength = encoder.encode(mixed, buffer, 0, CAPACITY); + final ObjectNode mixedDecoded = sbeJson.newDecoder().decodeCopy(buffer, 0, mixedLength); + assertEquals(50f, mixedDecoded.get("floats").get(1).floatValue()); + assertEquals(-1d, mixedDecoded.get("doubles").get(1).doubleValue()); + assertRejects(sbeJson, "Arrays", arrays(), ErrorCode.OUT_OF_RANGE, "Arrays.floats", + a -> a.putArray("floats").add(-1.1f).add(101f).add(0f)); + assertRejects(sbeJson, "Arrays", arrays(), ErrorCode.OUT_OF_RANGE, "Arrays.floats", + a -> a.putArray("floats").add(-1f).add(0f).add(0f)); + assertRejects(sbeJson, "Arrays", arrays(), ErrorCode.OUT_OF_RANGE, "Arrays.doubles", + a -> a.putArray("doubles").add(-2d).add(0d)); + } + + @Test + void loneSurrogatesAreRejectedInFixedArraysAndVarDataAlike() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + final String lone = "ab\uD800c"; + + assertRejects(sbeJson, "Arrays", arrays(), ErrorCode.TYPE_MISMATCH, "Arrays.utf8Name", + a -> a.put("utf8Name", lone)); + assertRejects(sbeJson, ErrorCode.TYPE_MISMATCH, "Edge.name16", e -> e.put("name16", lone)); + assertRejects(sbeJson, ErrorCode.TYPE_MISMATCH, "Edge.text16", e -> e.put("text16", lone)); + assertRejects(sbeJson, ErrorCode.TYPE_MISMATCH, "Edge.text16", e -> e.put("text16", "\uDC00")); + + // Paired surrogates are ordinary text. + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Arrays"); + final ObjectNode paired = arrays().put("utf8Name", "a\uD83D\uDE80"); + final int length = encoder.encode(paired, buffer, 0, CAPACITY); + assertEquals("a\uD83D\uDE80", sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("utf8Name").textValue()); + assertFalse(sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("floats").isNull()); + } + + @Test + void paddedBase64FitsExactPayloadLimits() + { + for (final int size : new int[]{ 1, 2, 3, 4, 10 }) + { + final SbeJson sbeJson = SbeJson.builder(IR) + .limits(Limits.builder().maxVarDataBytes(size).build()).build(); + final byte[] payload = new byte[size]; + final String base64 = Base64.getEncoder().encodeToString(payload); + // Jackson accepts whitespace between base64 units; it consumes no payload bytes. + final String spaced = " \n" + base64.substring(0, 4) + " \n" + base64.substring(4); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Edge"); + final int expectedLength = encoder.encodedLength(edge().set("blob", F.binaryNode(payload))); + for (final String text : new String[]{ base64, spaced }) + { + final ObjectNode node = edge().put("blob", text); + assertEquals(expectedLength, encoder.encodedLength(node)); + final UnsafeBuffer buffer = TestMessages.newBuffer(expectedLength); + assertEquals(expectedLength, encoder.encode(node, buffer, 0, expectedLength)); + assertEquals(F.binaryNode(payload), + sbeJson.newDecoder().decodeCopy(buffer, 0, expectedLength).get("blob")); + } + + // One payload byte over the limit, padded: rejected on the counted length before any decoding. + final String oneOver = Base64.getEncoder().encodeToString(new byte[size + 1]); + assertRejects(sbeJson, ErrorCode.LIMIT_EXCEEDED, "Edge.blob", + e -> e.set("blob", new UndecodableText(oneOver))); + assertRejects(sbeJson, ErrorCode.LIMIT_EXCEEDED, "Edge.blob", + e -> e.set("blob", new UndecodableText(" " + oneOver.substring(0, 4) + "\n" + oneOver.substring(4)))); + } + } + + @Test + void messagesBeyondTheIntRangeAreRejectedBeforeArithmeticWraps() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + final ObjectNode one = F.objectNode(); + one.putArray("rows").addObject().put("pad", ""); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Wide"); + assertEquals(HEADER + 4 + 65000, encoder.encodedLength(one)); + + // 34000 x 65000 = 2.21e9 > Integer.MAX_VALUE: the sizing walk must reject, not wrap to a small positive. + final ObjectNode wide = F.objectNode(); + final ArrayNode rows = wide.putArray("rows"); + for (int i = 0; i < 34000; i++) + { + rows.addObject().put("pad", ""); + } + + final SbeJsonException sizing = assertThrows(SbeJsonException.class, () -> encoder.encodedLength(wide)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, sizing.code()); + assertTrue(sizing.path().startsWith("Wide.rows["), sizing.path()); + + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final SbeJsonException writing = assertThrows( + SbeJsonException.class, () -> encoder.encode(wide, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, writing.code()); + } + + @Test + void trailingEmptyGroupDecodesFromAnExactFrame() + { + final SbeJson sbeJson = SbeJson.builder(IR).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final ObjectNode empty = F.objectNode(); + empty.putArray("rows"); + + final int length = encode(sbeJson, "Wide", empty, buffer); + assertEquals(HEADER + 4, length); + assertEquals(65000, buffer.getShort(HEADER, LITTLE_ENDIAN) & 0xFFFF); + + final ObjectNode decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertEquals(0, decoded.get("rows").size()); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length - 1)); + assertEquals(ErrorCode.FRAME_OVERFLOW, ex.code()); + assertEquals("Wide.rows", ex.path()); + } + + static ObjectNode edge() + { + final ObjectNode node = F.objectNode(); + node.put("bigEnum", "ONE"); + node.put("bigSet", 0); + node.put("name16", ""); + node.putArray("items"); + node.put("text16", ""); + node.set("blob", F.binaryNode(new byte[0])); + + return node; + } + + private static int encode( + final SbeJson sbeJson, final String message, final ObjectNode node, final UnsafeBuffer dst) + { + final SbeJsonEncoder encoder = sbeJson.newEncoder(message); + final int length = encoder.encode(node, dst, 0, dst.capacity()); + assertEquals(length, encoder.encodedLength(node)); + + return length; + } + + private static ObjectNode arrays() + { + return F.objectNode().put("utf8Name", "n"); + } + + private static void assertRejects( + final SbeJson sbeJson, final ErrorCode code, final String path, final Consumer mutation) + { + assertRejects(sbeJson, "Edge", edge(), code, path, mutation); + } + + /** + * Base64 text whose payload must never be materialised: the limits are checked on the counted length first. + */ + private static final class UndecodableText extends TextNode + { + private static final long serialVersionUID = 1L; + + UndecodableText(final String base64) + { + super(base64); + } + + public byte[] binaryValue() + { + throw new AssertionError("base64 payload materialised before the var-data limits were checked"); + } + + public byte[] getBinaryValue(final Base64Variant variant) + { + throw new AssertionError("base64 payload materialised before the var-data limits were checked"); + } + } + + private static void assertRejects( + final SbeJson sbeJson, + final String message, + final ObjectNode node, + final ErrorCode code, + final String path, + final Consumer mutation) + { + mutation.accept(node); + final SbeJsonEncoder encoder = sbeJson.newEncoder(message); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> encoder.encode(node, buffer, 0, CAPACITY), path); + assertEquals(code, ex.code(), ex.getMessage()); + assertEquals(path, ex.path(), ex.getMessage()); + + final SbeJsonException sizing = assertThrows(SbeJsonException.class, () -> encoder.encodedLength(node), path); + assertEquals(code, sizing.code(), sizing.getMessage()); + assertEquals(path, sizing.path(), sizing.getMessage()); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EncodeValidationTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EncodeValidationTest.java new file mode 100644 index 0000000000..93a6c2ba23 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EncodeValidationTest.java @@ -0,0 +1,421 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import uk.co.real_logic.sbe.ir.Ir; + +import java.lang.reflect.Field; +import java.math.BigInteger; +import java.util.Arrays; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Every reject row of DESIGN.md section 7 and every encode-reachable {@link ErrorCode}. The decode-only codes + * ({@code FRAME_OVERFLOW}, {@code FIELD_OUTSIDE_BLOCK}, {@code UNSUPPORTED_VERSION}) are covered by + * {@link VersioningTest} and {@link LimitsTest}; {@code SECTION_OUT_OF_ORDER} belongs to the parser path + * (DESIGN.md section 13 step 6). + */ +class EncodeValidationTest +{ + private static final int CAPACITY = 4096; + + private Ir ir; + private SbeJson sbeJson; + private ObjectNode car; + private UnsafeBuffer buffer; + + @BeforeEach + void setUp() throws Exception + { + ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + sbeJson = SbeJson.builder(ir).build(); + car = (ObjectNode)JsonNodes.MAPPER.readTree(EncoderOracleTest.BASELINE_CAR_JSON); + buffer = TestMessages.newBuffer(CAPACITY); + } + + @Test + void missingRequired() + { + assertRejects(ErrorCode.MISSING_REQUIRED, "Car.modelYear", c -> c.remove("modelYear")); + assertRejects(ErrorCode.MISSING_REQUIRED, "Car.modelYear", c -> c.putNull("modelYear")); + assertRejects(ErrorCode.MISSING_REQUIRED, "Car.engine", c -> c.remove("engine")); + assertRejects(ErrorCode.MISSING_REQUIRED, "Car.engine.capacity", c -> engine(c).remove("capacity")); + assertRejects(ErrorCode.MISSING_REQUIRED, "Car.fuelFigures[1].speed", + c -> ((ObjectNode)c.get("fuelFigures").get(1)).remove("speed")); + assertRejects(ErrorCode.MISSING_REQUIRED, "Car.fuelFigures[2]", + c -> c.withArray("fuelFigures").set(2, c.nullNode())); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newEncoder("Car").encode((JsonNode)null, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.MISSING_REQUIRED, ex.code()); + } + + @Test + void typeMismatch() + { + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.modelYear", c -> c.put("modelYear", 5.0)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.modelYear", c -> c.put("modelYear", "2013")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.serialNumber", c -> c.put("serialNumber", 1.5)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.serialNumber", c -> c.put("serialNumber", "abc")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.fuelFigures", c -> c.putObject("fuelFigures")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.fuelFigures[0]", c -> c.withArray("fuelFigures").set(0, 1)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.engine", c -> c.put("engine", 5)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.someNumbers", c -> c.put("someNumbers", 3)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.someNumbers", + c -> c.withArray("someNumbers").set(1, 1.5)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.extras", c -> c.put("extras", "abc")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.extras", + c -> c.putObject("extras").put("sunRoof", 1)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.vehicleCode", c -> c.put("vehicleCode", "abcdé")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.vehicleCode", c -> c.put("vehicleCode", 123456)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.code", c -> c.put("code", 1.5)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.manufacturer", c -> c.put("manufacturer", 7)); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.fuelFigures[0].mpg", + c -> ((ObjectNode)c.get("fuelFigures").get(0)).put("mpg", "fast")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.performanceFigures[1].acceleration[2].seconds", + c -> ((ObjectNode)c.get("performanceFigures").get(1).get("acceleration").get(2)).put("seconds", true)); + } + + @Test + void outOfRange() + { + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.modelYear", c -> c.put("modelYear", 70000)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.modelYear", c -> c.put("modelYear", -1)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.serialNumber", c -> c.put("serialNumber", -1)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.serialNumber", + c -> c.put("serialNumber", BigInteger.ONE.shiftLeft(64))); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.serialNumber", + c -> c.put("serialNumber", "18446744073709551615")); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.vehicleCode", c -> c.put("vehicleCode", "abcdefg")); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.someNumbers", c -> c.withArray("someNumbers").add(5)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.someNumbers", + c -> c.withArray("someNumbers").set(0, Long.MAX_VALUE)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.extras", c -> c.put("extras", 256)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.code", c -> c.put("code", 300)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.performanceFigures[0].octaneRating", + c -> ((ObjectNode)c.get("performanceFigures").get(0)).put("octaneRating", 89)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.fuelFigures", c -> + { + final ObjectNode entry = (ObjectNode)c.get("fuelFigures").get(0); + for (int i = 0; i < 252; i++) + { + c.withArray("fuelFigures").add(entry); + } + }); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.manufacturer", c -> c.put("manufacturer", "x".repeat(255))); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.fuelFigures[0].mpg", + c -> ((ObjectNode)c.get("fuelFigures").get(0)).put("mpg", 1e300)); + } + + @Test + void unknownEnumAndChoice() + { + assertRejects(ErrorCode.UNKNOWN_ENUM, "Car.available", c -> c.put("available", "Maybe")); + assertRejects(ErrorCode.UNKNOWN_ENUM, "Car.code", c -> c.put("code", "D")); + assertRejects(ErrorCode.UNKNOWN_CHOICE, "Car.extras", c -> c.putObject("extras").put("turbo", true)); + } + + @Test + void unknownEnumRawValueIsAcceptedForLosslessRoundTrip() + { + car.put("available", 7); + final int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertEquals(7, sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("available").intValue()); + } + + @Test + void unknownPropertyPerObjectByDefaultAndIgnoredOnRequest() + { + assertRejects(ErrorCode.UNKNOWN_PROPERTY, "Car", c -> c.put("colour", "red")); + assertRejects(ErrorCode.UNKNOWN_PROPERTY, "Car.engine", c -> engine(c).put("turbo", true)); + assertRejects(ErrorCode.UNKNOWN_PROPERTY, "Car.fuelFigures[1]", + c -> ((ObjectNode)c.get("fuelFigures").get(1)).put("units", "mpg")); + assertRejects(ErrorCode.UNKNOWN_PROPERTY, "Car.performanceFigures[0].acceleration[1]", + c -> ((ObjectNode)c.get("performanceFigures").get(0).get("acceleration").get(1)).put("kph", 1)); + + final SbeJsonException ex = assertThrows(SbeJsonException.class, () -> + { + car.put("colour", "red"); + sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + }); + assertTrue(ex.getMessage().contains("'colour'"), ex.getMessage()); + + final UnsafeBuffer reference = TestMessages.newBuffer(CAPACITY); + final int expectedLength = TestMessages.encodeBaselineCar(reference, 0); + final SbeJson lenient = SbeJson.builder(ir).unknownProperties(UnknownProperties.IGNORE).build(); + engine(car).put("turbo", true); + ((ObjectNode)car.get("fuelFigures").get(1)).put("units", "mpg"); + final int length = lenient.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertEquals(expectedLength, length); + assertArrayEquals(Arrays.copyOf(reference.byteArray(), length), Arrays.copyOf(buffer.byteArray(), length)); + } + + @Test + void constantsMayBeOmittedButMustMatchWhenSupplied() throws Exception + { + final UnsafeBuffer reference = TestMessages.newBuffer(CAPACITY); + final int expectedLength = TestMessages.encodeBaselineCar(reference, 0); + + engine(car).put("maxRpm", 9000); + engine(car).put("fuel", "Petrol"); + final int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertArrayEquals( + Arrays.copyOf(reference.byteArray(), expectedLength), Arrays.copyOf(buffer.byteArray(), length)); + + assertRejects(ErrorCode.CONSTANT_MISMATCH, "Car.engine.maxRpm", c -> engine(c).put("maxRpm", 9001)); + assertRejects(ErrorCode.CONSTANT_MISMATCH, "Car.engine.maxRpm", c -> engine(c).put("maxRpm", "9000")); + assertRejects(ErrorCode.CONSTANT_MISMATCH, "Car.engine.fuel", c -> engine(c).put("fuel", "Diesel")); + + final SbeJson extension = SbeJson.builder(TestMessages.ir(TestMessages.EXTENSION_SCHEMA)).build(); + final ObjectNode extended = (ObjectNode)JsonNodes.MAPPER.readTree(EncoderOracleTest.EXTENSION_CAR_JSON); + extended.put("discountedModel", "A"); + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> extension.newEncoder("Car").encode(extended, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.CONSTANT_MISMATCH, ex.code()); + assertEquals("Car.discountedModel", ex.path()); + extended.put("discountedModel", (int)'C'); + extension.newEncoder("Car").encode(extended, buffer, 0, CAPACITY); + } + + @Test + void destinationOverflow() + { + final SbeJsonEncoder encoder = sbeJson.newEncoder("Car"); + final int needed = encoder.encodedLength(car); + + assertEquals(needed, encoder.encode(car, buffer, 0, needed)); + + final SbeJsonException tail = assertThrows( + SbeJsonException.class, () -> encoder.encode(car, buffer, 0, needed - 1)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, tail.code()); + assertEquals("Car.activationCode", tail.path()); + + final SbeJsonException header = assertThrows( + SbeJsonException.class, () -> encoder.encode(car, buffer, 0, 3)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, header.code()); + + final SbeJsonException block = assertThrows( + SbeJsonException.class, () -> encoder.encode(car, buffer, 0, 20)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, block.code()); + + final SbeJsonException group = assertThrows( + SbeJsonException.class, () -> encoder.encode(car, buffer, 0, 8 + 62 + 4)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, group.code()); + assertTrue(group.path().startsWith("Car.fuelFigures"), group.path()); + + final SbeJsonException capacity = assertThrows( + SbeJsonException.class, () -> encoder.encode(car, buffer, CAPACITY - 10, needed)); + assertEquals(ErrorCode.DESTINATION_OVERFLOW, capacity.code()); + } + + @Test + void unknownTemplateFromEncoderFactory() + { + final SbeJsonException byName = assertThrows(SbeJsonException.class, () -> sbeJson.newEncoder("Nope")); + assertEquals(ErrorCode.UNKNOWN_TEMPLATE, byName.code()); + + final SbeJsonException byId = assertThrows(SbeJsonException.class, () -> sbeJson.newEncoder(99)); + assertEquals(ErrorCode.UNKNOWN_TEMPLATE, byId.code()); + assertEquals(99, byId.templateId()); + } + + @Test + void optionalMissingOrNullEncodesTheNullSentinel() + { + car.remove("uuid"); + car.putNull("cupHolderCount"); + car.remove("performanceFigures"); + car.remove("model"); + final int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + + final ObjectNode decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertTrue(decoded.get("uuid").isArray()); + assertEquals(Long.MIN_VALUE, decoded.get("uuid").get(0).longValue()); + assertTrue(decoded.get("cupHolderCount").isNull()); + assertEquals(0, decoded.get("performanceFigures").size()); + assertEquals("", decoded.get("model").textValue()); + + // The decoded sentinels (uuid elements at Long.MIN_VALUE, below the schema minimum) must re-encode. + final UnsafeBuffer again = TestMessages.newBuffer(CAPACITY); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Car"); + final int written = encoder.encode(decoded, again, 0, CAPACITY); + assertEquals(length, written); + assertEquals(length, encoder.encodedLength(decoded)); + assertArrayEquals(Arrays.copyOf(buffer.byteArray(), length), Arrays.copyOf(again.byteArray(), written)); + + // A sentinel in one element of an otherwise ordinary array, and an out-of-range non-sentinel, still differ. + car.putArray("uuid").add(Long.MIN_VALUE).add(7L); + final int mixed = encoder.encode(car, buffer, 0, CAPACITY); + final JsonNode uuid = sbeJson.newDecoder().decodeCopy(buffer, 0, mixed).get("uuid"); + assertEquals(Long.MIN_VALUE, uuid.get(0).longValue()); + assertEquals(7L, uuid.get(1).longValue()); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.modelYear", c -> c.put("modelYear", 65535)); + } + + @Test + void optionalScalarsRejectTheirNumericSentinel() + { + // uint8 cupHolderCount: 255 is the null sentinel, above the schema maximum of 254. + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.cupHolderCount", c -> c.put("cupHolderCount", 255)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.cupHolderCount", c -> c.put("cupHolderCount", 256)); + assertRejects(ErrorCode.OUT_OF_RANGE, "Car.cupHolderCount", c -> c.put("cupHolderCount", -1)); + + // Omission and JSON null still select the sentinel. + car.put("cupHolderCount", 254); + int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertEquals(254, sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("cupHolderCount").intValue()); + car.putNull("cupHolderCount"); + length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertTrue(sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("cupHolderCount").isNull()); + } + + @Test + void utf8VarDataRejectsLoneSurrogates() + { + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.model", c -> c.put("model", "ab\uD800c")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.model", c -> c.put("model", "\uDC00")); + assertRejects(ErrorCode.TYPE_MISMATCH, "Car.model", c -> c.put("model", "\uDC00\uD800")); + + car.put("model", "a\uD83D\uDE97z"); + final int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertEquals("a\uD83D\uDE97z", sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("model").textValue()); + } + + @Test + void encoderReleasesTheDestinationAfterEachCall() throws Exception + { + final SbeJsonEncoder encoder = sbeJson.newEncoder("Car"); + assertEquals(encoder.encodedLength(car), encoder.encode(car, buffer, 0, CAPACITY)); + assertNull(retainedDestination(encoder)); + + car.put("modelYear", 65535); + assertThrows(SbeJsonException.class, () -> encoder.encode(car, buffer, 0, CAPACITY)); + assertNull(retainedDestination(encoder)); + } + + private static Object retainedDestination(final SbeJsonEncoder encoder) throws Exception + { + final Field context = SbeJsonEncoder.class.getDeclaredField("context"); + context.setAccessible(true); + final WalkContext ctx = (WalkContext)context.get(encoder); + assertNotNull(ctx.treeEncoder); + final Field dst = PlanTreeEncoder.class.getDeclaredField("dst"); + dst.setAccessible(true); + + return dst.get(ctx.treeEncoder); + } + + @Test + void explicitNullForAConstantIsAConstantMismatch() + { + assertRejects(ErrorCode.CONSTANT_MISMATCH, "Car.engine.maxRpm", c -> engine(c).putNull("maxRpm")); + assertRejects(ErrorCode.CONSTANT_MISMATCH, "Car.engine.fuel", c -> engine(c).putNull("fuel")); + } + + @Test + void uint64AcceptsBigIntegerAndDecimalString() + { + car.put("serialNumber", new BigInteger("18446744073709551614")); + int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertEquals( + new BigInteger("18446744073709551614"), + sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("serialNumber").bigIntegerValue()); + + car.put("serialNumber", "9223372036854775808"); + length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + assertEquals( + new BigInteger("9223372036854775808"), + sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("serialNumber").bigIntegerValue()); + } + + @Test + void floatingPointAcceptsNanAndInfinityStrings() + { + ((ObjectNode)car.get("fuelFigures").get(0)).put("mpg", "NaN"); + ((ObjectNode)car.get("fuelFigures").get(1)).put("mpg", "Infinity"); + ((ObjectNode)car.get("fuelFigures").get(2)).put("mpg", "-Infinity"); + final int length = sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY); + + final JsonNode fuelFigures = sbeJson.newDecoder().decodeCopy(buffer, 0, length).get("fuelFigures"); + assertTrue(Float.isNaN(fuelFigures.get(0).get("mpg").floatValue())); + assertEquals(Float.POSITIVE_INFINITY, fuelFigures.get(1).get("mpg").floatValue()); + assertEquals(Float.NEGATIVE_INFINITY, fuelFigures.get(2).get("mpg").floatValue()); + } + + @Test + void asciiVarDataRejectsNonAscii() throws Exception + { + final SbeJson extension = SbeJson.builder(TestMessages.ir(TestMessages.EXTENSION_SCHEMA)).build(); + final ObjectNode extended = (ObjectNode)JsonNodes.MAPPER.readTree(EncoderOracleTest.EXTENSION_CAR_JSON); + extended.put("activationCode", "café"); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> extension.newEncoder("Car").encode(extended, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.TYPE_MISMATCH, ex.code()); + assertEquals("Car.activationCode", ex.path()); + } + + @Test + void exceptionCarriesTemplateIdAndOptionallyNoStackTrace() + { + final SbeJson quiet = SbeJson.builder(ir).exceptionStackTraces(false).build(); + car.put("modelYear", 5.5); + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> quiet.newEncoder("Car").encode(car, buffer, 0, CAPACITY)); + + assertEquals(1, ex.templateId()); + assertEquals("Car.modelYear", ex.path()); + assertEquals(8 + 8, ex.byteOffset()); + assertEquals(0, ex.getStackTrace().length); + + final SbeJsonException loud = assertThrows( + SbeJsonException.class, () -> sbeJson.newEncoder("Car").encode(car, buffer, 0, CAPACITY)); + assertTrue(loud.getStackTrace().length > 0); + } + + private void assertRejects(final ErrorCode code, final String path, final Consumer mutation) + { + final ObjectNode mutated = car.deepCopy(); + mutation.accept(mutated); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Car"); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> encoder.encode(mutated, buffer, 0, CAPACITY), path); + assertEquals(code, ex.code(), ex.getMessage()); + assertEquals(path, ex.path(), ex.getMessage()); + + final SbeJsonException sizing = assertThrows( + SbeJsonException.class, () -> encoder.encodedLength(mutated), path); + assertEquals(code, sizing.code(), sizing.getMessage()); + } + + private static ObjectNode engine(final ObjectNode car) + { + return (ObjectNode)car.get("engine"); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EncoderOracleTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EncoderOracleTest.java new file mode 100644 index 0000000000..d02988014f --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/EncoderOracleTest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import uk.co.real_logic.sbe.ir.Ir; + +import java.util.Arrays; +import java.util.function.ToIntBiFunction; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Bytes from {@link SbeJsonEncoder#encode} must equal the bytes the generated encoders produce for the same + * logical content. sbe-tool has no OTF encoder, so the generated codecs are the oracle. + */ +class EncoderOracleTest +{ + private static final int CAPACITY = 4096; + + static final String BASELINE_CAR_JSON = + "{\"serialNumber\":1234,\"modelYear\":2013,\"available\":\"T\",\"code\":\"A\"," + + "\"someNumbers\":[0,1,2,3,4],\"vehicleCode\":\"ab\\\"def\",\"extras\":6," + + "\"engine\":{\"capacity\":2000,\"numCylinders\":4,\"manufacturerCode\":\"123\"}," + + "\"uuid\":[7,3],\"cupHolderCount\":5," + + "\"fuelFigures\":[{\"speed\":30,\"mpg\":35.9},{\"speed\":55,\"mpg\":49.0},{\"speed\":75,\"mpg\":40.0}]," + + "\"performanceFigures\":[" + + "{\"octaneRating\":95,\"acceleration\":[{\"mph\":30,\"seconds\":4.0},{\"mph\":60,\"seconds\":7.5}," + + "{\"mph\":100,\"seconds\":12.2}]}," + + "{\"octaneRating\":99,\"acceleration\":[{\"mph\":30,\"seconds\":3.8},{\"mph\":60,\"seconds\":7.1}," + + "{\"mph\":100,\"seconds\":11.8}]}]," + + "\"manufacturer\":\"Honda\",\"model\":\"Civic VTi\",\"activationCode\":\"315\\\\8\"}"; + + static final String EXTENSION_CAR_JSON = + "{\"serialNumber\":1234,\"modelYear\":2013,\"available\":\"T\",\"code\":\"A\"," + + "\"someNumbers\":[1,2,3,4],\"vehicleCode\":\"abcdef\"," + + "\"extras\":{\"sportsPack\":true,\"cruiseControl\":true,\"sunRoof\":false}," + + "\"discountedModel\":\"C\"," + + "\"engine\":{\"capacity\":2000,\"numCylinders\":4,\"maxRpm\":9000,\"manufacturerCode\":\"123\"," + + "\"fuel\":\"Petrol\",\"efficiency\":35,\"boosterEnabled\":\"T\"," + + "\"booster\":{\"BoostType\":\"NITROUS\",\"horsePower\":200}}," + + "\"uuid\":[7,3],\"cupHolderCount\":5," + + "\"fuelFigures\":[{\"speed\":30,\"mpg\":35.9,\"usageDescription\":\"Urban Cycle\"}," + + "{\"speed\":55,\"mpg\":49.0,\"usageDescription\":\"Combined Cycle\"}]," + + "\"performanceFigures\":[" + + "{\"octaneRating\":95,\"acceleration\":[{\"mph\":30,\"seconds\":4.0},{\"mph\":60,\"seconds\":7.5}]}," + + "{\"octaneRating\":99,\"acceleration\":[]}]," + + "\"manufacturer\":\"Honda\",\"model\":\"Civic VTi éè 🚗\"," + + "\"activationCode\":\"abcdef\"}"; + + static final String COMPOSITE_ELEMENTS_JSON = + "{\"structure\":{\"enumOne\":\"Value10\",\"zeroth\":42,\"setOne\":{\"Bit0\":true,\"Bit26\":true}," + + "\"inner\":{\"first\":101,\"second\":-202}}}"; + + static final String GROUP_WITH_DATA_JSON = + "{\"Tag1\":99,\"Entries\":[" + + "{\"TagGroup1\":\"ABCDEFGHI\",\"NestedEntries\":[{\"TagGroup2\":1,\"varDataFieldNested\":\"nested one\"}," + + "{\"TagGroup2\":2,\"varDataFieldNested\":\"\"}],\"varDataField\":\"outer one\"}," + + "{\"TagGroup1\":\"JKLMNOPQR\",\"NestedEntries\":[],\"varDataField\":\"outer two\"}]}"; + + @Test + void baselineCarBytesEqualGeneratedEncoder() throws Exception + { + assertBytesEqualOracle(TestMessages.BASELINE_SCHEMA, "Car", BASELINE_CAR_JSON, TestMessages::encodeBaselineCar); + } + + @Test + void extensionCarBytesEqualGeneratedEncoder() throws Exception + { + assertBytesEqualOracle( + TestMessages.EXTENSION_SCHEMA, "Car", EXTENSION_CAR_JSON, TestMessages::encodeExtensionCar); + } + + @Test + void compositeElementsBytesEqualGeneratedEncoder() throws Exception + { + assertBytesEqualOracle( + TestMessages.COMPOSITE_ELEMENTS_SCHEMA, "Msg", COMPOSITE_ELEMENTS_JSON, + TestMessages::encodeCompositeElements); + } + + @Test + void groupWithDataBytesEqualGeneratedEncoder() throws Exception + { + assertBytesEqualOracle( + TestMessages.GROUP_WITH_DATA_SCHEMA, "TestMessage3", GROUP_WITH_DATA_JSON, + TestMessages::encodeGroupWithData); + } + + @Test + void encodeAtOffsetWithExactAvailableSucceeds() throws Exception + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJsonEncoder encoder = SbeJson.builder(ir).build().newEncoder(1); + final JsonNode car = JsonNodes.MAPPER.readTree(BASELINE_CAR_JSON); + final int needed = encoder.encodedLength(car); + + final UnsafeBuffer expected = TestMessages.newBuffer(CAPACITY); + final int expectedLength = TestMessages.encodeBaselineCar(expected, 0); + assertEquals(expectedLength, needed); + + final UnsafeBuffer actual = TestMessages.newBuffer(CAPACITY); + final int written = encoder.encode(car, actual, 200, needed); + assertEquals(needed, written); + assertArrayEquals( + Arrays.copyOfRange(expected.byteArray(), 0, expectedLength), + Arrays.copyOfRange(actual.byteArray(), 200, 200 + written)); + assertEquals("Car", encoder.messageName()); + assertEquals(1, encoder.templateId()); + } + + private static void assertBytesEqualOracle( + final String schema, + final String messageName, + final String json, + final ToIntBiFunction oracle) throws Exception + { + final Ir ir = TestMessages.ir(schema); + final UnsafeBuffer expected = TestMessages.newBuffer(CAPACITY); + final int expectedLength = oracle.applyAsInt(expected, 0); + + final SbeJsonEncoder encoder = SbeJson.builder(ir).build().newEncoder(messageName); + final JsonNode tree = JsonNodes.MAPPER.readTree(json); + final UnsafeBuffer actual = TestMessages.newBuffer(CAPACITY); + final int actualLength = encoder.encode(tree, actual, 0, CAPACITY); + + assertEquals(expectedLength, actualLength); + assertEquals(expectedLength, encoder.encodedLength(tree)); + assertArrayEquals( + Arrays.copyOf(expected.byteArray(), expectedLength), Arrays.copyOf(actual.byteArray(), actualLength)); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/JsonNodes.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/JsonNodes.java new file mode 100644 index 0000000000..c60ac71530 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/JsonNodes.java @@ -0,0 +1,161 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.util.Iterator; + +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Semantic comparison of stock Jackson trees for tests. Stock {@code equals} is class based ({@code IntNode} + * never equals {@code LongNode}, {@code FloatNode} never equals {@code DoubleNode}), so numbers are compared by + * value: integral by {@code bigIntegerValue()}, floating point by {@code double} with {@code float} widening when + * either side is a {@code FloatNode}, NaN equal to NaN. + */ +final class JsonNodes +{ + static final ObjectMapper MAPPER = new ObjectMapper(); + + private JsonNodes() + { + } + + static void assertSemanticEquals(final JsonNode expected, final JsonNode actual) + { + final String diff = firstDifference(expected, actual, "$"); + if (null != diff) + { + fail(diff + "\nexpected: " + expected + "\nactual: " + actual); + } + } + + static boolean semanticEquals(final JsonNode a, final JsonNode b) + { + return null == firstDifference(a, b, "$"); + } + + static String firstDifference(final JsonNode a, final JsonNode b, final String path) + { + if (a == null || b == null) + { + return a == b ? null : path + ": one side is missing"; + } + if (a.isNumber() && b.isNumber()) + { + return numbersEqual(a, b) ? null : path + ": " + a + " != " + b; + } + if (a.getNodeType() != b.getNodeType()) + { + return path + ": node type " + a.getNodeType() + " != " + b.getNodeType(); + } + + switch (a.getNodeType()) + { + case OBJECT: + { + if (a.size() != b.size()) + { + return path + ": object size " + a.size() + " != " + b.size() + " (" + fieldNames(a) + " vs " + + fieldNames(b) + ")"; + } + final Iterator names = a.fieldNames(); + while (names.hasNext()) + { + final String name = names.next(); + if (!b.has(name)) + { + return path + "." + name + ": missing on the right"; + } + final String diff = firstDifference(a.get(name), b.get(name), path + "." + name); + if (null != diff) + { + return diff; + } + } + return null; + } + + case ARRAY: + { + if (a.size() != b.size()) + { + return path + ": array size " + a.size() + " != " + b.size(); + } + for (int i = 0; i < a.size(); i++) + { + final String diff = firstDifference(a.get(i), b.get(i), path + "[" + i + "]"); + if (null != diff) + { + return diff; + } + } + return null; + } + + case BINARY: + { + try + { + return java.util.Arrays.equals(a.binaryValue(), b.binaryValue()) ? null : path + ": binary differs"; + } + catch (final java.io.IOException ex) + { + return path + ": " + ex; + } + } + + default: + return a.equals(b) ? null : path + ": " + a + " != " + b; + } + } + + private static boolean numbersEqual(final JsonNode a, final JsonNode b) + { + if (a.isIntegralNumber() && b.isIntegralNumber()) + { + return a.bigIntegerValue().equals(b.bigIntegerValue()); + } + if (a.isFloat() || b.isFloat()) + { + final float x = (float)a.doubleValue(); + final float y = (float)b.doubleValue(); + return Float.compare(x, y) == 0 || x == y; + } + final double x = a.doubleValue(); + final double y = b.doubleValue(); + + return Double.compare(x, y) == 0 || x == y; + } + + private static String fieldNames(final JsonNode node) + { + final StringBuilder sb = new StringBuilder("["); + final Iterator names = node.fieldNames(); + while (names.hasNext()) + { + sb.append(names.next()); + if (names.hasNext()) + { + sb.append(", "); + } + } + + return sb.append(']').toString(); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/LimitsTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/LimitsTest.java new file mode 100644 index 0000000000..302fc29e0c --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/LimitsTest.java @@ -0,0 +1,203 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import uk.co.real_logic.sbe.ir.Ir; + +import java.nio.ByteOrder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LimitsTest +{ + private static final int CAPACITY = 4096; + private static final int HEADER_LENGTH = 8; + private static final int CAR_BLOCK_LENGTH = 62; + + @Test + void hostileNestedNumInGroupStopsAtMaxGroupEntries() + { + final Ir ir = TestMessages.ir(TestMessages.NESTED_GROUP_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).limits(Limits.builder().maxGroupEntries(300).build()).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + TestMessages.encodeNestedGroups(buffer, 0); + buffer.putByte(HEADER_LENGTH + 16 + 1, (byte)200); + buffer.putByte(HEADER_LENGTH + 16 + 4, (byte)200); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, CAPACITY)); + + assertEquals(ErrorCode.LIMIT_EXCEEDED, ex.code()); + assertEquals("Top.x[0].y", ex.path()); + assertTrue(ex.getMessage().contains("maxGroupEntries 300"), ex.getMessage()); + } + + @Test + void schemaDeeperThanMaxDepthIsRejectedAtBuildTime() + { + final Ir ir = TestMessages.ir(TestMessages.NESTED_GROUP_SCHEMA); + final SbeJsonException ex = assertThrows( + SbeJsonException.class, + () -> SbeJson.builder(ir).limits(Limits.builder().maxDepth(2).build()).build()); + assertEquals(ErrorCode.LIMIT_EXCEEDED, ex.code()); + + assertNotNull(SbeJson.builder(ir).limits(Limits.builder().maxDepth(3).build()).build()); + } + + @Test + void varDataOverMaxVarDataBytesIsRejected() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).limits(Limits.builder().maxVarDataBytes(10).build()).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + + assertEquals(ErrorCode.LIMIT_EXCEEDED, ex.code()); + assertEquals("Car.model", ex.path()); + } + + @Test + void frameShorterThanMessageIsRejectedEvenThoughBufferHoldsIt() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + + final SbeJsonException tail = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length - 3)); + assertEquals(ErrorCode.FRAME_OVERFLOW, tail.code()); + assertEquals("Car.activationCode", tail.path()); + + final SbeJsonException block = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, HEADER_LENGTH + 10)); + assertEquals(ErrorCode.FRAME_OVERFLOW, block.code()); + + final SbeJsonException header = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, HEADER_LENGTH - 1)); + assertEquals(ErrorCode.FRAME_OVERFLOW, header.code()); + + final SbeJsonException capacity = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, CAPACITY - 4, length)); + assertEquals(ErrorCode.FRAME_OVERFLOW, capacity.code()); + } + + @Test + void numInGroupAtNullSentinelIsOutOfRange() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + buffer.putByte(HEADER_LENGTH + CAR_BLOCK_LENGTH + 2, (byte)0xFF); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + + assertEquals(ErrorCode.OUT_OF_RANGE, ex.code()); + assertEquals("Car.fuelFigures", ex.path()); + } + + @Test + void varDataLengthPastFrameIsFrameOverflow() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + final int manufacturerLengthOffset = length - 5 - 1 - 9 - 1 - 5 - 1; + assertEquals(5, buffer.getByte(manufacturerLengthOffset)); + buffer.putByte(manufacturerLengthOffset, (byte)200); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + + assertEquals(ErrorCode.FRAME_OVERFLOW, ex.code()); + assertEquals("Car.manufacturer", ex.path()); + } + + @Test + void groupBlockLengthPastFrameIsFrameOverflow() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + buffer.putShort(HEADER_LENGTH + CAR_BLOCK_LENGTH, (short)0x7FFF, ByteOrder.LITTLE_ENDIAN); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + + assertEquals(ErrorCode.FRAME_OVERFLOW, ex.code()); + assertTrue(ex.path().startsWith("Car.fuelFigures"), ex.path()); + } + + @Test + void unknownTemplateAndForeignSchemaIdAreRejected() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + + buffer.putShort(2, (short)99, ByteOrder.LITTLE_ENDIAN); + final SbeJsonException template = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + assertEquals(ErrorCode.UNKNOWN_TEMPLATE, template.code()); + assertEquals(99, template.templateId()); + + buffer.putShort(2, (short)1, ByteOrder.LITTLE_ENDIAN); + buffer.putShort(4, (short)5, ByteOrder.LITTLE_ENDIAN); + final SbeJsonException schema = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + assertEquals(ErrorCode.UNKNOWN_TEMPLATE, schema.code()); + assertTrue(schema.getMessage().contains("schema id 5"), schema.getMessage()); + } + + @Test + void encodeEnforcesGroupEntryAndVarDataBudgets() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + final ObjectNode car = SbeJson.builder(ir).build().newDecoder().decodeCopy(buffer, 0, length); + + final SbeJson groups = SbeJson.builder(ir).limits(Limits.builder().maxGroupEntries(5).build()).build(); + final SbeJsonException groupEx = assertThrows( + SbeJsonException.class, () -> groups.newEncoder("Car").encode(car, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.LIMIT_EXCEEDED, groupEx.code()); + assertEquals("Car.performanceFigures[0].acceleration", groupEx.path()); + + final SbeJson varData = SbeJson.builder(ir).limits(Limits.builder().maxVarDataBytes(13).build()).build(); + final SbeJsonException varDataEx = assertThrows( + SbeJsonException.class, () -> varData.newEncoder("Car").encode(car, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.LIMIT_EXCEEDED, varDataEx.code()); + assertEquals("Car.model", varDataEx.path()); + + final SbeJsonException sizingEx = assertThrows( + SbeJsonException.class, () -> varData.newEncoder("Car").encodedLength(car)); + assertEquals(ErrorCode.LIMIT_EXCEEDED, sizingEx.code()); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/PlanCompilerTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/PlanCompilerTest.java new file mode 100644 index 0000000000..be8fcc53d1 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/PlanCompilerTest.java @@ -0,0 +1,353 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Token; +import uk.co.real_logic.sbe.otf.OtfHeaderDecoder; +import uk.co.real_logic.sbe.otf.OtfMessageDecoder; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PlanCompilerTest +{ + private static final int CAPACITY = 4096; + + @ParameterizedTest + @ValueSource(ints = { 2, 1, 0 }) + void baselineCarPlanVisitsWhatOtfDecoderVisits(final int actingVersion) + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + TestMessages.encodeBaselineCar(buffer, 0); + + assertWalksMatch(ir, buffer, actingVersion); + } + + @ParameterizedTest + @ValueSource(ints = { 2, 1, 0 }) + void extensionCarPlanVisitsWhatOtfDecoderVisits(final int actingVersion) + { + final Ir ir = TestMessages.ir(TestMessages.EXTENSION_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + TestMessages.encodeExtensionCar(buffer, 0); + + assertWalksMatch(ir, buffer, actingVersion); + } + + @Test + void compositeElementsPlanVisitsWhatOtfDecoderVisits() + { + final Ir ir = TestMessages.ir(TestMessages.COMPOSITE_ELEMENTS_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + TestMessages.encodeCompositeElements(buffer, 0); + + assertWalksMatch(ir, buffer, 0); + } + + @Test + void groupWithDataPlanVisitsWhatOtfDecoderVisits() + { + final Ir ir = TestMessages.ir(TestMessages.GROUP_WITH_DATA_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + TestMessages.encodeGroupWithData(buffer, 0); + + assertWalksMatch(ir, buffer, 0); + } + + @Test + void nestedGroupsPlanVisitsWhatOtfDecoderVisits() + { + final Ir ir = TestMessages.ir(TestMessages.NESTED_GROUP_SCHEMA); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + TestMessages.encodeNestedGroups(buffer, 0); + + final List events = assertWalksMatch(ir, buffer, 0); + assertTrue(events.contains("encoding d@33 len=1"), events.toString()); + assertTrue(events.contains("encoding b@34 len=1"), events.toString()); + } + + @Test + void shouldCompileBaselineCarShape() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final MessagePlan plan = PlanCompiler.compile(ir, ir.getMessage(1)); + + assertEquals(1, plan.templateId); + assertEquals("Car", plan.name); + assertEquals(1, plan.schemaId); + assertEquals(2, plan.schemaVersion); + assertEquals(ir.getMessage(1).get(0).encodedLength(), plan.blockLength); + assertEquals(2, plan.maxGroupDepth); + assertEquals(0, plan.rootStart); + + final List rootNames = new ArrayList<>(); + for (int i = plan.rootStart; i < plan.rootEnd; i++) + { + rootNames.add(plan.fields[i].name); + } + assertEquals( + List.of( + "serialNumber", "modelYear", "available", "code", "someNumbers", "vehicleCode", "extras", "engine", + "uuid", "cupHolderCount", "fuelFigures", "performanceFigures", + "manufacturer", "model", "activationCode"), + rootNames); + + final FieldPlan serialNumber = field(plan, "serialNumber"); + assertEquals(FieldPlan.KIND_UINT64, serialNumber.kind); + assertEquals(0, serialNumber.offset); + assertEquals(8, serialNumber.encodedLength); + assertFalse(serialNumber.optional); + + final FieldPlan modelYear = field(plan, "modelYear"); + assertEquals(FieldPlan.KIND_INT, modelYear.kind); + assertEquals(8, modelYear.offset); + assertEquals(PrimitiveType.UINT16, modelYear.primitiveType); + assertEquals(65534, modelYear.maxValueLong); + + final FieldPlan available = field(plan, "available"); + assertEquals(FieldPlan.KIND_ENUM, available.kind); + assertEquals(10, available.offset); + assertArrayEquals(new long[]{ 0, 1 }, available.enumValues); + assertArrayEquals(new String[]{ "F", "T" }, available.enumNames); + assertEquals(1, available.enumNameToIndex.getValue("T")); + assertEquals(-1, available.enumNameToIndex.getValue("X")); + assertEquals(255, available.nullValueLong); + + final FieldPlan code = field(plan, "code"); + assertEquals(PrimitiveType.CHAR, code.primitiveType); + assertEquals('A', code.enumValues[0]); + assertEquals(0, code.enumIndexOf('A')); + assertEquals(2, code.enumIndexOf('C')); + assertEquals(-1, code.enumIndexOf('D')); + + final FieldPlan someNumbers = field(plan, "someNumbers"); + assertEquals(FieldPlan.KIND_NUMERIC_ARRAY, someNumbers.kind); + assertEquals(5, someNumbers.arrayLength); + assertEquals(20, someNumbers.encodedLength); + assertEquals(12, someNumbers.offset); + + final FieldPlan vehicleCode = field(plan, "vehicleCode"); + assertEquals(FieldPlan.KIND_CHAR_ARRAY, vehicleCode.kind); + assertEquals(6, vehicleCode.arrayLength); + assertEquals(FieldPlan.ENC_ASCII, vehicleCode.characterEncodingTag); + + final FieldPlan extras = field(plan, "extras"); + assertEquals(FieldPlan.KIND_BIT_SET, extras.kind); + assertArrayEquals(new int[]{ 0, 1, 2 }, extras.choiceBits); + assertArrayEquals(new String[]{ "sunRoof", "sportsPack", "cruiseControl" }, extras.choiceNames); + assertEquals(7, extras.knownMask); + assertEquals(0xFF, extras.maxValueLong); + + final FieldPlan uuid = field(plan, "uuid"); + assertEquals(2, uuid.sinceVersion); + assertTrue(uuid.optional); + assertEquals(2, uuid.arrayLength); + assertEquals(Long.MIN_VALUE, uuid.nullValueLong); + } + + @Test + void shouldCompileCompositeMembersWithAbsoluteOffsetsInScope() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final MessagePlan plan = PlanCompiler.compile(ir, ir.getMessage(1)); + + final FieldPlan engine = field(plan, "engine"); + assertEquals(FieldPlan.KIND_COMPOSITE, engine.kind); + assertEquals(39, engine.offset); + assertEquals(6, engine.encodedLength); + assertEquals(5, engine.childEnd - engine.childStart); + + final FieldPlan capacity = plan.fields[engine.childStart]; + assertEquals("capacity", capacity.name); + assertEquals(39, capacity.offset); + final FieldPlan numCylinders = plan.fields[engine.childStart + 1]; + assertEquals("numCylinders", numCylinders.name); + assertEquals(41, numCylinders.offset); + final FieldPlan maxRpm = plan.fields[engine.childStart + 2]; + assertEquals("maxRpm", maxRpm.name); + assertTrue(maxRpm.constant); + assertEquals(FieldPlan.KIND_INT, maxRpm.kind); + assertEquals(9000, maxRpm.constLong); + final FieldPlan manufacturerCode = plan.fields[engine.childStart + 3]; + assertEquals("manufacturerCode", manufacturerCode.name); + assertEquals(42, manufacturerCode.offset); + assertEquals(FieldPlan.KIND_CHAR_ARRAY, manufacturerCode.kind); + assertEquals(3, manufacturerCode.arrayLength); + final FieldPlan fuel = plan.fields[engine.childStart + 4]; + assertEquals("fuel", fuel.name); + assertTrue(fuel.constant); + assertEquals(FieldPlan.KIND_CHAR_ARRAY, fuel.kind); + assertEquals("Petrol", fuel.constString); + } + + @Test + void shouldCompileGroupAndVarDataLayouts() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final MessagePlan plan = PlanCompiler.compile(ir, ir.getMessage(1)); + + final FieldPlan fuelFigures = field(plan, "fuelFigures"); + assertEquals(FieldPlan.KIND_GROUP, fuelFigures.kind); + assertEquals(6, fuelFigures.blockLength); + assertEquals(3, fuelFigures.dimensionSize); + assertEquals(PrimitiveType.UINT16, fuelFigures.blockLengthType); + assertEquals(0, fuelFigures.blockLengthOffset); + assertEquals(PrimitiveType.UINT8, fuelFigures.numInGroupType); + assertEquals(2, fuelFigures.numInGroupOffset); + assertEquals(0, fuelFigures.numInGroupMin); + assertEquals(254, fuelFigures.numInGroupMax); + assertEquals(2, fuelFigures.childEnd - fuelFigures.childStart); + assertEquals("speed", plan.fields[fuelFigures.childStart].name); + assertEquals(0, plan.fields[fuelFigures.childStart].offset); + assertEquals("mpg", plan.fields[fuelFigures.childStart + 1].name); + assertEquals(2, plan.fields[fuelFigures.childStart + 1].offset); + assertEquals(FieldPlan.KIND_FLOAT, plan.fields[fuelFigures.childStart + 1].kind); + + final FieldPlan performanceFigures = field(plan, "performanceFigures"); + final FieldPlan acceleration = plan.fields[performanceFigures.childStart + 1]; + assertEquals("acceleration", acceleration.name); + assertEquals(FieldPlan.KIND_GROUP, acceleration.kind); + assertEquals(6, acceleration.blockLength); + + final FieldPlan manufacturer = field(plan, "manufacturer"); + assertEquals(FieldPlan.KIND_VAR_DATA, manufacturer.kind); + assertEquals(PrimitiveType.UINT8, manufacturer.lengthType); + assertEquals(0, manufacturer.lengthOffset); + assertEquals(254, manufacturer.lengthMax); + assertEquals(1, manufacturer.dataOffset); + assertEquals(FieldPlan.ENC_UTF8, manufacturer.characterEncodingTag); + + final MessagePlan credentials = PlanCompiler.compile(ir, ir.getMessage(2)); + final FieldPlan encryptedPassword = field(credentials, "encryptedPassword"); + assertEquals(FieldPlan.ENC_BINARY, encryptedPassword.characterEncodingTag); + assertEquals(PrimitiveType.UINT32, encryptedPassword.lengthType); + assertEquals(1073741824L, encryptedPassword.lengthMax); + assertEquals(4, encryptedPassword.dataOffset); + } + + @Test + void shouldCompileEffectiveLayoutTable() + { + final Ir baseline = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final MessagePlan car = PlanCompiler.compile(baseline, baseline.getMessage(1)); + assertArrayEquals(new int[]{ 0, 2 }, car.versionThresholds); + assertEquals(3, car.layoutCount()); + assertEquals(1, car.layoutIndex(0)); + assertEquals(1, car.layoutIndex(1)); + assertEquals(2, car.layoutIndex(2)); + assertEquals(2, car.layoutIndex(99)); + + final Ir extension = TestMessages.ir(TestMessages.EXTENSION_SCHEMA); + final MessagePlan extended = PlanCompiler.compile(extension, extension.getMessage(1)); + assertArrayEquals(new int[]{ 0, 1, 2 }, extended.versionThresholds); + assertEquals(4, extended.layoutCount()); + assertEquals(2, extended.layoutIndex(1)); + } + + @Test + void shouldCompileConstantEnumAndNestedCompositeInExtensionSchema() + { + final Ir ir = TestMessages.ir(TestMessages.EXTENSION_SCHEMA); + final MessagePlan plan = PlanCompiler.compile(ir, ir.getMessage(1)); + + final FieldPlan discountedModel = field(plan, "discountedModel"); + assertEquals(FieldPlan.KIND_ENUM, discountedModel.kind); + assertTrue(discountedModel.constant); + assertEquals("C", discountedModel.constString); + assertEquals('C', discountedModel.constLong); + + final FieldPlan engine = field(plan, "engine"); + final FieldPlan booster = plan.fields[engine.childEnd - 1]; + assertEquals("booster", booster.name); + assertEquals(FieldPlan.KIND_COMPOSITE, booster.kind); + assertEquals(engine.offset + 8, booster.offset); + final FieldPlan boostType = plan.fields[booster.childStart]; + assertEquals("BoostType", boostType.name); + assertEquals(FieldPlan.KIND_ENUM, boostType.kind); + assertEquals(booster.offset, boostType.offset); + final FieldPlan horsePower = plan.fields[booster.childStart + 1]; + assertEquals("horsePower", horsePower.name); + assertEquals(booster.offset + 1, horsePower.offset); + + final FieldPlan boosterEnabled = plan.fields[engine.childEnd - 2]; + assertEquals("boosterEnabled", boosterEnabled.name); + assertEquals(FieldPlan.KIND_ENUM, boosterEnabled.kind); + + final FieldPlan fuelFigures = field(plan, "fuelFigures"); + final FieldPlan mpg = plan.fields[fuelFigures.childStart + 1]; + assertEquals("mpg", mpg.name); + assertEquals(2, mpg.sinceVersion); + final FieldPlan usageDescription = plan.fields[fuelFigures.childStart + 2]; + assertEquals("usageDescription", usageDescription.name); + assertEquals(FieldPlan.KIND_VAR_DATA, usageDescription.kind); + assertEquals(FieldPlan.ENC_ASCII, usageDescription.characterEncodingTag); + } + + @Test + void shouldInternFieldNames() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final MessagePlan plan = PlanCompiler.compile(ir, ir.getMessage(1)); + assertSame("serialNumber", field(plan, "serialNumber").name); + assertSame("Car", plan.name); + } + + private static List assertWalksMatch(final Ir ir, final UnsafeBuffer buffer, final int actingVersion) + { + final OtfHeaderDecoder headerDecoder = new OtfHeaderDecoder(ir.headerStructure()); + final int templateId = headerDecoder.getTemplateId(buffer, 0); + final int blockLength = headerDecoder.getBlockLength(buffer, 0); + final int bodyOffset = headerDecoder.encodedLength(); + final List tokens = ir.getMessage(templateId); + + final SpyTokenListener spy = new SpyTokenListener(actingVersion); + OtfMessageDecoder.decode(buffer, bodyOffset, actingVersion, blockLength, tokens, spy); + + final MessagePlan plan = PlanCompiler.compile(ir, tokens); + final List planEvents = PlanWalker.walk(plan, buffer, bodyOffset, blockLength, actingVersion); + + assertEquals(spy.events(), planEvents); + assertTrue(planEvents.size() > 2); + + return planEvents; + } + + private static FieldPlan field(final MessagePlan plan, final String name) + { + for (int i = plan.rootStart; i < plan.rootEnd; i++) + { + if (plan.fields[i].name.equals(name)) + { + return plan.fields[i]; + } + } + + throw new IllegalArgumentException("no root field " + name); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/PlanWalker.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/PlanWalker.java new file mode 100644 index 0000000000..818bec20f4 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/PlanWalker.java @@ -0,0 +1,156 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.DirectBuffer; +import uk.co.real_logic.sbe.otf.Types; + +import java.util.ArrayList; +import java.util.List; + +/** + * Test-side walk of a {@link MessagePlan} over an encoded message producing the same event record as + * {@link SpyTokenListener}, so the compiled offsets, sizes and dimension layouts can be compared with what + * {@link uk.co.real_logic.sbe.otf.OtfMessageDecoder} visits. + */ +final class PlanWalker +{ + private final MessagePlan plan; + private final DirectBuffer buffer; + private final int actingVersion; + private final List events = new ArrayList<>(); + + private PlanWalker(final MessagePlan plan, final DirectBuffer buffer, final int actingVersion) + { + this.plan = plan; + this.buffer = buffer; + this.actingVersion = actingVersion; + } + + static List walk( + final MessagePlan plan, + final DirectBuffer buffer, + final int bodyOffset, + final int actingBlockLength, + final int actingVersion) + { + final PlanWalker walker = new PlanWalker(plan, buffer, actingVersion); + walker.events.add("beginMessage " + plan.name); + walker.walkScope(plan.rootStart, plan.rootEnd, bodyOffset, actingBlockLength); + walker.events.add("endMessage"); + + return walker.events; + } + + private int walkScope(final int childStart, final int childEnd, final int entryBase, final int actingBlockLength) + { + final FieldPlan[] fields = plan.fields; + int cursor = entryBase + actingBlockLength; + + for (int i = childStart; i < childEnd; i++) + { + final FieldPlan f = fields[i]; + if (f.isBlockField()) + { + if (f.sinceVersion <= actingVersion) + { + walkBlockField(f, entryBase); + } + } + else if (FieldPlan.KIND_GROUP == f.kind) + { + cursor = walkGroup(f, cursor); + } + else + { + cursor = walkVarData(f, cursor); + } + } + + return cursor; + } + + private void walkBlockField(final FieldPlan f, final int entryBase) + { + final int index = entryBase + f.offset; + switch (f.kind) + { + case FieldPlan.KIND_COMPOSITE: + events.add("beginComposite " + f.name); + for (int i = f.childStart; i < f.childEnd; i++) + { + final FieldPlan member = plan.fields[i]; + if (member.sinceVersion <= actingVersion) + { + walkBlockField(member, entryBase); + } + } + events.add("endComposite"); + break; + + case FieldPlan.KIND_ENUM: + events.add("enum " + f.name + "@" + index + " len=" + f.encodedLength); + break; + + case FieldPlan.KIND_BIT_SET: + events.add("bitSet " + f.name + "@" + index + " len=" + f.encodedLength); + break; + + default: + events.add("encoding " + f.name + "@" + index + " len=" + f.encodedLength); + break; + } + } + + private int walkGroup(final FieldPlan g, final int start) + { + int cursor = start; + if (g.sinceVersion > actingVersion) + { + events.add("group " + g.name + " n=0"); + return cursor; + } + + final int blockLength = Types.getInt(buffer, cursor + g.blockLengthOffset, g.blockLengthType, g.byteOrder); + final int numInGroup = Types.getInt(buffer, cursor + g.numInGroupOffset, g.numInGroupType, g.byteOrder); + cursor += g.dimensionSize; + events.add("group " + g.name + " n=" + numInGroup); + + for (int i = 0; i < numInGroup; i++) + { + events.add("beginGroup " + i); + cursor = walkScope(g.childStart, g.childEnd, cursor, blockLength); + events.add("endGroup " + i); + } + + return cursor; + } + + private int walkVarData(final FieldPlan v, final int start) + { + if (v.sinceVersion > actingVersion) + { + events.add("varData " + v.name + "@" + start + " len=0"); + return start; + } + + final int length = Types.getInt(buffer, start + v.lengthOffset, v.lengthType, v.byteOrder); + final int dataIndex = start + v.dataOffset; + events.add("varData " + v.name + "@" + dataIndex + " len=" + length); + + return dataIndex + length; + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/ProgrammaticIrTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/ProgrammaticIrTest.java new file mode 100644 index 0000000000..2e690a2299 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/ProgrammaticIrTest.java @@ -0,0 +1,160 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.json.JsonPrinter; +import uk.co.real_logic.sbe.otf.OtfHeaderDecoder; + +import java.nio.ByteOrder; + +import static java.nio.ByteOrder.BIG_ENDIAN; +import static java.nio.ByteOrder.LITTLE_ENDIAN; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Layouts only a hand-built IR can express: versioned composite members, mixed byte orders and uint32 headers. + */ +class ProgrammaticIrTest +{ + private static final int CAPACITY = 256; + private static final JsonNodeFactory F = JsonNodeFactory.instance; + + @Test + void compositeMembersAreBoundsCheckedIndividuallyAgainstTheActingBlock() + { + final Ir ir = ProgrammaticIrs.versionedComposite(); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + // A version 0 producer: block length 4, only member 'a' on the wire. The compiled composite is 8 bytes. + writeHeader(buffer, 4, 1, ProgrammaticIrs.SCHEMA_ID, 0); + buffer.putInt(8, 42, LITTLE_ENDIAN); + final ObjectNode v0 = sbeJson.newDecoder().decodeCopy(buffer, 0, 12); + assertEquals(42, v0.get("comp").get("a").intValue()); + assertFalse(v0.get("comp").has("b")); + + // Version 1 with block length 8: both members. + writeHeader(buffer, 8, 1, ProgrammaticIrs.SCHEMA_ID, 1); + buffer.putInt(12, 43, LITTLE_ENDIAN); + final ObjectNode v1 = sbeJson.newDecoder().decodeCopy(buffer, 0, 16); + assertEquals(42, v1.get("comp").get("a").intValue()); + assertEquals(43, v1.get("comp").get("b").intValue()); + + // Version 1 claiming block length 4: member 'b' is present in the version but outside the block. + writeHeader(buffer, 4, 1, ProgrammaticIrs.SCHEMA_ID, 1); + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, 16)); + assertEquals(ErrorCode.FIELD_OUTSIDE_BLOCK, ex.code()); + assertEquals("Msg.comp.b", ex.path()); + assertEquals(12, ex.byteOffset()); + } + + @Test + void headerAndGroupDimensionMembersKeepTheirOwnByteOrders() throws Exception + { + final Ir ir = ProgrammaticIrs.mixedByteOrders(); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + final ObjectNode message = F.objectNode(); + message.put("x", 7); + message.putArray("g").addObject().put("y", -2); + message.put("d", "hi"); + + final SbeJsonEncoder encoder = sbeJson.newEncoder("Mixed"); + final int length = encoder.encode(message, buffer, 0, CAPACITY); + assertEquals(22, length); + assertEquals(length, encoder.encodedLength(message)); + + // Header: blockLength LE, templateId BE, schemaId LE, version BE. + assertEquals(4, buffer.getShort(0, LITTLE_ENDIAN)); + assertEquals(1, buffer.getShort(2, BIG_ENDIAN)); + assertEquals(ProgrammaticIrs.SCHEMA_ID, buffer.getShort(4, LITTLE_ENDIAN)); + assertEquals(0, buffer.getShort(6, BIG_ENDIAN)); + // Group dimensions: blockLength LE, numInGroup BE; entry field y BE; var-data length BE. + assertEquals(2, buffer.getShort(12, LITTLE_ENDIAN)); + assertEquals(1, buffer.getShort(14, BIG_ENDIAN)); + assertEquals(-2, buffer.getShort(16, BIG_ENDIAN)); + assertEquals(2, buffer.getShort(18, BIG_ENDIAN)); + + // Oracle: the OTF header decoder and JsonPrinter honour per-member encodings. + final OtfHeaderDecoder otfHeader = new OtfHeaderDecoder(ir.headerStructure()); + assertEquals(4, otfHeader.getBlockLength(buffer, 0)); + assertEquals(1, otfHeader.getTemplateId(buffer, 0)); + assertEquals(ProgrammaticIrs.SCHEMA_ID, otfHeader.getSchemaId(buffer, 0)); + assertEquals(0, otfHeader.getSchemaVersion(buffer, 0)); + + final StringBuilder printed = new StringBuilder(); + new JsonPrinter(ir).print(printed, buffer, 0); + final JsonNode oracle = JsonNodes.MAPPER.readTree(printed.toString()); + assertEquals(7, oracle.get("x").intValue()); + assertEquals(1, oracle.get("g").size()); + assertEquals(-2, oracle.get("g").get(0).get("y").intValue()); + assertEquals("hi", oracle.get("d").textValue()); + + final ObjectNode decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + JsonNodes.assertSemanticEquals(message, decoded); + } + + @Test + void uint32HeaderMembersAtOrAbove2Pow31AreRejectedNotThrownAsIllegalState() + { + final Ir ir = ProgrammaticIrs.uint32Header(); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + + buffer.putInt(0, 1, LITTLE_ENDIAN); + buffer.putInt(4, 1, LITTLE_ENDIAN); + buffer.putInt(8, ProgrammaticIrs.SCHEMA_ID, LITTLE_ENDIAN); + buffer.putInt(12, 0, LITTLE_ENDIAN); + buffer.putByte(16, (byte)9); + assertEquals(9, sbeJson.newDecoder().decodeCopy(buffer, 0, 17).get("a").intValue()); + + buffer.putInt(4, 0x80000000, LITTLE_ENDIAN); + assertThrows( + IllegalStateException.class, () -> new OtfHeaderDecoder(ir.headerStructure()).getTemplateId(buffer, 0)); + + final SbeJsonException templateId = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, 17)); + assertEquals(ErrorCode.OUT_OF_RANGE, templateId.code()); + assertEquals(4, templateId.byteOffset()); + + buffer.putInt(4, 1, LITTLE_ENDIAN); + buffer.putInt(0, 0xFFFFFFFF, LITTLE_ENDIAN); + final SbeJsonException blockLength = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, 17)); + assertEquals(ErrorCode.OUT_OF_RANGE, blockLength.code()); + assertEquals(0, blockLength.byteOffset()); + } + + private static void writeHeader( + final UnsafeBuffer buffer, final int blockLength, final int templateId, final int schemaId, final int version) + { + final ByteOrder order = LITTLE_ENDIAN; + buffer.putShort(0, (short)blockLength, order); + buffer.putShort(2, (short)templateId, order); + buffer.putShort(4, (short)schemaId, order); + buffer.putShort(6, (short)version, order); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/ProgrammaticIrs.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/ProgrammaticIrs.java new file mode 100644 index 0000000000..b0e73d7d7e --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/ProgrammaticIrs.java @@ -0,0 +1,279 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import uk.co.real_logic.sbe.PrimitiveType; +import uk.co.real_logic.sbe.PrimitiveValue; +import uk.co.real_logic.sbe.ir.Encoding; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.ir.Signal; +import uk.co.real_logic.sbe.ir.Token; + +import java.nio.ByteOrder; +import java.util.ArrayList; +import java.util.List; + +import static java.nio.ByteOrder.BIG_ENDIAN; +import static java.nio.ByteOrder.LITTLE_ENDIAN; + +/** + * Hand-built IRs for layouts the XML front end cannot express: composite members with their own + * {@code sinceVersion}, per-member byte orders in headers and group dimensions, and uint32 header members. + * Token shapes follow {@code IrGenerator}: a field is {@code BEGIN_FIELD, type tokens, END_FIELD}; a group is + * {@code BEGIN_GROUP (size = blockLength), dimension composite, fields, END_GROUP}; var-data is + * {@code BEGIN_VAR_DATA, composite(length, varData), END_VAR_DATA}. + */ +final class ProgrammaticIrs +{ + static final int SCHEMA_ID = 77; + + private static final Encoding NONE = new Encoding.Builder().build(); + + private ProgrammaticIrs() + { + } + + static Ir edgeCases() + { + // The XML enum validator compares signed longs, so install the unsigned encoding directly in the IR: + // only the BigEnum tokens (BEGIN_ENUM .. END_ENUM) are widened, in every message that uses the enum + // (bigEnum and constEnum in Edge: 2 x 5 tokens) and in the type map that Ir captured on addMessage. + final Ir ir = TestMessages.ir(TestMessages.EDGE_SCHEMA); + int rewritten = 0; + for (final List message : ir.messages()) + { + rewritten += widenBigEnum(message); + } + if (10 != rewritten) + { + throw new IllegalStateException("expected 10 BigEnum tokens across the messages but rewrote " + rewritten); + } + if (5 != widenBigEnum(ir.getType("BigEnum"))) + { + throw new IllegalStateException("expected the BigEnum type entry to hold 5 tokens"); + } + + return ir; + } + + private static int widenBigEnum(final List tokens) + { + int rewritten = 0; + boolean inside = false; + for (int i = 0; i < tokens.size(); i++) + { + final Token token = tokens.get(i); + if (Signal.BEGIN_ENUM == token.signal() && "BigEnum".equals(token.name())) + { + inside = true; + } + if (!inside) + { + continue; + } + + final PrimitiveValue value = Signal.VALID_VALUE == token.signal() && "HIGH".equals(token.name()) ? + PrimitiveValue.parse("9223372036854775808", PrimitiveType.UINT64) : + token.encoding().constValue(); + tokens.set(i, new Token.Builder() + .signal(token.signal()).name(token.name()).id(token.id()).version(token.version()) + .size(token.encodedLength()).offset(token.offset()) + .componentTokenCount(token.componentTokenCount()) + .encoding(new Encoding.Builder() + .primitiveType(PrimitiveType.UINT64).byteOrder(token.encoding().byteOrder()) + .presence(token.encoding().presence()).constValue(value).build()) + .build()); + rewritten++; + + if (Signal.END_ENUM == token.signal() && "BigEnum".equals(token.name())) + { + inside = false; + } + } + + return rewritten; + } + + /** + * Schema version 1: message {@code Msg} (template 1, block length 8) with one composite field {@code comp} + * whose member {@code a} exists since version 0 and member {@code b} since version 1. + * + * @return the IR. + */ + static Ir versionedComposite() + { + final List msg = new ArrayList<>(); + msg.add(token(Signal.BEGIN_MESSAGE, "Msg", 1, 8, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_FIELD, "comp", 1, 0, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_COMPOSITE, "Comp", 0, 8, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "a", 0, 4, 0, 0, encoding(PrimitiveType.INT32, LITTLE_ENDIAN))); + msg.add(token(Signal.ENCODING, "b", 0, 4, 4, 1, encoding(PrimitiveType.INT32, LITTLE_ENDIAN))); + msg.add(token(Signal.END_COMPOSITE, "Comp", 0, 8, 0, 0, NONE)); + msg.add(token(Signal.END_FIELD, "comp", 1, 0, 0, 0, NONE)); + msg.add(token(Signal.END_MESSAGE, "Msg", 1, 8, 0, 0, NONE)); + + return ir(1, header(PrimitiveType.UINT16, LITTLE_ENDIAN, LITTLE_ENDIAN, LITTLE_ENDIAN, LITTLE_ENDIAN), msg); + } + + /** + * Header members alternate little and big endian (blockLength LE, templateId BE, schemaId LE, version BE). + * Message {@code Mixed} (template 1, block length 4): {@code x} int32 LE; group {@code g} with dimensions + * blockLength uint16 LE and numInGroup uint16 BE holding {@code y} int16 BE; var-data {@code d} with a + * uint16 BE length prefix and UTF-8 payload. + * + * @return the IR. + */ + static Ir mixedByteOrders() + { + final List msg = new ArrayList<>(); + msg.add(token(Signal.BEGIN_MESSAGE, "Mixed", 1, 4, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_FIELD, "x", 1, 0, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "int32", 0, 4, 0, 0, encoding(PrimitiveType.INT32, LITTLE_ENDIAN))); + msg.add(token(Signal.END_FIELD, "x", 1, 0, 0, 0, NONE)); + + msg.add(token(Signal.BEGIN_GROUP, "g", 2, 2, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_COMPOSITE, "groupSizeEncoding", 0, 4, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "blockLength", 0, 2, 0, 0, encoding(PrimitiveType.UINT16, LITTLE_ENDIAN))); + msg.add(token(Signal.ENCODING, "numInGroup", 0, 2, 2, 0, encoding(PrimitiveType.UINT16, BIG_ENDIAN))); + msg.add(token(Signal.END_COMPOSITE, "groupSizeEncoding", 0, 4, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_FIELD, "y", 3, 0, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "int16", 0, 2, 0, 0, encoding(PrimitiveType.INT16, BIG_ENDIAN))); + msg.add(token(Signal.END_FIELD, "y", 3, 0, 0, 0, NONE)); + msg.add(token(Signal.END_GROUP, "g", 2, 2, 0, 0, NONE)); + + msg.add(token(Signal.BEGIN_VAR_DATA, "d", 4, 0, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_COMPOSITE, "varStringEncoding", 0, Token.VARIABLE_LENGTH, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "length", 0, 2, 0, 0, encoding(PrimitiveType.UINT16, BIG_ENDIAN))); + msg.add(token(Signal.ENCODING, "varData", 0, Token.VARIABLE_LENGTH, 2, 0, + new Encoding.Builder() + .primitiveType(PrimitiveType.UINT8) + .byteOrder(LITTLE_ENDIAN) + .characterEncoding("UTF-8") + .build())); + msg.add(token(Signal.END_COMPOSITE, "varStringEncoding", 0, Token.VARIABLE_LENGTH, 0, 0, NONE)); + msg.add(token(Signal.END_VAR_DATA, "d", 4, 0, 0, 0, NONE)); + msg.add(token(Signal.END_MESSAGE, "Mixed", 1, 4, 0, 0, NONE)); + + return ir(0, header(PrimitiveType.UINT16, LITTLE_ENDIAN, BIG_ENDIAN, LITTLE_ENDIAN, BIG_ENDIAN), msg); + } + + /** + * Message {@code Spy} (template 1, block length 8): {@code name} is {@code char[8]} and {@code text} is + * var-data with a uint16 length prefix, both in {@link SpyCharset} so a test can observe how much scratch + * the encoder hands the charset. + * + * @return the IR. + */ + static Ir spyCharset() + { + final Encoding spyChars = new Encoding.Builder() + .primitiveType(PrimitiveType.CHAR) + .byteOrder(LITTLE_ENDIAN) + .characterEncoding(SpyCharset.NAME) + .build(); + final Encoding spyBytes = new Encoding.Builder() + .primitiveType(PrimitiveType.UINT8) + .byteOrder(LITTLE_ENDIAN) + .characterEncoding(SpyCharset.NAME) + .build(); + + final List msg = new ArrayList<>(); + msg.add(token(Signal.BEGIN_MESSAGE, "Spy", 1, 8, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_FIELD, "name", 1, 0, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "char", 0, 8, 0, 0, spyChars)); + msg.add(token(Signal.END_FIELD, "name", 1, 0, 0, 0, NONE)); + + msg.add(token(Signal.BEGIN_VAR_DATA, "text", 2, 0, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_COMPOSITE, "varSpyEncoding", 0, Token.VARIABLE_LENGTH, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "length", 0, 2, 0, 0, encoding(PrimitiveType.UINT16, LITTLE_ENDIAN))); + msg.add(token(Signal.ENCODING, "varData", 0, Token.VARIABLE_LENGTH, 2, 0, spyBytes)); + msg.add(token(Signal.END_COMPOSITE, "varSpyEncoding", 0, Token.VARIABLE_LENGTH, 0, 0, NONE)); + msg.add(token(Signal.END_VAR_DATA, "text", 2, 0, 0, 0, NONE)); + msg.add(token(Signal.END_MESSAGE, "Spy", 1, 8, 0, 0, NONE)); + + return ir(0, header(PrimitiveType.UINT16, LITTLE_ENDIAN, LITTLE_ENDIAN, LITTLE_ENDIAN, LITTLE_ENDIAN), msg); + } + + /** + * Header with four uint32 members (16 bytes) and message {@code M} (template 1) holding one uint8 {@code a}. + * + * @return the IR. + */ + static Ir uint32Header() + { + final List msg = new ArrayList<>(); + msg.add(token(Signal.BEGIN_MESSAGE, "M", 1, 1, 0, 0, NONE)); + msg.add(token(Signal.BEGIN_FIELD, "a", 1, 0, 0, 0, NONE)); + msg.add(token(Signal.ENCODING, "uint8", 0, 1, 0, 0, encoding(PrimitiveType.UINT8, LITTLE_ENDIAN))); + msg.add(token(Signal.END_FIELD, "a", 1, 0, 0, 0, NONE)); + msg.add(token(Signal.END_MESSAGE, "M", 1, 1, 0, 0, NONE)); + + return ir(0, header(PrimitiveType.UINT32, LITTLE_ENDIAN, LITTLE_ENDIAN, LITTLE_ENDIAN, LITTLE_ENDIAN), msg); + } + + private static List header( + final PrimitiveType type, + final ByteOrder blockLengthOrder, + final ByteOrder templateIdOrder, + final ByteOrder schemaIdOrder, + final ByteOrder versionOrder) + { + final int size = type.size(); + final List tokens = new ArrayList<>(); + tokens.add(token(Signal.BEGIN_COMPOSITE, "messageHeader", 0, 4 * size, 0, 0, NONE)); + tokens.add(token(Signal.ENCODING, "blockLength", 0, size, 0, 0, encoding(type, blockLengthOrder))); + tokens.add(token(Signal.ENCODING, "templateId", 0, size, size, 0, encoding(type, templateIdOrder))); + tokens.add(token(Signal.ENCODING, "schemaId", 0, size, 2 * size, 0, encoding(type, schemaIdOrder))); + tokens.add(token(Signal.ENCODING, "version", 0, size, 3 * size, 0, encoding(type, versionOrder))); + tokens.add(token(Signal.END_COMPOSITE, "messageHeader", 0, 4 * size, 0, 0, NONE)); + + return tokens; + } + + private static Ir ir(final int version, final List header, final List message) + { + Ir.updateComponentTokenCounts(header); + final Ir ir = new Ir("test", "test", SCHEMA_ID, version, "programmatic", "1.0", LITTLE_ENDIAN, header); + ir.addMessage(message.get(0).id(), message); + + return ir; + } + + private static Encoding encoding(final PrimitiveType type, final ByteOrder byteOrder) + { + return new Encoding.Builder().primitiveType(type).byteOrder(byteOrder).build(); + } + + private static Token token( + final Signal signal, + final String name, + final int id, + final int size, + final int offset, + final int version, + final Encoding encoding) + { + return new Token.Builder() + .signal(signal) + .name(name) + .id(id) + .size(size) + .offset(offset) + .version(version) + .encoding(encoding) + .build(); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/RoundTripPropertyTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/RoundTripPropertyTest.java new file mode 100644 index 0000000000..4fa5a29c3b Binary files /dev/null and b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/RoundTripPropertyTest.java differ diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/RoundTripTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/RoundTripTest.java new file mode 100644 index 0000000000..100b5caaa3 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/RoundTripTest.java @@ -0,0 +1,221 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import uk.co.real_logic.sbe.ir.Ir; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.ToIntBiFunction; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Generated-encoder bytes to {@code decodeCopy} to {@code encode} must reproduce the bytes (padding is zero in + * both), and {@code decodeCopy} to {@code encode} to {@code decodeCopy} must be a semantic fixed point, under + * every combination of presentation policies. + */ +class RoundTripTest +{ + private static final int CAPACITY = 4096; + + static Stream samples() + { + final List arguments = new ArrayList<>(); + final Object[][] messages = { + { TestMessages.BASELINE_SCHEMA, (ToIntBiFunction)TestMessages::encodeBaselineCar }, + { TestMessages.EXTENSION_SCHEMA, (ToIntBiFunction)TestMessages::encodeExtensionCar }, + { + TestMessages.COMPOSITE_ELEMENTS_SCHEMA, + (ToIntBiFunction)TestMessages::encodeCompositeElements + }, + { + TestMessages.GROUP_WITH_DATA_SCHEMA, + (ToIntBiFunction)TestMessages::encodeGroupWithData + }, + { + TestMessages.NESTED_GROUP_SCHEMA, + (ToIntBiFunction)TestMessages::encodeNestedGroups + }, + }; + + for (final Object[] message : messages) + { + for (final EnumStyle enumStyle : EnumStyle.values()) + { + for (final BitSetStyle bitSetStyle : BitSetStyle.values()) + { + for (final CharArrayStyle charArrayStyle : CharArrayStyle.values()) + { + arguments.add(Arguments.of(message[0], message[1], enumStyle, bitSetStyle, charArrayStyle)); + } + } + } + } + + return arguments.stream(); + } + + @ParameterizedTest(name = "{0} {2} {3} {4}") + @MethodSource("samples") + void bytesSurviveDecodeEncode( + final String schema, + final ToIntBiFunction oracle, + final EnumStyle enumStyle, + final BitSetStyle bitSetStyle, + final CharArrayStyle charArrayStyle) + { + final Ir ir = TestMessages.ir(schema); + final SbeJson sbeJson = SbeJson.builder(ir) + .enumStyle(enumStyle) + .bitSetStyle(bitSetStyle) + .charArrayStyle(charArrayStyle) + .build(); + + final UnsafeBuffer original = TestMessages.newBuffer(CAPACITY); + final int length = oracle.applyAsInt(original, 0); + final SbeJsonDecoder decoder = sbeJson.newDecoder(); + final ObjectNode tree = decoder.decodeCopy(original, 0, length); + final SbeJsonEncoder encoder = sbeJson.newEncoder(decoder.lastHeader().templateId()); + + final UnsafeBuffer reencoded = TestMessages.newBuffer(CAPACITY); + final int written = encoder.encode(tree, reencoded, 0, CAPACITY); + + assertEquals(length, written); + assertEquals(length, encoder.encodedLength(tree)); + assertArrayEquals( + Arrays.copyOf(original.byteArray(), length), Arrays.copyOf(reencoded.byteArray(), written)); + + final ObjectNode again = sbeJson.newDecoder().decodeCopy(reencoded, 0, written); + JsonNodes.assertSemanticEquals(tree, again); + assertEquals(tree, again); + } + + @ParameterizedTest(name = "{0} {2} {3} {4}") + @MethodSource("samples") + void propertyOrderDoesNotChangeBytes( + final String schema, + final ToIntBiFunction oracle, + final EnumStyle enumStyle, + final BitSetStyle bitSetStyle, + final CharArrayStyle charArrayStyle) + { + final Ir ir = TestMessages.ir(schema); + final SbeJson sbeJson = SbeJson.builder(ir) + .enumStyle(enumStyle) + .bitSetStyle(bitSetStyle) + .charArrayStyle(charArrayStyle) + .build(); + + final UnsafeBuffer original = TestMessages.newBuffer(CAPACITY); + final int length = oracle.applyAsInt(original, 0); + final SbeJsonDecoder decoder = sbeJson.newDecoder(); + final ObjectNode tree = decoder.decodeCopy(original, 0, length); + final ObjectNode shuffled = shuffle(tree); + + final UnsafeBuffer reencoded = TestMessages.newBuffer(CAPACITY); + final int written = sbeJson.newEncoder(decoder.lastHeader().templateId()) + .encode(shuffled, reencoded, 0, CAPACITY); + + assertArrayEquals( + Arrays.copyOf(original.byteArray(), length), Arrays.copyOf(reencoded.byteArray(), written)); + } + + @ParameterizedTest(name = "numInGroup {0}") + @ValueSource(ints = { 0, 1, 254 }) + void groupCountBoundariesRoundTrip(final int count) + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer original = TestMessages.newBuffer(CAPACITY); + final int originalLength = TestMessages.encodeBaselineCar(original, 0); + final ObjectNode car = sbeJson.newDecoder().decodeCopy(original, 0, originalLength); + + final com.fasterxml.jackson.databind.node.ArrayNode fuelFigures = car.putArray("fuelFigures"); + for (int i = 0; i < count; i++) + { + fuelFigures.addObject().put("speed", i).put("mpg", i * 0.5f); + } + car.putArray("performanceFigures"); + + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final SbeJsonEncoder encoder = sbeJson.newEncoder("Car"); + final int length = encoder.encode(car, buffer, 0, CAPACITY); + assertEquals(length, encoder.encodedLength(car)); + // fuelFigures dimensions follow the 62 byte root block: uint16 blockLength then uint8 numInGroup. + assertEquals(count, buffer.getByte(8 + 62 + 2) & 0xFF); + + final ObjectNode decoded = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertEquals(count, decoded.get("fuelFigures").size()); + assertEquals(0, decoded.get("performanceFigures").size()); + JsonNodes.assertSemanticEquals(car, decoded); + + // One past the numInGroup maximum of the IR (uint8 max 254) is rejected on encode. + fuelFigures.addObject().put("speed", 1).put("mpg", 1.0f); + while (fuelFigures.size() < 255) + { + fuelFigures.addObject().put("speed", 1).put("mpg", 1.0f); + } + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> encoder.encode(car, buffer, 0, CAPACITY)); + assertEquals(ErrorCode.OUT_OF_RANGE, ex.code()); + assertEquals("Car.fuelFigures", ex.path()); + } + + static ObjectNode shuffle(final ObjectNode node) + { + final List names = new ArrayList<>(); + node.fieldNames().forEachRemaining(names::add); + Collections.reverse(names); + + final ObjectNode result = node.objectNode(); + for (final String name : names) + { + final JsonNode child = node.get(name); + if (child.isObject()) + { + result.set(name, shuffle((ObjectNode)child)); + } + else if (child.isArray() && child.size() > 0 && child.get(0).isObject()) + { + final com.fasterxml.jackson.databind.node.ArrayNode array = result.arrayNode(child.size()); + for (final JsonNode element : child) + { + array.add(shuffle((ObjectNode)element)); + } + result.set(name, array); + } + else + { + result.set(name, child); + } + } + + return result; + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SbeJsonExceptionTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SbeJsonExceptionTest.java new file mode 100644 index 0000000000..81bbfb6033 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SbeJsonExceptionTest.java @@ -0,0 +1,67 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class SbeJsonExceptionTest +{ + @Test + void shouldFormatMessageWithAllParts() + { + final SbeJsonException ex = new SbeJsonException( + ErrorCode.OUT_OF_RANGE, 1, 42, "Car.engine.capacity", "value 70000 > 65534"); + + assertEquals(ErrorCode.OUT_OF_RANGE, ex.code()); + assertEquals(1, ex.templateId()); + assertEquals(42, ex.byteOffset()); + assertEquals("Car.engine.capacity", ex.path()); + assertEquals("OUT_OF_RANGE templateId=1 path=Car.engine.capacity offset=42: value 70000 > 65534", + ex.getMessage()); + } + + @Test + void shouldOmitAbsentParts() + { + final SbeJsonException ex = new SbeJsonException( + ErrorCode.FRAME_OVERFLOW, SbeJsonException.NO_TEMPLATE_ID, SbeJsonException.NO_OFFSET, null, null); + + assertEquals("FRAME_OVERFLOW", ex.getMessage()); + assertNull(ex.path()); + } + + @Test + void shouldSuppressStackTraceWhenRequested() + { + final SbeJsonException ex = new SbeJsonException( + ErrorCode.LIMIT_EXCEEDED, 1, SbeJsonException.NO_OFFSET, "Car.fuelFigures", "too many", false); + + assertEquals(0, ex.getStackTrace().length); + assertFalse(ex.getMessage().isEmpty()); + } + + @Test + void limitsBuilderRejectsNegativeValues() + { + assertThrows(IllegalArgumentException.class, () -> Limits.builder().maxDepth(-1)); + assertEquals(Limits.DEFAULT_MAX_DEPTH, Limits.defaults().maxDepth()); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyCharset.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyCharset.java new file mode 100644 index 0000000000..eabd63b864 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyCharset.java @@ -0,0 +1,102 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * UTF-16BE under the name {@value #NAME}, registered through {@link SpyCharsetProvider}, whose encoder records + * the capacity of every output buffer it is handed. Tests use it to prove that the encoder sizes its scratch + * from the var-data budget rather than from the text length. + */ +final class SpyCharset extends Charset +{ + static final String NAME = "X-SBE-JACKSON-SPY"; + + private static final AtomicInteger MAX_OUTPUT_CAPACITY = new AtomicInteger(-1); + private static final AtomicInteger ENCODE_CALLS = new AtomicInteger(); + + SpyCharset() + { + super(NAME, new String[0]); + } + + static void reset() + { + MAX_OUTPUT_CAPACITY.set(-1); + ENCODE_CALLS.set(0); + } + + /** + * Largest output buffer any encoder saw since {@link #reset()}, or -1 when no encoder ran. + * + * @return the capacity in bytes. + */ + static int maxOutputCapacity() + { + return MAX_OUTPUT_CAPACITY.get(); + } + + static int encodeCalls() + { + return ENCODE_CALLS.get(); + } + + public boolean contains(final Charset cs) + { + return cs instanceof SpyCharset; + } + + public CharsetDecoder newDecoder() + { + return StandardCharsets.UTF_16BE.newDecoder(); + } + + public CharsetEncoder newEncoder() + { + return new SpyEncoder(this); + } + + private static final class SpyEncoder extends CharsetEncoder + { + private final CharsetEncoder inner = StandardCharsets.UTF_16BE.newEncoder(); + + SpyEncoder(final Charset charset) + { + super(charset, 2.0f, 2.0f, new byte[]{ (byte)0xFF, (byte)0xFD }); + } + + protected CoderResult encodeLoop(final CharBuffer in, final ByteBuffer out) + { + ENCODE_CALLS.incrementAndGet(); + MAX_OUTPUT_CAPACITY.accumulateAndGet(out.capacity(), Math::max); + + return inner.encode(in, out, false); + } + + protected void implReset() + { + inner.reset(); + } + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyCharsetProvider.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyCharsetProvider.java new file mode 100644 index 0000000000..97612fd548 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyCharsetProvider.java @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import java.nio.charset.Charset; +import java.nio.charset.spi.CharsetProvider; +import java.util.Collections; +import java.util.Iterator; + +/** + * Registers {@link SpyCharset} through {@code META-INF/services} so that {@link Charset#forName} resolves it + * from a schema {@code characterEncoding}. + */ +public final class SpyCharsetProvider extends CharsetProvider +{ + private static final Charset SPY = new SpyCharset(); + + /** + * Required by {@link java.util.ServiceLoader}. + */ + public SpyCharsetProvider() + { + } + + public Iterator charsets() + { + return Collections.singletonList(SPY).iterator(); + } + + public Charset charsetForName(final String charsetName) + { + return SpyCharset.NAME.equalsIgnoreCase(charsetName) ? SPY : null; + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyTokenListener.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyTokenListener.java new file mode 100644 index 0000000000..3b676b0828 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/SpyTokenListener.java @@ -0,0 +1,152 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.DirectBuffer; +import uk.co.real_logic.sbe.ir.Token; +import uk.co.real_logic.sbe.otf.TokenListener; + +import java.util.ArrayList; +import java.util.List; + +/** + * Records what {@link uk.co.real_logic.sbe.otf.OtfMessageDecoder} visits: property names (resolved the way + * {@code JsonTokenListener} resolves them), absolute buffer indices, encoded lengths, group counts and var-data + * lengths. Fields not present in the acting version are skipped so the record describes the effective layout. + */ +final class SpyTokenListener implements TokenListener +{ + private final List events = new ArrayList<>(); + private final int actingVersion; + private int compositeLevel; + + SpyTokenListener(final int actingVersion) + { + this.actingVersion = actingVersion; + } + + List events() + { + return events; + } + + public void onBeginMessage(final Token token) + { + events.add("beginMessage " + token.name()); + } + + public void onEndMessage(final Token token) + { + events.add("endMessage"); + } + + public void onEncoding( + final Token fieldToken, + final DirectBuffer buffer, + final int bufferIndex, + final Token typeToken, + final int actingVersion) + { + if (fieldToken.version() > this.actingVersion) + { + return; + } + final String name = compositeLevel > 0 ? typeToken.name() : fieldToken.name(); + events.add("encoding " + name + "@" + bufferIndex + " len=" + typeToken.encodedLength()); + } + + public void onEnum( + final Token fieldToken, + final DirectBuffer buffer, + final int bufferIndex, + final List tokens, + final int fromIndex, + final int toIndex, + final int actingVersion) + { + if (fieldToken.version() > this.actingVersion) + { + return; + } + final String name = compositeLevel > 0 ? tokens.get(fromIndex).name() : fieldToken.name(); + events.add("enum " + name + "@" + bufferIndex + " len=" + tokens.get(fromIndex).encodedLength()); + } + + public void onBitSet( + final Token fieldToken, + final DirectBuffer buffer, + final int bufferIndex, + final List tokens, + final int fromIndex, + final int toIndex, + final int actingVersion) + { + if (fieldToken.version() > this.actingVersion) + { + return; + } + final String name = compositeLevel > 0 ? tokens.get(fromIndex).name() : fieldToken.name(); + events.add("bitSet " + name + "@" + bufferIndex + " len=" + tokens.get(fromIndex).encodedLength()); + } + + public void onBeginComposite( + final Token fieldToken, final List tokens, final int fromIndex, final int toIndex) + { + ++compositeLevel; + if (fieldToken.version() > this.actingVersion) + { + return; + } + final String name = compositeLevel > 1 ? tokens.get(fromIndex).name() : fieldToken.name(); + events.add("beginComposite " + name); + } + + public void onEndComposite( + final Token fieldToken, final List tokens, final int fromIndex, final int toIndex) + { + --compositeLevel; + if (fieldToken.version() > this.actingVersion) + { + return; + } + events.add("endComposite"); + } + + public void onGroupHeader(final Token token, final int numInGroup) + { + events.add("group " + token.name() + " n=" + numInGroup); + } + + public void onBeginGroup(final Token token, final int groupIndex, final int numInGroup) + { + events.add("beginGroup " + groupIndex); + } + + public void onEndGroup(final Token token, final int groupIndex, final int numInGroup) + { + events.add("endGroup " + groupIndex); + } + + public void onVarData( + final Token fieldToken, + final DirectBuffer buffer, + final int bufferIndex, + final int length, + final Token typeToken) + { + events.add("varData " + fieldToken.name() + "@" + bufferIndex + " len=" + length); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/TestMessages.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/TestMessages.java new file mode 100644 index 0000000000..626086ecfd --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/TestMessages.java @@ -0,0 +1,267 @@ +/** + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.MutableDirectBuffer; +import org.agrona.concurrent.UnsafeBuffer; +import uk.co.real_logic.sbe.ir.Ir; +import uk.co.real_logic.sbe.xml.IrGenerator; +import uk.co.real_logic.sbe.xml.MessageSchema; +import uk.co.real_logic.sbe.xml.ParserOptions; +import uk.co.real_logic.sbe.xml.XmlSchemaParser; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +/** + * Schema loading and sample message encoding shared by the tests. Sample messages are produced by the codecs + * generated from the sbe-tool test schemas so that they are an independent oracle for this module. + */ +final class TestMessages +{ + static final String BASELINE_SCHEMA = "json-printer-test-schema.xml"; + static final String EXTENSION_SCHEMA = "example-extension-schema.xml"; + static final String COMPOSITE_ELEMENTS_SCHEMA = "composite-elements-schema.xml"; + static final String GROUP_WITH_DATA_SCHEMA = "group-with-data-schema.xml"; + static final String NESTED_GROUP_SCHEMA = "nested-group-schema.xml"; + static final String VERSIONED_V1_SCHEMA = "versioned-message-v1.xml"; + static final String VERSIONED_V2_SCHEMA = "versioned-message-v2.xml"; + static final String EDGE_SCHEMA = "edge-cases-schema.xml"; + + private TestMessages() + { + } + + static Ir ir(final String resourceName) + { + try (InputStream in = TestMessages.class.getClassLoader().getResourceAsStream(resourceName)) + { + if (null == in) + { + throw new IllegalArgumentException("resource not found: " + resourceName); + } + final MessageSchema schema = XmlSchemaParser.parse(in, ParserOptions.DEFAULT); + return new IrGenerator().generate(schema); + } + catch (final Exception ex) + { + throw new RuntimeException(ex); + } + } + + static UnsafeBuffer newBuffer(final int capacity) + { + return new UnsafeBuffer(new byte[capacity]); + } + + /* + * The message from {@code EncodedCarTestBase} in sbe-tool, encoded with the generated {@code baseline} codecs. + * + * @return total encoded length including the header. + */ + static int encodeBaselineCar(final MutableDirectBuffer buffer, final int offset) + { + final baseline.MessageHeaderEncoder header = new baseline.MessageHeaderEncoder(); + final baseline.CarEncoder car = new baseline.CarEncoder(); + + car.wrapAndApplyHeader(buffer, offset, header) + .serialNumber(1234) + .modelYear(2013) + .available(baseline.BooleanType.T) + .code(baseline.Model.A) + .putVehicleCode("ab\"def".getBytes(StandardCharsets.US_ASCII), 0); + + for (int i = 0, size = baseline.CarEncoder.someNumbersLength(); i < size; i++) + { + car.someNumbers(i, i); + } + + car.extras() + .clear() + .cruiseControl(true) + .sportsPack(true) + .sunRoof(false); + + car.engine() + .capacity(2000) + .numCylinders((short)4) + .putManufacturerCode("123".getBytes(StandardCharsets.US_ASCII), 0); + + car.putUuid(7L, 3L) + .cupHolderCount((byte)5); + + car.fuelFiguresCount(3) + .next().speed(30).mpg(35.9f) + .next().speed(55).mpg(49.0f) + .next().speed(75).mpg(40.0f); + + final baseline.CarEncoder.PerformanceFiguresEncoder perfFigures = car.performanceFiguresCount(2); + perfFigures.next() + .octaneRating((short)95) + .accelerationCount(3) + .next().mph(30).seconds(4.0f) + .next().mph(60).seconds(7.5f) + .next().mph(100).seconds(12.2f); + perfFigures.next() + .octaneRating((short)99) + .accelerationCount(3) + .next().mph(30).seconds(3.8f) + .next().mph(60).seconds(7.1f) + .next().mph(100).seconds(11.8f); + + car.manufacturer("Honda"); + car.model("Civic VTi"); + car.activationCode("315\\8"); + + return header.encodedLength() + car.encodedLength(); + } + + /* + * A message from the {@code extension} schema (nested composites, constant enum, var-data inside a group, + * two var-data encodings), encoded with the generated {@code extension} codecs. + */ + static int encodeExtensionCar(final MutableDirectBuffer buffer, final int offset) + { + final extension.MessageHeaderEncoder header = new extension.MessageHeaderEncoder(); + final extension.CarEncoder car = new extension.CarEncoder(); + + car.wrapAndApplyHeader(buffer, offset, header) + .serialNumber(1234) + .modelYear(2013) + .available(extension.BooleanType.T) + .code(extension.Model.A) + .putSomeNumbers(1, 2, 3, 4) + .vehicleCode("abcdef"); + + car.extras() + .clear() + .cruiseControl(true) + .sportsPack(true) + .sunRoof(false); + + final extension.EngineEncoder engine = car.engine(); + engine.capacity(2000) + .numCylinders((short)4) + .manufacturerCode("123") + .efficiency((byte)35) + .boosterEnabled(extension.BooleanType.T) + .booster().boostType(extension.BoostType.NITROUS).horsePower((short)200); + + car.putUuid(7L, 3L) + .cupHolderCount((short)5); + + car.fuelFiguresCount(2) + .next().speed(30).mpg(35.9f).usageDescription("Urban Cycle") + .next().speed(55).mpg(49.0f).usageDescription("Combined Cycle"); + + final extension.CarEncoder.PerformanceFiguresEncoder perfFigures = car.performanceFiguresCount(2); + perfFigures.next() + .octaneRating((short)95) + .accelerationCount(2) + .next().mph(30).seconds(4.0f) + .next().mph(60).seconds(7.5f); + perfFigures.next() + .octaneRating((short)99) + .accelerationCount(0); + + car.manufacturer("Honda"); + car.model("Civic VTi éè 🚗"); + car.activationCode("abcdef"); + + return header.encodedLength() + car.encodedLength(); + } + + /* + * Composite with an enum, a set and a nested composite as members. + */ + static int encodeCompositeElements(final MutableDirectBuffer buffer, final int offset) + { + final composite.elements.MessageHeaderEncoder header = new composite.elements.MessageHeaderEncoder(); + final composite.elements.MsgEncoder msg = new composite.elements.MsgEncoder(); + + final composite.elements.OuterEncoder outer = msg.wrapAndApplyHeader(buffer, offset, header).structure(); + outer.enumOne(composite.elements.EnumOne.Value10); + outer.zeroth((short)42); + outer.setOne().clear().bit0(true).bit26(true); + outer.inner().first(101L).second(-202L); + + return header.encodedLength() + msg.encodedLength(); + } + + /* + * Group containing a nested group containing var-data, plus var-data at the outer group level. + */ + static int encodeGroupWithData(final MutableDirectBuffer buffer, final int offset) + { + final group.with.data.MessageHeaderEncoder header = new group.with.data.MessageHeaderEncoder(); + final group.with.data.TestMessage3Encoder msg = new group.with.data.TestMessage3Encoder(); + + msg.wrapAndApplyHeader(buffer, offset, header).tag1(99); + final group.with.data.TestMessage3Encoder.EntriesEncoder entries = msg.entriesCount(2); + entries.next().tagGroup1("ABCDEFGHI"); + entries.nestedEntriesCount(2) + .next().tagGroup2(1L).varDataFieldNested("nested one") + .next().tagGroup2(2L).varDataFieldNested(""); + entries.varDataField("outer one"); + + entries.next().tagGroup1("JKLMNOPQR"); + entries.nestedEntriesCount(0); + entries.varDataField("outer two"); + + return header.encodedLength() + msg.encodedLength(); + } + + /* + * Hand-encoded message for {@code nested-group-schema.xml} (three group levels, all uint8, root block length + * 16 as declared by the schema; the schema has no legal Java package so there are no generated codecs). + * 

+     * a=7
+     * x[2]: {b=1, y[1]: {c=11, z[2]: {d=21}, {d=22}}}, {b=2, y[0]}
+     *
+ */ + static int encodeNestedGroups(final MutableDirectBuffer buffer, final int offset) + { + int pos = offset; + buffer.putShort(pos, (short)16); + pos += 2; + buffer.putShort(pos, (short)1); + pos += 2; + buffer.putShort(pos, (short)2); + pos += 2; + buffer.putShort(pos, (short)0); + pos += 2; + + buffer.putByte(pos, (byte)7); + pos += 16; + buffer.putByte(pos++, (byte)1); + buffer.putByte(pos++, (byte)2); + + buffer.putByte(pos++, (byte)1); + buffer.putByte(pos++, (byte)1); + buffer.putByte(pos++, (byte)1); + buffer.putByte(pos++, (byte)11); + buffer.putByte(pos++, (byte)1); + buffer.putByte(pos++, (byte)2); + buffer.putByte(pos++, (byte)21); + buffer.putByte(pos++, (byte)22); + + buffer.putByte(pos++, (byte)2); + buffer.putByte(pos++, (byte)1); + buffer.putByte(pos++, (byte)0); + + return pos - offset; + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/Utf8Test.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/Utf8Test.java new file mode 100644 index 0000000000..8771b4b596 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/Utf8Test.java @@ -0,0 +1,74 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class Utf8Test +{ + @Test + void encodedLengthMatchesTheJdkForMixedWidthText() + { + final String text = "aé中🚗z"; + assertEquals(text.getBytes(StandardCharsets.UTF_8).length, Utf8.encodedLength(text)); + } + + @Test + void loneSurrogatesAreReportedByEncodedLengthAndRefusedByEncode() + { + for (final String text : new String[]{ "a\uDC00z", "\uD800", "x\uD800\uD800y", "\uDC00\uD800" }) + { + assertEquals(-1, Utf8.encodedLength(text), text); + final UnsafeBuffer buffer = TestMessages.newBuffer(16); + assertThrows(IllegalArgumentException.class, () -> Utf8.encode(text, buffer, 0), text); + } + } + + @Test + void encodedLengthDoesNotWrapWhenTheEncodingExceedsIntRange() + { + // Just enough three-byte characters for the byte count to pass Integer.MAX_VALUE. + final int chars = Integer.MAX_VALUE / 3 + 2; + final CharSequence huge = new CharSequence() + { + public int length() + { + return chars; + } + + public char charAt(final int index) + { + return '中'; + } + + public CharSequence subSequence(final int start, final int end) + { + throw new UnsupportedOperationException(); + } + }; + + final long length = Utf8.encodedLength(huge); + assertEquals(3L * chars, length); + assertTrue(length > Integer.MAX_VALUE); + } +} diff --git a/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/VersioningTest.java b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/VersioningTest.java new file mode 100644 index 0000000000..74bb9b3744 --- /dev/null +++ b/sbe-jackson/src/test/java/uk/co/real_logic/sbe/jackson/VersioningTest.java @@ -0,0 +1,192 @@ +/* + * Copyright 2013-2025 Real Logic Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package uk.co.real_logic.sbe.jackson; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.agrona.concurrent.UnsafeBuffer; +import org.junit.jupiter.api.Test; +import uk.co.real_logic.sbe.ir.Ir; + +import java.nio.ByteOrder; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class VersioningTest +{ + private static final int CAPACITY = 4096; + private static final int HEADER_LENGTH = 8; + + @Test + void olderMessageDecodesWithNewerSchemaOmittingAbsentFields() throws Exception + { + final SbeJson v1 = SbeJson.builder(TestMessages.ir("versioned-group-v1.xml")).build(); + final SbeJson v2 = SbeJson.builder(TestMessages.ir("versioned-group-v2.xml")).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final JsonNode order = JsonNodes.MAPPER.readTree( + "{\"id\":42,\"legs\":[{\"qty\":5},{\"qty\":6}],\"note\":\"hi\"}"); + final int length = v1.newEncoder("Order").encode(order, buffer, 0, CAPACITY); + + final SbeJsonDecoder decoder = v2.newDecoder(); + final ObjectNode decoded = decoder.decodeCopy(buffer, 0, length); + + assertEquals(1, decoder.lastHeader().actingVersion()); + assertEquals(4, decoder.lastHeader().blockLength()); + JsonNodes.assertSemanticEquals(order, decoded); + assertFalse(decoded.has("price")); + assertFalse(decoded.has("fills")); + assertFalse(decoded.has("memo")); + assertFalse(decoded.get("legs").get(0).has("side")); + } + + @Test + void newerMessageIsRejected() throws Exception + { + final SbeJson v1 = SbeJson.builder(TestMessages.ir("versioned-group-v1.xml")).build(); + final SbeJson v2 = SbeJson.builder(TestMessages.ir("versioned-group-v2.xml")).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final JsonNode order = JsonNodes.MAPPER.readTree( + "{\"id\":42,\"price\":null,\"legs\":[{\"qty\":5,\"side\":1}],\"fills\":[{\"px\":7}]," + + "\"note\":\"hi\",\"memo\":\"m\"}"); + final int length = v2.newEncoder("Order").encode(order, buffer, 0, CAPACITY); + JsonNodes.assertSemanticEquals(order, v2.newDecoder().decodeCopy(buffer, 0, length)); + + final SbeJsonDecoder decoder = v1.newDecoder(); + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> decoder.decodeCopy(buffer, 0, length)); + + assertEquals(ErrorCode.UNSUPPORTED_VERSION, ex.code()); + assertEquals(1, ex.templateId()); + assertTrue(decoder.lastHeader().populated()); + assertEquals(2, decoder.lastHeader().actingVersion()); + } + + @Test + void extensionCarWithOlderActingVersionOmitsVersionedFieldsAndSkipsTrailingGroupBytes() + { + final Ir ir = TestMessages.ir(TestMessages.EXTENSION_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeExtensionCar(buffer, 0); + final ObjectNode v2 = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertTrue(v2.has("uuid")); + assertTrue(v2.get("fuelFigures").get(0).has("mpg")); + + buffer.putShort(6, (short)1, ByteOrder.LITTLE_ENDIAN); + final ObjectNode v1 = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertTrue(v1.has("uuid")); + assertTrue(v1.has("cupHolderCount")); + assertFalse(v1.get("fuelFigures").get(0).has("mpg")); + assertEquals("Urban Cycle", v1.get("fuelFigures").get(0).get("usageDescription").textValue()); + + buffer.putShort(6, (short)0, ByteOrder.LITTLE_ENDIAN); + final ObjectNode v0 = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + assertFalse(v0.has("uuid")); + assertFalse(v0.has("cupHolderCount")); + assertFalse(v0.get("fuelFigures").get(0).has("mpg")); + + final ObjectNode expected = v2.deepCopy(); + expected.remove("uuid"); + expected.remove("cupHolderCount"); + expected.withArray("fuelFigures").forEach(entry -> ((ObjectNode)entry).remove("mpg")); + JsonNodes.assertSemanticEquals(expected, v0); + } + + @Test + void largerHeaderBlockLengthSkipsTrailingRootBytes() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + final ObjectNode reference = sbeJson.newDecoder().decodeCopy(buffer, 0, length); + final int blockLength = ir.getMessage(1).get(0).encodedLength(); + + final int padding = 4; + final UnsafeBuffer padded = TestMessages.newBuffer(CAPACITY); + padded.putBytes(0, buffer, 0, HEADER_LENGTH + blockLength); + padded.putBytes( + HEADER_LENGTH + blockLength + padding, buffer, HEADER_LENGTH + blockLength, + length - HEADER_LENGTH - blockLength); + padded.putShort(0, (short)(blockLength + padding), ByteOrder.LITTLE_ENDIAN); + + final SbeJsonDecoder decoder = sbeJson.newDecoder(); + final ObjectNode decoded = decoder.decodeCopy(padded, 0, length + padding); + JsonNodes.assertSemanticEquals(reference, decoded); + assertEquals(blockLength + padding, decoder.lastHeader().blockLength()); + } + + @Test + void smallerHeaderBlockLengthThanFieldsNeedFailsFieldOutsideBlock() + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJson sbeJson = SbeJson.builder(ir).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + buffer.putShort(0, (short)40, ByteOrder.LITTLE_ENDIAN); + + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> sbeJson.newDecoder().decodeCopy(buffer, 0, length)); + + assertEquals(ErrorCode.FIELD_OUTSIDE_BLOCK, ex.code()); + assertEquals("Car.engine.capacity", ex.path()); + assertEquals(HEADER_LENGTH + 39, ex.byteOffset()); + } + + @Test + void sbeToolVersionedSchemasWithoutSinceVersionFailFieldOutsideBlock() throws Exception + { + final SbeJson v1 = SbeJson.builder(TestMessages.ir(TestMessages.VERSIONED_V1_SCHEMA)).build(); + final SbeJson v2 = SbeJson.builder(TestMessages.ir(TestMessages.VERSIONED_V2_SCHEMA)).build(); + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final JsonNode message = JsonNodes.MAPPER.readTree("{\"FieldA1\":1,\"FieldB1\":2,\"String1\":\"s\"}"); + final int length = v1.newEncoder("VersionedMessageV1").encode(message, buffer, 0, CAPACITY); + JsonNodes.assertSemanticEquals(message, v1.newDecoder().decodeCopy(buffer, 0, length)); + + // The v2 schema declares FieldC2..E2 without sinceVersion, so a v1 block cannot hold them. + final SbeJsonException ex = assertThrows( + SbeJsonException.class, () -> v2.newDecoder().decodeCopy(buffer, 0, length)); + assertEquals(ErrorCode.FIELD_OUTSIDE_BLOCK, ex.code()); + assertEquals("VersionedMessageV2.FieldC2", ex.path()); + } + + @Test + void lastHeaderIsPopulatedAfterDecodeCopyAndOverwrittenByTheNext() throws Exception + { + final Ir ir = TestMessages.ir(TestMessages.BASELINE_SCHEMA); + final SbeJsonDecoder decoder = SbeJson.builder(ir).build().newDecoder(); + assertFalse(decoder.lastHeader().populated()); + + final UnsafeBuffer buffer = TestMessages.newBuffer(CAPACITY); + final int length = TestMessages.encodeBaselineCar(buffer, 0); + decoder.decodeCopy(buffer, 0, length); + assertTrue(decoder.lastHeader().populated()); + assertEquals(1, decoder.lastHeader().templateId()); + + buffer.putShort(2, (short)2, ByteOrder.LITTLE_ENDIAN); + buffer.putShort(0, (short)0, ByteOrder.LITTLE_ENDIAN); + buffer.putShort(HEADER_LENGTH, (short)0, ByteOrder.LITTLE_ENDIAN); + buffer.putInt(HEADER_LENGTH + 1, 0, ByteOrder.LITTLE_ENDIAN); + final ObjectNode credentials = decoder.decodeCopy(buffer, 0, HEADER_LENGTH + 1 + 4); + assertEquals(2, decoder.lastHeader().templateId()); + assertEquals(0, decoder.lastHeader().blockLength()); + assertEquals("", credentials.get("login").textValue()); + assertEquals(0, credentials.get("encryptedPassword").binaryValue().length); + } +} diff --git a/sbe-jackson/src/test/resources/META-INF/services/java.nio.charset.spi.CharsetProvider b/sbe-jackson/src/test/resources/META-INF/services/java.nio.charset.spi.CharsetProvider new file mode 100644 index 0000000000..9f40ffef4d --- /dev/null +++ b/sbe-jackson/src/test/resources/META-INF/services/java.nio.charset.spi.CharsetProvider @@ -0,0 +1 @@ +uk.co.real_logic.sbe.jackson.SpyCharsetProvider diff --git a/sbe-jackson/src/test/resources/edge-cases-schema.xml b/sbe-jackson/src/test/resources/edge-cases-schema.xml new file mode 100644 index 0000000000..085d7865d4 --- /dev/null +++ b/sbe-jackson/src/test/resources/edge-cases-schema.xml @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + 18446744073709551614 + + 1 + -9223372036854775807 + -2 + + + 0 + 63 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sbe-jackson/src/test/resources/versioned-group-v1.xml b/sbe-jackson/src/test/resources/versioned-group-v1.xml new file mode 100644 index 0000000000..c6a7305141 --- /dev/null +++ b/sbe-jackson/src/test/resources/versioned-group-v1.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/sbe-jackson/src/test/resources/versioned-group-v2.xml b/sbe-jackson/src/test/resources/versioned-group-v2.xml new file mode 100644 index 0000000000..8244e72c17 --- /dev/null +++ b/sbe-jackson/src/test/resources/versioned-group-v2.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/settings.gradle b/settings.gradle index 19ec0d7679..eab9ea097a 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,4 @@ -include 'sbe-tool', 'sbe-samples', 'sbe-benchmarks', 'sbe-all' +include 'sbe-tool', 'sbe-samples', 'sbe-benchmarks', 'sbe-all', 'sbe-jackson' rootProject.name = 'sbe' project(':sbe-all').projectDir.mkdirs()