Implements {@link AutoCloseable}: closing flushes pending bytes to the underlying stream. */ public class PbjWriter implements AutoCloseable { - private byte[] buf; + private byte[] buf, ownedBuf; private int pos, cap; private int offset, err; private RuntimeException cause; private OutputStream output; - private boolean reuseable; private boolean mayGrow = true; + /* + * If a project doesn't care about stacktraces, setting pbj.ReaderWriter.useStackTrace to false + * will throw a premade exception that doesn't have the correct stacktrace. It's fast and good against DOS attacks + */ private static final boolean useStacktrace = !"false".equalsIgnoreCase(System.getProperty("pbj.ReaderWriter.useStackTrace")); public static final int EOF = PbjReader.EOF, @@ -56,6 +59,10 @@ public class PbjWriter implements AutoCloseable { private static final RuntimeException premadeRuntimeException; + /* + * Some projects may not want exceptions, but their test expects them. + * Here are premade exceptions that are created once and thrown potentially many times + */ static { premadeRuntimeException = new RuntimeException("Stacktrace not enabled in PbjWriter"); } @@ -68,16 +75,16 @@ public class PbjWriter implements AutoCloseable { */ public PbjWriter(@NonNull OutputStream output) { this.output = output; - buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + ownedBuf = buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache cap = buf.length; - reuseable = true; } /** * Creates a writer backed by a {@link ByteBuffer}. * - *
If the buffer has a backing array it is used directly. Otherwise an internal 16 KB - * streaming buffer is used and bytes are forwarded to the {@link ByteBuffer} on flush. + *
If the buffer has a backing array it is used directly and this will not grow beyond + * it. Otherwise an internal 16 KB streaming buffer is used and bytes are forwarded to the + * {@link ByteBuffer} on flush. * * @param buffer the target byte buffer */ @@ -86,6 +93,7 @@ public PbjWriter(ByteBuffer buffer) { buf = buffer.array(); pos = buffer.arrayOffset() + buffer.position(); cap = buffer.arrayOffset() + buffer.limit(); + mayGrow = false; } else { this.output = new OutputStream() { @Override @@ -98,9 +106,8 @@ public void write(@NonNull byte[] b, int off, int len) { buffer.put(b, off, len); } }; - buf = new byte[16 << 10]; + ownedBuf = buf = new byte[16 << 10]; cap = buf.length; - reuseable = true; } } @@ -115,6 +122,7 @@ public PbjWriter(byte[] buffer, int pos) { this.buf = buffer; this.pos = pos; this.cap = buffer.length; + mayGrow = false; } /** @@ -142,9 +150,8 @@ public void write(@NonNull byte[] b, int off, int len) { * No backing output stream is attached; use {@link #toByteArray()} to retrieve the written bytes. */ public PbjWriter() { - buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + ownedBuf = buf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache cap = buf.length; - reuseable = true; } /** @@ -156,12 +163,12 @@ public PbjWriter() { * {@code false} to keep the buffer fixed at {@code reserveSize} bytes */ public PbjWriter(int reserveSize, boolean mayGrow) { - if (mayGrow) buf = new byte[Math.max(reserveSize, 16 << 10)]; // 16k is friendly to x86-64 L1 cache + if (mayGrow) ownedBuf = new byte[Math.max(reserveSize, 16 << 10)]; // 16k is friendly to x86-64 L1 cache else { - buf = new byte[reserveSize]; + ownedBuf = new byte[reserveSize]; } + buf = ownedBuf; cap = buf.length; - reuseable = true; this.mayGrow = mayGrow; } @@ -242,10 +249,11 @@ public void writeByte(byte b) { buf[pos++] = b; return; } - writeByteInternal(b); + // seperated so the above is likely to inline + writeByte_cold(b); } - private void writeByteInternal(byte b) { + private void writeByte_cold(byte b) { flushOrGrow(1); buf[pos++] = b; } @@ -263,10 +271,10 @@ public void writeByte2(byte b1, byte b2) { pos += 2; return; } - writeByte2Internal(b1, b2); + writeByte2_cold(b1, b2); } - private void writeByte2Internal(byte b1, byte b2) { + private void writeByte2_cold(byte b1, byte b2) { flushOrGrow(2); buf[pos] = b1; buf[pos + 1] = b2; @@ -288,10 +296,10 @@ public void writeByte3(byte b1, byte b2, byte b3) { pos += 3; return; } - writeByte3Internal(b1, b2, b3); + writeByte3_cold(b1, b2, b3); } - private void writeByte3Internal(byte b1, byte b2, byte b3) { + private void writeByte3_cold(byte b1, byte b2, byte b3) { flushOrGrow(3); buf[pos] = b1; buf[pos + 1] = b2; @@ -316,10 +324,10 @@ public void writeByte4(byte b1, byte b2, byte b3, byte b4) { pos += 4; return; } - writeByte4Internal(b1, b2, b3, b4); + writeByte4_cold(b1, b2, b3, b4); } - private void writeByte4Internal(byte b1, byte b2, byte b3, byte b4) { + private void writeByte4_cold(byte b1, byte b2, byte b3, byte b4) { flushOrGrow(4); buf[pos] = b1; buf[pos + 1] = b2; @@ -343,10 +351,10 @@ public void writeBytes(@NonNull final BufferedData src) { pos += len; return; } - writeBytesBDInternal(src, len, srcPos); + writeBytesBD_cold(src, len, srcPos); } - private void writeBytesBDInternal(BufferedData src, int len, long srcPos) { + private void writeBytesBD_cold(BufferedData src, int len, long srcPos) { if (output == null) { flushOrGrow(len); // to grow at least to pos + len src.getBytes(srcPos, buf, pos, len); @@ -397,10 +405,11 @@ public void writeBytes(@NonNull byte[] src, int offset, int length) { pos += length; return; } - writeBytesInternal(src, offset, length); + writeBytes_cold(src, offset, length); } - private void writeBytesInternal(byte[] src, int srcOffset, int length) { + private void writeBytes_cold(byte[] src, int srcOffset, int length) { + // the 2048 was picked out of the air, it's 1/8th of the buffer size if (output != null && length >= 2048) { if (pos > 0) { try { @@ -437,10 +446,10 @@ public void writeBytes(@NonNull RandomAccessData src) { pos += len; return; } - writeBytesRAInternal(src, len); + writeBytesRA_cold(src, len); } - private void writeBytesRAInternal(RandomAccessData src, int len) { + private void writeBytesRA_cold(RandomAccessData src, int len) { if (output == null) { flushOrGrow(len); src.getBytes(0, buf, pos, len); @@ -491,10 +500,12 @@ public void writeIntBE(int value) { pos += 4; return; } - writeIntBEInternal(value); + // Don't inline the below. The above is very likely to inline + // while the below is not (and calls large methods like flushOrGrow()) + writeIntBE_cold(value); } - private void writeIntBEInternal(int value) { + private void writeIntBE_cold(int value) { flushOrGrow(4); buf[pos] = (byte) (value >>> 24); buf[pos + 1] = (byte) (value >>> 16); @@ -517,10 +528,12 @@ public void writeIntLE(int value) { pos += 4; return; } - writeIntLEInternal(value); + // Don't inline the below. The above is very likely to inline + // while the below is not (and calls large methods like flushOrGrow()) + writeIntLE_cold(value); } - private void writeIntLEInternal(int value) { + private void writeIntLE_cold(int value) { flushOrGrow(4); buf[pos] = (byte) value; buf[pos + 1] = (byte) (value >>> 8); @@ -547,10 +560,12 @@ public void writeLongLE(long value) { pos += 8; return; } - writeLongLEInternal(value); + // Don't inline the below. The above is very likely to inline + // while the below is not (and calls large methods like flushOrGrow()) + writeLongLE_cold(value); } - private void writeLongLEInternal(long value) { + private void writeLongLE_cold(long value) { flushOrGrow(8); buf[pos] = (byte) value; buf[pos + 1] = (byte) (value >>> 8); @@ -644,10 +659,12 @@ public void writeLongBE(long value) { pos += 8; return; } - writeLongBEInternal(value); + // Don't inline the below. The above is very likely to inline + // while the below is not (and calls large methods like flushOrGrow()) + writeLongBE_cold(value); } - private void writeLongBEInternal(long value) { + private void writeLongBE_cold(long value) { flushOrGrow(8); buf[pos] = (byte) (value >>> 56); buf[pos + 1] = (byte) (value >>> 48); @@ -703,7 +720,7 @@ public void writeVarLong(long value, boolean zigZag) { writeVarLongNoZZ(v); } - private void writeVarLongInternal(long v) { + private void writeVarLong_cold(long v) { flushOrGrow(10); while ((v & ~0x7FL) != 0) { buf[pos++] = (byte) (((int) v & 0x7F) | 0x80); @@ -737,7 +754,7 @@ public void writeVarLongNoZZ(long v) { buf[pos++] = (byte) v; return; } - writeVarLongInternal(v); + writeVarLong_cold(v); } /** @@ -781,6 +798,11 @@ public void close() { } } + /* + * called from every *_cold path when the buffer doesn't have enough room for minLength bytes. + * If streaming, flushes buf to the output stream; otherwise grows buf to the next power of two + * that fits minLength, but only if mayGrow is true + */ private void flushOrGrow(int minLength) { if (output != null) { if (minLength > cap) { @@ -794,11 +816,11 @@ private void flushOrGrow(int minLength) { } offset += pos; pos = 0; - } else if (reuseable && mayGrow) { + } else if (mayGrow) { int power2Capacity = (int) 2L << (63 - Long.numberOfLeadingZeros(Math.max(buf.length, pos + minLength))); byte[] newBuf = new byte[power2Capacity]; System.arraycopy(buf, 0, newBuf, 0, pos); - buf = newBuf; + ownedBuf = buf = newBuf; cap = buf.length; } // A possible else case is using a byte array and trying to reserve (or grow) past the length of it @@ -827,17 +849,21 @@ public void resetWithNull() { /** * Resets this writer and redirects output to a new {@link OutputStream}. - * Only valid on writers that were originally created with an output stream. - * Sets the error code to {@link #USAGE_ERROR} if called on a non-reuseable writer. + * + *
If this writer was backed by a caller-supplied array or {@link ByteBuffer}, that memory + * is abandoned in favor of a freshly (or previously) owned internal buffer, which becomes + * growable going forward. * * @param out the new output stream */ public void resetWith(OutputStream out) { reset(); - if (!reuseable) { - setError(USAGE_ERROR, "resetWith on non-reuseable PbjWriter"); - return; + if (ownedBuf == null) { + ownedBuf = new byte[16 << 10]; // 16k is friendly to x86-64 L1 cache + mayGrow = true; } + buf = ownedBuf; + cap = buf.length; output = out; } diff --git a/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java new file mode 100644 index 00000000..8d36ec5d --- /dev/null +++ b/pbj-core/pbj-runtime/src/test/java/com/hedera/pbj/runtime/io/buffer/PbjReaderWriterTest.java @@ -0,0 +1,1590 @@ +// SPDX-License-Identifier: Apache-2.0 +package com.hedera.pbj.runtime.io.buffer; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.ReadableSequentialData; +import com.hedera.pbj.runtime.io.WritableSequentialData; +import com.hedera.pbj.runtime.io.stream.ReadableStreamingData; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +public class PbjReaderWriterTest { + + @Test + void WriteConstructorBufferSizeConsistent() { + final int expectedSize = new PbjWriter().internalArray().length; + assertEquals(expectedSize, new PbjWriter((OutputStream) null).internalArray().length); + assertEquals(expectedSize, new PbjWriter((WritableSequentialData) null).internalArray().length); + assertEquals(expectedSize, new PbjWriter(128, true).internalArray().length); + // Does not apply to ByteBuffer, byte[], or reserve when large, or the below + assertEquals(128, new PbjWriter(128, false).internalArray().length); + } + + @Test + void writerConstructorCanReserveLarge() { + PbjWriter writer = new PbjWriter(2 << 20, true); + assertEquals(2 << 20, writer.internalArray().length); + } + + @Test + void writeByteBufferConstructorHeap() { + ByteBuffer bb = ByteBuffer.allocate(32); + PbjWriter writer = new PbjWriter(bb); + writer.writeByte3((byte) 11, (byte) 22, (byte) 33); + assertEquals(11, bb.array()[bb.arrayOffset()]); + assertEquals(22, bb.array()[bb.arrayOffset() + 1]); + assertEquals(33, bb.array()[bb.arrayOffset() + 2]); + assertEquals(3, writer.position()); + } + + @Test + void writeByteBufferConstructorDirect() { + ByteBuffer bb = ByteBuffer.allocateDirect(32); + PbjWriter writer = new PbjWriter(bb); + writer.writeByte3((byte) 11, (byte) 22, (byte) 33); + writer.flush(); + bb.flip(); + assertEquals(11, bb.get()); + assertEquals(22, bb.get()); + assertEquals(33, bb.get()); + } + + @Test + void writeBytesFromRandomAccessData() { + Bytes src = Bytes.wrap(new byte[] {10, 20, 30, 40}); + PbjWriter writer = new PbjWriter(); + writer.writeBytes(src); + assertArrayEquals(new byte[] {10, 20, 30, 40}, writer.toByteArray()); + } + + @Test + void writeBytesFromBufferedData() { + BufferedData src = BufferedData.wrap(new byte[] {10, 20, 30, 40}); + PbjWriter writer = new PbjWriter(); + writer.writeBytes(src); + assertArrayEquals(new byte[] {10, 20, 30, 40}, writer.toByteArray()); + } + + @Test + void closeFlushesDataToOutputStream() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + writer.writeByte2((byte) 55, (byte) 66); + assertArrayEquals(new byte[] {}, baos.toByteArray()); + writer.close(); + assertArrayEquals(new byte[] {55, 66}, baos.toByteArray()); + } + + @Test + void closeIsNoopWithoutOutputStream() { + PbjWriter writer = new PbjWriter(); + writer.writeByte((byte) 1); + writer.close(); + assertEquals(1, writer.position()); + } + + @Test + void toByteArrayWrappedErrorsOnStreamingWriter() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + assertEquals(Bytes.EMPTY, writer.internalArrayWrapped()); + assertEquals(Bytes.EMPTY, writer.toByteArrayWrapped()); + assertEquals(PbjWriter.USAGE_ERROR, writer.error()); + assertThrows(RuntimeException.class, () -> writer.throwOnError()); + } + + @Test + void toByteArrayErrorsOnStreamingWriter() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + assertEquals(null, writer.toByteArray()); + assertEquals(PbjWriter.USAGE_ERROR, writer.error()); + assertThrows(RuntimeException.class, () -> writer.throwOnError()); + } + + @Test + void writerRelativeReserve() { + PbjWriter writer = new PbjWriter(); + byte[] origArray = writer.internalArray(); + writer.reserveRel(origArray.length); + byte[] arr2 = writer.internalArray(); + assertEquals(origArray, arr2); // same object + + writer.reserveRel(origArray.length + 1); + byte[] arr3 = writer.internalArray(); + assertNotEquals(origArray, arr3); // diff object + + writer.skip(arr3.length - 1); + writer.reserveRel(1); + byte[] arr4 = writer.internalArray(); + assertEquals(arr3, arr4); + + writer.skip(1); + writer.reserveRel(0); + byte[] arr5 = writer.internalArray(); + assertEquals(arr4, arr5); + + writer.reserveRel(1); + byte[] arr6 = writer.internalArray(); + assertNotEquals(arr4, arr6); + } + + @Test + void writerUsesByteArray() { + byte[] bytes = new byte[128]; + PbjWriter writer = new PbjWriter(bytes, 0); + writer.writeByte3((byte) 10, (byte) 20, (byte) 30); + assertEquals(bytes[0], 10); + assertEquals(bytes[1], 20); + assertEquals(bytes[2], 30); + } + + @Test + void manyStringRoundtrip() { + byte[] bytes = new byte[128]; + PbjWriter writer = new PbjWriter(bytes, 0); + String strings[] = { + "a", + "\u0100", + "\u2603", + "\uD800\uDC00", + "\uE000", + "a\u0100\u2603\uE000\uD800\uDC00", + "\uD800\uDC00\uE000\u2603\u0100a", + "\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uE000\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100\uD800\uDC00\u2603\u0100a" + }; + for (String str : strings) { + writer.reset(); + writer.writeStringWithTag(str); + String res = writer.toPbjReader().readString(128); + assertEquals(str, res); + } + } + + @ParameterizedTest + @ValueSource( + strings = { + "Test Ascii", + "UTF16 ☃", + "Hangul Syllable Hwen 휀", + "Private Use E000 \uE000", + "Linear B Syllable \uD800\uDC00", + "4 byte char \uDB40\uDDEF", + "☃ UTF16", + "휀Hangul Syllable Hwen", + "\uE000Private Use E000", + "\uD800\uDC00Linear B Syllable", + "\uDB40\uDDEF4 byte char", + "\u007F", + "\u013F" + }) + void testReadString_readString_unicode(final String expected) { + byte[] utf8 = expected.getBytes(StandardCharsets.UTF_8); + BufferedData data = BufferedData.allocate(utf8.length + 5); + data.writeVarInt(utf8.length, false); + data.writeBytes(utf8); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + assertEquals(expected, reader.readString(Integer.MAX_VALUE)); + } + + @Test + void testReadString_readString_malformed_utf8() { + BufferedData data = BufferedData.allocate(128); + byte[][] manyBadEncoding = { + {(byte) 0xED, (byte) 0xA0, (byte) 0x80}, + {(byte) 0xED, (byte) 0x9F, (byte) 0xC0}, + {(byte) 0xE2, (byte) 0x98, (byte) 0x03}, + {(byte) 0xE2, (byte) 0x18, (byte) 0x83}, + {(byte) 0x82, (byte) 0x98, (byte) 0x83}, + {(byte) 0xC4, (byte) 0x3F}, + {(byte) 0x84, (byte) 0x3F}, + {(byte) 0x3F, (byte) 0x84, 32}, + {(byte) 0xF0, (byte) 0x90, (byte) 0x84, (byte) 0x3F}, + {(byte) 0xF0, (byte) 0x90, (byte) 0x04, (byte) 0xBF}, + {(byte) 0xF0, (byte) 0x10, (byte) 0x84, (byte) 0xBF}, + {(byte) 0x70, (byte) 0x90, (byte) 0x84, (byte) 0xBF}, + + // Largest Legal Value is 10FFFF (F4 8F BF BF) + {(byte) 0xF4, (byte) 0x90, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF4, (byte) 0x8F, (byte) 0xBF, (byte) 0xC0}, + {(byte) 0xF5, (byte) 0x80, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF5, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF6, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF7, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + {(byte) 0xF8, (byte) 0xBF, (byte) 0x80, (byte) 0x80}, + + // Check if the non tail codepath checks for these + {(byte) 0xF4, (byte) 0x90, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF4, (byte) 0x8F, (byte) 0xBF, (byte) 0xC0, 65}, + {(byte) 0xF5, (byte) 0x80, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF5, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF6, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF7, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + {(byte) 0xF8, (byte) 0xBF, (byte) 0x80, (byte) 0x80, 65}, + }; + + // First check the code is encoding correctly + { + byte[] ok = {(byte) 0xED, (byte) 0x9F, (byte) 0xBF}; + data.reset(); + data.writeVarInt(ok.length, false); + data.writeBytes(ok); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + String oksz = reader.readString(Integer.MAX_VALUE); + assertEquals("\uD7FF", oksz); + assert (reader.error() <= 0); + } + + for (var bad : manyBadEncoding) { + data.reset(); + data.writeVarInt(bad.length, false); + data.writeBytes(bad); + data.flip(); + PbjReader reader = new PbjReader(data.toInputStream()); + String badsz = reader.readString(Integer.MAX_VALUE); + assertEquals("", badsz); + assertEquals(reader.error(), PbjReader.PARSE); + } + } + + @Test + void toByteArrayDoesAClone() { + byte[] bytes = new byte[128]; + PbjWriter writer = new PbjWriter(bytes, 0); + for (int i = 0; i < 128; i++) { + writer.writeVarIntNoZZ(i); + } + assertEquals(128, writer.position()); + assertEquals(bytes, writer.internalArray()); + byte[] arr1 = writer.toByteArray(); + assertNotEquals(bytes, arr1); + Bytes arr2 = writer.toByteArrayWrapped(); + for (int i = 0; i < 128; i++) { + assertEquals(i, arr1[i]); + assertEquals(i, arr2.getByte(i)); + } + } + + @Test + void writeResetCheck() { + PbjWriter writer = new PbjWriter(); + writer.writeByte((byte) 10); + assertEquals(1, writer.position()); + writer.reset(); + assertEquals(0, writer.position()); + writer.writeByte2((byte) 99, (byte) 88); + assertEquals(2, writer.position()); + assertEquals(99, writer.internalArray()[0]); + assertEquals(88, writer.internalArray()[1]); + assertArrayEquals(new byte[] {99, 88}, writer.toByteArray()); + } + + @Test + void throwOnErrorThrows_NoPrevOverwrite() { + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag("\uD800"); // lone surrogate sets MalformString + assertEquals(PbjWriter.MALFORM_STRING, writer.error()); + assertThrows(RuntimeException.class, writer::throwOnError); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer2 = new PbjWriter(baos); + writer2.writeStringWithTag("\uD800"); // lone surrogate sets MalformString + assertEquals(PbjWriter.MALFORM_STRING, writer2.error()); + assertThrows(RuntimeException.class, writer2::throwOnError); + // Confirm error didn't change + assertEquals(Bytes.EMPTY, writer2.toByteArrayWrapped()); // if no error, this would set usage error + assertEquals(PbjWriter.MALFORM_STRING, writer2.error()); + + baos = new ByteArrayOutputStream(); + PbjWriter writer3 = new PbjWriter(baos); + assertEquals(Bytes.EMPTY, writer3.toByteArrayWrapped()); + assertEquals(PbjWriter.USAGE_ERROR, writer3.error()); + } + + @Test + void encodeUtf8HighSurrogateFollowedByHighSurrogate() { + PbjWriter writer = new PbjWriter(); + writer.writeStringNoTag("\uD800\uD801"); + assertEquals(0, writer.position()); + } + + @Test + void doesntThrowOnNoError() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4}); + assertEquals(0x04030201, reader.readIntLE()); + assertEquals(0, reader.error()); + assertDoesNotThrow(() -> reader.throwOnError()); // must not throw + + PbjWriter writer = new PbjWriter(); + writer.writeByte((byte) 1); + assertEquals(0, writer.error()); + writer.throwOnError(); // must not throw + } + + @Test + void manyRoundtrip() { + PbjWriter w = new PbjWriter(); + + w.writeInt(0x01020304); + w.writeIntBE(0x05060708); + w.writeIntLE(0x090A0B0C); + + w.writeLong(0x0102030405060708L); + w.writeLongBE(0x090A0B0C0D0E0F10L); + w.writeLongLE(0x1112131415161718L); + + w.writeFloat(1.5f); + w.writeFloatBE(2.5f); + w.writeFloatLE(3.5f); + + w.writeDouble(1.25); + w.writeDoubleBE(2.25); + w.writeDoubleLE(3.25); + + w.writeBoolean(true); + w.writeBoolean(false); + w.writeByte((byte) 127); + + byte[] arr1 = {10, 20, 30, 40, 50}; + w.writeBytes(arr1); + w.writeBytes(arr1, 1, 3); + w.writeBytes(Bytes.wrap(new byte[] {60, 70})); + BufferedData bd = BufferedData.allocate(2); + bd.writeByte((byte) 80); + bd.writeByte((byte) 90); + bd.flip(); + w.writeBytes(bd); + + w.writeStringWithTag("hello"); + w.writeStringWithTag("Ā☃"); + + PbjReader reader = w.toPbjReader(); + + assertEquals(0x01020304, reader.readInt()); + assertEquals(0x05060708, reader.readIntBE()); + assertEquals(0x090A0B0C, reader.readIntLE()); + + assertEquals(0x0102030405060708L, reader.readLong()); + assertEquals(0x090A0B0C0D0E0F10L, reader.readLongBE()); + assertEquals(0x1112131415161718L, reader.readLongLE()); + + assertEquals(1.5f, reader.readFloat(), 0f); + assertEquals(2.5f, reader.readFloat(), 0f); + assertEquals(3.5f, reader.readFloatLE(), 0f); + + assertEquals(1.25, reader.readDouble(), 0.0); + assertEquals(2.25, reader.readDouble(), 0.0); + assertEquals(3.25, reader.readDoubleLE(), 0.0); + + assertEquals(true, reader.readBoolean()); + assertEquals(false, reader.readBoolean()); + assertEquals(127, reader.readByte()); + + byte[] dst1 = new byte[5]; + reader.readBytes(dst1); + assertArrayEquals(arr1, dst1); + + byte[] dst2 = new byte[3]; + reader.readBytes(dst2); + assertArrayEquals(new byte[] {20, 30, 40}, dst2); + + byte[] dst3 = new byte[2]; + reader.readBytes(dst3); + assertArrayEquals(new byte[] {60, 70}, dst3); + + byte[] dst4 = new byte[2]; + reader.readBytes(dst4); + assertArrayEquals(new byte[] {80, 90}, dst4); + + assertEquals("hello", reader.readString(100)); + assertEquals("Ā☃", reader.readString(100)); + + assertFalse(reader.hasRemaining()); + assertEquals(0, w.error()); + } + + @Test + void writeVarIntZigZagRoundtrip() { + int[] values = {0, 1, -1, Integer.MAX_VALUE, Integer.MIN_VALUE, 100, -100}; + PbjWriter writer = new PbjWriter(); + for (int v : values) { + writer.reset(); + writer.writeVarInt(v, true); + assertEquals(v, writer.toPbjReader().readVarIntZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarInt(v, false); + assertEquals(v, writer.toPbjReader().readVarIntNoZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarIntZZ(v); + assertEquals(v, writer.toPbjReader().readVarIntZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarIntNoZZ(v); + assertEquals(v, writer.toPbjReader().readVarIntNoZZ(), "Failed for value " + v); + } + } + + @Test + void writeVarLongZigZagRoundtrip() { + long[] values = {0L, 1L, -1L, Long.MAX_VALUE, Long.MIN_VALUE, 1_000_000_000L, -1_000_000_000L}; + PbjWriter writer = new PbjWriter(); + for (long v : values) { + writer.reset(); + writer.writeVarLong(v, true); + assertEquals(v, writer.toPbjReader().readVarLongZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarLong(v, false); + assertEquals(v, writer.toPbjReader().readVarLongNoZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarLongZZ(v); + assertEquals(v, writer.toPbjReader().readVarLongZZ(), "Failed for value " + v); + + writer.reset(); + writer.writeVarLongNoZZ(v); + int noZZLen = writer.position(); + assertEquals(v, writer.toPbjReader().readVarLongNoZZ(), "Failed for value " + v); + + // edge test + writer.reset(); + int len = writer.internalArray().length; + writer.skip(len); + writer.writeVarLongNoZZ(v); + byte[] buf = writer.internalArray(); + for (int i = 0; i < noZZLen; i++) { + assertEquals(buf[i], buf[i + len]); + } + } + } + + @Test + void testWriteByteAtEdge() { + PbjWriter writer = new PbjWriter(); + int defaultLen = writer.internalArray().length; + writer.skip(defaultLen - 1); + writer.writeByte((byte) 1); + byte[] internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 1]); + writer.writeByte((byte) 1); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 1]); + assertEquals((byte) 1, internalArray[defaultLen]); + + writer = new PbjWriter(); + writer.skip(defaultLen - 2); + writer.writeByte2((byte) 1, (byte) 2); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 2]); + assertEquals((byte) 2, internalArray[defaultLen - 1]); + writer.skip(-1); + writer.writeByte2((byte) 1, (byte) 2); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 1]); + assertEquals((byte) 2, internalArray[defaultLen]); + + writer = new PbjWriter(); + writer.skip(defaultLen - 3); + writer.writeByte3((byte) 1, (byte) 2, (byte) 3); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 3]); + assertEquals((byte) 2, internalArray[defaultLen - 2]); + assertEquals((byte) 3, internalArray[defaultLen - 1]); + writer.skip(-2); + writer.writeByte3((byte) 1, (byte) 2, (byte) 3); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 2]); + assertEquals((byte) 2, internalArray[defaultLen - 1]); + assertEquals((byte) 3, internalArray[defaultLen]); + + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeByte4((byte) 1, (byte) 2, (byte) 3, (byte) 4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 4]); + assertEquals((byte) 2, internalArray[defaultLen - 3]); + assertEquals((byte) 3, internalArray[defaultLen - 2]); + assertEquals((byte) 4, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeByte4((byte) 1, (byte) 2, (byte) 3, (byte) 4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 1, internalArray[defaultLen - 3]); + assertEquals((byte) 2, internalArray[defaultLen - 2]); + assertEquals((byte) 3, internalArray[defaultLen - 1]); + assertEquals((byte) 4, internalArray[defaultLen]); + + // writeInt writes big-endian: 4 = {0, 0, 0, 4} + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeInt(4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 0, internalArray[defaultLen - 4]); + assertEquals((byte) 0, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 4, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeInt(4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + assertEquals((byte) 4, internalArray[defaultLen]); + + // writeIntLE writes little-endian: 4 = {4, 0, 0, 0} + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeIntLE(4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 4, internalArray[defaultLen - 4]); + assertEquals((byte) 0, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeIntLE(4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 4, internalArray[defaultLen - 3]); + assertEquals((byte) 0, internalArray[defaultLen - 2]); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + assertEquals((byte) 0, internalArray[defaultLen]); + + // writeLong writes big-endian: 8 = {0, 0, 0, 0, 0, 0, 0, 8} + writer = new PbjWriter(); + writer.skip(defaultLen - 8); + writer.writeLong(8); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 8, internalArray[defaultLen - 1]); + writer.skip(-7); + writer.writeLong(8); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0, internalArray[defaultLen - 1]); + assertEquals((byte) 8, internalArray[defaultLen]); + + // writeFloatLE writes little-endian: 4.0f = 0x40800000 = {0x00, 0x00, 0x80, 0x40} + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeFloatLE(4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x00, internalArray[defaultLen - 4]); + assertEquals((byte) 0x00, internalArray[defaultLen - 3]); + assertEquals((byte) 0x80, internalArray[defaultLen - 2]); + assertEquals((byte) 0x40, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeFloatLE(4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x00, internalArray[defaultLen - 3]); + assertEquals((byte) 0x00, internalArray[defaultLen - 2]); + assertEquals((byte) 0x80, internalArray[defaultLen - 1]); + assertEquals((byte) 0x40, internalArray[defaultLen]); + + // writeDoubleLE writes little-endian: 8.0 = 0x4020000000000000 = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, + // 0x40} + writer = new PbjWriter(); + writer.skip(defaultLen - 8); + writer.writeDoubleLE(8); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x20, internalArray[defaultLen - 2]); + assertEquals((byte) 0x40, internalArray[defaultLen - 1]); + writer.skip(-7); + writer.writeDoubleLE(8); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 0x20, internalArray[defaultLen - 1]); + assertEquals((byte) 0x40, internalArray[defaultLen]); + + byte[] array4 = new byte[] {10, 20, 30, 40}; + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeBytes(array4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 4]); + assertEquals((byte) 20, internalArray[defaultLen - 3]); + assertEquals((byte) 30, internalArray[defaultLen - 2]); + assertEquals((byte) 40, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeBytes(array4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 3]); + assertEquals((byte) 20, internalArray[defaultLen - 2]); + assertEquals((byte) 30, internalArray[defaultLen - 1]); + assertEquals((byte) 40, internalArray[defaultLen]); + + Bytes bytes4 = Bytes.wrap(new byte[] {10, 20, 30, 40}); + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeBytes(bytes4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 4]); + assertEquals((byte) 20, internalArray[defaultLen - 3]); + assertEquals((byte) 30, internalArray[defaultLen - 2]); + assertEquals((byte) 40, internalArray[defaultLen - 1]); + writer.skip(-3); + writer.writeBytes(bytes4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 3]); + assertEquals((byte) 20, internalArray[defaultLen - 2]); + assertEquals((byte) 30, internalArray[defaultLen - 1]); + assertEquals((byte) 40, internalArray[defaultLen]); + + BufferedData bb4 = BufferedData.wrap(new byte[] {10, 20, 30, 40}); + writer = new PbjWriter(); + writer.skip(defaultLen - 4); + writer.writeBytes(bb4); + internalArray = writer.internalArray(); + assertEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 4]); + assertEquals((byte) 20, internalArray[defaultLen - 3]); + assertEquals((byte) 30, internalArray[defaultLen - 2]); + assertEquals((byte) 40, internalArray[defaultLen - 1]); + writer.skip(-3); + bb4.resetPosition(); + writer.writeBytes(bb4); + assertEquals(defaultLen + 1, writer.position()); + internalArray = writer.internalArray(); + assertNotEquals(defaultLen, internalArray.length); + assertEquals((byte) 10, internalArray[defaultLen - 3]); + assertEquals((byte) 20, internalArray[defaultLen - 2]); + assertEquals((byte) 30, internalArray[defaultLen - 1]); + assertEquals((byte) 40, internalArray[defaultLen]); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + writer = new PbjWriter(baos); + writer.skip(defaultLen - 4); + bb4.resetPosition(); + writer.writeBytes(bb4); + assertEquals(0, baos.size()); + + writer.skip(-3); + bb4.resetPosition(); + writer.writeBytes(bb4); + assertEquals(defaultLen + 1, writer.position()); + + assertEquals(defaultLen, baos.size()); + byte[] data = baos.toByteArray(); + assertEquals(defaultLen, data.length); + assertEquals((byte) 10, data[defaultLen - 4]); + + assertEquals((byte) 10, data[defaultLen - 3]); + assertEquals((byte) 20, data[defaultLen - 2]); + assertEquals((byte) 30, data[defaultLen - 1]); + // last byte inside internal buffer + + // Again but the Bytes path + baos = new ByteArrayOutputStream(); + writer = new PbjWriter(baos); + writer.skip(defaultLen - 4); + writer.writeBytes(bytes4); + assertEquals(0, baos.size()); + + writer.skip(-3); + writer.writeBytes(bytes4); + assertEquals(defaultLen + 1, writer.position()); + + assertEquals(defaultLen, baos.size()); + data = baos.toByteArray(); + assertEquals(defaultLen, data.length); + assertEquals((byte) 10, data[defaultLen - 4]); + + assertEquals((byte) 10, data[defaultLen - 3]); + assertEquals((byte) 20, data[defaultLen - 2]); + assertEquals((byte) 30, data[defaultLen - 1]); + // last byte inside internal buffer + } + + @Test + void utf8EncodingTest() { + PbjWriter writer = new PbjWriter(); + char arr[] = new char[6 << 10]; + for (int i = 0; i < 127; i++) { + arr[i] = 'a'; + } + + writer.writeStringWithTag(new String(arr, 0, 127)); + assertEquals(128, writer.position()); + + arr[50] = 'Ā'; + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 127)); + assertEquals(130, writer.position()); + + for (int i = 0; i < 127; i++) { + arr[i] = 'Ā'; + } + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 127)); + assertEquals(127 * 2 + 2, writer.position()); + + for (int i = 0; i < 5460; i++) { + arr[i] = 'b'; + } + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 5460)); + assertEquals(5460 + 2, writer.position()); + + for (int i = 0; i < 5460; i++) { + arr[i] = 'ā'; + } + + writer.reset(); + writer.writeStringWithTag(new String(arr, 0, 5460)); + assertEquals(5460 * 2 + 2, writer.position()); + + for (int i = 0; i < 6 << 10; i++) { + arr[i] = (char) (32 + i); + } + + writer.reset(); + String str = new String(arr); + writer.writeStringWithTag(str); + String res = writer.toPbjReader().readString(1 << 20); + assertEquals(str, res); + } + + @Test + void readerLimit() { + byte[] data = new byte[] {1, 2, 3, 4, 5, 6, 7, 8}; + PbjReader reader = new PbjReader(data); + + assertTrue(reader.hasRemaining()); + assertEquals(0, reader.position()); + assertEquals(data.length, reader.limit()); + reader.limit(0); + assertTrue(!reader.hasRemaining()); + assertEquals(0, reader.position()); + + reader.limit(4); + assertTrue(reader.hasRemaining()); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + assertEquals(4, reader.position()); + assertEquals(0, reader.readIntBE()); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void readerByteBufferConstructor() { + PbjReader reader = new PbjReader(ByteBuffer.wrap(new byte[] {1, 2, 3, 4})); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void readerResetWithByteBuffer() { + PbjReader reader = new PbjReader(ByteBuffer.wrap(new byte[] {1, 2, 3, 4})); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + reader.resetWith(ByteBuffer.wrap(new byte[] {5, 6, 7, 8})); + assertEquals(0, reader.position()); + assertEquals(0x05060708, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + assertEquals(0, reader.error()); + } + + @Test + void readerBufferBytesConstructor() { + PbjReader reader = new PbjReader(Bytes.wrap(new byte[] {0x0A, 0x0B, 0x0C, 0x0D})); + assertEquals(0x0A0B0C0D, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void readerBufferInputStreamConstructor() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4})); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void readerSkipAdvancesPosition() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4, 5}); + reader.skip(3); + assertEquals(3, reader.position()); + assertTrue(reader.hasRemaining()); + } + + @Test + void readerSkipBeyondDataSetsBufferUnderflow() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4}); + reader.skip(5); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void readerResetAllowsReRead() { + byte[] data = {1, 2, 3, 4}; + PbjReader reader = new PbjReader(data); + assertEquals(0x01020304, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + reader.resetWith(data); + assertEquals(0, reader.position()); + assertTrue(reader.hasRemaining()); + assertEquals(0x01020304, reader.readIntBE()); + } + + @Test + void readerResetWithReplacesBuffer() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4}); + assertEquals(0x01020304, reader.readIntBE()); + reader.resetWith(new byte[] {5, 6, 7, 8}); + assertEquals(0x05060708, reader.readIntBE()); + assertFalse(reader.hasRemaining()); + } + + @Test + void writeLargeBytesBypassInternalBuffer() { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + writer.writeByte((byte) 7); + byte[] large = new byte[4096]; + large[0] = 11; + large[4095] = 22; + writer.writeBytes(large); + + byte[] result = baos.toByteArray(); + assertEquals(4097, result.length); + assertEquals(7, result[0]); + assertEquals(11, result[1]); + assertEquals(22, result[4096]); + + byte[] internalArray = writer.internalArray(); + assertEquals(7, internalArray[0]); + assertEquals(0, internalArray[1]); + } + + @Test + void writeBytesRandomAccessDataZeroLengthWritesNothing() { + PbjWriter writer = new PbjWriter(); + writer.writeBytes(Bytes.wrap(new byte[0])); + assertEquals(0, writer.position()); + writer.writeBytes(Bytes.EMPTY); + assertEquals(0, writer.position()); + } + + @Test + void writeBytesBufferedDataZeroRemainingWritesNothing() { + PbjWriter writer = new PbjWriter(); + writer.writeBytes(BufferedData.wrap(new byte[0])); + assertEquals(0, writer.position()); + + BufferedData bd = BufferedData.wrap(new byte[] {1, 2, 3}); + bd.skip(3); + writer.writeBytes(bd); + assertEquals(0, writer.position()); + } + + @Test + void writeBytesArrayZeroAndNegativeLengthWritesNothing() { + byte[] src = new byte[] {1, 2, 3, 4}; + PbjWriter writer = new PbjWriter(); + writer.writeBytes(src, 0, 0); + assertEquals(0, writer.position()); + writer.writeBytes(src, 0, -2); + assertEquals(0, writer.position()); + } + + @Test + void quickTest() { + byte[] a = {2, 3}; + var by = Bytes.wrap(a); + var adap = by.toReadableSequentialData(); + var r = new PbjReader(adap); + var w = new PbjWriter(); + w.setError(4, ""); + int q = 0; + } + + @Test + void largeWriteBypassCorrectness() { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(out); + + byte[] fewBytes = {1, 2, 3, 4, 5}; + byte[] manyBytes = new byte[8_000]; + Arrays.fill(manyBytes, (byte) 0x7F); + + writer.writeBytes(fewBytes); + writer.writeBytes(manyBytes); + writer.flush(); + + byte[] expected = new byte[fewBytes.length + manyBytes.length]; + System.arraycopy(fewBytes, 0, expected, 0, fewBytes.length); + System.arraycopy(manyBytes, 0, expected, fewBytes.length, manyBytes.length); + + assertArrayEquals( + expected, + out.toByteArray(), + "Buffered prefix bytes must not be dropped when a large payload bypasses the buffer"); + } + + @Test + void writeBytesArrayFastPathBoundaryWithOutputStream() { + + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + byte[] src = new byte[2047]; + src[0] = 11; + src[2046] = 22; + writer.writeBytes(src, 0, 2047); + assertEquals(11, writer.internalArray()[0]); + assertEquals(0, baos.toByteArray().length); + writer.reset(); + baos.reset(); + int internalLen = writer.internalArray().length; + writer.skip(internalLen - 1); + writer.writeBytes(src, 0, 2047); + assertEquals(internalLen - 1, baos.toByteArray().length); + } + { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(baos); + byte[] src = new byte[2048]; + src[0] = 11; + src[2047] = 22; + writer.writeBytes(src, 0, 2048); + writer.flush(); + assertEquals(0, writer.internalArray()[0]); // this length bypasses buffer + byte[] result = baos.toByteArray(); + assertEquals(2048, result.length); + assertEquals((byte) 11, result[0]); + assertEquals((byte) 22, result[2047]); + byte[] internalArray = writer.internalArray(); + assertEquals(0, internalArray[0]); + assertEquals(0, internalArray[2047]); + } + } + + @Test + void writeBytesBufferedDataFlushThrowsPropagated() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("Called write"); + } + }; + PbjWriter writer = new PbjWriter(failing); + byte[] chunk = new byte[1024]; // 2k is a fastpath + for (int i = 0; i < 16; i++) writer.writeBytes(chunk); + UncheckedIOException ex = + assertThrows(UncheckedIOException.class, () -> writer.writeBytes(BufferedData.wrap(new byte[] {1}))); + assertEquals("java.io.IOException: Called write", ex.getMessage()); + } + + @Test + void writeLargeByteArrayFlushThrowsPropagated() { + OutputStream failOnLarge = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + if (len == 1) return; + if (len >= 2048) throw new IOException("2k write alt path"); + throw new RuntimeException("Called write"); + } + }; + PbjWriter writer = new PbjWriter(failOnLarge); + writer.writeBoolean(true); + UncheckedIOException ex = assertThrows(UncheckedIOException.class, () -> writer.writeBytes(new byte[2048])); + assertEquals("java.io.IOException: 2k write alt path", ex.getMessage()); + } + + @Test + void writeBytesRandomAccessDataFlushThrowsPropagated() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("writeBytesRAInternal write"); + } + }; + PbjWriter writer = new PbjWriter(failing); + byte[] chunk = new byte[1024]; + for (int i = 0; i < 16; i++) writer.writeBytes(chunk); // fill 16k internal buffer + UncheckedIOException ex = + assertThrows(UncheckedIOException.class, () -> writer.writeBytes(Bytes.wrap(new byte[] {1}))); + assertEquals("java.io.IOException: writeBytesRAInternal write", ex.getMessage()); + } + + @Test + void flushOrGrowFlushThrowsPropagated() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("Called write"); + } + }; + PbjWriter writer = new PbjWriter(failing); + byte[] chunk = new byte[1024]; + for (int i = 0; i < 16; i++) writer.writeBytes(chunk); // fill 16k internal buffer + UncheckedIOException ex = assertThrows(UncheckedIOException.class, () -> writer.writeByte((byte) 1)); + assertEquals("java.io.IOException: Called write", ex.getMessage()); + } + + @Test + void flushPropagatesIOExceptionAsUnchecked() { + OutputStream failing = new OutputStream() { + @Override + public void write(int b) {} + + @Override + public void write(byte[] b, int off, int len) throws IOException { + throw new IOException("fake disk full"); + } + }; + PbjWriter writer = new PbjWriter(failing); + writer.writeByte((byte) 1); + UncheckedIOException ex = assertThrows(UncheckedIOException.class, writer::flush); + assertEquals("java.io.IOException: fake disk full", ex.getMessage()); + } + + @Test + void constructorWithWritableSequentialDataFlushesThrough() { + BufferedData bd = BufferedData.allocate(16); + PbjWriter writer = new PbjWriter(bd); + writer.writeByte2((byte) 10, (byte) 20); + writer.flush(); + assertEquals(10, bd.getByte(0)); + assertEquals(20, bd.getByte(1)); + assertEquals(2, bd.position()); + } + + @Test + void resetWithWritableSequentialDataSwitchesOutput() { + ByteArrayOutputStream out1 = new ByteArrayOutputStream(); + BufferedData bd = BufferedData.allocate(16); + PbjWriter writer = new PbjWriter(out1); + writer.writeByte((byte) 5); + writer.resetWith(bd); + writer.writeByte((byte) 7); + writer.flush(); + assertArrayEquals(new byte[] {5}, out1.toByteArray()); // flushed to out1 during reset + assertEquals(7, bd.getByte(0)); + assertEquals(1, bd.position()); + } + + @Test + void resetWithFlushesAndSwitchesOutput() { + ByteArrayOutputStream out1 = new ByteArrayOutputStream(); + ByteArrayOutputStream out2 = new ByteArrayOutputStream(); + PbjWriter writer = new PbjWriter(out1); + writer.writeByte((byte) 10); + assertEquals(0, out1.size()); + writer.resetWith(out2); // flushes before reset + assertArrayEquals(new byte[] {10}, out1.toByteArray()); + + assertEquals(0, writer.position()); + writer.writeByte((byte) 20); + writer.flush(); + assertArrayEquals(new byte[] {20}, out2.toByteArray()); + writer.writeByte((byte) 5); + writer.resetWithNull(); + assertArrayEquals(new byte[] {20, 5}, out2.toByteArray()); + writer.writeByte((byte) 7); + assertArrayEquals(new byte[] {7}, writer.toByteArray()); + assertArrayEquals(new byte[] {20, 5}, out2.toByteArray()); + } + + @Test + void resetWithOnFixedArrayWriterLazilyOwnsABuffer() { + byte[] buf = new byte[64]; + PbjWriter writer = new PbjWriter(buf, 0); + writer.writeByte((byte) 1); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + writer.resetWith(out); + assertEquals(0, writer.error()); + assertEquals(1, buf[0]); + + writer.writeByte((byte) 9); + writer.flush(); + assertArrayEquals(new byte[] {9}, out.toByteArray()); + assertEquals(1, buf[0]); + } + + @Test + void bufferMoreBeyondAbsoluteLimitSetsBufferUnderflow() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5})); + reader.limit(3); + assertEquals(1, reader.readByte()); + assertEquals(2, reader.readByte()); + assertEquals(3, reader.readByte()); + assertEquals(false, reader.hasRemaining()); + reader.readByte(); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void limitPropagatedToUnderlyingReadableSequentialData() { + BufferedData bd = BufferedData.allocate(8); + for (byte b = 0; b < 8; b++) bd.writeByte(b); + bd.flip(); + PbjReader reader = new PbjReader(bd); + reader.limit(4); + reader.readByte(); + assertEquals(4, bd.position()); + assertEquals(1, reader.readByte()); + assertEquals(2, reader.readByte()); + assertEquals(3, reader.readByte()); + assertFalse(reader.hasRemaining()); + } + + @Test + void skipTriggersUnderflow() { + byte[] data = {1, 2, 3, 4, 5}; + PbjReader reader = new PbjReader(data); + assertEquals(1, reader.readByte()); + reader.skip(5); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void skipInternalAccountsForBytesRemainingInBuffer() { + byte[] data = {1, 2, 3, 4, 5}; + PbjReader reader = new PbjReader(new ByteArrayInputStream(data)); + assertEquals(1, reader.readByte()); + reader.skip(5); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void readVarLongWithMoreThanTenBytesSetsDataEncoding() { + byte[] malformed = new byte[16]; + for (int i = 0; i < 16; i++) malformed[i] = (byte) 0xFF; + PbjReader reader = new PbjReader(malformed); + reader.readVarLongNoZZ(); + assertEquals(PbjReader.DATA_ENCODING, reader.error()); + reader.resetWith(malformed); + reader.readVarLongBytes(); + assertEquals(PbjReader.DATA_ENCODING, reader.error()); + } + + @Test + void skipInternalSuccessfullySkipsFromInputStream() { + InputStream in = new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5}); + PbjReader reader = new PbjReader(in); + reader.skip(3); + assertEquals(3, reader.position()); + assertEquals(4, reader.readByte()); + } + + @Test + void skipInternalSuccessfullySkipsFromReadableSequentialData() { + BufferedData bd = BufferedData.allocate(8); + for (int i = 0; i < 5; i++) { + bd.writeByte((byte) i); + } + bd.flip(); + PbjReader reader = new PbjReader(bd); + reader.skip(3); + assertEquals(3, reader.position()); + assertEquals(3, reader.readByte()); + } + + @Test + void skipInternalSetsIOErrorWhenInputStreamSkipThrows() { + InputStream in = new InputStream() { + @Override + public int read() { + return -1; + } + + @Override + public long skip(long n) throws IOException { + throw new IOException("skip failed"); + } + }; + PbjReader reader = new PbjReader(in); + reader.skip(5); + assertEquals(PbjReader.IO_ERROR, reader.error()); + } + + @Test + void skipInternalCallsUnderlyingInputSkipForReadableSequentialData() { + BufferedData bd = BufferedData.allocate(8); + for (byte b = 0; b < 8; b++) bd.writeByte(b); + bd.flip(); + PbjReader reader = new PbjReader(bd); + reader.skip(3); + assertEquals(3, reader.position()); + assertEquals(3, bd.position()); + assertEquals(3, reader.readByte()); + } + + @Test + void readVarLongBytesReturnsWrappedBytesForValidVarint() { + PbjReader reader = new PbjReader(new byte[] {(byte) 0xAC, 0x02}); + Bytes result = reader.readVarLongBytes(); + assertEquals(2, result.length()); + assertEquals((byte) 0xAC, result.getByte(0)); + assertEquals((byte) 0x02, result.getByte(1)); + assertEquals(0, reader.error()); + } + + @Test + void readLargeStringCantBeBuffered() { + int len = (16 << 10) + 1; + String str = "a".repeat(len); + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag(str); + PbjReader reader = writer.toPbjReader(); + assertEquals(len, reader.readVarIntNoZZ()); + assertEquals(0, reader.error()); + } + + @Test + void readStringBufferedInternalSuccessPath() { + String str = "a".repeat(16384); + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag(str); + PbjReader reader = new PbjReader(new ByteArrayInputStream(writer.toByteArray())); + assertEquals(str, reader.readString(16384 + 10)); + assertEquals(0, reader.error()); + } + + @Test + void readStringBufferedInternalSuccessPathLarger() { + String str = "a".repeat(16385); + PbjWriter writer = new PbjWriter(); + writer.writeStringWithTag(str); + PbjReader reader = new PbjReader(new ByteArrayInputStream(writer.toByteArray())); + assertEquals(str, reader.readString(16385 + 10)); + assertEquals(0, reader.error()); + } + + @Test + void readFromInputSetsIOErrorWhenInputStreamReadThrows() { + InputStream failingOnSecondRead = new InputStream() { + private boolean doThrow = false; + + @Override + public int read() { + return 1; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (doThrow) throw new IOException("io error"); + doThrow = true; + b[off] = 25; + return 1; + } + }; + PbjReader reader = new PbjReader(failingOnSecondRead); + reader.readByte(); + assertEquals(PbjReader.IO_ERROR, reader.error()); + } + + @Test + void readStringLengthExceedsMaxSize() { + var reader = new PbjReader(new byte[] {5, 'h', 'e', 'l', 'l', 'o'}); + assertEquals("", reader.readString(2)); + assertEquals(PbjReader.PARSE, reader.error()); + } + + @Test + void readStringNegativeLengthSetsParseError() { + var reader = new PbjReader(new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, 0x0F}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.PARSE, reader.error()); + } + + @Test + void readStringInvalidUtf8SetParseError() { + var reader = new PbjReader(new byte[] {1, (byte) 0x80}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.PARSE, reader.error()); + } + + @Test + void readStringRejectsOverlong2ByteEncoding() { + var reader = new PbjReader(new byte[] {2, (byte) 0xC0, (byte) 0xAF}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.PARSE, reader.error()); + } + + @Test + void readStringRejectsOverlong3ByteEncoding() { + var reader = new PbjReader(new byte[] {3, (byte) 0xE0, (byte) 0x80, (byte) 0xAF}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.PARSE, reader.error()); + } + + @Test + void readStringRejectsOverlong4ByteEncoding() { + var reader = new PbjReader(new byte[] {4, (byte) 0xF0, (byte) 0x80, (byte) 0x80, (byte) 0xAF}); + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.PARSE, reader.error()); + } + + @Test + void readStringReturnsEmptyOnInsufficientData() { + var reader = new PbjReader(new byte[] {5, 'a', 'b'}); // length=5 but only 2 bytes follow + assertEquals("", reader.readString(Long.MAX_VALUE)); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + ///// Reset ///// + + private static final byte[] DATA = {1, 2, 3, 4, 5}; + + @Test + void resetWithByteArraySequentialData_readsCorrectBytes() throws ParseException { + ReadableSequentialData byteArraySeq = Bytes.wrap(DATA).toReadableSequentialData(); + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + + reader.resetWith(byteArraySeq); + + for (byte expected : DATA) { + assertEquals(expected, reader.readByte()); + } + reader.throwOnError(); + } + + @Test + void resetWithNonByteArraySequentialData_readsCorrectBytes() throws ParseException { + ReadableSequentialData buffered = BufferedData.wrap(DATA); + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + + reader.resetWith(buffered); + + for (byte expected : DATA) { + assertEquals(expected, reader.readByte()); + } + reader.throwOnError(); + } + + @Test + void resetWithInputStream_readsCorrectBytes() throws ParseException { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + + reader.resetWith(new ByteArrayInputStream(DATA)); + + for (byte expected : DATA) { + assertEquals(expected, reader.readByte()); + } + reader.throwOnError(); + } + + @Test + void resetWithNull_throwsOnError() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[0])); + reader.resetWith((ReadableSequentialData) null); + assertThrows(ParseException.class, reader::throwOnError); + + PbjReader reader2 = new PbjReader(new ByteArrayInputStream(new byte[0])); + reader2.resetWith((ReadableSequentialData) null); + assertThrows(ParseException.class, reader2::throwOnError); + } + + @Test + void resetFromBytesToStream() { + var reader = new PbjReader(new byte[] {1}); + assertEquals((byte) 1, reader.readByte()); + reader.resetWith(new ByteArrayInputStream(new byte[] {7, 8})); + assertEquals((byte) 7, reader.readByte()); + assertEquals((byte) 8, reader.readByte()); + assertEquals(0, reader.error()); + } + + // + + @Test + void asInputStreamReturnsUnderlyingStreamIfNeverRead() { + var stream = new ByteArrayInputStream(new byte[] {1, 2, 3}); + var reader = new PbjReader(stream); + assertSame(stream, reader.asInputStream()); + } + + @Test + void asInputStreamDelegatesToInputIfNeverRead() throws Exception { + var innerStream = new ByteArrayInputStream(new byte[] {21}); + var reader = new PbjReader(new ReadableStreamingData(innerStream)); + var is = reader.asInputStream(); + assertNotNull(is); + assertEquals(21, is.read()); + } + + @Test + void asInputStreamForByteArrayReader() throws Exception { + var reader = new PbjReader(new byte[] {1, 2, 3}); + var is = reader.asInputStream(); + assertEquals(1, is.read()); + assertEquals(2, is.read()); + assertEquals(3, is.read()); + } + + @Test + void readIntLESlowPathSuccess() { + var reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4})); + assertEquals(0x04030201, reader.readIntLE()); + assertEquals(0, reader.error()); + } + + @Test + void readLongLESlowPathSuccess() { + var reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5, 6, 7, 8})); + assertEquals(0x0807060504030201L, reader.readLongLE()); + assertEquals(0, reader.error()); + } + + @Test + void readIntLEUnderflow() { + var reader = new PbjReader(new byte[] {1, 2, 3}); // only 3 bytes, need 4 + assertEquals(0, reader.readIntLE()); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void readLongLEUnderflow() { + var reader = new PbjReader(new byte[] {1, 2, 3, 4, 5, 6, 7}); // only 7 bytes, need 8 + assertEquals(0L, reader.readLongLE()); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + ///// readBytes ///// + + @Test + void readBytesArrayOffsetLen_writesIntoCorrectSlice() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4, 5}); + byte[] dst = new byte[7]; + long n = reader.readBytes(dst, 2, 3); + assertEquals(3, n); + assertArrayEquals(new byte[] {0, 0, 1, 2, 3, 0, 0}, dst); + } + + @Test + void readBytesIntoByteBuffer_dataReadAndPositionAdvances() { + PbjReader reader = new PbjReader(new byte[] {10, 20, 30}); + ByteBuffer bb = ByteBuffer.allocate(5); + long n = reader.readBytes(bb); + assertEquals(3, n); + assertEquals(3, bb.position()); + assertArrayEquals(new byte[] {10, 20, 30, 0, 0}, bb.array()); + } + + @Test + void readBytesIntoByteBuffer_positionUnchangedWhenError() { + PbjReader reader = new PbjReader(new byte[] {1, 2}); + reader.skip(10); + assertTrue(reader.error() > 0); + ByteBuffer bb = ByteBuffer.allocate(5); + long n = reader.readBytes(bb); + assertEquals(-1, n); + assertEquals(0, bb.position()); + } + + @Test + void readBytesInt_fastPath_bytesBuffered() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3, 4, 5}); + Bytes result = reader.readBytes(3); + assertEquals(3, result.length()); + assertArrayEquals(new byte[] {1, 2, 3}, result.toByteArray()); + assertEquals(0, reader.error()); + } + + @Test + void readBytesInt_slowPath_triggersReadBytesInternal() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {7, 8, 9})); + Bytes result = reader.readBytes(3); + assertEquals(3, result.length()); + assertArrayEquals(new byte[] {7, 8, 9}, result.toByteArray()); + assertEquals(0, reader.error()); + } + + @Test + void readBytesInternal_zeroLength_returnsEmpty() { + PbjReader reader = new PbjReader(new byte[] {1, 2}); + reader.skip(10); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + assertSame(Bytes.EMPTY, reader.readBytes(2)); + } + + @Test + void readBytesInternal_notEnoughData_setsBufferUnderflow() { + PbjReader reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2})); + Bytes result = reader.readBytes(5); + assertSame(Bytes.EMPTY, result); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void readLongBEInternal_streamingReaderSucceeds() { + byte[] bytes = {0, 0, 0, 0, 0, 0, 0, 7}; + PbjReader reader = new PbjReader(new ByteArrayInputStream(bytes)); + assertEquals(7L, reader.readLongBE()); + assertEquals(0, reader.error()); + } + + @Test + void readLongBEInternal_notEnoughData_setsBufferUnderflow() { + PbjReader reader = new PbjReader(new byte[] {1, 2, 3}); + reader.readLongBE(); + assertEquals(PbjReader.BUFFER_UNDERFLOW, reader.error()); + } + + @Test + void readBytesNegativeLengthSetsIllegalArgument() { + var reader = new PbjReader(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5})); + reader.readByte(); + reader.readByte(); + reader.limit(0); + var result = reader.readBytes(-1); + assertEquals(Bytes.EMPTY, result); + assertEquals(PbjReader.ILLEGAL_ARGUMENT, reader.error()); + } +}