From 47c28c4b85313f4e5c446cfcbb01717eafe6a7c5 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Thu, 3 Sep 2026 19:20:43 +0100 Subject: [PATCH] Fix DividingCoder buffer size under-allocation for large inputs Compute reduction factor as a double (log2(L)/8) and use Ceil for safe encode/decode sizes instead of a truncated 1000-scaled integer, which lost precision and under-allocated the decode buffer on long, high-value strings. Add standalone DividingCoderTests covering edge-buffer roundtrips (0xFF, zero, leading-zero patterns) and a size-sufficiency regression guard across Base58/Base62/Base36; update the Base58 safe-char-count expectation. --- .../SimpleBaseLib.Tests.Mobile.dpr | 3 +- .../SimpleBaseLib.Tests.Mobile.dproj | 1 + .../Delphi.Tests/SimpleBaseLib.Tests.dpr | 3 +- .../FreePascal.Tests/SimpleBaseLib.Tests.lpi | 10 +- .../FreePascal.Tests/SimpleBaseLib.lpr | 2 +- .../FreePascal.Tests/SimpleBaseLibConsole.lpi | 6 +- .../FreePascal.Tests/SimpleBaseLibConsole.lpr | 2 +- .../src/Base58/Base58Tests.pas | 2 +- .../src/Coders/DividingCoderTests.pas | 173 ++++++++++++++++++ SimpleBaseLib/src/Coders/SbpDividingCoder.pas | 8 +- 10 files changed, 197 insertions(+), 13 deletions(-) create mode 100644 SimpleBaseLib.Tests/src/Coders/DividingCoderTests.pas diff --git a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dpr b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dpr index 313c837..672b2a4 100644 --- a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dpr +++ b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dpr @@ -98,7 +98,8 @@ uses Base8Tests in '..\src\Base8\Base8Tests.pas', Base8StreamRegressionTests in '..\src\Base8\Base8StreamRegressionTests.pas', Base64Tests in '..\src\Base64\Base64Tests.pas', - Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas'; + Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas', + DividingCoderTests in '..\src\Coders\DividingCoderTests.pas'; begin Application.Initialize; diff --git a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dproj b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dproj index 858083e..2a33279 100644 --- a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dproj +++ b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dproj @@ -439,6 +439,7 @@ + Base diff --git a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr index de555e8..0aeaee4 100644 --- a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr +++ b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr @@ -115,7 +115,8 @@ uses Base8Tests in '..\src\Base8\Base8Tests.pas', Base8StreamRegressionTests in '..\src\Base8\Base8StreamRegressionTests.pas', Base64Tests in '..\src\Base64\Base64Tests.pas', - Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas'; + Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas', + DividingCoderTests in '..\src\Coders\DividingCoderTests.pas'; begin diff --git a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.Tests.lpi b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.Tests.lpi index a908a26..49a0c10 100644 --- a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.Tests.lpi +++ b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.Tests.lpi @@ -23,7 +23,7 @@ - + @@ -71,7 +71,7 @@ - + @@ -173,6 +173,10 @@ + + + + @@ -183,7 +187,7 @@ - + diff --git a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.lpr b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.lpr index 732fcfd..5b48b35 100644 --- a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.lpr +++ b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.lpr @@ -11,7 +11,7 @@ EncodingAlphabetTests, CodingAlphabetTests, BitsTests, Base8Tests, Base8StreamRegressionTests, Base62Tests, Base45Tests, Base45StreamRegressionTests, Base2StreamRegressionTests, Base2Tests, - Base10Tests, Base36Tests; + Base10Tests, Base36Tests, DividingCoderTests; {$R *.res} diff --git a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpi b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpi index e2103ff..a7c9c40 100644 --- a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpi +++ b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpi @@ -131,6 +131,10 @@ + + + + @@ -141,7 +145,7 @@ - + diff --git a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpr b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpr index 500906d..cbbf0b7 100644 --- a/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpr +++ b/SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpr @@ -12,7 +12,7 @@ Base8Tests, Base8StreamRegressionTests, Base85Tests, Base85StreamRegressionTests, EncodingAlphabetTests, CodingAlphabetTests, BitsTests, MultibaseTests, - Base64Tests, Base64StreamRegressionTests; + Base64Tests, Base64StreamRegressionTests, DividingCoderTests; type diff --git a/SimpleBaseLib.Tests/src/Base58/Base58Tests.pas b/SimpleBaseLib.Tests/src/Base58/Base58Tests.pas index 24ba5ed..d70223b 100644 --- a/SimpleBaseLib.Tests/src/Base58/Base58Tests.pas +++ b/SimpleBaseLib.Tests/src/Base58/Base58Tests.pas @@ -533,7 +533,7 @@ procedure TTestBase58.Test_Alphabet_GetSafeCharCountForEncoding_Works; LInput: TSimpleBaseLibByteArray; begin LInput := TSimpleBaseLibByteArray.Create(0, 0, 0, 0, 1, 2, 3, 4); - CheckEquals(10, TBase58.Bitcoin.GetSafeCharCountForEncoding(LInput)); + CheckEquals(11, TBase58.Bitcoin.GetSafeCharCountForEncoding(LInput)); end; procedure TTestBase58.Test_Instances_AreIsolated; diff --git a/SimpleBaseLib.Tests/src/Coders/DividingCoderTests.pas b/SimpleBaseLib.Tests/src/Coders/DividingCoderTests.pas new file mode 100644 index 0000000..790e5d6 --- /dev/null +++ b/SimpleBaseLib.Tests/src/Coders/DividingCoderTests.pas @@ -0,0 +1,173 @@ +unit DividingCoderTests; + +{$IFDEF FPC} +{$MODE DELPHI} +{$HINTS OFF} +{$WARNINGS OFF} +{$ENDIF FPC} + +interface + +uses + SysUtils, + Math, +{$IFDEF FPC} + fpcunit, + testregistry, +{$ELSE} + TestFramework, +{$ENDIF FPC} + SbpIDividingCoder, + SbpBase58, + SbpBase62, + SbpBase36, + SbpArrayUtilities, + SbpSimpleBaseLibTypes, + SimpleBaseLibTestBase; + +type + TTestDividingCoder = class(TSimpleBaseLibTestCase) + strict private + procedure CheckRoundtrip(const ACoder: IDividingCoder; const AName: String; + const ABytes: TSimpleBaseLibByteArray); + procedure CheckEdgeBuffers(const ACoder: IDividingCoder; const AName: String); + published + procedure Test_Base58_Roundtrip_EdgeBuffers; + procedure Test_Base62_Roundtrip_EdgeBuffers; + procedure Test_Base36_Roundtrip_EdgeBuffers; + procedure Test_SafeSizes_NeverUnderAllocate; + end; + +implementation + +function MakeFilled(ALen: Int32; AValue: Byte): TSimpleBaseLibByteArray; +begin + System.SetLength(Result, ALen); + if ALen > 0 then + TArrayUtilities.Fill(Result, 0, ALen, AValue); +end; + +function MakeLeadingZeros(AZeroLen, ADataLen: Int32): TSimpleBaseLibByteArray; +var + LI: Int32; +begin + System.SetLength(Result, AZeroLen + ADataLen); + for LI := 0 to AZeroLen - 1 do + Result[LI] := 0; + for LI := 0 to ADataLen - 1 do + Result[AZeroLen + LI] := Byte(LI + 1); +end; + +procedure TTestDividingCoder.CheckRoundtrip(const ACoder: IDividingCoder; + const AName: String; const ABytes: TSimpleBaseLibByteArray); +var + LEncoded: String; + LDecoded, LDecodeBuf: TSimpleBaseLibByteArray; + LEncodeBuf: TSimpleBaseLibCharArray; + LCharsWritten, LBytesWritten, LSafeChars, LSafeBytes: Int32; +begin + // allocating API + LEncoded := ACoder.Encode(ABytes); + LDecoded := ACoder.Decode(LEncoded); + CheckTrue(AreEqual(ABytes, LDecoded), AName + ' Encode/Decode roundtrip failed'); + + // non-allocating API within reported safe sizes + LSafeChars := ACoder.GetSafeCharCountForEncoding(ABytes); + CheckTrue(LSafeChars >= System.Length(LEncoded), + AName + ' GetSafeCharCountForEncoding under-allocated'); + System.SetLength(LEncodeBuf, LSafeChars); + CheckTrue(ACoder.TryEncode(ABytes, LEncodeBuf, LCharsWritten), + AName + ' TryEncode failed'); + CheckEquals(LEncoded, CharsToString(LEncodeBuf, LCharsWritten), + AName + ' TryEncode output mismatch'); + + LSafeBytes := ACoder.GetSafeByteCountForDecoding(LEncoded); + CheckTrue(LSafeBytes >= System.Length(LDecoded), + AName + ' GetSafeByteCountForDecoding under-allocated'); + System.SetLength(LDecodeBuf, LSafeBytes); + CheckTrue(ACoder.TryDecode(LEncoded, LDecodeBuf, LBytesWritten), + AName + ' TryDecode failed'); + CheckTrue(AreEqual(ABytes, System.Copy(LDecodeBuf, 0, LBytesWritten)), + AName + ' TryDecode output mismatch'); +end; + +procedure TTestDividingCoder.CheckEdgeBuffers(const ACoder: IDividingCoder; + const AName: String); +var + LLens: TSimpleBaseLibInt32Array; + LI: Int32; +begin + LLens := TSimpleBaseLibInt32Array.Create(1, 2, 7, 32, 255, 256, 1024, 4096); + for LI := 0 to High(LLens) do + begin + CheckRoundtrip(ACoder, AName + ' 0xFF', MakeFilled(LLens[LI], $FF)); + CheckRoundtrip(ACoder, AName + ' zeros', MakeFilled(LLens[LI], 0)); + CheckRoundtrip(ACoder, AName + ' leading-zeros', + MakeLeadingZeros(LLens[LI], LLens[LI])); + end; +end; + +procedure TTestDividingCoder.Test_Base58_Roundtrip_EdgeBuffers; +begin + CheckEdgeBuffers(TBase58.Bitcoin, 'Base58.Bitcoin'); +end; + +procedure TTestDividingCoder.Test_Base62_Roundtrip_EdgeBuffers; +begin + CheckEdgeBuffers(TBase62.Default, 'Base62.Default'); +end; + +procedure TTestDividingCoder.Test_Base36_Roundtrip_EdgeBuffers; +begin + CheckEdgeBuffers(TBase36.LowerCase, 'Base36.LowerCase'); +end; + +procedure TTestDividingCoder.Test_SafeSizes_NeverUnderAllocate; +const + Lens: array[0..6] of Int32 = (1, 100, 1000, 10000, 20000, 50000, 100000); + + procedure CheckOne(const ACoder: IDividingCoder; const AName: String); + var + LI, LN, LLen, LMinBytes, LMinChars: Int32; + LMaxChar: Char; + LMaxText: String; + LBytes: TSimpleBaseLibByteArray; + LBitsPerChar: Double; + LAlpha: String; + begin + LAlpha := ACoder.Alphabet.Value; + LMaxChar := LAlpha[System.Length(LAlpha)]; + LBitsPerChar := Log2(ACoder.Alphabet.Length); + for LI := 0 to High(Lens) do + begin + LN := Lens[LI]; + // longest value expressible in LN chars needs Ceil(LN * log2(L) / 8) bytes + LMaxText := StringOfChar(LMaxChar, LN); + LMinBytes := Ceil(LN * LBitsPerChar / 8); + CheckTrue(ACoder.GetSafeByteCountForDecoding(LMaxText) >= LMinBytes, + AName + ' decode size under-allocated at len ' + IntToStr(LN)); + + // longest value in LN bytes needs Ceil(LN * 8 / log2(L)) chars + LLen := LN; + LBytes := MakeFilled(LLen, $FF); + LMinChars := Ceil(LLen * 8 / LBitsPerChar); + CheckTrue(ACoder.GetSafeCharCountForEncoding(LBytes) >= LMinChars, + AName + ' encode size under-allocated at len ' + IntToStr(LN)); + end; + end; + +begin + CheckOne(TBase58.Bitcoin, 'Base58.Bitcoin'); + CheckOne(TBase62.Default, 'Base62.Default'); + CheckOne(TBase36.LowerCase, 'Base36.LowerCase'); +end; + +initialization + +{$IFDEF FPC} + RegisterTest(TTestDividingCoder); +{$ELSE} + RegisterTest(TTestDividingCoder.Suite); +{$ENDIF FPC} + +end. diff --git a/SimpleBaseLib/src/Coders/SbpDividingCoder.pas b/SimpleBaseLib/src/Coders/SbpDividingCoder.pas index 5b6c2fd..4c0434f 100644 --- a/SimpleBaseLib/src/Coders/SbpDividingCoder.pas +++ b/SimpleBaseLib/src/Coders/SbpDividingCoder.pas @@ -52,7 +52,7 @@ TRangeWritten = record var FAlphabet: ICodingAlphabet; - FReductionFactor: Int32; + FReductionFactor: Double; FZeroChar: Char; function GetSafeByteCountForDecodingInternal(ATextLen: Int32; AZeroPrefixLen: Int32): Int32; inline; @@ -106,7 +106,7 @@ constructor TDividingCoder.Create(const AAlphabet: ICodingAlphabet); inherited Create; FAlphabet := AAlphabet; LAlphabetValue := FAlphabet.Value; - FReductionFactor := Trunc(1000 * Log2(System.Length(LAlphabetValue)) / 8); + FReductionFactor := Log2(System.Length(LAlphabetValue)) / 8; FZeroChar := LAlphabetValue[1]; end; @@ -134,13 +134,13 @@ class function TDividingCoder.CountPrefixChars( function TDividingCoder.GetSafeByteCountForDecodingInternal( ATextLen: Int32; AZeroPrefixLen: Int32): Int32; begin - Result := AZeroPrefixLen + ((ATextLen - AZeroPrefixLen) * FReductionFactor div 1000) + 1; + Result := AZeroPrefixLen + Ceil((ATextLen - AZeroPrefixLen) * FReductionFactor) + 1; end; function TDividingCoder.GetSafeCharCountForEncodingInternal( ABytesLen: Int32; AZeroPrefixLen: Int32): Int32; begin - Result := AZeroPrefixLen + ((ABytesLen - AZeroPrefixLen) * 1000 div FReductionFactor) + 1; + Result := AZeroPrefixLen + Ceil((ABytesLen - AZeroPrefixLen) / FReductionFactor) + 1; end; function TDividingCoder.GetSafeByteCountForDecoding(