Skip to content

Fix DividingCoder buffer size under-allocation for large inputs #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Xor-el merged 1 commit into from
Sep 3, 2026
+197 −13
Merged

Fix DividingCoder buffer size under-allocation for large inputs #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dpr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,8 @@ uses
Base8Tests in '..\src\Base8\Base8Tests.pas',
Base8StreamRegressionTests in '..\src\Base8\Base8StreamRegressionTests.pas',
Base64Tests in '..\src\Base64\Base64Tests.pas',
Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas';
Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas',
DividingCoderTests in '..\src\Coders\DividingCoderTests.pas';

begin
Application.Initialize;
Expand Down
1 change: 1 addition & 0 deletions SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.Mobile.dproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -439,6 +439,7 @@
<DCCReference Include="..\src\Base8\Base8StreamRegressionTests.pas"/>
<DCCReference Include="..\src\Base64\Base64Tests.pas"/>
<DCCReference Include="..\src\Base64\Base64StreamRegressionTests.pas"/>
<DCCReference Include="..\src\Coders\DividingCoderTests.pas"/>
<BuildConfiguration Include="Base">
<Key>Base</Key>
</BuildConfiguration>
Expand Down
3 changes: 2 additions & 1 deletion SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,8 @@ uses
Base8Tests in '..\src\Base8\Base8Tests.pas',
Base8StreamRegressionTests in '..\src\Base8\Base8StreamRegressionTests.pas',
Base64Tests in '..\src\Base64\Base64Tests.pas',
Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas';
Base64StreamRegressionTests in '..\src\Base64\Base64StreamRegressionTests.pas',
DividingCoderTests in '..\src\Coders\DividingCoderTests.pas';

begin

Expand Down
10 changes: 7 additions & 3 deletions SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.Tests.lpi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
<PathDelim Value="\"/>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<OtherUnitFiles Value="..\src\Base16;..\src\Base32;..\src\Base58;..\src\Base64;..\src\Base85;..\src;..\src\Multibase;..\src\Misc;..\src\Base8;..\src\Base62;..\src\Base45;..\src\Base2;..\src\Base10;..\src\Base36"/>
<OtherUnitFiles Value="..\src\Base16;..\src\Base32;..\src\Base58;..\src\Base64;..\src\Base85;..\src;..\src\Multibase;..\src\Misc;..\src\Base8;..\src\Base62;..\src\Base45;..\src\Base2;..\src\Base10;..\src\Base36;..\src\Coders"/>
</SearchPaths>
<Parsing>
<SyntaxOptions>
Expand Down Expand Up @@ -71,7 +71,7 @@
<PackageName Value="FCL"/>
</Item4>
</RequiredPackages>
<Units Count="25">
<Units Count="26">
<Unit0>
<Filename Value="SimpleBaseLib.lpr"/>
<IsPartOfProject Value="True"/>
Expand Down Expand Up @@ -173,6 +173,10 @@
<Filename Value="..\src\Base36\Base36Tests.pas"/>
<IsPartOfProject Value="True"/>
</Unit24>
<Unit25>
<Filename Value="..\src\Coders\DividingCoderTests.pas"/>
<IsPartOfProject Value="True"/>
</Unit25>
</Units>
</ProjectOptions>
<CompilerOptions>
Expand All @@ -183,7 +187,7 @@
</Target>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<OtherUnitFiles Value="..\src\Base16;..\src\Base32;..\src\Base58;..\src\Base64;..\src\Base85;..\src;..\src\Multibase;..\src\Misc;..\src\Base8;..\src\Base62;..\src\Base45;..\src\Base2;..\src\Base10;..\src\Base36"/>
<OtherUnitFiles Value="..\src\Base16;..\src\Base32;..\src\Base58;..\src\Base64;..\src\Base85;..\src;..\src\Multibase;..\src\Misc;..\src\Base8;..\src\Base62;..\src\Base45;..\src\Base2;..\src\Base10;..\src\Base36;..\src\Coders"/>
</SearchPaths>
<Linking>
<Debugging>
Expand Down
2 changes: 1 addition & 1 deletion SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLib.lpr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
EncodingAlphabetTests, CodingAlphabetTests, BitsTests, Base8Tests,
Base8StreamRegressionTests, Base62Tests, Base45Tests,
Base45StreamRegressionTests, Base2StreamRegressionTests, Base2Tests,
Base10Tests, Base36Tests;
Base10Tests, Base36Tests, DividingCoderTests;

{$R *.res}

Expand Down
6 changes: 5 additions & 1 deletion SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpi
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@
<Filename Value="..\src\Base36\Base36Tests.pas"/>
<IsPartOfProject Value="True"/>
</Unit>
<Unit>
<Filename Value="..\src\Coders\DividingCoderTests.pas"/>
<IsPartOfProject Value="True"/>
</Unit>
</Units>
</ProjectOptions>
<CompilerOptions>
Expand All @@ -141,7 +145,7 @@
</Target>
<SearchPaths>
<IncludeFiles Value="$(ProjOutDir)"/>
<OtherUnitFiles Value="..\src;..\src\Base64;..\src\Multibase;..\src\Misc;..\src\Base85;..\src\Base8;..\src\Base62;..\src\Base58;..\src\Base45;..\src\Base32;..\src\Base2;..\src\Base16;..\src\Base10;..\src\Base36"/>
<OtherUnitFiles Value="..\src;..\src\Base64;..\src\Multibase;..\src\Misc;..\src\Base85;..\src\Base8;..\src\Base62;..\src\Base58;..\src\Base45;..\src\Base32;..\src\Base2;..\src\Base16;..\src\Base10;..\src\Base36;..\src\Coders"/>
<UnitOutputDirectory Value="lib\$(TargetCPU)-$(TargetOS)"/>
</SearchPaths>
<Other>
Expand Down
2 changes: 1 addition & 1 deletion SimpleBaseLib.Tests/FreePascal.Tests/SimpleBaseLibConsole.lpr
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
Base8Tests, Base8StreamRegressionTests, Base85Tests,
Base85StreamRegressionTests,
EncodingAlphabetTests, CodingAlphabetTests, BitsTests, MultibaseTests,
Base64Tests, Base64StreamRegressionTests;
Base64Tests, Base64StreamRegressionTests, DividingCoderTests;

type

Expand Down
2 changes: 1 addition & 1 deletion SimpleBaseLib.Tests/src/Base58/Base58Tests.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -533,7 +533,7 @@ procedure TTestBase58.Test_Alphabet_GetSafeCharCountForEncoding_Works;
LInput: TSimpleBaseLibByteArray;
begin
LInput := TSimpleBaseLibByteArray.Create(0, 0, 0, 0, 1, 2, 3, 4);
CheckEquals(10, TBase58.Bitcoin.GetSafeCharCountForEncoding(LInput));
CheckEquals(11, TBase58.Bitcoin.GetSafeCharCountForEncoding(LInput));
end;

procedure TTestBase58.Test_Instances_AreIsolated;
Expand Down
173 changes: 173 additions & 0 deletions SimpleBaseLib.Tests/src/Coders/DividingCoderTests.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
unit DividingCoderTests;

{$IFDEF FPC}
{$MODE DELPHI}
{$HINTS OFF}
{$WARNINGS OFF}
{$ENDIF FPC}

interface

uses
SysUtils,
Math,
{$IFDEF FPC}
fpcunit,
testregistry,
{$ELSE}
TestFramework,
{$ENDIF FPC}
SbpIDividingCoder,
SbpBase58,
SbpBase62,
SbpBase36,
SbpArrayUtilities,
SbpSimpleBaseLibTypes,
SimpleBaseLibTestBase;

type
TTestDividingCoder = class(TSimpleBaseLibTestCase)
strict private
procedure CheckRoundtrip(const ACoder: IDividingCoder; const AName: String;
const ABytes: TSimpleBaseLibByteArray);
procedure CheckEdgeBuffers(const ACoder: IDividingCoder; const AName: String);
published
procedure Test_Base58_Roundtrip_EdgeBuffers;
procedure Test_Base62_Roundtrip_EdgeBuffers;
procedure Test_Base36_Roundtrip_EdgeBuffers;
procedure Test_SafeSizes_NeverUnderAllocate;
end;

implementation

function MakeFilled(ALen: Int32; AValue: Byte): TSimpleBaseLibByteArray;
begin
System.SetLength(Result, ALen);
if ALen > 0 then
TArrayUtilities.Fill<Byte>(Result, 0, ALen, AValue);
end;

function MakeLeadingZeros(AZeroLen, ADataLen: Int32): TSimpleBaseLibByteArray;
var
LI: Int32;
begin
System.SetLength(Result, AZeroLen + ADataLen);
for LI := 0 to AZeroLen - 1 do
Result[LI] := 0;
for LI := 0 to ADataLen - 1 do
Result[AZeroLen + LI] := Byte(LI + 1);
end;

procedure TTestDividingCoder.CheckRoundtrip(const ACoder: IDividingCoder;
const AName: String; const ABytes: TSimpleBaseLibByteArray);
var
LEncoded: String;
LDecoded, LDecodeBuf: TSimpleBaseLibByteArray;
LEncodeBuf: TSimpleBaseLibCharArray;
LCharsWritten, LBytesWritten, LSafeChars, LSafeBytes: Int32;
begin
// allocating API
LEncoded := ACoder.Encode(ABytes);
LDecoded := ACoder.Decode(LEncoded);
CheckTrue(AreEqual(ABytes, LDecoded), AName + ' Encode/Decode roundtrip failed');

// non-allocating API within reported safe sizes
LSafeChars := ACoder.GetSafeCharCountForEncoding(ABytes);
CheckTrue(LSafeChars >= System.Length(LEncoded),
AName + ' GetSafeCharCountForEncoding under-allocated');
System.SetLength(LEncodeBuf, LSafeChars);
CheckTrue(ACoder.TryEncode(ABytes, LEncodeBuf, LCharsWritten),
AName + ' TryEncode failed');
CheckEquals(LEncoded, CharsToString(LEncodeBuf, LCharsWritten),
AName + ' TryEncode output mismatch');

LSafeBytes := ACoder.GetSafeByteCountForDecoding(LEncoded);
CheckTrue(LSafeBytes >= System.Length(LDecoded),
AName + ' GetSafeByteCountForDecoding under-allocated');
System.SetLength(LDecodeBuf, LSafeBytes);
CheckTrue(ACoder.TryDecode(LEncoded, LDecodeBuf, LBytesWritten),
AName + ' TryDecode failed');
CheckTrue(AreEqual(ABytes, System.Copy(LDecodeBuf, 0, LBytesWritten)),
AName + ' TryDecode output mismatch');
end;

procedure TTestDividingCoder.CheckEdgeBuffers(const ACoder: IDividingCoder;
const AName: String);
var
LLens: TSimpleBaseLibInt32Array;
LI: Int32;
begin
LLens := TSimpleBaseLibInt32Array.Create(1, 2, 7, 32, 255, 256, 1024, 4096);
for LI := 0 to High(LLens) do
begin
CheckRoundtrip(ACoder, AName + ' 0xFF', MakeFilled(LLens[LI], $FF));
CheckRoundtrip(ACoder, AName + ' zeros', MakeFilled(LLens[LI], 0));
CheckRoundtrip(ACoder, AName + ' leading-zeros',
MakeLeadingZeros(LLens[LI], LLens[LI]));
end;
end;

procedure TTestDividingCoder.Test_Base58_Roundtrip_EdgeBuffers;
begin
CheckEdgeBuffers(TBase58.Bitcoin, 'Base58.Bitcoin');
end;

procedure TTestDividingCoder.Test_Base62_Roundtrip_EdgeBuffers;
begin
CheckEdgeBuffers(TBase62.Default, 'Base62.Default');
end;

procedure TTestDividingCoder.Test_Base36_Roundtrip_EdgeBuffers;
begin
CheckEdgeBuffers(TBase36.LowerCase, 'Base36.LowerCase');
end;

procedure TTestDividingCoder.Test_SafeSizes_NeverUnderAllocate;
const
Lens: array[0..6] of Int32 = (1, 100, 1000, 10000, 20000, 50000, 100000);

procedure CheckOne(const ACoder: IDividingCoder; const AName: String);
var
LI, LN, LLen, LMinBytes, LMinChars: Int32;
LMaxChar: Char;
LMaxText: String;
LBytes: TSimpleBaseLibByteArray;
LBitsPerChar: Double;
LAlpha: String;
begin
LAlpha := ACoder.Alphabet.Value;
LMaxChar := LAlpha[System.Length(LAlpha)];
LBitsPerChar := Log2(ACoder.Alphabet.Length);
for LI := 0 to High(Lens) do
begin
LN := Lens[LI];
// longest value expressible in LN chars needs Ceil(LN * log2(L) / 8) bytes
LMaxText := StringOfChar(LMaxChar, LN);
LMinBytes := Ceil(LN * LBitsPerChar / 8);
CheckTrue(ACoder.GetSafeByteCountForDecoding(LMaxText) >= LMinBytes,
AName + ' decode size under-allocated at len ' + IntToStr(LN));

// longest value in LN bytes needs Ceil(LN * 8 / log2(L)) chars
LLen := LN;
LBytes := MakeFilled(LLen, $FF);
LMinChars := Ceil(LLen * 8 / LBitsPerChar);
CheckTrue(ACoder.GetSafeCharCountForEncoding(LBytes) >= LMinChars,
AName + ' encode size under-allocated at len ' + IntToStr(LN));
end;
end;

begin
CheckOne(TBase58.Bitcoin, 'Base58.Bitcoin');
CheckOne(TBase62.Default, 'Base62.Default');
CheckOne(TBase36.LowerCase, 'Base36.LowerCase');
end;

initialization

{$IFDEF FPC}
RegisterTest(TTestDividingCoder);
{$ELSE}
RegisterTest(TTestDividingCoder.Suite);
{$ENDIF FPC}

end.
8 changes: 4 additions & 4 deletions SimpleBaseLib/src/Coders/SbpDividingCoder.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ TRangeWritten = record

var
FAlphabet: ICodingAlphabet;
FReductionFactor: Int32;
FReductionFactor: Double;
FZeroChar: Char;

function GetSafeByteCountForDecodingInternal(ATextLen: Int32; AZeroPrefixLen: Int32): Int32; inline;
Expand Down Expand Up @@ -106,7 +106,7 @@ constructor TDividingCoder.Create(const AAlphabet: ICodingAlphabet);
inherited Create;
FAlphabet := AAlphabet;
LAlphabetValue := FAlphabet.Value;
FReductionFactor := Trunc(1000 * Log2(System.Length(LAlphabetValue)) / 8);
FReductionFactor := Log2(System.Length(LAlphabetValue)) / 8;
FZeroChar := LAlphabetValue[1];
end;

Expand Down Expand Up @@ -134,13 +134,13 @@ class function TDividingCoder.CountPrefixChars(
function TDividingCoder.GetSafeByteCountForDecodingInternal(
ATextLen: Int32; AZeroPrefixLen: Int32): Int32;
begin
Result := AZeroPrefixLen + ((ATextLen - AZeroPrefixLen) * FReductionFactor div 1000) + 1;
Result := AZeroPrefixLen + Ceil((ATextLen - AZeroPrefixLen) * FReductionFactor) + 1;
end;

function TDividingCoder.GetSafeCharCountForEncodingInternal(
ABytesLen: Int32; AZeroPrefixLen: Int32): Int32;
begin
Result := AZeroPrefixLen + ((ABytesLen - AZeroPrefixLen) * 1000 div FReductionFactor) + 1;
Result := AZeroPrefixLen + Ceil((ABytesLen - AZeroPrefixLen) / FReductionFactor) + 1;
end;

function TDividingCoder.GetSafeByteCountForDecoding(
Expand Down