Skip to content

Commit 0df4357

Browse files
GenboxGenbox
committed
Support encoding reduction
1 parent 5f875e0 commit 0df4357

47 files changed

Lines changed: 313 additions & 248 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ There are many benefits gained from generating data structures at compile time:
222222
* No virtual dispatching (virtual method calls & inheritance) and no unnecessary branching
223223
* Modulo operations are known constants and compilers optimize it to bitwise operations
224224
* Data can be stored in smaller data types (e.g. `byte` instead of `int`) in values permit it
225+
* Data can be encoding reduced. That is, if all characters are ASCII, they can be stored as single bytes, which saves memory and improves performance.
225226

226227
### Data analysis
227228

‎Src/FastData.Generator.CPlusPlus.Benchmarks/Program.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,13 @@ private static void Main()
5252
private static string PrintQueries(ITestData data, string identifier)
5353
{
5454
CPlusPlusLanguageDef langDef = new CPlusPlusLanguageDef();
55-
TypeHelper helper = new TypeHelper(new TypeMap(langDef.TypeDefinitions));
55+
TypeMap map = new TypeMap(langDef.TypeDefinitions, langDef.Encoding);
5656

5757
StringBuilder sb = new StringBuilder();
5858

5959
for (int i = 0; i < 25; i++)
6060
{
61-
sb.AppendLine(CultureInfo.InvariantCulture, $" DoNotOptimize({identifier}::contains({data.GetValueLabel(helper)}));");
61+
sb.AppendLine(CultureInfo.InvariantCulture, $" DoNotOptimize({identifier}::contains({data.GetValueLabel(map)}));");
6262
}
6363

6464
return sb.ToString();

‎Src/FastData.Generator.CPlusPlus.Tests/VectorTests.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
using System.Diagnostics.CodeAnalysis;
2+
using Genbox.FastData.Enums;
23
using Genbox.FastData.Generator.CPlusPlus.Internal.Framework;
34
using Genbox.FastData.Generator.CPlusPlus.Shared;
5+
using Genbox.FastData.Generator.Extensions;
46
using Genbox.FastData.Generator.Framework;
7+
using Genbox.FastData.Generators;
58
using Genbox.FastData.InternalShared;
69
using Genbox.FastData.InternalShared.TestClasses;
710
using Genbox.FastData.InternalShared.TestClasses.TheoryData;
@@ -25,7 +28,7 @@ await Verify(spec.Source)
2528
.DisableDiff();
2629

2730
CPlusPlusLanguageDef langDef = new CPlusPlusLanguageDef();
28-
TypeHelper helper = new TypeHelper(new TypeMap(langDef.TypeDefinitions));
31+
TypeMap map = new TypeMap(langDef.TypeDefinitions, spec.Flags.HasFlag(GeneratorFlags.AllAreASCII) ? GeneratorEncoding.ASCII : langDef.Encoding);
2932

3033
string source = $$"""
3134
#include <string>
@@ -36,7 +39,7 @@ await Verify(spec.Source)
3639
int main(int argc, char* argv[])
3740
{
3841
{{FormatList(data.Values, x => $"""
39-
if (!{spec.Identifier}::contains({helper.ToValueLabel(x)}))
42+
if (!{spec.Identifier}::contains({map.ToValueLabel(x)}))
4043
return false;
4144
""", "\n")}}
4245

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/ArrayStructure_String_100.verified.txt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@
88

99
class ArrayStructure_String_100 final
1010
{
11-
static constexpr std::array<std::u32string_view, 100> entries = {
12-
U"0", U"1", U"2", U"3", U"4", U"5", U"6", U"7", U"8", U"9",
13-
U"10", U"11", U"12", U"13", U"14", U"15", U"16", U"17", U"18", U"19",
14-
U"20", U"21", U"22", U"23", U"24", U"25", U"26", U"27", U"28", U"29",
15-
U"30", U"31", U"32", U"33", U"34", U"35", U"36", U"37", U"38", U"39",
16-
U"40", U"41", U"42", U"43", U"44", U"45", U"46", U"47", U"48", U"49",
17-
U"50", U"51", U"52", U"53", U"54", U"55", U"56", U"57", U"58", U"59",
18-
U"60", U"61", U"62", U"63", U"64", U"65", U"66", U"67", U"68", U"69",
19-
U"70", U"71", U"72", U"73", U"74", U"75", U"76", U"77", U"78", U"79",
20-
U"80", U"81", U"82", U"83", U"84", U"85", U"86", U"87", U"88", U"89",
21-
U"90", U"91", U"92", U"93", U"94", U"95", U"96", U"97", U"98", U"99"
11+
static constexpr std::array<std::string_view, 100> entries = {
12+
"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
13+
"10", "11", "12", "13", "14", "15", "16", "17", "18", "19",
14+
"20", "21", "22", "23", "24", "25", "26", "27", "28", "29",
15+
"30", "31", "32", "33", "34", "35", "36", "37", "38", "39",
16+
"40", "41", "42", "43", "44", "45", "46", "47", "48", "49",
17+
"50", "51", "52", "53", "54", "55", "56", "57", "58", "59",
18+
"60", "61", "62", "63", "64", "65", "66", "67", "68", "69",
19+
"70", "71", "72", "73", "74", "75", "76", "77", "78", "79",
20+
"80", "81", "82", "83", "84", "85", "86", "87", "88", "89",
21+
"90", "91", "92", "93", "94", "95", "96", "97", "98", "99"
2222
};
2323

2424
public:
2525
[[nodiscard]]
26-
static bool contains(const std::u32string_view value) noexcept
26+
static bool contains(const std::string_view value) noexcept
2727
{
2828
if (const size_t len = value.length(); len < 1u || len > 2u)
2929
return false;

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/ArrayStructure_String_3.verified.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@
88

99
class ArrayStructure_String_3 final
1010
{
11-
static constexpr std::array<std::u32string_view, 3> entries = {
12-
U"a", U"item", U"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
11+
static constexpr std::array<std::string_view, 3> entries = {
12+
"a", "item", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1313
};
1414

1515
public:
1616
[[nodiscard]]
17-
static bool contains(const std::u32string_view value) noexcept
17+
static bool contains(const std::string_view value) noexcept
1818
{
1919
if (const size_t len = value.length(); len < 1u || len > 255u)
2020
return false;

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/BinarySearchStructure_String_100.verified.txt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@
88

99
class BinarySearchStructure_String_100 final
1010
{
11-
static constexpr std::array<std::u32string_view, 100> entries = {
12-
U"0", U"1", U"10", U"11", U"12", U"13", U"14", U"15", U"16", U"17",
13-
U"18", U"19", U"2", U"20", U"21", U"22", U"23", U"24", U"25", U"26",
14-
U"27", U"28", U"29", U"3", U"30", U"31", U"32", U"33", U"34", U"35",
15-
U"36", U"37", U"38", U"39", U"4", U"40", U"41", U"42", U"43", U"44",
16-
U"45", U"46", U"47", U"48", U"49", U"5", U"50", U"51", U"52", U"53",
17-
U"54", U"55", U"56", U"57", U"58", U"59", U"6", U"60", U"61", U"62",
18-
U"63", U"64", U"65", U"66", U"67", U"68", U"69", U"7", U"70", U"71",
19-
U"72", U"73", U"74", U"75", U"76", U"77", U"78", U"79", U"8", U"80",
20-
U"81", U"82", U"83", U"84", U"85", U"86", U"87", U"88", U"89", U"9",
21-
U"90", U"91", U"92", U"93", U"94", U"95", U"96", U"97", U"98", U"99"
11+
static constexpr std::array<std::string_view, 100> entries = {
12+
"0", "1", "10", "11", "12", "13", "14", "15", "16", "17",
13+
"18", "19", "2", "20", "21", "22", "23", "24", "25", "26",
14+
"27", "28", "29", "3", "30", "31", "32", "33", "34", "35",
15+
"36", "37", "38", "39", "4", "40", "41", "42", "43", "44",
16+
"45", "46", "47", "48", "49", "5", "50", "51", "52", "53",
17+
"54", "55", "56", "57", "58", "59", "6", "60", "61", "62",
18+
"63", "64", "65", "66", "67", "68", "69", "7", "70", "71",
19+
"72", "73", "74", "75", "76", "77", "78", "79", "8", "80",
20+
"81", "82", "83", "84", "85", "86", "87", "88", "89", "9",
21+
"90", "91", "92", "93", "94", "95", "96", "97", "98", "99"
2222
};
2323

2424
public:
2525
[[nodiscard]]
26-
static bool contains(const std::u32string_view value) noexcept
26+
static bool contains(const std::string_view value) noexcept
2727
{
2828
if (const size_t len = value.length(); len < 1u || len > 2u)
2929
return false;

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/BinarySearchStructure_String_3.verified.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@
88

99
class BinarySearchStructure_String_3 final
1010
{
11-
static constexpr std::array<std::u32string_view, 3> entries = {
12-
U"a", U"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", U"item"
11+
static constexpr std::array<std::string_view, 3> entries = {
12+
"a", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "item"
1313
};
1414

1515
public:
1616
[[nodiscard]]
17-
static bool contains(const std::u32string_view value) noexcept
17+
static bool contains(const std::string_view value) noexcept
1818
{
1919
if (const size_t len = value.length(); len < 1u || len > 255u)
2020
return false;

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/ConditionalStructure_String_100.verified.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@ class ConditionalStructure_String_100 final
1010
{
1111
public:
1212
[[nodiscard]]
13-
static bool contains(const std::u32string_view value) noexcept
13+
static bool contains(const std::string_view value) noexcept
1414
{
1515
if (const size_t len = value.length(); len < 1u || len > 2u)
1616
return false;
1717

18-
if (value == U"0" || value == U"1" || value == U"2" || value == U"3" || value == U"4" || value == U"5" || value == U"6" || value == U"7" || value == U"8" || value == U"9" || value == U"10" || value == U"11" || value == U"12" || value == U"13" || value == U"14" || value == U"15" || value == U"16" || value == U"17" || value == U"18" || value == U"19" || value == U"20" || value == U"21" || value == U"22" || value == U"23" || value == U"24" || value == U"25" || value == U"26" || value == U"27" || value == U"28" || value == U"29" || value == U"30" || value == U"31" || value == U"32" || value == U"33" || value == U"34" || value == U"35" || value == U"36" || value == U"37" || value == U"38" || value == U"39" || value == U"40" || value == U"41" || value == U"42" || value == U"43" || value == U"44" || value == U"45" || value == U"46" || value == U"47" || value == U"48" || value == U"49" || value == U"50" || value == U"51" || value == U"52" || value == U"53" || value == U"54" || value == U"55" || value == U"56" || value == U"57" || value == U"58" || value == U"59" || value == U"60" || value == U"61" || value == U"62" || value == U"63" || value == U"64" || value == U"65" || value == U"66" || value == U"67" || value == U"68" || value == U"69" || value == U"70" || value == U"71" || value == U"72" || value == U"73" || value == U"74" || value == U"75" || value == U"76" || value == U"77" || value == U"78" || value == U"79" || value == U"80" || value == U"81" || value == U"82" || value == U"83" || value == U"84" || value == U"85" || value == U"86" || value == U"87" || value == U"88" || value == U"89" || value == U"90" || value == U"91" || value == U"92" || value == U"93" || value == U"94" || value == U"95" || value == U"96" || value == U"97" || value == U"98" || value == U"99")
18+
if (value == "0" || value == "1" || value == "2" || value == "3" || value == "4" || value == "5" || value == "6" || value == "7" || value == "8" || value == "9" || value == "10" || value == "11" || value == "12" || value == "13" || value == "14" || value == "15" || value == "16" || value == "17" || value == "18" || value == "19" || value == "20" || value == "21" || value == "22" || value == "23" || value == "24" || value == "25" || value == "26" || value == "27" || value == "28" || value == "29" || value == "30" || value == "31" || value == "32" || value == "33" || value == "34" || value == "35" || value == "36" || value == "37" || value == "38" || value == "39" || value == "40" || value == "41" || value == "42" || value == "43" || value == "44" || value == "45" || value == "46" || value == "47" || value == "48" || value == "49" || value == "50" || value == "51" || value == "52" || value == "53" || value == "54" || value == "55" || value == "56" || value == "57" || value == "58" || value == "59" || value == "60" || value == "61" || value == "62" || value == "63" || value == "64" || value == "65" || value == "66" || value == "67" || value == "68" || value == "69" || value == "70" || value == "71" || value == "72" || value == "73" || value == "74" || value == "75" || value == "76" || value == "77" || value == "78" || value == "79" || value == "80" || value == "81" || value == "82" || value == "83" || value == "84" || value == "85" || value == "86" || value == "87" || value == "88" || value == "89" || value == "90" || value == "91" || value == "92" || value == "93" || value == "94" || value == "95" || value == "96" || value == "97" || value == "98" || value == "99")
1919
return true;
2020

2121
return false;

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/ConditionalStructure_String_3.verified.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ class ConditionalStructure_String_3 final
1010
{
1111
public:
1212
[[nodiscard]]
13-
static bool contains(const std::u32string_view value) noexcept
13+
static bool contains(const std::string_view value) noexcept
1414
{
1515

1616

17-
if (value == U"a" || value == U"item" || value == U"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
17+
if (value == "a" || value == "item" || value == "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
1818
return true;
1919

2020
return false;

‎Src/FastData.Generator.CPlusPlus.Tests/Verify/EytzingerSearchStructure_String_100.verified.txt‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@
88

99
class EytzingerSearchStructure_String_100 final
1010
{
11-
static constexpr std::array<std::u32string_view, 100> entries = {
12-
U"66", U"37", U"85", U"22", U"51", U"78", U"92", U"15", U"3", U"44",
13-
U"59", U"73", U"81", U"89", U"96", U"11", U"19", U"26", U"33", U"40",
14-
U"48", U"55", U"62", U"7", U"76", U"8", U"83", U"87", U"90", U"94",
15-
U"98", U"1", U"13", U"17", U"20", U"24", U"28", U"31", U"35", U"39",
16-
U"42", U"46", U"5", U"53", U"57", U"60", U"64", U"68", U"71", U"75",
17-
U"77", U"79", U"80", U"82", U"84", U"86", U"88", U"9", U"91", U"93",
18-
U"95", U"97", U"99", U"0", U"10", U"12", U"14", U"16", U"18", U"2",
19-
U"21", U"23", U"25", U"27", U"29", U"30", U"32", U"34", U"36", U"38",
20-
U"4", U"41", U"43", U"45", U"47", U"49", U"50", U"52", U"54", U"56",
21-
U"58", U"6", U"61", U"63", U"65", U"67", U"69", U"70", U"72", U"74"
11+
static constexpr std::array<std::string_view, 100> entries = {
12+
"66", "37", "85", "22", "51", "78", "92", "15", "3", "44",
13+
"59", "73", "81", "89", "96", "11", "19", "26", "33", "40",
14+
"48", "55", "62", "7", "76", "8", "83", "87", "90", "94",
15+
"98", "1", "13", "17", "20", "24", "28", "31", "35", "39",
16+
"42", "46", "5", "53", "57", "60", "64", "68", "71", "75",
17+
"77", "79", "80", "82", "84", "86", "88", "9", "91", "93",
18+
"95", "97", "99", "0", "10", "12", "14", "16", "18", "2",
19+
"21", "23", "25", "27", "29", "30", "32", "34", "36", "38",
20+
"4", "41", "43", "45", "47", "49", "50", "52", "54", "56",
21+
"58", "6", "61", "63", "65", "67", "69", "70", "72", "74"
2222
};
2323

2424
public:
2525
[[nodiscard]]
26-
static bool contains(const std::u32string_view value) noexcept
26+
static bool contains(const std::string_view value) noexcept
2727
{
2828
if (const size_t len = value.length(); len < 1u || len > 2u)
2929
return false;

0 commit comments

Comments
 (0)