Add more robust deduplication

Genbox · Genbox · commit cd28633163d2 · 2025-12-28T23:56:51.000+01:00
diff --git a/Src/FastData.Tests/FastDataGeneratorTests.cs b/Src/FastData.Tests/FastDataGeneratorTests.cs
@@ -9,11 +9,82 @@ namespace Genbox.FastData.Tests;
 
 public class FastDataGeneratorTests
 {
-    [Fact]
-    public void Generate_ThrowOnDuplicate()
+    [Theory]
+    [InlineData(DeduplicationMode.HashSetThrowOnDup)]
+    [InlineData(DeduplicationMode.SortThrowOnDup)]
+    public void Generate_ThrowOnDuplicate(DeduplicationMode mode)
     {
         FastDataConfig config = new FastDataConfig();
+        config.DeduplicationMode = mode;
+
         Assert.Throws<InvalidOperationException>(() => FastDataGenerator.Generate(["item", "item"], config, new DummyGenerator()));
+        Assert.Throws<InvalidOperationException>(() => FastDataGenerator.Generate([1, 2, 2], config, new DummyGenerator()));
+    }
+
+    [Theory]
+    [InlineData(DeduplicationMode.Disabled)]
+    [InlineData(DeduplicationMode.HashSet)]
+    [InlineData(DeduplicationMode.Sort)]
+    public void Generate_NoThrowOnDuplicates(DeduplicationMode mode)
+    {
+        FastDataConfig config = new FastDataConfig();
+        config.DeduplicationMode = mode;
+
+        FastDataGenerator.Generate(["item", "item"], config, new DummyGenerator());
+        FastDataGenerator.Generate([1, 2, 2], config, new DummyGenerator());
+    }
+
+    [Theory]
+    [InlineData(DeduplicationMode.HashSet)]
+    [InlineData(DeduplicationMode.Sort)]
+    public void GenerateKeyed_StringDeduplication_RemovesDuplicates(DeduplicationMode mode)
+    {
+        string[] keys = ["b", "a", "b", "c"];
+        string[] values = ["vb", "va", "vb", "vc"];
+
+        FastDataConfig config = new FastDataConfig(StructureType.Array);
+        config.DeduplicationMode = mode;
+
+        ContextCaptureGenerator generator = new ContextCaptureGenerator();
+        FastDataGenerator.GenerateKeyed(keys, values, config, generator);
+
+        ArrayContext<string, string> ctx = Assert.IsType<ArrayContext<string, string>>(generator.Context);
+        if (mode == DeduplicationMode.HashSet)
+        {
+            Assert.True(ctx.Keys.Span.SequenceEqual(["b", "a", "c"]));
+            Assert.True(ctx.Values.Span.SequenceEqual(["vb", "va", "vc"]));
+        }
+        else
+        {
+            Assert.True(ctx.Keys.Span.SequenceEqual(["a", "b", "c"]));
+            Assert.True(ctx.Values.Span.SequenceEqual(["va", "vb", "vc"]));
+        }
+    }
+
+    [Theory]
+    [InlineData(DeduplicationMode.HashSet)]
+    [InlineData(DeduplicationMode.Sort)]
+    public void GenerateKeyed_NumericDeduplication_RemovesDuplicates(DeduplicationMode mode)
+    {
+        int[] keys = [3, 1, 3, 2];
+        string[] values = ["v3", "v1", "v3", "v2"];
+        FastDataConfig config = new FastDataConfig(StructureType.Array);
+        config.DeduplicationMode = mode;
+        ContextCaptureGenerator generator = new ContextCaptureGenerator();
+
+        FastDataGenerator.GenerateKeyed(keys, values, config, generator);
+
+        ArrayContext<int, string> ctx = Assert.IsType<ArrayContext<int, string>>(generator.Context);
+        if (mode == DeduplicationMode.HashSet)
+        {
+            Assert.True(ctx.Keys.Span.SequenceEqual([3, 1, 2]));
+            Assert.True(ctx.Values.Span.SequenceEqual(["v3", "v1", "v2"]));
+        }
+        else
+        {
+            Assert.True(ctx.Keys.Span.SequenceEqual([1, 2, 3]));
+            Assert.True(ctx.Values.Span.SequenceEqual(["v1", "v2", "v3"]));
+        }
     }
 
     [Fact]
diff --git a/Src/FastData/DeduplicationMode.cs b/Src/FastData/DeduplicationMode.cs
@@ -0,0 +1,10 @@
+namespace Genbox.FastData;
+
+public enum DeduplicationMode : byte
+{
+    Disabled = 0,
+    HashSet,
+    HashSetThrowOnDup,
+    Sort,
+    SortThrowOnDup
+}
diff --git a/Src/FastData/FastDataConfig.cs b/Src/FastData/FastDataConfig.cs
@@ -9,6 +9,9 @@ public sealed class FastDataConfig(StructureType structureType = StructureType.A
     /// <summary>The type of structure to create. Defaults to Auto.</summary>
     public StructureType StructureType { get; set; } = structureType;
 
+    /// <summary>When true, duplicates will be eliminated from the input.</summary>
+    public DeduplicationMode DeduplicationMode { get; set; } = DeduplicationMode.HashSet;
+
     /// <summary>For hash-based structures, you can set this factor higher or lower to control how many slots are used. A factor higher than 1 will use more memory, but can improve performance by reducing collisions. A factor lower than 1 will use less memory, but can increase collisions and thus reduce performance.</summary>
     public int HashCapacityFactor { get; set; } = 1;
 
diff --git a/Src/FastData/FastDataGenerator.Logging.cs b/Src/FastData/FastDataGenerator.Logging.cs
@@ -5,8 +5,11 @@ namespace Genbox.FastData;
 
 public static partial class FastDataGenerator
 {
-    [LoggerMessage(LogLevel.Information, "There are {Count} unique items")]
-    internal static partial void LogUniqueItems(ILogger logger, int count);
+    [LoggerMessage(LogLevel.Information, "There are {Count} keys")]
+    internal static partial void LogNumberOfKeys(ILogger logger, int count);
+
+    [LoggerMessage(LogLevel.Information, "There are {Count} keys, but only {UniqueCount} unique")]
+    internal static partial void LogNumberOfUniqueKeys(ILogger logger, int count, int uniqueCount);
 
     [LoggerMessage(LogLevel.Information, "Data consists of {KeyType}")]
     internal static partial void LogKeyType(ILogger logger, KeyType keyType);
diff --git a/Src/FastData/FastDataGenerator.cs b/Src/FastData/FastDataGenerator.cs
@@ -76,24 +76,21 @@ private static string GenerateStringInternal<TValue>(ReadOnlyMemory<string> keys
         if (!values.IsEmpty && keys.Length != values.Length)
             throw new InvalidOperationException("The number of values does not match the number of keys.");
 
-        ReadOnlyMemory<string> keyMemory = keys;
-        ReadOnlySpan<string> keySpan = keyMemory.Span;
-
         factory ??= NullLoggerFactory.Instance;
 
-        //Validate that we only have unique data
-        HashSet<string> uniq = new HashSet<string>(StringComparer.Ordinal);
-
-        for (int i = 0; i < keySpan.Length; i++)
-        {
-            string val = keySpan[i];
-            if (!uniq.Add(val))
-                throw new InvalidOperationException($"Duplicate data found: {val}");
-        }
-
         ILogger logger = factory.CreateLogger(typeof(FastDataGenerator));
         LogUserStructureType(logger, fdCfg.StructureType);
-        LogUniqueItems(logger, uniq.Count);
+
+        int oldCount = keys.Length;
+
+        DeduplicateKeys(fdCfg, keys, values, StringComparer.Ordinal, StringComparer.Ordinal, out keys, out values, out int newCount);
+
+        if (oldCount == newCount)
+            LogNumberOfKeys(logger, newCount);
+        else
+            LogNumberOfUniqueKeys(logger, oldCount, newCount);
+
+        ReadOnlySpan<string> keySpan = keys.Span;
 
         const KeyType keyType = KeyType.String;
         LogKeyType(logger, keyType);
@@ -108,14 +105,14 @@ private static string GenerateStringInternal<TValue>(ReadOnlyMemory<string> keys
         {
             trimPrefix = strProps.DeltaData.Prefix;
             trimSuffix = strProps.DeltaData.Suffix;
-            keyMemory = SubStringKeys(keySpan, strProps);
-            keySpan = keyMemory.Span;
+            keys = SubStringKeys(keySpan, strProps);
+            keySpan = keys.Span;
         }
 
         LogMinMaxLength(logger, strProps.LengthData.LengthMap.Min, strProps.LengthData.LengthMap.Max);
 
         HashDetails hashDetails = new HashDetails();
-        TempStringState<string, TValue> tempState = new TempStringState<string, TValue>(keyMemory, values, fdCfg, generator, strProps, hashDetails, trimPrefix, trimSuffix);
+        TempStringState<string, TValue> tempState = new TempStringState<string, TValue>(keys, values, fdCfg, generator, strProps, hashDetails, trimPrefix, trimSuffix);
 
         switch (fdCfg.StructureType)
         {
@@ -189,28 +186,25 @@ private static string GenerateNumericInternal<TKey, TValue>(ReadOnlyMemory<TKey>
         if (!values.IsEmpty && keys.Length != values.Length)
             throw new InvalidOperationException("The number of values does not match the number of keys.");
 
-        ReadOnlySpan<TKey> keySpan = keys.Span;
-
         Type type = typeof(TKey);
 
         if (type != typeof(char) && type != typeof(sbyte) && type != typeof(byte) && type != typeof(short) && type != typeof(ushort) && type != typeof(int) && type != typeof(uint) && type != typeof(long) && type != typeof(ulong) && type != typeof(float) && type != typeof(double))
             throw new InvalidOperationException($"Unsupported data type: {type.Name}");
 
         factory ??= NullLoggerFactory.Instance;
+        ILogger logger = factory.CreateLogger(typeof(FastDataGenerator));
 
-        //Validate that we only have unique data
-        HashSet<TKey> uniq = new HashSet<TKey>();
+        int oldCount = keys.Length;
+        DeduplicateKeys(fdCfg, keys, values, EqualityComparer<TKey>.Default, Comparer<TKey>.Default, out keys, out values, out int newCount);
 
-        for (int i = 0; i < keySpan.Length; i++)
-        {
-            TKey key = keySpan[i];
-            if (!uniq.Add(key))
-                throw new InvalidOperationException($"Duplicate data found: {key}");
-        }
+        if (oldCount == newCount)
+            LogNumberOfKeys(logger, newCount);
+        else
+            LogNumberOfUniqueKeys(logger, oldCount, newCount);
+
+        ReadOnlySpan<TKey> keySpan = keys.Span;
 
-        ILogger logger = factory.CreateLogger(typeof(FastDataGenerator));
         LogUserStructureType(logger, fdCfg.StructureType);
-        LogUniqueItems(logger, uniq.Count);
 
         KeyType keyType = (KeyType)Enum.Parse(typeof(KeyType), type.Name, false);
         LogKeyType(logger, keyType);
@@ -266,6 +260,113 @@ private static string GenerateNumericInternal<TKey, TValue>(ReadOnlyMemory<TKey>
         }
     }
 
+    private static void DeduplicateKeys<TKey, TValue>(FastDataConfig fdCfg, ReadOnlyMemory<TKey> keys, ReadOnlyMemory<TValue> values, IEqualityComparer<TKey> equalityComparer, IComparer<TKey> sortComparer, out ReadOnlyMemory<TKey> newKeys, out ReadOnlyMemory<TValue> newValues, out int uniqueCount)
+    {
+        if (fdCfg.DeduplicationMode == DeduplicationMode.Disabled)
+        {
+            TKey[] keyCopy = new TKey[keys.Length];
+            keys.CopyTo(keyCopy);
+            newKeys = keyCopy;
+
+            TValue[] valueCopy = new TValue[values.Length];
+            values.CopyTo(valueCopy);
+            newValues = valueCopy;
+
+            uniqueCount = keyCopy.Length;
+            return;
+        }
+
+        ReadOnlySpan<TKey> keySpan = keys.Span;
+        ReadOnlySpan<TValue> valueSpan = values.Span;
+        bool hasValues = !values.IsEmpty;
+
+        if (fdCfg.DeduplicationMode is DeduplicationMode.HashSet or DeduplicationMode.HashSetThrowOnDup)
+        {
+            HashSet<TKey> uniq = new HashSet<TKey>(equalityComparer);
+            TKey[] keyCopy = new TKey[keys.Length];
+            TValue[] valueCopy = hasValues ? new TValue[values.Length] : [];
+
+            int offset = 0;
+            for (int i = 0; i < keySpan.Length; i++)
+            {
+                TKey key = keySpan[i];
+
+                if (!uniq.Add(key))
+                {
+                    if (fdCfg.DeduplicationMode == DeduplicationMode.HashSetThrowOnDup)
+                        throw new InvalidOperationException($"Duplicate key found: {key}");
+
+                    continue;
+                }
+
+                keyCopy[offset] = key;
+
+                if (hasValues)
+                    valueCopy[offset] = valueSpan[i];
+
+                offset++;
+            }
+
+            newKeys = keyCopy.AsMemory(0, offset);
+            newValues = hasValues ? valueCopy.AsMemory(0, offset) : ReadOnlyMemory<TValue>.Empty;
+            uniqueCount = offset;
+            return;
+        }
+
+        if (fdCfg.DeduplicationMode is DeduplicationMode.Sort or DeduplicationMode.SortThrowOnDup)
+        {
+            int[] map = new int[keys.Length];
+
+            for (int i = 0; i < keys.Length; i++)
+                map[i] = i;
+
+            TKey[] keyCopy = new TKey[keys.Length];
+            keys.CopyTo(keyCopy);
+            Array.Sort(keyCopy, map, sortComparer);
+
+            TValue[] valueCopy = hasValues ? new TValue[values.Length] : [];
+
+            // Handle the first key/value manually to avoid branching inside the for loop below
+            int firstIndex = map[0];
+            TKey last = keySpan[firstIndex]!;
+
+            keyCopy[0] = last;
+
+            if (hasValues)
+                valueCopy[0] = valueSpan[firstIndex];
+
+            int offset = 1;
+            for (int i = 1; i < keys.Length; i++)
+            {
+                int sourceIndex = map[i];
+                TKey key = keySpan[sourceIndex]!;
+
+                if (equalityComparer.Equals(key, last))
+                {
+                    if (fdCfg.DeduplicationMode == DeduplicationMode.SortThrowOnDup)
+                        throw new InvalidOperationException($"Duplicate key found: {key}");
+
+                    continue;
+                }
+
+                keyCopy[offset] = key;
+
+                if (hasValues)
+                    valueCopy[offset] = valueSpan[sourceIndex];
+
+                last = key;
+                offset++;
+            }
+
+            newKeys = keyCopy.AsMemory(0, offset);
+            newValues = hasValues ? valueCopy.AsMemory(0, offset) : ReadOnlyMemory<TValue>.Empty;
+            uniqueCount = offset;
+            return;
+        }
+
+        throw new InvalidOperationException("Unsupported deduplication mode: " + fdCfg.DeduplicationMode);
+    }
+
     internal static string[] SubStringKeys(ReadOnlySpan<string> keys, StringKeyProperties props)
     {
         int prefix = props.DeltaData.Prefix.Length;
