Add benchmark for deduplication as well as docs on options

Author: Genbox

Src/FastData.Benchmarks/Benchmarks/DeduplicationBenchmarks.cs

@@ -0,0 +1,43 @@
+namespace Genbox.FastData.Benchmarks.Benchmarks;
+
+[MemoryDiagnoser]
+public class DeduplicationBenchmarks
+{
+    private int[] _intKeys = [];
+    private string[] _stringKeys = [];
+
+    [Params(DeduplicationMode.Disabled, DeduplicationMode.HashSet, DeduplicationMode.Sort)]
+    public DeduplicationMode Mode { get; set; }
+
+    [GlobalSetup]
+    public void Setup()
+    {
+        Random rng = new Random(42);
+
+        _intKeys = new int[1000];
+        for (int i = 0; i < _intKeys.Length; i++)
+            _intKeys[i] = rng.Next(0, 200);
+
+        _stringKeys = new string[1000];
+        for (int i = 0; i < _stringKeys.Length; i++)
+            _stringKeys[i] = "key" + rng.Next(0, 200);
+    }
+
+    [Benchmark]
+    public void IntKeys()
+    {
+        FastDataConfig cfg = new FastDataConfig();
+        cfg.DeduplicationMode = Mode;
+
+        FastDataGenerator.DeduplicateKeys(cfg, _intKeys, ReadOnlyMemory<int>.Empty, EqualityComparer<int>.Default, Comparer<int>.Default, out _, out _, out int _);
+    }
+
+    [Benchmark]
+    public void StringKeys()
+    {
+        FastDataConfig cfg = new FastDataConfig();
+        cfg.DeduplicationMode = Mode;
+
+        FastDataGenerator.DeduplicateKeys(cfg, _stringKeys, ReadOnlyMemory<int>.Empty, StringComparer.Ordinal, StringComparer.Ordinal, out _, out _, out int _);
+    }
+}

Src/FastData/DeduplicationMode.cs

@@ -2,9 +2,28 @@ namespace Genbox.FastData;
 
 public enum DeduplicationMode : byte
 {
+    /// <summary>
+    /// No deduplication is performed. However, if there is a duplicate in the input, it might cause undefined behavior.
+    /// </summary>
     Disabled = 0,
+
+    /// <summary>
+    /// Uses a hash set to deduplicate data. It is faster than sorting, but uses more memory. It does not change the order of keys.
+    /// </summary>
     HashSet,
+
+    /// <summary>
+    /// Same as <seealso cref="HashSet"/>, but throws an exception when it finds a duplicate.
+    /// </summary>
     HashSetThrowOnDup,
+
+    /// <summary>
+    /// Uses sorting to deduplicate data. It is not as fast as <seealso cref="HashSet"/>, but it uses about half the memory. As a side effect, it changes the order of keys, which might be a desired side effect under certain circumstances.
+    /// </summary>
     Sort,
+
+    /// <summary>
+    /// Same as <seealso cref="Sort"/>, but throws an exception when it finds a duplicate.
+    /// </summary>
     SortThrowOnDup
 }

Src/FastData/FastDataGenerator.cs

@@ -260,7 +260,7 @@ private static string GenerateNumericInternal<TKey, TValue>(ReadOnlyMemory<TKey>
         }
     }
 
-    private static void DeduplicateKeys<TKey, TValue>(FastDataConfig fdCfg, ReadOnlyMemory<TKey> keys, ReadOnlyMemory<TValue> values, IEqualityComparer<TKey> equalityComparer, IComparer<TKey> sortComparer, out ReadOnlyMemory<TKey> newKeys, out ReadOnlyMemory<TValue> newValues, out int uniqueCount)
+    internal static void DeduplicateKeys<TKey, TValue>(FastDataConfig fdCfg, ReadOnlyMemory<TKey> keys, ReadOnlyMemory<TValue> values, IEqualityComparer<TKey> equalityComparer, IComparer<TKey> sortComparer, out ReadOnlyMemory<TKey> newKeys, out ReadOnlyMemory<TValue> newValues, out int uniqueCount)
     {
         if (fdCfg.DeduplicationMode == DeduplicationMode.Disabled)
         {
@@ -518,4 +518,4 @@ private static void Benchmark(byte[] data, int iterations, Candidate candidate)
 
     private readonly record struct TempStringState<TKey, TValue>(ReadOnlyMemory<TKey> Keys, ReadOnlyMemory<TValue> Values, FastDataConfig Config, ICodeGenerator Generator, StringKeyProperties StringKeyProperties, HashDetails HashDetails, string TrimPrefix, string TrimSuffix);
     private readonly record struct TempNumericState<TKey, TValue>(ReadOnlyMemory<TKey> Keys, ReadOnlyMemory<TValue> Values, FastDataConfig Config, ICodeGenerator Generator, NumericKeyProperties<TKey> NumericKeyProperties, HashDetails HashDetails, KeyType KeyType);
-}
+}