Add SOUNDEX and DIFFERENCE scalar functions for phonetic string matching (#18293)

Author: xiangfu0

pinot-common/src/main/java/org/apache/pinot/common/function/scalar/StringFunctions.java

@@ -26,6 +26,7 @@
 import java.util.Base64;
 import java.util.UUID;
 import javax.annotation.Nullable;
+import org.apache.commons.codec.language.Soundex;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.pinot.common.utils.URIUtils;
 import org.apache.pinot.spi.annotations.ScalarFunction;
@@ -44,6 +45,8 @@ public class StringFunctions {
   private StringFunctions() {
   }
 
+  private static final Soundex SOUNDEX = new Soundex();
+
   /**
    * @see StringUtils#reverse(String)
    * @param input
@@ -908,4 +911,37 @@ public static boolean isJson(String inputStr) {
       return false;
     }
   }
+
+  /**
+   * Returns the Soundex code for a string. Empty string returns "0000" (SQL standard behaviour).
+   */
+  @Nullable
+  @ScalarFunction(nullableParameters = true)
+  public static String soundex(@Nullable String input) {
+    if (input == null) {
+      return null;
+    }
+    if (input.isEmpty()) {
+      return "0000";
+    }
+    return SOUNDEX.soundex(input);
+  }
+
+  /**
+   * Returns an integer 0-4 indicating how similar two strings sound based on their Soundex codes.
+   * 4 means the codes are identical; 0 means they share no common code characters.
+   * The framework null-propagates when either argument is null.
+   */
+  @ScalarFunction
+  public static int difference(String input1, String input2) {
+    String code1 = soundex(input1);
+    String code2 = soundex(input2);
+    int matches = 0;
+    for (int i = 0; i < 4; i++) {
+      if (code1.charAt(i) == code2.charAt(i)) {
+        matches++;
+      }
+    }
+    return matches;
+  }
 }

pinot-common/src/test/java/org/apache/pinot/common/function/scalar/StringFunctionsTest.java

@@ -24,6 +24,7 @@
 import org.testng.annotations.Test;
 
 import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNull;
 
 
 public class StringFunctionsTest {
@@ -349,6 +350,27 @@ public void testHammingDistance() {
     assertEquals(StringFunctions.levenshteinDistance("cat", "cats"), 1); // Levenshtein can handle different lengths
   }
 
+  @Test
+  public void testSoundex() {
+    assertEquals(StringFunctions.soundex("Robert"), "R163");
+    assertEquals(StringFunctions.soundex("Rupert"), "R163");
+    assertEquals(StringFunctions.soundex("Ashcraft"), "A261");
+    // Empty string returns SQL-standard fallback code
+    assertEquals(StringFunctions.soundex(""), "0000");
+    assertNull(StringFunctions.soundex(null));
+  }
+
+  @Test
+  public void testDifference() {
+    assertEquals(StringFunctions.difference("Robert", "Rupert"), 4);
+    assertEquals(StringFunctions.difference("Smith", "Johnson"), 1);
+    assertEquals(StringFunctions.difference("Ann", "Ann"), 4);
+    // "0000" vs "0000" — both encode to the standard empty fallback, all 4 positions match
+    assertEquals(StringFunctions.difference("", ""), 4);
+    // "R163" vs "0000" — first characters differ, no positions match
+    assertEquals(StringFunctions.difference("Robert", ""), 0);
+  }
+
   @Test
   public void encodeUrl() {
     assertEquals(StringFunctions.encodeUrl(""), "");