bundle UINT collating sequence

Author: mlin

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
     name: clang-format
     language: system
     files: \.(c|cc|cxx|cpp|h|hpp|hxx)$
-    exclude: json1\.c
+    exclude: (json1|uint)\.c
     verbose: true
     entry: clang-format
     args: [-i,'-style={IndentWidth: 4, ColumnLimit: 100, BreakStringLiterals: false, ReflowComments: false}']

CMakeLists.txt

@@ -36,7 +36,7 @@ execute_process(COMMAND git describe --tags --always --dirty
                 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGIT_REVISION=\"\\\"${GIT_REVISION}\\\"\"")
 
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
-add_library(genomicsqlite SHARED src/genomicsqlite.cc include/genomicsqlite.h src/json1.c src/hardcoded_refseq.hpp)
+add_library(genomicsqlite SHARED src/genomicsqlite.cc include/genomicsqlite.h src/json1.c src/uint.c src/hardcoded_refseq.hpp)
 if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     find_library(LIBZSTD_A NAMES libzstd.a REQUIRED)
     target_link_libraries(genomicsqlite PRIVATE SQLiteCpp ${LIBZSTD_A})

docs/guide.md

@@ -184,7 +184,7 @@ Range positions are considered [**zero-based & half-open**](http://www.cs.utexas
 The extension provides routines to populate a small `_gri_refseq` table describing the genomic reference sequences, which other tables can reference by integer ID ("rid") instead of storing a column with textual sequence names like 'chr10'. This convention is not required, as the GRI can index either chromosome name or rid columns, but reasons to observe it include:
 
 * Integers are more compact and faster to look up.
-* Results sort properly with `ORDER BY rid` instead of considering e.g. `'chr10'` < `'chr2'` lexicographically.
+* Results sort properly with `ORDER BY rid` instead of considering e.g. `'chr10'` < `'chr2'` lexicographically. (See also the UINT collating sequence, below)
 * A table with chromosome names can be reconstructed easily by joining with `_gri_refseq`.
 
 ### Create GRI
@@ -1102,9 +1102,11 @@ Given a two-bit-encoded BLOB value, return the length of the *decoded* sequence
 
 Given a TEXT value, return its byte length. Given NULL, return NULL. Any other input is an error.
 
-#### JSON functions
+#### JSON1 and UINT extensions
 
-The Genomics Extension bundles the SQLite developers' [JSON1 extension](https://www.sqlite.org/json1.html) and enables it automatically. By convention, JSON object columns should be named *_json and JSON array columns should be named *_jsarray. The JSON1 functions can be used with [generated columns](https://sqlite.org/gencol.html) to effectively allow indexing of JSON-embedded fields.
+The Genomics Extension bundles the SQLite developers' [JSON1 extension](https://www.sqlite.org/json1.html) and enables it automatically. By convention, JSON object columns should be named \*_json and JSON array columns should be named \*_jsarray. The JSON1 functions can be used with [generated columns](https://sqlite.org/gencol.html) to effectively allow indexing of JSON-embedded fields.
+
+The [UINT collating sequence](https://www.sqlite.org/uintcseq.html) is also bundled. This can be useful to make e.g. `ORDER BY chromosome COLLATE UINT` put 'chr2' before 'chr10'.
 
 #### Genomics Extension version
 

src/genomicsqlite.cc

@@ -1594,6 +1594,7 @@ static void sqlfn_parse_genomic_range_end(sqlite3_context *ctx, int argc, sqlite
  **************************************************************************************************/
 
 extern "C" int genomicsqliteJson1Register(sqlite3 *db);
+extern "C" int genomicsqlite_uint_init(sqlite3 *db);
 
 static int register_genomicsqlite_functions(sqlite3 *db, const char **pzErrMsg,
                                             const sqlite3_api_routines *pApi) {
@@ -1671,16 +1672,22 @@ static int register_genomicsqlite_functions(sqlite3 *db, const char **pzErrMsg,
         }
         return rc;
     }
-    // genomicsqliteJson1Register() may return SQLITE_BUSY if JSON1 (possibly another version
-    // thereof) is already loaded, and the extension is being loaded by SELECT load_extension().
-    // That is tolerable.
+    // other extensions may return SQLITE_BUSY if another version is already loaded; that is
+    // tolerable.
     rc = genomicsqliteJson1Register(db);
     if (rc != SQLITE_OK && rc != SQLITE_BUSY) {
         if (pzErrMsg) {
             *pzErrMsg = sqlite3_mprintf("Genomics Extension failed to register JSON1");
         }
         return rc;
     }
+    rc = genomicsqlite_uint_init(db);
+    if (rc != SQLITE_OK && rc != SQLITE_BUSY) {
+        if (pzErrMsg) {
+            *pzErrMsg = sqlite3_mprintf("Genomics Extension failed to register UINT collation");
+        }
+        return rc;
+    }
     return SQLITE_OK;
 }
 

src/uint.c

@@ -0,0 +1,97 @@
+/*
+** GenomicSQLite bundles the UINT extension: https://www.sqlite.org/uintcseq.html
+**
+** This is the (public domain) upstream source file from:
+**   https://sqlite.org/src/file/ext/misc/uint.c
+** with minor edits deferring to genomicsqlite.cc:sqlite3_genomicsqlite_init for extension init
+*/
+
+/*
+** 2020-04-14
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This SQLite extension implements the UINT collating sequence.
+**
+** UINT works like BINARY for text, except that embedded strings
+** of digits compare in numeric order.
+**
+**     *   Leading zeros are handled properly, in the sense that
+**         they do not mess of the maginitude comparison of embedded
+**         strings of digits.  "x00123y" is equal to "x123y".
+**
+**     *   Only unsigned integers are recognized.  Plus and minus
+**         signs are ignored.  Decimal points and exponential notation
+**         are ignored.
+**
+**     *   Embedded integers can be of arbitrary length.  Comparison
+**         is *not* limited integers that can be expressed as a
+**         64-bit machine integer.
+*/
+#include "sqlite3ext.h"
+/* instead of SQLITE_EXTENSION_INIT1, link to the one in genomicsqlite.cc: */
+extern const sqlite3_api_routines *sqlite3_api;
+#include <assert.h>
+#include <string.h>
+#include <ctype.h>
+
+/*
+** Compare text in lexicographic order, except strings of digits
+** compare in numeric order.
+*/
+static int uintCollFunc(
+  void *notUsed,
+  int nKey1, const void *pKey1,
+  int nKey2, const void *pKey2
+){
+  const unsigned char *zA = (const unsigned char*)pKey1;
+  const unsigned char *zB = (const unsigned char*)pKey2;
+  int i=0, j=0, x;
+  (void)notUsed;
+  while( i<nKey1 && j<nKey2 ){
+    x = zA[i] - zB[j];
+    if( isdigit(zA[i]) ){
+      int k;
+      if( !isdigit(zB[j]) ) return x;
+      while( i<nKey1 && zA[i]=='0' ){ i++; }
+      while( j<nKey2 && zB[j]=='0' ){ j++; }
+      k = 0;
+      while( i+k<nKey1 && isdigit(zA[i+k])
+             && j+k<nKey2 && isdigit(zB[j+k]) ){
+        k++;
+      }
+      if( i+k<nKey1 && isdigit(zA[i+k]) ){
+        return +1;
+      }else if( j+k<nKey2 && isdigit(zB[j+k]) ){
+        return -1;
+      }else{
+        x = memcmp(zA+i, zB+j, k);
+        if( x ) return x;
+        i += k;
+        j += k;
+      }
+    }else if( x ){
+      return x;
+    }else{
+      i++;
+      j++;
+    }
+  }
+  return (nKey1 - i) - (nKey2 - j);
+}
+
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int genomicsqlite_uint_init(
+  sqlite3 *db
+){
+  return sqlite3_create_collation(db, "uint", SQLITE_UTF8, 0, uintCollFunc);
+}

test/test_ext.py

@@ -0,0 +1,19 @@
+import genomicsqlite
+
+
+def test_bundled_extensions():
+    con = genomicsqlite.connect(":memory:")
+
+    con.executescript(
+        """
+        CREATE TABLE test(json TEXT);
+        INSERT INTO test(json) VALUES('{"chrom":"chr10"}');
+        INSERT INTO test(json) VALUES('{"chrom":"chr2"}');
+        INSERT INTO test(json) VALUES('{"chrom":"chrMT"}');
+        """
+    )
+
+    results = con.execute(
+        "SELECT json_extract(json,'$.chrom') AS chrom FROM test ORDER BY chrom COLLATE UINT"
+    )
+    assert list(results) == [("chr2",), ("chr10",), ("chrMT",)]