"safe mode" for range queries

Author: mlin

bindings/python/genomicsqlite.py

@@ -85,9 +85,12 @@ def genomic_range_rowids_sql(
     qrid: Optional[str] = None,
     qbeg: Optional[str] = None,
     qend: Optional[str] = None,
+    safe: bool = False,
 ) -> str:
     return _execute1(
-        conn, "SELECT genomic_range_rowids_sql(?,?,?,?)", (indexed_table, qrid, qbeg, qend)
+        conn,
+        f"SELECT genomic_range_rowids_{'safe_' if safe else ''}sql(?,?,?,?)",
+        (indexed_table, qrid, qbeg, qend),
     )
 
 

include/genomicsqlite.h

@@ -40,10 +40,32 @@ int genomicsqlite_open(const char *dbfile, sqlite3 **ppDb, int flags, const char
 char *genomicsqlite_vacuum_into_sql(const char *destfile, const char *config_json);
 
 /*
- * Genomic range indexing
+ * Generate SQL script to create a genomic range index on the specified table.
+ * rid: column name for the reference sequence (chromosome) ID of each row
+ * beg: column name or simple SQL expression for the interval begin position
+ * end: column name or simple SQL expression for the interval end position
  */
 char *create_genomic_range_index_sql(const char *table, const char *rid, const char *beg,
                                      const char *end, int max_depth);
+
+/*
+ * Generate parenthesized SELECT statement to query the existing genomic range index of the
+ * specified table. The query finds table rows which overlap the query range, producing one column
+ * of _rowid_'s.
+ * dbconn: If provided, the procedure first inspects the index to collect information that may
+ *         allow it to optimize the generated query SQL. Recommended if the generated SQL will be
+ *         used repeatedly, which is the typical case.
+ *         A query optimized in this way must not be used after rows are added or updated, as it's
+ *         then liable to produce incomplete results. The SQL should be regenerated after such
+ *         changes.
+ *         Alternatively, if dbconn is NULL then a less-efficient version of the query is generated
+ *         which is safe to use across updates (also saving the small upfront cost of inspection).
+ * qrid: query reference sequence (chromosome) ID; defaults to "?1" i.e. the first parameter of the
+ *       compiled statement. One could substitute a different parameter, a constant value, or a SQL
+ *       expression referring to columns of a joined table.
+ * qbeg: query range begin position; defaults to "?2"
+ * qend: query range end position; defaults to "?3"
+ */
 char *genomic_range_rowids_sql(const char *indexed_table, sqlite3 *dbconn, const char *qrid,
                                const char *qbeg, const char *qend);
 
@@ -78,6 +100,7 @@ int GenomicSQLiteOpen(const std::string &dbfile, sqlite3 **ppDb, int flags,
 std::unique_ptr<SQLite::Database> GenomicSQLiteOpen(const std::string &dbfile, int flags,
                                                     const std::string &config_json = "{}");
 #endif
+
 std::string GenomicSQLiteVacuumIntoSQL(const std::string &dbfile,
                                        const std::string &config_json = "{}");
 
@@ -96,6 +119,7 @@ std::string PutGenomicReferenceSequenceSQL(const std::string &name, sqlite3_int6
                                            sqlite3_int64 rid = -1,
                                            const std::string &attached_schema = "");
 
+/* Lookup helpers for stored reference sequence metadata (assumes it's finalized) */
 struct gri_refseq_t {
     unsigned long long rid, length;
     std::string name, assembly, refget_id;

src/genomicsqlite.cc

@@ -646,6 +646,17 @@ static void sqlfn_genomic_range_rowids_sql(sqlite3_context *ctx, int argc, sqlit
         GenomicRangeRowidsSQL(indexed_table, sqlite3_context_db_handle(ctx), qrid, qbeg, qend))
 }
 
+static void sqlfn_genomic_range_rowids_safe_sql(sqlite3_context *ctx, int argc,
+                                                sqlite3_value **argv) {
+    string indexed_table, qrid = "?1", qbeg = "?2", qend = "?3";
+    assert(argc >= 1 && argc <= 4);
+    ARG_TEXT(indexed_table, 0)
+    ARG_TEXT_OPTIONAL(qrid, 1)
+    ARG_TEXT_OPTIONAL(qbeg, 2)
+    ARG_TEXT_OPTIONAL(qend, 3)
+    SQL_WRAPPER(GenomicRangeRowidsSQL(indexed_table, nullptr, qrid, qbeg, qend))
+}
+
 /**************************************************************************************************
  * reference sequence metadata (__gri_refseq) helpers
  **************************************************************************************************/
@@ -1034,6 +1045,10 @@ static int register_genomicsqlite_functions(sqlite3 *db, const char **pzErrMsg,
                  {FPNM(genomic_range_rowids_sql), 2, 0},
                  {FPNM(genomic_range_rowids_sql), 3, 0},
                  {FPNM(genomic_range_rowids_sql), 4, 0},
+                 {FPNM(genomic_range_rowids_safe_sql), 1, 0},
+                 {FPNM(genomic_range_rowids_safe_sql), 2, 0},
+                 {FPNM(genomic_range_rowids_safe_sql), 3, 0},
+                 {FPNM(genomic_range_rowids_safe_sql), 4, 0},
                  {FPNM(put_genomic_reference_sequence_sql), 2, 0},
                  {FPNM(put_genomic_reference_sequence_sql), 3, 0},
                  {FPNM(put_genomic_reference_sequence_sql), 4, 0},

test/test_gri.py

@@ -160,6 +160,8 @@ def fanout(query):
     assert " / 4096)" in query  # level 6
     assert fanout(query) == 2
 
+    assert fanout(genomicsqlite.genomic_range_rowids_sql(con, "features", safe=True)[1:-1]) == 9
+
     con.executescript(
         """
         INSERT INTO features VALUES(43, NULL, 10000000000);

test/test_txdb.py

@@ -83,15 +83,15 @@ def test_txdbquery(genomicsqlite_txdb):
         control_query = f"SELECT _rowid_ FROM {tbl} NOT INDEXED WHERE {pfx}_chrom = ? AND NOT ({pfx}_end < ? OR {pfx}_start > ?) ORDER BY _rowid_"
 
         total_results = 0
-        for _ in range(1000):
+        for _ in range(2000):
             chrom = random.choice(chroms)
             beg = random.randint(0, chrom[1] - 65536)
             end = beg + random.randint(1, random.choice([16, 256, 4096, 65536]))
             ids = list(row[0] for row in conn.execute(query, (chrom[0], beg, end)))
             control_ids = list(row[0] for row in conn.execute(control_query, (chrom[0], beg, end)))
             assert ids == control_ids
             total_results += len(control_ids)
-        assert total_results in (3802, 1341)
+        assert total_results in (7341, 2660)
 
     # join cds to exon
     cds_exon_counts = (