_gri_rid and JSON schema tweaks

Author: mlin

bindings/python/genomicsqlite.py

@@ -6,7 +6,7 @@
 import sys
 import json
 import sqlite3
-from typing import Optional, NamedTuple, Dict
+from typing import Optional, NamedTuple, Dict, Any
 from ctypes.util import find_library
 
 # One-time global initialization -- load the extension shared-library
@@ -106,13 +106,15 @@ def put_reference_sequence_sql(
     length: int,
     assembly: Optional[str] = None,
     refget_id: Optional[str] = None,
+    meta: Optional[Dict[str, Any]] = None,
     rid: Optional[int] = None,
     schema: Optional[str] = None,
 ):
+    meta_json = json.dumps(meta) if meta else None
     return _execute1(
         conn,
-        "SELECT put_genomic_reference_sequence_sql(?,?,?,?,?,?)",
-        (name, length, assembly, refget_id, rid, schema),
+        "SELECT put_genomic_reference_sequence_sql(?,?,?,?,?,?,?)",
+        (name, length, assembly, refget_id, meta_json, rid, schema),
     )
 
 
@@ -131,7 +133,7 @@ def get_reference_sequences_by_rid(
     if schema:
         table = f"{schema}.{table}"
     sql = (
-        "SELECT gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id FROM "
+        "SELECT _gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id FROM "
         + table
     )
     params = []

include/genomicsqlite.h

@@ -75,7 +75,8 @@ char *genomic_range_rowids_sql(const char *indexed_table, sqlite3 *dbconn, const
 char *put_genomic_reference_assembly_sql(const char *assembly, const char *attached_schema);
 char *put_genomic_reference_sequence_sql(const char *name, sqlite3_int64 length,
                                          const char *assembly, const char *refget_id,
-                                         sqlite3_int64 rid, const char *attached_schema);
+                                         const char *meta_json, sqlite3_int64 rid,
+                                         const char *attached_schema);
 
 /*
  * C++ bindings: are liable to throw exceptions except where marked noexcept
@@ -116,6 +117,7 @@ std::string PutGenomicReferenceAssemblySQL(const std::string &assembly,
 std::string PutGenomicReferenceSequenceSQL(const std::string &name, sqlite3_int64 length,
                                            const std::string &assembly = "",
                                            const std::string &refget_id = "",
+                                           const std::string &meta_json = "{}",
                                            sqlite3_int64 rid = -1,
                                            const std::string &attached_schema = "");
 

loaders/sam_into_sqlite.cc

@@ -30,7 +30,7 @@ map<string, int> import_readgroups(const string &table_prefix, sam_hdr_t *hdr, S
                                    bool progress) {
     string ddl =
         "CREATE TABLE " + table_prefix +
-        "readgroups(rg_id INTEGER PRIMARY KEY, rg_name TEXT NOT NULL UNIQUE, rg_tags_json TEXT NOT NULL)";
+        "readgroups(rg_id INTEGER PRIMARY KEY, rg_name TEXT NOT NULL UNIQUE, rg_tags_json TEXT NOT NULL DEFAULT '{}')";
     progress &&cerr << ddl << endl;
     db.exec(ddl);
 
@@ -228,7 +228,7 @@ int main(int argc, char *argv[]) {
             if (sam_hdr_find_tag_pos(hdr.get(), "SQ", rid, "M5", ks.get()) == 0) {
                 m5 = ks->s;
             }
-            string sql = PutGenomicReferenceSequenceSQL(name, length, "", m5, rid);
+            string sql = PutGenomicReferenceSequenceSQL(name, length, "", m5, "{}", rid);
             if (rid == 0) {
                 progress &&cerr << sql << endl;
             } else if (rid == 1) {
@@ -244,7 +244,7 @@ int main(int argc, char *argv[]) {
         // TODO: allow --append
         string ddl =
             "CREATE TABLE " + table_prefix +
-            "reads(rowid INTEGER PRIMARY KEY, flag INTEGER NOT NULL, rid INTEGER REFERENCES __gri_refseq(gri_rid), pos INTEGER, endpos INTEGER, "
+            "reads(rowid INTEGER PRIMARY KEY, flag INTEGER NOT NULL, rid INTEGER REFERENCES __gri_refseq(_gri_rid), pos INTEGER, endpos INTEGER, "
             "mapq INTEGER, cigar TEXT, rnext INTEGER, pnext INTEGER, tlen INTEGER, "
             "rg_id INTEGER REFERENCES " +
             table_prefix + "readgroups(rg_id))";
@@ -253,7 +253,7 @@ int main(int argc, char *argv[]) {
                "reads(rowid), qname TEXT, seq TEXT, qual TEXT)";
         ddl += ";\nCREATE TABLE " + table_prefix +
                "reads_tags(rowid INTEGER PRIMARY KEY REFERENCES " + table_prefix +
-               "reads(rowid), tags_json TEXT)";
+               "reads(rowid), tags_json TEXT NOT NULL DEFAULT '{}')";
 
         progress &&cerr << ddl << endl;
         db->exec(ddl);

loaders/vcf_into_sqlite.cc

@@ -49,9 +49,9 @@ string schemaDDL(const string &table_prefix, vector<map<string, string>> &info_h
 
     OStringStream out;
     out << "CREATE TABLE " << table_prefix
-        << "variants (variant_rowid INTEGER NOT NULL PRIMARY KEY, rid INTEGER NOT NULL REFERENCES __gri_refseq(gri_rid), "
-           "POS INTEGER NOT NULL, rlen INTEGER NOT NULL, ID_jsarray TEXT, REF TEXT NOT NULL, "
-           "ALT_jsarray TEXT, QUAL REAL, FILTER_jsarray";
+        << "variants (variant_rowid INTEGER NOT NULL PRIMARY KEY, rid INTEGER NOT NULL REFERENCES __gri_refseq(_gri_rid), "
+           "POS INTEGER NOT NULL, rlen INTEGER NOT NULL, ID_jsarray TEXT DEFAULT '[]', REF TEXT NOT NULL, "
+           "ALT_jsarray TEXT NOT NULL DEFAULT '[]', QUAL REAL, FILTER_jsarray";
 
     // INFO columns
     for (auto &hrec : info_hrecs) {
@@ -60,7 +60,7 @@ string schemaDDL(const string &table_prefix, vector<map<string, string>> &info_h
             out << " INTEGER NOT NULL";
         } else if (hrec["Number"] != "1" &&
                    (hrec["Type"] == "Integer" || hrec["Type"] == "Float")) {
-            out << "_jsarray TEXT";
+            out << "_jsarray TEXT DEFAULT '[]'";
         } else if (hrec["Type"] == "Integer") {
             out << " INTEGER";
         } else if (hrec["Type"] == "Float") {
@@ -102,7 +102,7 @@ string schemaDDL(const string &table_prefix, vector<map<string, string>> &info_h
                 out << "\n, " << hrec["ID"];
                 if (hrec["Number"] != "1" &&
                     (hrec["Type"] == "Integer" || hrec["Type"] == "Float")) {
-                    out << "_jsarray TEXT";
+                    out << "_jsarray TEXT DEFAULT '[]'";
                 } else if (hrec["Type"] == "Integer") {
                     out << " INTEGER";
                 } else if (hrec["Type"] == "Float") {
@@ -736,7 +736,7 @@ int main(int argc, char *argv[]) {
             if (length <= 0 || errno) {
                 throw runtime_error("invalid contig length in VCF header");
             }
-            string sql = PutGenomicReferenceSequenceSQL(ctg["ID"], length, assembly, "", rid);
+            string sql = PutGenomicReferenceSequenceSQL(ctg["ID"], length, assembly, "", "{}", rid);
             if (rid == 0) {
                 progress &&cerr << sql << endl;
             } else if (rid == 1) {

src/genomicsqlite.cc

@@ -434,8 +434,9 @@ static string gri_refseq_ddl(const string &schema) {
     }
     ostringstream out;
     out << "CREATE TABLE IF NOT EXISTS " << schema_prefix << "__gri_refseq"
-        << "(gri_rid INTEGER NOT NULL PRIMARY KEY, gri_refseq_name TEXT NOT NULL, gri_assembly TEXT,"
-        << " gri_refget_id TEXT UNIQUE, gri_refseq_length INTEGER NOT NULL, UNIQUE(gri_assembly,gri_refseq_name))"
+        << "(_gri_rid INTEGER NOT NULL PRIMARY KEY, gri_refseq_name TEXT NOT NULL, gri_assembly TEXT,"
+        << " gri_refget_id TEXT UNIQUE, gri_refseq_length INTEGER NOT NULL, gri_refseq_meta_json TEXT NOT NULL DEFAULT '{}', "
+        << "UNIQUE(gri_assembly,gri_refseq_name))"
         << ";\nCREATE INDEX IF NOT EXISTS " << schema_prefix << "__gri_refseq_name ON "
         << schema_prefix << "__gri_refseq(gri_refseq_name)";
     return out.str();
@@ -663,7 +664,8 @@ static void sqlfn_genomic_range_rowids_safe_sql(sqlite3_context *ctx, int argc,
 
 string PutGenomicReferenceSequenceSQL(const string &name, sqlite3_int64 length,
                                       const string &assembly, const string &refget_id,
-                                      sqlite3_int64 rid, const string &schema, bool with_ddl) {
+                                      const string &meta_json, sqlite3_int64 rid,
+                                      const string &schema, bool with_ddl) {
     string schema_prefix;
     if (!schema.empty()) {
         schema_prefix = schema + ".";
@@ -673,40 +675,45 @@ string PutGenomicReferenceSequenceSQL(const string &name, sqlite3_int64 length,
         out << gri_refseq_ddl(schema) << ";\n";
     }
     out << "INSERT INTO " << schema_prefix
-        << "__gri_refseq(gri_rid,gri_refseq_name,gri_assembly,gri_refget_id,gri_refseq_length) VALUES("
+        << "__gri_refseq(_gri_rid,gri_refseq_name,gri_assembly,gri_refget_id,gri_refseq_length,gri_refseq_meta_json) VALUES("
         << (rid >= 0 ? std::to_string(rid) : "NULL") << "," << sqlquote(name) << ","
         << (assembly.empty() ? "NULL" : sqlquote(assembly)) << ","
         << (refget_id.empty() ? "NULL" : sqlquote(refget_id)) << "," << std::to_string(length)
-        << ")";
+        << "," << sqlquote(meta_json.empty() ? string("{}") : meta_json) << ")";
     return out.str();
 }
 
 string PutGenomicReferenceSequenceSQL(const string &name, sqlite3_int64 length,
                                       const string &assembly, const string &refget_id,
-                                      sqlite3_int64 rid, const string &schema) {
-    return PutGenomicReferenceSequenceSQL(name, length, assembly, refget_id, rid, schema, true);
+                                      const string &meta_json, sqlite3_int64 rid,
+                                      const string &schema) {
+    return PutGenomicReferenceSequenceSQL(name, length, assembly, refget_id, meta_json, rid, schema,
+                                          true);
 }
 
 extern "C" char *put_genomic_reference_sequence_sql(const char *name, sqlite3_int64 length,
                                                     const char *assembly, const char *refget_id,
-                                                    sqlite3_int64 rid, const char *schema) {
-    C_WRAPPER(PutGenomicReferenceSequenceSQL(name, length, assembly ? assembly : "",
-                                             refget_id ? refget_id : "", rid, schema ? schema : "",
-                                             true));
+                                                    const char *meta_json, sqlite3_int64 rid,
+                                                    const char *schema) {
+    C_WRAPPER(PutGenomicReferenceSequenceSQL(
+        name, length, assembly ? assembly : "", refget_id ? refget_id : "",
+        meta_json ? meta_json : "{}", rid, schema ? schema : "", true));
 }
 
 static void sqlfn_put_genomic_reference_sequence_sql(sqlite3_context *ctx, int argc,
                                                      sqlite3_value **argv) {
-    string name, assembly, refget_id, schema;
+    string name, assembly, refget_id, meta_json = "{}", schema;
     sqlite3_int64 length, rid = -1;
-    assert(argc >= 2 && argc <= 6);
+    assert(argc >= 2 && argc <= 7);
     ARG_TEXT(name, 0)
     ARG(length, 1, SQLITE_INTEGER, int64)
     ARG_TEXT_OPTIONAL(assembly, 2)
     ARG_TEXT_OPTIONAL(refget_id, 3)
-    ARG_OPTIONAL(rid, 4, SQLITE_INTEGER, int64)
-    ARG_TEXT_OPTIONAL(schema, 5);
-    SQL_WRAPPER(PutGenomicReferenceSequenceSQL(name, length, assembly, refget_id, rid, schema))
+    ARG_TEXT_OPTIONAL(meta_json, 4)
+    ARG_OPTIONAL(rid, 5, SQLITE_INTEGER, int64)
+    ARG_TEXT_OPTIONAL(schema, 6);
+    SQL_WRAPPER(
+        PutGenomicReferenceSequenceSQL(name, length, assembly, refget_id, meta_json, rid, schema))
 }
 
 struct hardcoded_refseq_t {
@@ -939,7 +946,7 @@ string PutGenomicReferenceAssemblySQL(const string &assembly, const string &sche
         const hardcoded_refseq_t &hcrs = hardcoded_refseqs[i];
         out << PutGenomicReferenceSequenceSQL(string(hcrs.name), hcrs.length, assembly,
                                               hcrs.refget_id ? string(hcrs.refget_id) : string(),
-                                              -1, schema, i == 0);
+                                              string("{}"), -1, schema, i == 0);
     }
     return out.str();
 }
@@ -963,7 +970,7 @@ GetGenomicReferenceSequencesByRid(sqlite3 *dbconn, const string &assembly, const
     string schema_prefix = schema.empty() ? "" : (schema + ".");
 
     string query =
-        "SELECT gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id FROM " +
+        "SELECT _gri_rid, gri_refseq_name, gri_refseq_length, gri_assembly, gri_refget_id FROM " +
         schema_prefix + "__gri_refseq";
     if (!assembly.empty()) {
         query += " WHERE gri_assembly = ?";
@@ -1054,6 +1061,7 @@ static int register_genomicsqlite_functions(sqlite3 *db, const char **pzErrMsg,
                  {FPNM(put_genomic_reference_sequence_sql), 4, 0},
                  {FPNM(put_genomic_reference_sequence_sql), 5, 0},
                  {FPNM(put_genomic_reference_sequence_sql), 6, 0},
+                 {FPNM(put_genomic_reference_sequence_sql), 7, 0},
                  {FPNM(put_genomic_reference_assembly_sql), 1, 0},
                  {FPNM(put_genomic_reference_assembly_sql), 2, 0}};
 

test/test_gri.py

@@ -218,7 +218,7 @@ def test_refseq():
     assert len(refseq_by_rid) == 195
 
     query = (
-        "SELECT gri_rid, beg, end, id FROM __gri_refseq, exons WHERE gri_refseq_name = rid AND exons._rowid_ IN "
+        "SELECT __gri_refseq._gri_rid, rid, beg, end, id FROM exons, __gri_refseq WHERE exons.rid = gri_refseq_name AND exons._rowid_ IN "
         + genomicsqlite.genomic_range_rowids_sql(con, "exons")
     )
     print("\n" + query)

test/test_vcf.py

@@ -30,8 +30,8 @@ def test_gnomad_sites_small(tmp_path):
     con = genomicsqlite.connect(dbfile, read_only=True)
 
     query = (
-        "SELECT variant_rowid, id_jsarray FROM (SELECT gri_rid FROM __gri_refseq WHERE gri_refseq_name=?1) AS chrom, gnomad_variants WHERE variant_rowid IN"
-        + genomicsqlite.genomic_range_rowids_sql(con, "gnomad_variants", "chrom.gri_rid")
+        "SELECT variant_rowid, id_jsarray FROM (SELECT _gri_rid AS rid FROM __gri_refseq WHERE gri_refseq_name=?1) AS query, gnomad_variants WHERE variant_rowid IN"
+        + genomicsqlite.genomic_range_rowids_sql(con, "gnomad_variants", "query.rid")
     )
     rs671 = ("chr12", 111803912, 111804012)
     print(query)
@@ -57,8 +57,8 @@ def test_gnomad_sites_small(tmp_path):
     results_rowids = set(vt[0] for vt in results)
     assert next(vt for vt in results if vt[1] and "rs671" in vt[1])
 
-    control = "SELECT variant_rowid FROM gnomad_variants WHERE NOT ((pos+rlen) < ? OR pos > ?)"
-    control_rowids = set(vt[0] for vt in con.execute(control, (rs671[1], rs671[2])))
+    control = "SELECT variant_rowid FROM gnomad_variants NATURAL JOIN __gri_refseq WHERE gri_refseq_name = ? AND NOT ((pos+rlen) < ? OR pos > ?)"
+    control_rowids = set(vt[0] for vt in con.execute(control, rs671))
     assert len(control_rowids) == 22
     assert results_rowids == control_rowids