support .dbi helper files to speed up web access (#19)

Author: mlin

CMakeLists.txt

@@ -18,7 +18,7 @@ endif()
 FetchContent_Declare(
     sqlite_zstd_vfs
     GIT_REPOSITORY  https://github.com/mlin/sqlite_zstd_vfs.git
-    GIT_TAG         e81ef25
+    GIT_TAG         eadb758
 )
 FetchContent_MakeAvailable(sqlite_zstd_vfs)
 FetchContent_MakeAvailable(sqlitecpp)

README.md

@@ -16,7 +16,7 @@ This October 2020 poster discusses the context and long-run ambitions:
 
 Our **[Colab notebook](https://colab.research.google.com/drive/1OlHPOcRQBhDmEnS1wtOdtUGDkcD7LtKx?usp=sharing)** demonstrates key features with Python, one of several language bindings.
 
-**USE AT YOUR OWN RISK:** The extension makes fundamental changes to the database storage layer. While designed to preserve ACID transaction safety, it's young and unlikely to have zero bugs. This project is not associated with the SQLite developers.
+**USE AT YOUR OWN RISK:** This project is not associated with the SQLite developers. The database storage extensions are designed to preserve ACID transaction safety, but they're young and unlikely to be totally bug-free.
 
 ## [Installation & Programming Guide](https://mlin.github.io/GenomicSQLite/)
 

bindings/python/genomicsqlite/__init__.py

@@ -202,13 +202,20 @@ def get_reference_sequences_by_name(
 
 sqlite3_ARG: passed through to sqlite3 (see `sqlite3 -help`)
 
+--
+
+Other tools:
+
+genomicsqlite DB_FILENAME --compact [options|--help]
+[Re]compress and defragment an existing SQLite3 or GenomicSQLite database. See `genomicsqlite --compact --help`
+
+genomicsqlite DB_FILENAME --dbi [options|--help]
+Generate a .dbi helper file to optimize web access for a given database. See `genomicsqlite --dbi --help`
 
-Usage: genomicsqlite DB_FILENAME --compact [options|--help]
-[Re]compress and defragment an existing SQLite3 or GenomicSQLite database. See --compact --help for options
 """
 
 
-def _cli():
+def _cli(argv=None):
     """
     Command-line entry point wrapping the `sqlite3` interactive CLI to open a GenomicSQLite
     compressed database file.
@@ -217,17 +224,22 @@ def _cli():
     language bindings.
     """
 
-    if "--compact" in sys.argv or "-compact" in sys.argv:
-        _compact(sys.argv[1], sys.argv[2:])
+    if argv is None:
+        argv = sys.argv
+    if "--compact" in argv or "-compact" in argv:
+        _compact(argv[1], argv[2:])
+        return
+    if "--dbi" in argv or "-dbi" in argv:
+        _dbi(argv[1], argv[2:])
         return
 
     if (
-        len(sys.argv) < 2
+        len(argv) < 2
         or not (
-            next((True for pfx in ("http:", "https:") if sys.argv[1].startswith(pfx)), False)
-            or os.path.isfile(sys.argv[1])
+            next((True for pfx in ("http:", "https:") if argv[1].startswith(pfx)), False)
+            or os.path.isfile(argv[1])
         )
-        or next((True for a in ("-h", "-help", "--help") if a in sys.argv), False)
+        or next((True for a in ("-h", "-help", "--help") if a in argv), False)
     ):
         print(
             "Usage: genomicsqlite DB_FILENAME [--readonly] [sqlite3_ARG ...]\n",
@@ -250,13 +262,13 @@ def _cli():
 
     cfg = {}
     try:
-        sys.argv[sys.argv.index("--readonly")] = "-readonly"
+        argv[argv.index("--readonly")] = "-readonly"
     except ValueError:
         pass
-    if "-readonly" in sys.argv:
+    if "-readonly" in argv:
         cfg["mode"] = "ro"
     cfg = json.dumps(cfg)
-    uri = _execute1(_MEMCONN, "SELECT genomicsqlite_uri(?,?)", (sys.argv[1], cfg))
+    uri = _execute1(_MEMCONN, "SELECT genomicsqlite_uri(?,?)", (argv[1], cfg))
     tuning_sql = _execute1(_MEMCONN, "SELECT genomicsqlite_tuning_sql(?)", (cfg,))
 
     cmd = [
@@ -274,7 +286,7 @@ def _cli():
         '.prompt "GenomicSQLite> "',
     ]
     if sys.stdout.isatty() and not next(
-        (arg for arg in sys.argv[2:] if not arg.startswith("-")), False
+        (arg for arg in argv[2:] if not arg.startswith("-")), False
     ):
         # interactive mode:
         cmd += [
@@ -284,7 +296,7 @@ def _cli():
             ".headers on",
         ]
     cmd.append(":memory:")  # placeholder so remaining positional args are recognized as such
-    cmd += sys.argv[2:]
+    cmd += argv[2:]
 
     if "DEBUG" in os.environ:
         print(
@@ -422,5 +434,12 @@ def _compact(dbfilename, argv):
         print("OK", file=sys.stderr)
 
 
+def _dbi(dbfilename, argv):
+    from .sqlite_web_dbi import main as dbi_main  # pylint: disable=E0401,E0611
+
+    argv = [elt for elt in argv if elt not in ("-dbi", "--dbi")]
+    dbi_main(["genomicsqlite DBFILE --dbi", dbfilename] + argv)
+
+
 if __name__ == "__main__":
-    _cli()
+    _cli(sys.argv)

bindings/python/genomicsqlite/__main__.py

@@ -0,0 +1,5 @@
+import sys
+from . import _cli
+
+if __name__ == "__main__":
+    _cli(sys.argv)

bindings/python/genomicsqlite/sqlite_web_dbi.py

@@ -0,0 +1 @@
+../../../build/_deps/sqlite_web_vfs-src/sqlite_web_dbi.py

docs/guide_db.md

@@ -200,12 +200,24 @@ Due to decompression overhead, the compaction procedure may be impractically slo
 
 The **GenomicSQLite Open** routine and the `genomicsqlite` shell also accept http: and https: URLs instead of local filenames, creating a connection to read the compressed file over the web directly. The database connection must be opened read-only in the appropriate manner for your language bindings (such as the flag `SQLITE_OPEN_READONLY`). The URL server must support [HTTP GET range](https://developer.mozilla.org/en-US/docs/Web/HTTP/Range_requests) requests, and the content must not change for the lifetime of the connection.
 
-Under the hood, the extension uses [libcurl](https://curl.se/libcurl/) to send web requests for necessary portions of the database file as queries proceed, with adaptive batching & prefetching to balance the number and size of these requests. This works well for point lookups and queries that scan largely-contiguous slices of tables and indexes (a modest number thereof). It's less suitable for big multi-way joins and other aggressively random access patterns; in such cases, it'd be better to download the database file upfront to open locally.
+Under the hood, the extension uses [libcurl](https://curl.se/libcurl/) to send web requests for necessary portions of the database file as queries proceed, with adaptive batching & prefetching to balance the number and size of these requests. This works well for point lookups and queries that scan largely-contiguous slices of tables and indexes (and a modest number thereof). It's less suitable for big multi-way joins and other aggressively random access patterns; in such cases, it'd be better to download the database file upfront to open locally.
 
-* The above-described `genomicsqlite DB_FILENAME --compact` tool can optimize a file's suitability for web access.
 * Reading large databases over the web, budget an additional ~600MiB of memory for HTTP prefetch buffers.
-* To disable TLS certificate and hostname verification, set web_insecure = true in the GenomicSQLite configuration, or SQLITE_WEB_INSECURE=1 in the environment.
 * The HTTP driver writes log messages to standard error when requests fail or had to be retried, which can be disabled by setting configuration web_log = 0 or environment SQLITE_WEB_LOG=0; or increased up to 5 to log every request and other details.
+* To disable TLS certificate and hostname verification, set web_insecure = true in the GenomicSQLite configuration, or SQLITE_WEB_INSECURE=1 in the environment.
+* The above-described `genomicsqlite DB_FILENAME --compact` optimizes a database for web access by making the request pattern more contiguous.
+
+### Web access optimization with .dbi helper files
+
+*Experimental feature*
+
+Optionally, web access can be further optimized by a small .dbi helper file served alongside the main database file. The client automatically probes for this by appending `.dbi` to the database URL (unless there's a query string). If that's not usable for any reason, the database falls back to direct access. Increase the web_log to 3 or higher to see which mode is used.
+
+Use `genomicsqlite DB_FILENAME --dbi` to generate the .dbi helper for an immutable database file, then publish them alongside each other. The .dbi must be regenerated if the database subsequently changes.
+
+To override the automatic probe, set configuration web_dbi_url to a different URL for the .dbi file, or to a local `file:/path/to.dbi` downloaded beforehand. Use the latter feature to save multiple connections from each having to fetch the .dbi separately. Lastly, set web_nodbi to true or environment SQLITE_WEB_NODBI=1 to disable dbi mode entirely.
+
+The .dbi helper is optional, but often beneficial for big databases accessed with high-latency requests. It collects bits of the main file that are key for navigating it, but typically scattered throughout (even after compaction). Prefetching them in the compact .dbi saves the reader from having to pluck them from all over the main file.
 
 ## Advice for big data
 

src/genomicsqlite.cc

@@ -69,7 +69,9 @@ std::string GenomicSQLiteDefaultConfigJSON() {
     "inner_page_KiB": 16,
     "outer_page_KiB": 32,
     "web_log": 2,
-    "web_insecure": false
+    "web_insecure": false,
+    "web_dbi_url": "",
+    "web_nodbi": false
 })";
 }
 
@@ -172,8 +174,18 @@ string GenomicSQLiteURI(const string &dbfile, const string &config_json = "") {
     uri << "file:" << (web ? "/__web__" : SQLiteNested::urlencode(dbfile, true)) << "?vfs=zstd";
     if (web) {
         uri << "&mode=ro&immutable=1&web_url=" << SQLiteNested::urlencode(dbfile)
-            << "&web_log=" << cfg.GetInt("$.web_log")
-            << "&web_insecure=" << (cfg.GetBool("$.web_insecure") ? 1 : 0);
+            << "&web_log=" << cfg.GetInt("$.web_log");
+        if (cfg.GetBool("$.web_insecure")) {
+            uri << "&web_insecure=1";
+        }
+        if (cfg.GetBool("$.web_nodbi")) {
+            uri << "&web_nodbi=1";
+        } else {
+            auto web_dbi_url = cfg.GetString("$.web_dbi_url");
+            if (!web_dbi_url.empty()) {
+                uri << "&web_dbi_url=" << SQLiteNested::urlencode(web_dbi_url);
+            }
+        }
     }
     int threads = cfg.GetInt("$.threads");
     uri << "&outer_cache_size=-65536"

test/genomicsqlite_big_tests.wdl

@@ -23,13 +23,21 @@ workflow genomicsqlite_big_tests {
         sam_into_sqlite = build.sam_into_sqlite
     }
 
-    call test_sam_web {
+    call test_sam_web as test_sam_web_nodbi {
         input:
         reads_db = test_sam.reads_db,
         libgenomicsqlite_so = select_first([libgenomicsqlite_so, build.libgenomicsqlite_so]),
         genomicsqlite_py = build.genomicsqlite_py
     }
 
+    call test_sam_web as test_sam_web_dbi {
+        input:
+        reads_db = test_sam.reads_db,
+        libgenomicsqlite_so = select_first([libgenomicsqlite_so, build.libgenomicsqlite_so]),
+        genomicsqlite_py = build.genomicsqlite_py,
+        sqlite_web_dbi_py = build.sqlite_web_dbi_py
+    }
+
     call test_vcf {
         input:
         variants = variants,
@@ -65,6 +73,7 @@ task build {
     output {
         File libgenomicsqlite_so = "GenomicSQLite/build/libgenomicsqlite.so"
         File genomicsqlite_py = "GenomicSQLite/genomicsqlite.py"
+        File sqlite_web_dbi_py = "GenomicSQLite/build/_deps/sqlite_web_vfs-src/sqlite_web_dbi.py"
         File sam_into_sqlite = "GenomicSQLite/build/loaders/sam_into_sqlite"
         File vcf_into_sqlite = "GenomicSQLite/build/loaders/vcf_into_sqlite"
     }
@@ -172,6 +181,7 @@ task test_sam_web {
         File reads_db
         File genomicsqlite_py
         File libgenomicsqlite_so
+        File? sqlite_web_dbi_py
     }
 
     command <<<
@@ -183,6 +193,14 @@ task test_sam_web {
         cp ~{libgenomicsqlite_so} /usr/local/lib/libgenomicsqlite.so
         ldconfig
 
+        READS_DB_DIR="$(dirname '~{reads_db}')"
+        if [ -n '~{sqlite_web_dbi_py}' ]; then
+            # generate .dbi to serve alongside database
+            cp '~{sqlite_web_dbi_py}' /usr/local/bin/sqlite_web_dbi.py
+            chmod +x /usr/local/bin/sqlite_web_dbi.py
+            sqlite_web_dbi.py '~{reads_db}'
+        fi
+
         # make self-signed cert
         openssl req -new -newkey rsa:4096 -days 365 -nodes -x509 \
             -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=www.example.com" \
@@ -191,7 +209,6 @@ task test_sam_web {
         # write nginx.config as heredoc
         # references: https://docs.nginx.com/nginx/admin-guide/web-server/serving-static-content/
         #             http://nginx.org/en/docs/http/configuring_https_servers.html
-        READS_DB_DIR="$(dirname '~{reads_db}')"
         cat << EOF > nginx.config
         worker_processes     8;
         error_log            stderr warn;