ld & scalability tweaks

Author: mlin

CMakeLists.txt

@@ -49,5 +49,5 @@ add_subdirectory(loaders)
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     include(CTest)
     enable_testing()
-    add_test(NAME pytest COMMAND env PYTHONPATH=$PYTHONPATH:${CMAKE_CURRENT_SOURCE_DIR}/bindings/python LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${CMAKE_BINARY_DIR} python3 -m pytest -xv -n 4 --tb=short ${CMAKE_CURRENT_SOURCE_DIR}/test)
+    add_test(NAME pytest COMMAND env PYTHONPATH=$PYTHONPATH:${CMAKE_CURRENT_SOURCE_DIR}/bindings/python LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${CMAKE_BINARY_DIR} GENOMICSQLITE_SYSTEM_LIBRARY=1 python3 -m pytest -xv -n 4 --tb=short ${CMAKE_CURRENT_SOURCE_DIR}/test)
 endif()

bindings/python/genomicsqlite/__init__.py

@@ -10,16 +10,18 @@
 from typing import Optional, NamedTuple, Dict, Any
 from ctypes.util import find_library
 
-HERE = os.path.dirname(__file__)
+_HERE = os.path.dirname(__file__)
+_YES = ("1", "true", "t", "yes", "y")
 
-# One-time global initialization -- locate shared library file; preferably the copy installed with
-# this package, otherwise look in the usual places.
+# Module initialization -- locate shared library file; preferably the copy installed with this
+# package, otherwise look in the usual places.
 _DLL = None
-if platform.system() == "Linux" and os.path.isfile(os.path.join(HERE, "libgenomicsqlite.so")):
-    _DLL = os.path.join(HERE, "libgenomicsqlite.so")
+if os.environ.get("GENOMICSQLITE_SYSTEM_LIBRARY", "").strip().lower() not in _YES:
+    if platform.system() == "Linux" and os.path.isfile(os.path.join(_HERE, "libgenomicsqlite.so")):
+        _DLL = os.path.join(_HERE, "libgenomicsqlite.so")
 if not _DLL:
     _DLL = find_library("genomicsqlite")
-assert _DLL, "couldn't locate genomicsqlite shared-library file"
+assert _DLL, "Unable to locate genomicsqlite shared-library file"
 # open a dummy connection to :memory: just for loading the extension.
 _MEMCONN = sqlite3.connect(":memory:")
 _MEMCONN.enable_load_extension(True)

loaders/sam_into_sqlite.cc

@@ -191,7 +191,7 @@ int main(int argc, char *argv[]) {
     hts_set_threads(sam.get(), max(2U, thread::hardware_concurrency() / 4));
     unique_ptr<sam_hdr_t, void (*)(sam_hdr_t *)> hdr(sam_hdr_read(sam.get()), &sam_hdr_destroy);
     if (!hdr) {
-        cerr << "sam_into_sqlite: failed reading VCF header from " << infilename << '\n';
+        cerr << "sam_into_sqlite: failed reading SAM header from " << infilename << '\n';
         return 1;
     }
 

src/genomicsqlite.cc

@@ -136,6 +136,7 @@ string GenomicSQLiteURI(const string &dbfile, const string &config_json = "") {
     uri << "file:" << dbfile << "?vfs=zstd";
     uri << "&threads=" << to_string(threads);
     uri << "&outer_page_size=" << to_string(outer_page_KiB * 1024);
+    uri << "&outer_cache_size=-65536"; // enlarge to hold index b-tree pages for large db's
     uri << "&level=" << to_string(zstd_level);
     if (unsafe_load) {
         uri << "&outer_unsafe";
@@ -237,7 +238,8 @@ string GenomicSQLiteTuningSQL(const string &config_json, const string &schema =
     }
     map<string, string> pragmas;
     pragmas[schema_prefix + "cache_size"] = to_string(-1024 * page_cache_MiB);
-    pragmas["threads"] = to_string(threads >= 0 ? threads : thread::hardware_concurrency());
+    pragmas["threads"] =
+        to_string(threads >= 0 ? threads : std::min(8, (int)thread::hardware_concurrency()));
     if (unsafe_load) {
         pragmas[schema_prefix + "journal_mode"] = "OFF";
         pragmas[schema_prefix + "synchronous"] = "OFF";

test/genomicsqlite_big_tests.wdl

@@ -130,9 +130,9 @@ task test_sam {
 
         # add a QNAME-sorted seqs table
         chmod +x /usr/lib/python3.8/genomicsqlite.py
-        time /usr/lib/python3.8/genomicsqlite.py "~{dbname}" "PRAGMA cache_size=-4194304; CREATE TABLE reads_seqs_by_qname AS SELECT * from reads_seqs NOT INDEXED ORDER BY qname"
+        time /usr/lib/python3.8/genomicsqlite.py "~{dbname}" "PRAGMA journal_mode=off; PRAGMA synchronous=off; PRAGMA cache_size=-4194304; CREATE TABLE reads_seqs_by_qname AS SELECT * from reads_seqs NOT INDEXED ORDER BY qname"
         >&2 ls -l "~{dbname}"
-        time /usr/lib/python3.8/genomicsqlite.py "~{dbname}" "PRAGMA journal_mode=off; DROP TABLE reads_seqs_by_qname"
+        time /usr/lib/python3.8/genomicsqlite.py "~{dbname}" "PRAGMA journal_mode=off; PRAGMA synchronous=off; DROP TABLE reads_seqs_by_qname"
         >&2 ls -l "~{dbname}"
     >>>