page size tuning experiment

Author: mlin

loaders/sam_into_sqlite.cc

@@ -175,6 +175,8 @@ void help() {
          << "  --table-prefix PREFIX  prefix to the name of each table created" << '\n'
          << "  --no-gri               skip genomic range indexing" << '\n'
          << "  --no-qname-index       skip QNAME indexing" << '\n'
+         << "  --inner-page-KiB N     inner page size; one of {1,2,4,8,16,32,64}" << '\n'
+         << "  --outer-page-KiB N     outer page size; one of {1,2,4,8,16,32,64}" << '\n'
          << "  -l,--level LEVEL       database compression level (-7 to 22, default 6)" << '\n'
          << "  -q,--quiet             suppress progress information on standard error" << '\n'
          << "  -h,--help              show this help message" << '\n'
@@ -184,18 +186,20 @@ void help() {
 int main(int argc, char *argv[]) {
     string table_prefix, infilename, outfilename;
     bool gri = true, qname_idx = true, progress = true;
-    int level = 6;
+    int level = 6, inner_page_KiB = 16, outer_page_KiB = 32;
 
     static struct option long_options[] = {{"help", no_argument, 0, 'h'},
                                            {"quiet", no_argument, 0, 'q'},
                                            {"level", required_argument, 0, 'l'},
+                                           {"inner-page-KiB", required_argument, 0, 'I'},
+                                           {"outer-page-KiB", required_argument, 0, 'O'},
                                            {"table-prefix", required_argument, 0, 't'},
                                            {"no-gri", no_argument, 0, 'G'},
                                            {"no-qname-index", no_argument, 0, 'Q'},
                                            {0, 0, 0, 0}};
 
     int c;
-    while (-1 != (c = getopt_long(argc, argv, "hqQGl:t:", long_options, nullptr))) {
+    while (-1 != (c = getopt_long(argc, argv, "hqQGl:t:I:O:", long_options, nullptr))) {
         switch (c) {
         case 'h':
             help();
@@ -216,7 +220,23 @@ int main(int argc, char *argv[]) {
             errno = 0;
             level = strtol(optarg, nullptr, 10);
             if (errno || level < -7 || level > 22) {
-                cerr << "sam_into_sqlite: couldn't parse --level in [-7,22]";
+                cerr << "sam_into_sqlite: couldn't parse --level in [-7,22]" << endl;
+                return -1;
+            }
+            break;
+        case 'I':
+            errno = 0;
+            inner_page_KiB = strtol(optarg, nullptr, 10);
+            if (errno || inner_page_KiB < 1 || inner_page_KiB > 64) {
+                cerr << "sam_into_sqlite: invalid --inner-page-KiB" << endl;
+                return -1;
+            }
+            break;
+        case 'O':
+            errno = 0;
+            outer_page_KiB = strtol(optarg, nullptr, 10);
+            if (errno || outer_page_KiB < 1 || outer_page_KiB > 64) {
+                cerr << "sam_into_sqlite: invalid --outer-page-KiB" << endl;
                 return -1;
             }
             break;
@@ -257,9 +277,12 @@ int main(int argc, char *argv[]) {
         // open output database
         sqlite3_config(SQLITE_CONFIG_MEMSTATUS, 0);
         sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 2048, 128);
+        string config_json = R"({"unsafe_load": true, "zstd_level":)" + to_string(level) +
+                             R"(,"inner_page_KiB":)" + to_string(inner_page_KiB) +
+                             R"(,"outer_page_KiB":)" + to_string(outer_page_KiB) + "}";
         auto db = GenomicSQLiteOpen(
             outfilename, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX,
-            R"( {"unsafe_load": true, "zstd_level": )" + to_string(level) + "}");
+            config_json);
 #ifndef NDEBUG
         db->exec("PRAGMA foreign_keys=ON");
 #endif

test/compression_tuning/Dockerfile

@@ -0,0 +1,3 @@
+FROM ubuntu:20.04
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get -qq update && apt-get install -y samtools zstd

test/compression_tuning/Makefile

@@ -0,0 +1,10 @@
+all:
+	docker build -t genomicsqlite_compression_tuning .
+	miniwdl run run.wdl \
+		trial.docker=genomicsqlite_compression_tuning \
+		trial.bam=../data/NA12878.chr21:20000000-22500000.bam \
+		trial.sam_into_sqlite=../../build/loaders/sam_into_sqlite \
+		trial.libgenomicsqlite_so=../../build/libgenomicsqlite.so \
+		--verbose --dir /tmp
+
+.PHONY: all

test/compression_tuning/run.wdl

@@ -0,0 +1,63 @@
+version 1.0
+
+struct Result {
+    Int inner_page_KiB
+    Int outer_page_KiB
+    Int db_size
+    Int efficiency
+}
+
+workflow genomicsqlite_compression_tuning {
+    input {}
+    Array[Int] page_sizes = [4, 8, 16, 32, 64]
+    scatter (inner_page_KiB in page_sizes) {
+        scatter(outer_page_KiB in page_sizes) {
+            call trial {
+                input:
+                inner_page_KiB = inner_page_KiB,
+                outer_page_KiB = outer_page_KiB
+            }
+            Result res = object {
+                inner_page_KiB: inner_page_KiB,
+                outer_page_KiB: outer_page_KiB,
+                db_size: trial.db_size,
+                efficiency: trial.efficiency
+            }
+        }
+    }
+
+    output {
+        Array[Result] results = flatten(res)
+    }
+}
+
+task trial {
+    input {
+        File bam
+        File sam_into_sqlite
+        File libgenomicsqlite_so
+        Int inner_page_KiB
+        Int outer_page_KiB
+        String docker
+    }
+
+    command <<<
+        set -euxo pipefail
+        cp ~{libgenomicsqlite_so} libgenomicsqlite.so
+        cp ~{sam_into_sqlite} sam_into_sqlite
+        chmod +x sam_into_sqlite
+
+        LD_LIBRARY_PATH=$(pwd) ./sam_into_sqlite --inner-page-KiB ~{inner_page_KiB} --outer-page-KiB ~{outer_page_KiB} ~{bam} reads.db
+        zstd -6 -q reads.db -o reads.db.zst
+    >>>
+
+    runtime {
+        docker: docker
+        cpu: 4
+    }
+
+    output {
+        Int db_size = round(size("reads.db"))
+        Int efficiency = round(100.0*size("reads.db.zst")/size("reads.db"))
+    }
+}