restored fast init-db

Author: tonsky

deps.edn

@@ -14,6 +14,7 @@
     :dev {
       :extra-paths ["dev" "target/classes"]
       :extra-deps {
+        org.clojure/clojurescript {:mvn/version "1.10.516"}
         org.clojure/tools.namespace {:mvn/version "0.2.11"}
       }
     }

src-java/datascript/ASortedSet.java

@@ -4,7 +4,7 @@
 import clojure.lang.*;
 
 @SuppressWarnings("unchecked")
-public abstract class ASortedSet implements IObj, Counted, IPersistentSet, ILookup, Iterable, Set, java.io.Serializable, IHashEq {
+public abstract class ASortedSet extends AFn implements IObj, Counted, IPersistentSet, ILookup, Iterable, Set, java.io.Serializable, IHashEq {
   int _hash;
   int _hasheq;
   final IPersistentMap _meta;

src-java/datascript/SortedSet.java

@@ -9,13 +9,13 @@ public class SortedSet extends ASortedSet implements IEditableCollection, ITrans
   static Leaf[] EARLY_EXIT = new Leaf[0],
                 UNCHANGED  = new Leaf[0];
 
-  static int MIN_LEN = 32, MAX_LEN = 64, EXTRA_LEN = 8;
+  public static int MIN_LEN = 32, MAX_LEN = 64, EXTRA_LEN = 8;
 
   public static final SortedSet EMPTY = new SortedSet();
 
-  static class Edit {
+  public static class Edit {
     public volatile boolean _value = false;
-    Edit(boolean value) { _value = value; }
+    public Edit(boolean value) { _value = value; }
     public boolean editable() { return _value; }
     public void setEditable(boolean value) { _value = value; }
   }
@@ -38,7 +38,7 @@ public SortedSet(IPersistentMap meta, Comparator cmp) {
     _count = 0;
   }
 
-  SortedSet(IPersistentMap meta, Comparator cmp, Leaf root, int count, Edit edit) {
+  public SortedSet(IPersistentMap meta, Comparator cmp, Leaf root, int count, Edit edit) {
     super(meta, cmp);
     _root  = root;
     _count = count;
@@ -252,18 +252,18 @@ public Iterator iterator() {
 
   // ===== LEAF =====
 
-  static class Leaf {
+  public static class Leaf {
     final Object[] _keys;
     int _len;
     final Edit _edit;
 
-    Leaf(Object[] keys, int len, Edit edit) {
+    public Leaf(Object[] keys, int len, Edit edit) {
       _keys = keys;
       _len  = len;
       _edit = edit;
     }
 
-    Object maxKey() {
+    public Object maxKey() {
       return _keys[_len-1];
     }
 
@@ -517,10 +517,10 @@ public String str(int lvl) {
 
   // ===== NODE =====
 
-  static class Node extends Leaf {
+  public static class Node extends Leaf {
     final Leaf[] _children;
 
-    Node(Object[] keys, Leaf[] children, int len, Edit edit) {
+    public Node(Object[] keys, Leaf[] children, int len, Edit edit) {
       super(keys, len, edit);
       _children = children;
     }

src-java/datascript/Stitch.java

@@ -1,6 +1,7 @@
 package datascript;
 
 import java.util.*;
+import clojure.lang.*;
 
 public class Stitch<T> {
 
@@ -9,6 +10,25 @@ public static <T> T[] copy(T[] src, int from, int to, T[] target, int offset) {
     return target;
   }
 
+  public static Object indexedToArray(Class type, Indexed coll, int from, int to) {
+    int len = to - from;
+    Object ret = java.lang.reflect.Array.newInstance(type, len);
+    for (int i = 0; i < len; ++i)
+      java.lang.reflect.Array.set(ret, i, coll.nth(i+from));
+    return ret;
+  }
+
+  public static int distinct(Comparator<Object> cmp, Object[] arr) {
+    int to = 0;
+    for (int idx = 1; idx < arr.length; ++idx) {
+      if (cmp.compare(arr[idx], arr[to]) != 0) {
+        ++to;
+        if (to != idx) arr[to] = arr[idx];
+      }
+    }
+    return to + 1;
+  }
+
   T[] target;
   int offset;
 

src/datascript/arrays.cljc

@@ -73,29 +73,47 @@
              (System/arraycopy ^{:tag "[[Ljava.lang.Object;"} b 0 res al bl)
              res)))
 
-(defn amap [f arr]
-  #?(:cljs (.map arr f)
-     :clj  (clojure.core/amap ^{:tag "[[Ljava.lang.Object;"} arr i res (f (aget arr i)))))
+#?(:cljs
+   (defn amap [f arr]
+     (.map arr f))
+   :clj
+   (defn amap 
+     ([f arr]
+      (amap f Object arr))
+     ([f type arr]
+      (let [res (clojure.core/make-array type (alength arr))]
+        (dotimes [i (alength arr)]
+          (aset res i (f (aget arr i))))
+        res))))
 
 (defn asort [arr cmp]
   #?(:cljs (.sort arr cmp)
-     :clj  (doto arr (java.util.Arrays/sort cmp))))
+     :clj  (doto arr (Arrays/parallelSort cmp))))
 
-#?(:cljs (defn ^boolean array? [x]
-           (if (identical? *target* "nodejs")
-             (.isArray js/Array x)
-             (instance? js/Array x)))
-   :clj  (defn array? [^Object x] (-> x .getClass .isArray)))
+#?(:cljs
+   (defn ^boolean array? [x]
+     (if (identical? *target* "nodejs")
+       (.isArray js/Array x)
+       (instance? js/Array x)))
+   :clj
+   (defn array? [^Object x]
+     (-> x .getClass .isArray)))
 
 #?(:clj
-    (defmacro alast [arr]
-      `(let [arr# ~arr]
-         (aget arr# (dec (alength arr#))))))
+   (defmacro alast [arr]
+     `(let [arr# ~arr]
+        (aget arr# (dec (alength arr#))))))
 
 #?(:clj
-  (defmacro half [x]
-    `(unsigned-bit-shift-right ~x 1)))
+   (defmacro half [x]
+     `(unsigned-bit-shift-right ~x 1)))
 
 #?(:clj
-  (defmacro not== [x y]
-    `(not (== ~x ~y))))
+   (defmacro not== [x y]
+     `(not (== ~x ~y))))
+
+#?(:clj
+  (def array-type
+    (memoize
+      (fn [type]
+        (.getClass ^Object (java.lang.reflect.Array/newInstance ^Class type 0))))))

src/datascript/btset.clj

@@ -1,16 +1,21 @@
 (ns ^{:no-doc true
       :author "Nikita Prokopov"}
   datascript.btset
+  (:require
+    [datascript.arrays :as da])
   (:import
-    [java.util Comparator]
-    [datascript SortedSet]))
+    [java.util Comparator Arrays]
+    [datascript SortedSet SortedSet$Leaf SortedSet$Node SortedSet$Edit Stitch]))
+
 
 (defn btset-conj [set key cmp]
   (.cons ^SortedSet set key cmp))
 
+
 (defn btset-disj [set key cmp]
   (.disjoin ^SortedSet set key cmp))
 
+
 (defn slice
   "`(slice set key)` returns iterator that contains all elements equal to the key.
    `(slice set from to)` returns iterator for all Xs where from <= X <= to.
@@ -22,6 +27,7 @@
   ([set from to cmp]
     (.slice ^SortedSet set from to ^Comparator cmp)))
 
+
 (defn rslice
   "`(rslice set from to)` returns backwards iterator for all Xs where from <= X <= to.
    `(rslice set from nil)` returns backwards iterator for all Xs where X <= from."
@@ -30,12 +36,69 @@
   ([set from to cmp]
     (.rslice ^SortedSet set from to ^Comparator cmp)))
 
+
+(defn- array-from-indexed [coll type from to]
+  (cond
+    (instance? clojure.lang.Indexed coll)
+    (Stitch/indexedToArray type coll from to)
+
+    (da/array? coll)
+    (Arrays/copyOfRange coll from to (da/array-type type))))
+
+
+(defn- split
+  ([coll to type avg max]
+   (persistent! (split (transient []) 0 coll to type avg max)))
+  ([res from coll to type avg max]
+   (let [len (- to from)]
+     (cond
+       (== 0 len)
+       res
+
+       (>= len (* 2 avg))
+       (recur (conj! res (array-from-indexed coll type from (+ from avg))) (+ from avg) coll to type avg max)
+
+       (<= len max)
+       (conj! res (array-from-indexed coll type from to))
+
+       :else
+       (-> res
+         (conj! (array-from-indexed coll type from (+ from (quot len 2))))
+         (conj! (array-from-indexed coll type (+ from (quot len 2)) to)))))))
+
+
+(defn from-sorted-array
+  ([cmp keys]
+   (from-sorted-array cmp keys (da/alength keys)))
+  ([cmp keys len]
+   (let [max    SortedSet/MAX_LEN
+         avg    (quot (+ SortedSet/MIN_LEN max) 2)
+         edit   (SortedSet$Edit. false)
+         ->Leaf (fn [keys]
+                  (SortedSet$Leaf. keys (count keys) edit))
+         ->Node (fn [children]
+                  (SortedSet$Node.
+                    (da/amap #(.maxKey ^SortedSet$Leaf %) Object children)
+                    children (count children) edit))]
+     (loop [nodes (mapv ->Leaf (split keys len Object avg max))]
+       (case (count nodes)
+         0 (SortedSet. cmp)
+         1 (SortedSet. {} cmp (first nodes) len edit)
+         (recur (mapv ->Node (split nodes (count nodes) SortedSet$Leaf avg max))))))))
+
+
+(defn ctor [^Comparator cmp keys]
+  (let [arr (to-array keys)
+        _   (da/asort arr cmp)
+        len (Stitch/distinct cmp arr)]
+    (from-sorted-array cmp arr len)))
+
+
 (defn btset-by
   ([cmp] (SortedSet. ^Comparator cmp))
-  ([cmp & keys]
-    (into (btset-by cmp) keys)))
+  ([cmp & keys] (ctor cmp keys)))
+
 
 (defn btset
   ([] (SortedSet.))
-  ([& keys]
-    (into (btset) keys)))
+  ([& keys] (ctor compare keys)))

src/datascript/btset.cljs

@@ -887,10 +887,10 @@
 ;; public interface
 
 (defn -btset-from-sorted-arr [arr cmp]
-  (let [leafs (->> arr
+  (let [leaves (->> arr
                    (arr-partition-approx min-len max-len)
                    (arr-map-inplace #(Leaf. %)))]
-    (loop [current-level leafs
+    (loop [current-level leaves
            shift 0]
       (case (count current-level)
         0 (BTSet. (Leaf. (da/array)) 0 0 cmp nil uninitialized-hash)

src/datascript/db.cljc

@@ -656,10 +656,15 @@
                avet    (btset/-btset-from-sorted-arr avet-datoms cmp-datoms-avet)
                max-eid (init-max-eid eavt)]
               :clj
-              [eavt        (apply btset/btset-by cmp-datoms-eavt datoms)
-               aevt        (apply btset/btset-by cmp-datoms-aevt datoms)
+              [arr         (to-array datoms)
+               _           (da/asort arr cmp-datoms-eavt-quick)
+               eavt        (btset/from-sorted-array cmp-datoms-eavt arr)
+               _           (da/asort arr cmp-datoms-aevt-quick)
+               aevt        (btset/from-sorted-array cmp-datoms-aevt arr)
                avet-datoms (filter (fn [^Datom d] (contains? indexed (.-a d))) datoms)
-               avet        (apply btset/btset-by cmp-datoms-avet avet-datoms)
+               avet-arr    (to-array avet-datoms)
+               _           (da/asort avet-arr cmp-datoms-avet-quick)
+               avet        (btset/from-sorted-array cmp-datoms-avet avet-arr)
                max-eid     (init-max-eid eavt)])
           max-tx (transduce (map (fn [^Datom d] (.-tx d))) max tx0 eavt)]
       (map->DB {