IReduce, IChunkedSeq, tests, kaocha-cljs

Author: tonsky

.gitignore

@@ -20,4 +20,7 @@ web/target-cljs
 .cpcache
 dev/playground.clj
 .DS_Store
-TODO.txt
+TODO.txt
+.cljs_node_repl
+node_modules
+package-lock.json

deps.edn

@@ -22,7 +22,8 @@
       :extra-paths ["test"]
       :extra-deps {
         org.clojure/clojurescript {:mvn/version "1.10.516"}
-        lambdaisland/kaocha {:mvn/version "0.0-389"}
+        lambdaisland/kaocha       {:mvn/version "0.0-389"}
+        lambdaisland/kaocha-cljs  {:mvn/version "0.0-16"}
       }
     }
 

project.clj

@@ -91,12 +91,13 @@
   ]}
 
   :profiles {
-    :1.9 { :dependencies [[org.clojure/clojure       "1.9.0"   :scope "provided"]
-                          [org.clojure/clojurescript "1.9.946" :scope "provided"]] }
+    :1.9 { :dependencies [[org.clojure/clojure         "1.9.0"   :scope "provided"]
+                          [org.clojure/clojurescript   "1.9.946" :scope "provided"]] }
     :dev { :source-paths ["bench/src" "test" "dev"]
-           :dependencies [[org.clojure/tools.nrepl "0.2.13"]
+           :dependencies [[org.clojure/tools.nrepl     "0.2.13"]
                           [org.clojure/tools.namespace "0.2.11"]
-                          [lambdaisland/kaocha "0.0-389"]] }
+                          [lambdaisland/kaocha         "0.0-389"]
+                          [lambdaisland/kaocha-cljs    "0.0-16"]] }
     :aot { :aot [#"datascript\.(?!query-v3).*"]
            :jvm-opts ["-Dclojure.compiler.direct-linking=true"] }
   }

src-java/datascript/SortedSet.java

@@ -23,7 +23,7 @@
 */
 
 @SuppressWarnings("unchecked")
-public class SortedSet extends ASortedSet implements IEditableCollection, ITransientCollection, Reversible {
+public class SortedSet extends ASortedSet implements IEditableCollection, ITransientCollection, Reversible, IReduce {
 
   static Leaf[] EARLY_EXIT = new Leaf[0],
                 UNCHANGED  = new Leaf[0];
@@ -168,10 +168,16 @@ public SortedSet withMeta(IPersistentMap meta) {
   // Reversible
   public ISeq rseq() { return rslice(null, null, _cmp); }
 
-  // IChunkedSeq
-  // public IChunk chunkedFirst() {}
-  // public ISeq chunkedNext() {}
-  // public ISeq chunkedMore() {}
+  // IReduce
+  public Object reduce(IFn f) {
+    Seq seq = seq();
+    return seq == null ? f.invoke() : seq.reduce(f);
+  }
+
+  public Object reduce(IFn f, Object start) {
+    Seq seq = seq();
+    return seq == null ? start : seq().reduce(f, start);
+  }
 
   // IPersistentCollection
   public SortedSet empty() {
@@ -870,9 +876,84 @@ public Object next() {
     }
   }
 
+  // ===== CHUNK =====
+  static class Chunk implements IChunk {
+    final Object[] _keys;
+    final int _idx, _end;
+    final boolean _asc;
+
+    Chunk(Seq seq) {
+      _asc  = seq._asc;
+      _idx  = seq._idx;
+      _keys = seq._node._keys;
+      if (_asc) {
+        int end = seq._node._len - 1;
+        if (seq._keyTo != null)
+          while (end > _idx && seq._cmp.compare(_keys[end], seq._keyTo) > 0)
+            --end;
+        _end = end;
+      } else {
+        int end = 0;
+        if (seq._keyTo != null)
+          while (end < _idx && seq._cmp.compare(_keys[end], seq._keyTo) < 0)
+            ++end;
+        _end = end;
+      }
+    }
+
+    Chunk(Object[] keys, int idx, int end, boolean asc) {
+      _keys = keys;
+      _idx  = idx;
+      _end  = end;
+      _asc  = asc;
+    }
+
+    public IChunk dropFirst() {
+      if (_idx == _end)
+        throw new IllegalStateException("dropFirst of empty chunk");
+      return new Chunk(_keys, _asc ? _idx+1 : _idx-1, _end, _asc);
+    }
+
+    public Object reduce(IFn f, Object start) {
+      Object ret = f.invoke(start, _keys[_idx]);
+      if (ret instanceof Reduced)
+        return ((Reduced) ret).deref();
+      if (_asc)
+        for (int x = _idx + 1; x <= _end; ++x) {
+          ret = f.invoke(ret, _keys[x]);
+          if (ret instanceof Reduced)
+            return ((Reduced) ret).deref();
+        }
+      else // !_asc
+        for (int x = _idx - 1; x >= _end; --x) {
+          ret = f.invoke(ret, _keys[x]);
+          if (ret instanceof Reduced)
+            return ((Reduced) ret).deref();
+        }
+      return ret;
+    }
+
+    public Object nth(int i) {
+      assert (i >= 0 && i < count());
+      return _asc ? _keys[_idx + i] : _keys[_idx - i];
+    }
+
+    public Object nth(int i, Object notFound) {
+      if (i >= 0 && i < count())
+        return nth(i);
+      return notFound;
+    }
+
+    public int count() {
+      if (_asc) return _end - _idx + 1;
+      else return _idx - _end + 1;
+    }
+  }
+
+
   // ===== SEQ =====
 
-  class Seq extends ASeq implements IReduce, Reversible {
+  class Seq extends ASeq implements IReduce, Reversible, IChunkedSeq {
     Seq  _parent;
     Leaf _node;
     int  _idx;
@@ -975,11 +1056,25 @@ public Object reduce(IFn f, Object start) {
     // Iterable
     public Iterator iterator() { return new JavaIter(clone()); }
 
-    // clojure.lang.IChunkedSeq
-    // (chunkedFirst [this] (iter-chunk this))
-    // (chunkedNext  [this] (iter-chunked-next this))
-    // (chunkedMore  [this] (or (.chunkedNext this) ()))
+    // IChunkedSeq
+    public Chunk chunkedFirst() { return new Chunk(this); }
+
+    public Seq chunkedNext() {
+      if (_parent == null) return null;
+      Seq nextParent = _parent.next();
+      if (nextParent == null) return null;
+      Leaf node = nextParent.child();
+      Seq seq = new Seq(meta(), nextParent, node, _asc ? 0 : node._len - 1, _keyTo, _cmp, _asc);
+      return seq.over() ? null : seq;
+    }
+
+    public ISeq chunkedMore() {
+      Seq seq = chunkedNext();
+      if (seq == null) return PersistentList.EMPTY;
+      return seq;
+    }
 
+    // Reversible
     boolean atBeginning() {
       return _idx == 0 && (_parent == null || _parent.atBeginning());
     }

src/datascript/btset.cljs

@@ -605,6 +605,39 @@
           right  (inc (-rpath (.-root set) (.-shift set)))]
       (iter set left right))))
 
+;; replace with cljs.core/ArrayChunk after https://dev.clojure.org/jira/browse/CLJS-2470
+(deftype Chunk [arr off end]
+  ICounted
+  (-count [_] (- end off))
+
+  IIndexed
+  (-nth [this i]
+    (aget arr (+ off i)))
+  (-nth [this i not-found]
+    (if (and (>= i 0) (< i (- end off)))
+      (aget arr (+ off i))
+      not-found))
+
+  IChunk
+  (-drop-first [this]
+    (if (== off end)
+      (throw (js/Error. "-drop-first of empty chunk"))
+      (ArrayChunk. arr (inc off) end)))
+
+  IReduce
+  (-reduce [this f]
+    (if (== off end)
+      (f)
+      (-reduce (-drop-first this) f (aget arr off))))
+  (-reduce [this f start]
+    (loop [val start, n off]
+      (if (< n end)
+        (let [val' (f val (aget arr n))]
+          (if (reduced? val')
+            @val'
+            (recur val' (inc n))))
+        val))))
+
 (defprotocol IIter
   (-copy [this left right]))
 
@@ -634,26 +667,26 @@
         ;; can use cached array to move forward
         (when (< (inc left) right)
           (Iter. set (inc left) right keys (inc idx)))
-        (let [left (next-path set left)]
-          (when (and (not= -1 left) (< left right))
-            (-copy this left right))))))
+        (let [left' (next-path set left)]
+          (when (and (not= -1 left') (< left' right))
+            (-copy this left' right))))))
 
   IChunkedSeq
   (-chunked-first [this]
     (let [end-idx (if (= (bit-or left path-mask)
-                        (bit-or right path-mask))
+                         (bit-or right path-mask))
                     (bit-and right path-mask)
                     (da/alength keys))]
-      (array-chunk keys idx end-idx)))
+      (Chunk. keys idx end-idx)))
 
   (-chunked-rest [this]
     (or (-chunked-next this) ()))
 
   IChunkedNext
   (-chunked-next [this]
-    (let [left (next-path set (+ left (- (da/alength keys) idx 1)))]
-      (when (and (not= -1 left) (< left right))
-        (-copy this left right))))
+    (let [left' (next-path set (+ left (- (da/alength keys) idx 1)))]
+      (when (and (not= -1 left') (< left' right))
+        (-copy this left' right))))
 
   IReduce
   (-reduce [this f]

test-java/datascript/Bench.java

@@ -357,7 +357,19 @@ public static void test() throws Exception {
     System.out.println("40   .. 30 % 10: " + s.rslice(40, 30, cmp));
     System.out.println("40   .. 30 % 10: " + s.slice(30, 40, cmp).rseq());
 
-    // System.out.println(s.slice(30, 30, cmp).reduce(new AFn() { public Object invoke(Object x, Object y) { return ((Integer)x)+((Integer)y); }}));
+    System.out.println("\nReduces");
+    // System.out.println(s.slice(null, null).reduce(new AFn() { public Object invoke(Object x, Object y) { return ((Integer)x)+((Integer)y); }}));
+    IFn plus = (IFn) Clojure.var("clojure.core", "+");
+    System.out.println("reduced sum(0 .. 100) 4950 = " + s.slice(null, null).reduce(plus));
+    System.out.println("reduced sum(100 .. 0) 4950 = " + s.rslice(null, null).reduce(plus));
+    System.out.println("reduced sum(10 .. 20) 165  = " + s.slice(10, 20).reduce(plus));
+    System.out.println("reduced sum(20 .. 10) 165  = " + s.rslice(20, 10).reduce(plus));
+
+    IFn reduce1 = (IFn) Clojure.var("clojure.core", "reduce1");
+    System.out.println("chunked sum(0 .. 100) 4950 = " + reduce1.invoke(plus, 0, s.slice(null, null)));
+    System.out.println("chunked sum(100 .. 0) 4950 = " + reduce1.invoke(plus, 0, s.rslice(null, null)));
+    System.out.println("chunked sum(10 .. 20) 165  = " + reduce1.invoke(plus, 0, s.slice(10, 20)));
+    System.out.println("chunked sum(20 .. 10) 165  = " + reduce1.invoke(plus, 0, s.rslice(20, 10)));
   }
 
   public static void main(String args[]) throws Exception {

test/datascript/test/btset.cljc

@@ -49,7 +49,7 @@
     (range from (inc to))
     (range from (dec to) -1)))
 
-(deftest slice-test
+(deftest test-slice
   (dotimes [i iters]
     (testing "straight 3 layers"
       (let [s (into (btset/btset) (shuffle (irange 0 5000)))]
@@ -236,7 +236,95 @@
           5000   0       
           4999   1     
           2500   2500  
-          2500.9 2500.1)))))
+          2500.9 2500.1)))
+
+    (testing "Slice with equal elements"
+      (let [cmp10 (fn [a b] (compare (quot a 10) (quot b 10)))
+            s10   (reduce #(btset/btset-conj %1 %2 compare) (btset/btset-by cmp10) (shuffle (irange 0 5000)))]
+        (are [from to expected] (= expected (btset/slice s10 from to))
+          30 30      (irange 30 39)
+          130 4970   (irange 130 4979)
+          -100 6000  (irange 0 5000))
+        (are [from to expected] (= expected (btset/rslice s10 from to))
+          30 30      (irange 39 30)
+          4970 130   (irange 4979 130)
+          6000 -100  (irange 5000 0)))
+
+      (let [cmp100 (fn [a b] (compare (quot a 100) (quot b 100)))
+            s100   (reduce #(btset/btset-conj %1 %2 compare) (btset/btset-by cmp100) (shuffle (irange 0 5000)))]
+        (are [from to expected] (= expected (btset/slice s100 from to))
+          30  30     (irange 0 99)
+          2550 2550  (irange 2500 2599)
+          130 4850   (irange 100 4899)
+          -100 6000  (irange 0 5000))
+        (are [from to expected] (= expected (btset/rslice s100 from to))
+          30 30      (irange 99 0)
+          2550 2550  (irange 2599 2500)
+          4850 130   (irange 4899 100)
+          6000 -100  (irange 5000 0))))
+))
+
+
+(defn ireduce
+  ([f coll] (#?(:clj .reduce :cljs -reduce) coll f))
+  ([f val coll] (#?(:clj .reduce :cljs -reduce) coll f val)))
+
+
+(defn reduce-chunked [f val coll]
+  (if-some [s (seq coll)]
+    (if (chunked-seq? s)
+      (recur f (#?(:clj .reduce :cljs -reduce) (chunk-first s) f val) (chunk-next s))
+      (recur f (f val (first s)) (next s)))
+    val))
+
+
+(deftest test-reduces
+  (testing "IReduced"
+    (testing "Empty"
+      (let [s (btset/btset)]
+        (is (= 0 (ireduce + s)))
+        (is (= 0 (ireduce + 0 s)))))
+
+    (testing "~3 layers"
+      (let [s (into (btset/btset) (irange 0 5000))]
+        (is (= 12502500 (ireduce +   s)))
+        (is (= 12502500 (ireduce + 0 s)))
+        (is (= 12502500 (ireduce +   (seq s))))
+        (is (= 12502500 (ireduce + 0 (seq s))))
+        (is (= 7502500  (ireduce +   (btset/slice s 1000 4000))))
+        (is (= 7502500  (ireduce + 0 (btset/slice s 1000 4000))))
+        #?@(:clj [(is (= 12502500 (ireduce +   (rseq s))))
+                  (is (= 12502500 (ireduce + 0 (rseq s))))
+                  (is (= 7502500  (ireduce +   (btset/rslice s 4000 1000))))
+                  (is (= 7502500  (ireduce + 0 (btset/rslice s 4000 1000))))])))
+
+    (testing "~1 layer"
+      (let [s (into (btset/btset) (irange 0 10))]
+        (is (= 55 (ireduce +   s)))
+        (is (= 55 (ireduce + 0 s)))
+        (is (= 55 (ireduce +   (seq s))))
+        (is (= 55 (ireduce + 0 (seq s))))
+        (is (= 35 (ireduce +   (btset/slice s 2 8))))
+        (is (= 35 (ireduce + 0 (btset/slice s 2 8))))
+        #?@(:clj [(is (= 55 (ireduce +   (rseq s))))
+                  (is (= 55 (ireduce + 0 (rseq s))))
+                  (is (= 35 (ireduce +   (btset/rslice s 8 2))))
+                  (is (= 35 (ireduce + 0 (btset/rslice s 8 2))))]))))
+
+  (testing "IChunkedSeq"
+    (testing "~3 layers"
+      (let [s (into (btset/btset) (irange 0 5000))]
+        (is (= 12502500 (reduce-chunked + 0 s)))
+        (is (= 7502500  (reduce-chunked + 0 (btset/slice s 1000 4000))))
+        (is (= 12502500 (reduce-chunked + 0 (rseq s))))
+        (is (= 7502500  (reduce-chunked + 0 (btset/rslice s 4000 1000))))))
+
+    (testing "~1 layer"
+      (let [s (into (btset/btset) (irange 0 10))]
+        (is (= 55 (reduce-chunked + 0 s)))
+        (is (= 35 (reduce-chunked + 0 (btset/slice s 2 8))))
+        (is (= 55 (reduce-chunked + 0 (rseq s))))
+        (is (= 35 (reduce-chunked + 0 (btset/rslice s 8 2))))))))
 
 
 (defn into-via-doseq [to from]
@@ -298,8 +386,4 @@
 #_(println "[ OK ] btset slice checked"))
 
 
-(deftest test-reduced
-  (is (= [1 2] (into [] (take 2) (btset/btset 1 2)))))
-
-
 ;; (t/test-ns 'datascript.test.btset)