Improve RLE performance; add progress bar

Author: patrislav1

bin2hpm/hpm_conv.py

@@ -30,11 +30,11 @@ def hpm_conv(img_data, compression_enable, **kwargs):
     # Compress data if compression enabled
     if compression_enable:
         enc_data = rle.encode(img_data)
-        print('Verifying compressed data...', end='')
+        print('Verifying compressed data...')
         if rle.decode(enc_data) != img_data:
             print(f'RLE compression verify mismatch', file=sys.stderr)
             sys.exit(-1)
-        print('OK\n')
+        print('RLE compression verify OK\n')
 
         img_comp_hdr = b'COMPRESSED\x00'
         img_comp_hdr += int.to_bytes(len(img_data), length=4, byteorder='big')

bin2hpm/rle.py

@@ -10,6 +10,8 @@
 #                                                                         #
 ###########################################################################
 
+from tqdm import tqdm
+
 RLE_MAX_BLKSIZE = 128
 
 
@@ -18,34 +20,40 @@ def to_byte(b):
 
 
 def encode(data):
-    def check_block(data):
+    def check_block(data, k):
         n = 1
-        while len(data) > n and data[n] == data[0] and n < RLE_MAX_BLKSIZE:
+        while (len(data) - k) > n and data[k+n] == data[k] and n < RLE_MAX_BLKSIZE:
             n += 1
-        return n, data[0]
+        return n, data[k]
 
     print('Performing RLE compression ...')
     orig_len = len(data)
 
     result = b''
-    while len(data):
-        tmp = b''
-        while len(data) and len(tmp) < RLE_MAX_BLKSIZE:
-            n, val = check_block(data)
-            if n > 2:
-                data = data[n:]
-                break
-            elif n == 2:
-                n = 1
-            data = data[n:]
-            tmp += to_byte(val)
-        if len(tmp):
-            result += to_byte(len(tmp) - 1)
-            result += tmp
-        if n < 2:
-            continue
-        result += to_byte((n - 2) | 0x80)
-        result += to_byte(val)
+    with tqdm(total=orig_len, unit='B', unit_scale=True, leave=False) as pbar:
+        k = 0
+        last_k = k
+        while k < len(data):
+            tmp = b''
+            while k < len(data) and len(tmp) < RLE_MAX_BLKSIZE:
+                n, val = check_block(data, k)
+                if n > 2:
+                    k += n
+                    break
+                elif n == 2:
+                    n = 1
+                k += n
+                tmp += to_byte(val)
+            pbar.update(k - last_k)
+            last_k = k
+
+            if len(tmp):
+                result += to_byte(len(tmp) - 1)
+                result += tmp
+            if n < 2:
+                continue
+            result += to_byte((n - 2) | 0x80)
+            result += to_byte(val)
 
     comp_len = len(result)
     comp_ratio = 100 * comp_len / orig_len
@@ -56,12 +64,18 @@ def check_block(data):
 
 def decode(data):
     result = b''
-    while (len(data)):
-        if data[0] & 0x80:
-            result += to_byte(data[1]) * ((data[0] & 0x7f) + 2)
-            data = data[2:]
-        else:
-            l = data[0] + 1
-            result += data[1:l+1]
-            data = data[l+1:]
+    with tqdm(total=len(data), unit='B', unit_scale=True, leave=False) as pbar:
+        k = 0
+        last_k = k
+        while k < len(data):
+            if data[k] & 0x80:
+                result += to_byte(data[k+1]) * ((data[k] & 0x7f) + 2)
+                k += 2
+            else:
+                l = data[k] + 1
+                result += data[k+1:k+l+1]
+                k += l+1
+            pbar.update(k - last_k)
+            last_k = k
+
     return result

requirements.txt

@@ -0,0 +1 @@
+tqdm>=4.61.0