is_array_index 支持 string order

补充 expand & restore 的测试

Author: alingse

fixture/files/raw.2.json

@@ -0,0 +1,2 @@
+["a", "ab", "b", "a", "ab", "b", "a", "ab", "b", "a", "ab", "b"]
+["c", "ab", "b", "a", "ab", "b", "a", "ab", "b", "a", "ab", "c"]

jsoncsv/jsontool.py

@@ -5,7 +5,6 @@
 from __future__ import unicode_literals
 
 import json
-
 from copy import deepcopy
 from itertools import groupby
 from operator import itemgetter
@@ -26,7 +25,6 @@ def gen_leaf(root, path=None):
     if path is None:
         path = []
 
-    # the leaf
     if not isinstance(root, (dict, list)) or not root:
         leaf = (path, root)
         yield leaf
@@ -36,7 +34,6 @@ def gen_leaf(root, path=None):
                 items = root.iteritems()
             else:
                 items = root.items()
-
         else:
             items = enumerate(root)
 
@@ -47,14 +44,17 @@ def gen_leaf(root, path=None):
                 yield leaf
 
 
-def is_array(keys, ensure_str=True):
-    copy_keys = list(deepcopy(keys))
-    int_keys = list(range(len(copy_keys)))
-    if copy_keys == int_keys:
+def is_array_index(keys, enable_str=True):
+    keys = list(deepcopy(keys))
+    # 不强调有序
+    key_map = {key: True for key in keys}
+    int_keys = range(len(keys))
+
+    if all(key in key_map for key in int_keys):
         return True
-    if ensure_str:
-        str_keys = list(map(str, int_keys))
-        if copy_keys == str_keys:
+
+    if enable_str:
+        if all(str(key) in key_map for key in int_keys):
             return True
     return False
 
@@ -84,7 +84,7 @@ def from_leaf(leafs):
         child.append((head, _child))
 
     child_keys = map(_get_head, child)
-    if is_array(child_keys):
+    if is_array_index(child_keys):
         child.sort(key=lambda x: int(x[0]))
         return list(map(_get_leaf, child))
 
@@ -140,6 +140,7 @@ def convert_json(fin, fout, func, separator=".", safe=False, json_array=False):
 
     # default: read json objects from each line
     def gen_objs():
+        print(type(fin))
         for line in fin:
             obj = json.loads(line)
             yield obj
@@ -161,6 +162,7 @@ def gen_objs_from_array():
         content = json.dumps(new, ensure_ascii=False)
         if PY2:
             fout.write(content.encode('utf-8'))
+            fout.write('\n'.encode('utf-8'))
         else:
             fout.write(content)
-        fout.write(str('\n'))
+            fout.write('\n')

jsoncsv/main.py

@@ -7,7 +7,15 @@
 from jsoncsv import dumptool
 from jsoncsv.dumptool import dump_excel
 from jsoncsv.jsontool import convert_json
-from jsoncsv.utils import separator_type
+from jsoncsv.utils import unit_char
+
+
+def separator_type(sep):
+    if len(sep) != 1:
+        raise click.BadOptionUsage('separator can only be a char')
+    if sep == unit_char:
+        raise click.BadOptionUsage('separator can not be `\\` ')
+    return sep
 
 
 @click.command()

jsoncsv/utils.py

@@ -2,21 +2,9 @@
 # author@alingse
 # 2016.11.20
 
-import click
-
-__all__ = []
-
 unit_char = '\\'
 
 
-def separator_type(sep):
-    if len(sep) > 1:
-        raise click.BadOptionUsage('separator can only be a char')
-    if sep == unit_char:
-        raise click.BadOptionUsage('separator can not be `\\` ')
-    return sep
-
-
 def encode_safe_key(path, separator):
     path = [p.replace(unit_char, unit_char * 2) for p in path]
     separator = unit_char + separator

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name='jsoncsv',
-    version='2.2.2',
+    version='2.2.3',
     url='https://github.com/alingse/jsoncsv',
     description='A command tool easily convert json file to csv or xlsx.',
     long_description=readme,

tests/test_jsoncsv.py

@@ -1,7 +1,8 @@
 # coding=utf-8
 # author@alingse
 # 2016.08.09
-
+import io
+import json
 import unittest
 from click.testing import CliRunner
 
@@ -21,3 +22,18 @@ def test_jsoncsv_expand_with_json_array(self):
         args = ['-e', 'fixture/files/raw.1.json', 'fixture/files/tmp.expand.1.json', '-A']
         result = runner.invoke(jsoncsv, args=args)
         assert result.exit_code == 0
+
+    def test_jsoncsv_expand_restore(self):
+        runner = CliRunner(echo_stdin=True)
+        result = runner.invoke(jsoncsv, args=['-e', 'fixture/files/raw.2.json', 'fixture/files/tmp.expand.2.json'])
+        assert result.exit_code == 0
+        result = runner.invoke(jsoncsv, args=['-r', 'fixture/files/tmp.expand.2.json', 'fixture/files/tmp.restore.2.json'])
+        assert result.exit_code == 0
+
+        with io.open('fixture/files/raw.2.json', 'r') as f:
+            input_data = [json.loads(line) for line in f]
+
+        with io.open('fixture/files/tmp.restore.2.json', 'r') as f:
+            resotre_data = [json.loads(line) for line in f]
+
+        self.assertEqual(input_data, resotre_data)

tests/test_jsontool.py

@@ -8,7 +8,7 @@
 
 from jsoncsv import PY2
 from jsoncsv.jsontool import expand, restore
-from jsoncsv.jsontool import is_array
+from jsoncsv.jsontool import is_array_index
 from jsoncsv.jsontool import convert_json
 
 
@@ -63,11 +63,13 @@ def test_complex(self):
 
         self.assertListEqual(s[4], _s[4])
 
-    def test_is_array(self):
-        assert is_array([0, 1, 2, 3])
-        assert is_array(['0', '1', '2', '3'])
-        assert not is_array([1, 2, 3])
-        assert not is_array(['0', 1, 2])
+    def test_is_array_index(self):
+        self.assertTrue(is_array_index([0, 1, 2, 3]))
+        self.assertTrue(is_array_index(['0', '1', '2', '3']))
+        # string order
+        self.assertTrue(is_array_index(['0', '1', '10', '2', '3', '4', '5', '6', '7', '8', '9']))
+        self.assertFalse(is_array_index([1, 2, 3]))
+        self.assertFalse(is_array_index(['0', 1, 2]))
 
     def test_unicode(self):
         data = [
@@ -90,6 +92,15 @@ def test_expand_with_safe(self):
         origin = restore(expobj, safe=True)
         self.assertEqual(origin, data)
 
+    def test_expand_and_restore(self):
+        data = ["a", "ab", "b"] * 4
+        expobj = expand(data)
+        self.assertEqual(expobj["0"], "a")
+        self.assertEqual(expobj["1"], "ab")
+
+        origin = restore(expobj)
+        self.assertEqual(data, origin)
+
 
 class TestConvertJSON(unittest.TestCase):