Merge pull request #28 from alingse/add-convert-json

add convert json

Author: alingse

.travis.yml

@@ -1,23 +1,24 @@
 language: python
+cache: pip
 sudo: false
 python:
     - "2.7"
     - "3.5"
     - "3.6"
-
 install:
     - pip install .
     - pip install flake8
     - pip install coverage
     - pip install coveralls
-
 script:
     - flake8 jsoncsv tests --max-line-length=200
     - nosetests --cover-package=jsoncsv --with-coverage
     - python setup.py install
     - cat fixture/files/raw.0.json|jsoncsv -e > fixture/files/tmp.expand.0.json
+    - cat fixture/files/raw.0.json|jsoncsv -e|jsoncsv -r|jsoncsv > fixture/files/tmp.expand.0.json
     - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls
     - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv
-
 after_success:
-  - coveralls
+    - coveralls
+notifications:
+    email: false

jsoncsv/dumptool.py

@@ -162,11 +162,13 @@ def on_finish(self):
         self.wb.save(self.fout)
 
 
-def dumpexcel(fin, fout, type_, **kwargs):
+def dump_excel(fin, fout, type_, **kwargs):
     if type_ == 'csv':
         DumpKlass = DumpCSV
     elif type_ == 'xls':
         DumpKlass = DumpXLS
+    else:
+        raise ValueError("unknow dumpexcel type")
 
     dump = DumpKlass(fin, fout, **kwargs)
     dump.dump()

jsoncsv/jsontool.py

@@ -2,6 +2,9 @@
 # author@alingse
 # 2016.05.27
 from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import json
 
 from copy import deepcopy
 from itertools import groupby
@@ -13,6 +16,7 @@
 
 
 __all__ = [
+    'convert_json',
     'expand',
     'restore',
     'gen_leaf',
@@ -129,3 +133,22 @@ def restore(expobj, separator='.', safe=False):
 
     origin = from_leaf(leafs)
     return origin
+
+
+def convert_json(fin, fout, type="expand", separator=".", safe=False):
+    if type == "expand":
+        func = expand
+    elif type == "restore":
+        func = restore
+    else:
+        raise ValueError("unknow convert_json type")
+
+    for line in fin:
+        obj = json.loads(line)
+        new = func(obj, separator=separator, safe=safe)
+        content = json.dumps(new, ensure_ascii=False)
+        if PY3:
+            fout.write(content)
+        else:
+            fout.write(content.encode('utf-8'))
+        fout.write(str('\n'))

jsoncsv/main.py

@@ -1,12 +1,10 @@
 # coding=utf-8
 
 import click
-import json
 import sys
 
-from jsoncsv import PY3
-from jsoncsv.jsontool import expand, restore
-from jsoncsv.dumptool import dumpexcel
+from jsoncsv.dumptool import dump_excel
+from jsoncsv.jsontool import convert_json
 from jsoncsv.utils import separator_type
 
 
@@ -18,17 +16,20 @@
     type=separator_type,
     default='.',
     help='separator')
-@click.option('--safe', is_flag=True, help='use safe mode')
+@click.option(
+    '--safe',
+    is_flag=True,
+    help='use safe mode')
 @click.option(
     '-r',
     '--restore',
-    'restore_',
+    'restore',
     is_flag=True,
     help='restore expanded json')
 @click.option(
     '-e',
     '--expand',
-    'expand_',
+    'expand',
     is_flag=True,
     help='expand json')
 @click.argument(
@@ -39,23 +40,15 @@
     'output',
     type=click.File('w'),
     default=sys.stdout)
-def jsoncsv(output, input, expand_, restore_, safe, separator):
-    if expand_ and restore_:
+def jsoncsv(output, input, expand, restore, safe, separator):
+    if expand and restore:
         raise click.UsageError('can not choose both, default is `-e`')
 
-    func = expand
-    if restore_:
-        func = restore
+    type = "expand"  # default
+    if restore:
+        type = "restore"
 
-    for line in input:
-        obj = json.loads(line)
-        new = func(obj, separator=separator, safe=safe)
-        content = json.dumps(new, ensure_ascii=False)
-        if PY3:
-            output.write(content)
-        else:
-            output.write(content.encode('utf-8'))
-        output.write('\n')
+    convert_json(input, output, type, separator, safe)
 
     input.close()
     output.close()
@@ -94,7 +87,7 @@ def mkexcel(output, input, sort_, row, type_):
     if output == sys.stdout and type_ == "xls":
         output = click.get_binary_stream('stdout')
 
-    dumpexcel(input, output, type_, read_row=row, sort_type=sort_)
+    dump_excel(input, output, type_, read_row=row, sort_type=sort_)
 
     input.close()
     output.close()

setup.py

@@ -12,7 +12,7 @@
 
 setup(
     name='jsoncsv',
-    version='2.0.9',
+    version='2.1.0a',
     url='https://github.com/alingse/jsoncsv',
     description='A command tool easily convert json file to csv or xlsx.',
     long_description=readme,

tests/test_escape.py

@@ -10,7 +10,7 @@
 class Testescape(unittest.TestCase):
 
     def test_all(self):
-        path = ['A', 'B', '..', '\.\\ww']
+        path = ['A', 'B', '..', '\\.\\ww']
 
         for sep in 'AB.w':
             key = encode_safe_key(path, sep)
@@ -23,10 +23,10 @@ def test_encode(self):
         sep = '.'
         key = encode_safe_key(path, sep)
 
-        self.assertEqual(key, 'A\.B\.C\.www.xxx.com')
+        self.assertEqual(key, 'A\\.B\\.C\\.www.xxx.com')
 
     def test_decode(self):
-        key = 'A\.B\.C\.www.xxx.com'
+        key = 'A\\.B\\.C\\.www.xxx.com'
         sep = '.'
         path = decode_safe_key(key, sep)
 

tests/test_jsoncsv.py

@@ -14,4 +14,5 @@ def test_jsoncsv_expand(self):
         runner = CliRunner()
         args = ['-e', 'fixture/files/raw.0.json', 'fixture/files/tmp.expand.0.json']
         result = runner.invoke(jsoncsv, args=args)
+        print(result)
         assert result.exit_code == 0

tests/test_jsontool.py

@@ -1,14 +1,18 @@
 # coding=utf-8
 # author@alingse
 # 2016.08.09
+from __future__ import unicode_literals
 
+import io
 import unittest
 
+from jsoncsv import PY3
 from jsoncsv.jsontool import expand, restore
 from jsoncsv.jsontool import is_array
+from jsoncsv.jsontool import convert_json
 
 
-class Testjsontool(unittest.TestCase):
+class TestJSONTool(unittest.TestCase):
 
     def test_string(self):
         s = "sss"
@@ -73,3 +77,46 @@ def test_unicode(self):
 
         expobj = expand(data)
         assert expobj
+
+    def test_expand_with_safe(self):
+        data = {
+            "www.a.com": {"qps": 100, "p95": 20},
+            "api.a.com": {"qps": 100, "p95": 20, "p99": 100},
+        }
+        expobj = expand(data, safe=True)
+        self.assertEqual(expobj['api.a.com\\.p95'], 20)
+        self.assertEqual(expobj['api.a.com\\.p99'], 100)
+
+        origin = restore(expobj, safe=True)
+        self.assertEqual(origin, data)
+
+
+class TestConvertJSON(unittest.TestCase):
+
+    def test_convert_expand(self):
+        fin = io.StringIO('{"a":{"b":3}}\n{"a":{"c":4}}\n')
+        if PY3:
+            fout = io.StringIO()
+        else:
+            fout = io.BytesIO()
+
+        convert_json(fin, fout)
+
+        self.assertEqual('{"a.b": 3}\n{"a.c": 4}\n', fout.getvalue())
+
+        fin.close()
+        fout.close()
+
+    def test_convert_restore(self):
+        fin = io.StringIO('{"a.b": 3}\n{"a.c": 4}\n')
+        if PY3:
+            fout = io.StringIO()
+        else:
+            fout = io.BytesIO()
+
+        convert_json(fin, fout, type="restore")
+
+        self.assertEqual('{"a": {"b": 3}}\n{"a": {"c": 4}}\n', fout.getvalue())
+
+        fin.close()
+        fout.close()

tests/test_mkexcel.py

@@ -4,7 +4,7 @@
 
 import unittest
 
-from jsoncsv.dumptool import dumpexcel
+from jsoncsv.dumptool import dump_excel
 
 
 class Testdumptool(unittest.TestCase):
@@ -14,7 +14,7 @@ def test_dumpexcel_csv(self):
         fin = open('./fixture/files/expand.1.json', 'r')
         fout = open('./fixture/files/tmp.output.1.csv', 'w')
 
-        dumpexcel(fin, fout, 'csv')
+        dump_excel(fin, fout, 'csv')
         fin.close()
         fout.close()
 
@@ -30,7 +30,7 @@ def test_dumpexcel_csv_with_sort(self):
         fin = open('./fixture/files/expand.1.json', 'r')
         fout = open('./fixture/files/tmp.output.1.sort.csv', 'w')
 
-        dumpexcel(fin, fout, 'csv', sort_type=True)
+        dump_excel(fin, fout, 'csv', sort_type=True)
         fin.close()
         fout.close()
 
@@ -46,7 +46,7 @@ def test_dumpcexcel_xls(self):
         fin = open('./fixture/files/expand.1.json', 'r')
         fout = open('./fixture/files/tmp.output.1.xls', 'wb')
 
-        dumpexcel(fin, fout, 'xls')
+        dump_excel(fin, fout, 'xls')
 
         fin.close()
         fout.close()