Merge pull request #31 from alingse/use-csv-library

use python csv package

Author: alingse

.travis.yml

@@ -16,8 +16,8 @@ script:
     - python setup.py install
     - cat fixture/files/raw.0.json|jsoncsv -e > fixture/files/tmp.expand.0.json
     - cat fixture/files/raw.0.json|jsoncsv -e|jsoncsv -r|jsoncsv > fixture/files/tmp.expand.0.json
-    - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls
-    - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv
+    - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls
+    - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv
 after_success:
     - coveralls
 notifications:

fixture/files/expand.2.json

@@ -0,0 +1,4 @@
+{"河流名字": "长江", "河流长度": "6000千米"}
+{"河流名字": "黄河", "河流长度": "5000千米", "Area": 300.0}
+{"河流名字": "珠江", "Out": "南海"}
+{"河流名字": "Hanjiang", "Source": "HanZhong", "Out": "长江"}

fixture/files/output.1.csv

@@ -1,5 +1,5 @@
-detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
-eat,2016,beijing,one,two,three,AAAA
-play,2016,湖北,one,two,three,BBBB
-,2015,chengdu,two,three,,CCCC
-happy,2015,,,,,DDDD
+detail.how,detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
+,eat,2016,beijing,one,two,three,AAAA
+offline,play,2016,湖北,one,two,three,BBBB
+,,2015,chengdu,two,three,,CCCC
+online,happy,2015,,,,,DDDD

fixture/files/output.1.sort.csv

@@ -1,5 +1,5 @@
-detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
-eat,2016,beijing,one,two,three,AAAA
-play,2016,湖北,one,two,three,BBBB
-,2015,chengdu,two,three,,CCCC
-happy,2015,,,,,DDDD
+detail.how,detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
+,eat,2016,beijing,one,two,three,AAAA
+offline,play,2016,湖北,one,two,three,BBBB
+,,2015,chengdu,two,three,,CCCC
+online,happy,2015,,,,,DDDD

fixture/files/raw.2.json

@@ -2,6 +2,8 @@
 # author@alingse
 # 2015.10.09
 
+import six
+import csv
 import json
 import xlwt
 
@@ -18,15 +20,6 @@ def __init__(self, fin, fout, **kwargs):
     def initialize(self, **kwargs):
         pass
 
-    def patch(self, value):
-        if value is None:
-            return ''
-
-        if value == {} or value == []:
-            return ''
-
-        return value
-
     def prepare(self):
         pass
 
@@ -51,7 +44,7 @@ def load_headers(fin, read_row=None, sort_type=None):
 
         # read
         if not read_row or read_row < 1:
-            read_row = 1
+            read_row = -1
 
         for line in fin:
             obj = json.loads(line)
@@ -100,35 +93,34 @@ class DumpCSV(DumpExcel):
 
     def initialize(self, **kwargs):
         super(DumpCSV, self).initialize(**kwargs)
-
-        self._separator = kwargs.get('separator', ',')
+        self.csv_writer = None
 
     def write_headers(self):
-        header = self._separator.join(self._headers)
-        if PY3:
-            self.fout.write(header)
-        else:
-            self.fout.write(header.encode('utf-8'))
-        self.fout.write('\n')
-
-    def patch(self, value):
-        value = super(DumpCSV, self).patch(value)
-        if PY3:
-            return str(value)
+        if not PY3:
+            # Python 2 csv does not support unicode
+            fieldnames = [header.encode('utf8') for header in self._headers]
         else:
-            return unicode(value)  # noqa
+            fieldnames = self._headers
+        self.csv_writer = csv.DictWriter(self.fout, fieldnames)
+        self.csv_writer.writeheader()
 
     def write_obj(self, obj):
-        values = [
-            self.patch(obj.get(head))
-            for head in self._headers
-        ]
-        content = self._separator.join(values)
-        if PY3:
-            self.fout.write(content)
-        else:
-            self.fout.write(content.encode('utf-8'))
-        self.fout.write('\n')
+        self.csv_writer.writerow(self.patch_obj(obj))
+
+    def patch_obj(self, obj):
+        new_obj = {}
+        for key, value in obj.items():
+            if value in [None, {}, []]:
+                value = ""
+
+            if not PY3:
+                # Python 2 csv does not support unicode
+                key = key.encode('utf8')
+                if isinstance(value, six.text_type):
+                    value = value.encode('utf8')
+
+            new_obj[key] = value
+        return new_obj
 
 
 class DumpXLS(DumpExcel):
@@ -152,7 +144,7 @@ def write_obj(self, obj):
         self.cloumn = 0
 
         for head in self._headers:
-            value = self.patch(obj.get(head))
+            value = obj.get(head)
             self.ws.write(self.row, self.cloumn, value)
             self.cloumn += 1
 

jsoncsv/dumptool.py

@@ -1,10 +1,12 @@
 # coding=utf-8
 
+import io
 import click
 import sys
 
 from jsoncsv import jsontool
 from jsoncsv import dumptool
+from jsoncsv import PY3
 from jsoncsv.dumptool import dump_excel
 from jsoncsv.jsontool import convert_json
 from jsoncsv.utils import separator_type
@@ -86,8 +88,11 @@ def jsoncsv(output, input, expand, restore, safe, separator):
     type=click.File('wb'),
     default=sys.stdout)
 def mkexcel(output, input, sort_, row, type_):
-    if output == sys.stdout and type_ == "xls":
+    if type_ == "xls" and output == sys.stdout:
         output = click.get_binary_stream('stdout')
+    if type_ == "csv" and output != sys.stdout:
+        if PY3:
+            output = io.TextIOWrapper(output)
 
     klass = dumptool.DumpCSV
     if type_ == "xls":

jsoncsv/main.py

@@ -12,7 +12,7 @@
 
 setup(
     name='jsoncsv',
-    version='2.1.0',
+    version='2.2.0a',
     url='https://github.com/alingse/jsoncsv',
     description='A command tool easily convert json file to csv or xlsx.',
     long_description=readme,

setup.py

@@ -9,7 +9,7 @@
 from jsoncsv.dumptool import dump_excel
 
 
-class Testdumptool(unittest.TestCase):
+class TestDumpTool(unittest.TestCase):
 
     # FIXME (使用虚拟文件)
     def test_dumpexcel_csv(self):
@@ -52,3 +52,21 @@ def test_dumpcexcel_xls(self):
 
         fin.close()
         fout.close()
+
+    def test_dump_csv_with_non_ascii(self):
+        fin = open('./fixture/files/expand.2.json', 'r')
+        fout = open('./fixture/files/tmp.output.2.csv', 'w')
+
+        dump_excel(fin, fout, DumpCSV)
+
+        fin.close()
+        fout.close()
+
+    def test_dump_xls_with_non_ascii(self):
+        fin = open('./fixture/files/expand.2.json', 'r')
+        fout = open('./fixture/files/tmp.output.2.xls', 'wb')
+
+        dump_excel(fin, fout, DumpXLS)
+
+        fin.close()
+        fout.close()