使用 python csv 包

alingse · alingse · commit 704fa503c068 · 2019-07-14T22:55:16.000+08:00
diff --git a/.travis.yml b/.travis.yml
@@ -16,8 +16,8 @@ script:
     - python setup.py install
     - cat fixture/files/raw.0.json|jsoncsv -e > fixture/files/tmp.expand.0.json
     - cat fixture/files/raw.0.json|jsoncsv -e|jsoncsv -r|jsoncsv > fixture/files/tmp.expand.0.json
-    - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls
-    - cat fixture/files/raw.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv
+    - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t xls > fixture/files/tmp.output.2.xls
+    - cat fixture/files/expand.2.json|jsoncsv|mkexcel -t csv > fixture/files/tmp.output.2..csv
 after_success:
     - coveralls
 notifications:
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,3 +1,7 @@
+2019.07.14
+
+- 2.2.0 use python csv package
+
 2019.06.12
 
 - 2.1.0 add convert_json && dump_excel to use as library
diff --git a/fixture/files/expand.0.json b/fixture/files/expand.0.json
@@ -1,4 +1,4 @@
-{"detail.when": 2016, "detail.what": "eat", "title": "AAAA", "detail.where": "beijing", "detail.who.2": "three", "detail.who.1": "two", "detail.who.0": "one"}
-{"detail.when": 2016, "detail.what": "play", "title": "BBBB", "detail.where": "湖北", "detail.who.2": "three", "detail.who.1": "two", "detail.who.0": "one", "detail.how": "offline"}
+{"名字": "A","detail.when": 2016, "detail.what": "eat", "title": "AAAA", "detail.where": "beijing", "detail.who.2": "three", "detail.who.1": "two", "detail.who.0": "one"}
+{"名字": "A""detail.when": 2016, "detail.what": "play", "title": "BBBB", "detail.where": "湖北", "detail.who.2": "three", "detail.who.1": "two", "detail.who.0": "one", "detail.how": "offline"}
 {"detail.when": 2015, "title": "CCCC", "detail.where": "chengdu", "detail.who.1": "three", "detail.who.0": "two"}
 {"detail.when": 2015, "detail.how": "online", "detail.what": "happy", "title": "DDDD"}
diff --git a/fixture/files/expand.2.json b/fixture/files/expand.2.json
@@ -0,0 +1,4 @@
+{"河流名字": "长江", "河流长度": "6000千米"}
+{"河流名字": "黄河", "河流长度": "5000千米", "Area": 300.0}
+{"河流名字": "珠江", "Out": "南海"}
+{"河流名字": "Hanjiang", "Source": "HanZhong", "Out": "长江"}
diff --git a/fixture/files/output.1.csv b/fixture/files/output.1.csv
@@ -1,5 +1,5 @@
-detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
-eat,2016,beijing,one,two,three,AAAA
-play,2016,湖北,one,two,three,BBBB
-,2015,chengdu,two,three,,CCCC
-happy,2015,,,,,DDDD
+detail.how,detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
+,eat,2016,beijing,one,two,three,AAAA
+offline,play,2016,湖北,one,two,three,BBBB
+,,2015,chengdu,two,three,,CCCC
+online,happy,2015,,,,,DDDD
diff --git a/fixture/files/output.1.sort.csv b/fixture/files/output.1.sort.csv
@@ -1,5 +1,5 @@
-detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
-eat,2016,beijing,one,two,three,AAAA
-play,2016,湖北,one,two,three,BBBB
-,2015,chengdu,two,three,,CCCC
-happy,2015,,,,,DDDD
+detail.how,detail.what,detail.when,detail.where,detail.who.0,detail.who.1,detail.who.2,title
+,eat,2016,beijing,one,two,three,AAAA
+offline,play,2016,湖北,one,two,three,BBBB
+,,2015,chengdu,two,three,,CCCC
+online,happy,2015,,,,,DDDD
diff --git a/fixture/files/raw.2.json b/fixture/files/raw.2.json
diff --git a/jsoncsv/dumptool.py b/jsoncsv/dumptool.py
@@ -2,6 +2,7 @@
 # author@alingse
 # 2015.10.09
 
+import csv
 import json
 import xlwt
 
@@ -18,15 +19,6 @@ def __init__(self, fin, fout, **kwargs):
     def initialize(self, **kwargs):
         pass
 
-    def patch(self, value):
-        if value is None:
-            return ''
-
-        if value == {} or value == []:
-            return ''
-
-        return value
-
     def prepare(self):
         pass
 
@@ -51,7 +43,7 @@ def load_headers(fin, read_row=None, sort_type=None):
 
         # read
         if not read_row or read_row < 1:
-            read_row = 1
+            read_row = -1
 
         for line in fin:
             obj = json.loads(line)
@@ -100,35 +92,31 @@ class DumpCSV(DumpExcel):
 
     def initialize(self, **kwargs):
         super(DumpCSV, self).initialize(**kwargs)
-
-        self._separator = kwargs.get('separator', ',')
+        self.csv_writer = None
 
     def write_headers(self):
-        header = self._separator.join(self._headers)
-        if PY3:
-            self.fout.write(header)
+        if not PY3:
+            fieldnames = [header.encode('utf8') for header in self._headers]
         else:
-            self.fout.write(header.encode('utf-8'))
-        self.fout.write('\n')
-
-    def patch(self, value):
-        value = super(DumpCSV, self).patch(value)
-        if PY3:
-            return str(value)
-        else:
-            return unicode(value)  # noqa
+            fieldnames = self._headers
+        self.csv_writer = csv.DictWriter(self.fout, fieldnames)
+        self.csv_writer.writeheader()
 
     def write_obj(self, obj):
-        values = [
-            self.patch(obj.get(head))
-            for head in self._headers
-        ]
-        content = self._separator.join(values)
-        if PY3:
-            self.fout.write(content)
-        else:
-            self.fout.write(content.encode('utf-8'))
-        self.fout.write('\n')
+        self.csv_writer.writerow(self.patch_obj(obj))
+
+    def patch_obj(self, obj):
+        new_obj = {}
+        for key, value in obj.items():
+            if value in [None, {}, []]:
+                value = ""
+            if not PY3:
+                key = key.encode('utf8')
+                if isinstance(value, unicode):
+                    value = value.encode('utf8')
+
+            new_obj[key] = value
+        return new_obj
 
 
 class DumpXLS(DumpExcel):
@@ -152,7 +140,7 @@ def write_obj(self, obj):
         self.cloumn = 0
 
         for head in self._headers:
-            value = self.patch(obj.get(head))
+            value = obj.get(head)
             self.ws.write(self.row, self.cloumn, value)
             self.cloumn += 1
 
diff --git a/setup.py b/setup.py
@@ -12,7 +12,7 @@
 
 setup(
     name='jsoncsv',
-    version='2.1.0',
+    version='2.2.0a',
     url='https://github.com/alingse/jsoncsv',
     description='A command tool easily convert json file to csv or xlsx.',
     long_description=readme,
diff --git a/tests/test_mkexcel.py b/tests/test_mkexcel.py
@@ -9,7 +9,7 @@
 from jsoncsv.dumptool import dump_excel
 
 
-class Testdumptool(unittest.TestCase):
+class TestDumpTool(unittest.TestCase):
 
     # FIXME (使用虚拟文件)
     def test_dumpexcel_csv(self):
@@ -52,3 +52,21 @@ def test_dumpcexcel_xls(self):
 
         fin.close()
         fout.close()
+
+    def test_dump_csv_with_non_ascii(self):
+        fin = open('./fixture/files/expand.2.json', 'r')
+        fout = open('./fixture/files/tmp.output.2.csv', 'wb')
+
+        dump_excel(fin, fout, DumpCSV)
+
+        fin.close()
+        fout.close()
+
+    def test_dump_xls_with_non_ascii(self):
+        fin = open('./fixture/files/expand.2.json', 'r')
+        fout = open('./fixture/files/tmp.output.2.xls', 'wb')
+
+        dump_excel(fin, fout, DumpXLS)
+
+        fin.close()
+        fout.close()
