fix #266

Author: Keyang

bin/options.json

@@ -13,7 +13,7 @@
             "type": "~object"
         },
         "--quote": {
-            "desc": "quote surrounding a column content containing delimiters. default double quote (\"). e.g. --quote=# ",
+            "desc": "quote surrounding a column content containing delimiters. To turn off quote, please use 'off' --quote=off. default double quote (\"). e.g. chage to hash: --quote=# ",
             "type": "string"
         },
         "--trim": {

package.json

@@ -297,8 +297,9 @@
     "test": "rm -Rf .ts-node && TS_NODE_CACHE_DIRECTORY=.ts-node mocha -r ts-node/register src/**/*.test.ts ./test/*.ts -R spec",
     "travis": "nyc --reporter lcov mocha -r ts-node/register src/**/*.test.ts ./test/*.ts -R spec",
     "test:debug": "mocha debug -r ts-node/register src/**/*.test.ts ./test/*.ts -R spec",
+    "test:function":"mocha -r ts-node/register test/**/*.ts",
     "test-all": "mocha  ./test -R spec && CSV_WORKER=3 mocha ./test -R spec ",
-    "test:unit": "mocha -r ts-node/register src/**/*.test.ts",
+    "test:unit": "mocha -r ts-node/register src/**/*.ts",
     "test:all:debug": "mocha debug ./testNew -R spec",
     "coverage": "nyc --reporter html  mocha -r ts-node/register src/**/*.test.ts ./test/*.ts -R spec",
     "coveralls": "cat ./coverage/lcov.info | ./node_modules/.bin/coveralls"

src/Converter.ts

@@ -13,7 +13,7 @@ import { ProcessorLocal } from "./ProcessorLocal";
 import { Result } from "./Result";
 import CSVError from "./CSVError";
 import { bufFromString } from "./util";
-export class Converter extends Transform {
+export class Converter extends Transform implements PromiseLike<Array<any>> {
   preRawData(onRawData: PreRawDataCallback): Converter {
     this.runtime.preRawDataHook = onRawData;
     return this;

src/rowSplit.test.ts

@@ -106,25 +106,38 @@ describe("RowSplit.parse function", function () {
     assert.equal(res.closed, true);
     assert.equal(res.cells[2], 'csvtojson,a"\nwesome');
   });
-  it ("should allow blank quotes",()=>{
-    const data="a|^^|^b^";
-    
+  it("should allow blank quotes", () => {
+    const data = "a|^^|^b^";
+
     const rowSplit = new RowSplit(new Converter({
       delimiter: '|',
       quote: '^',
       noheader: true
     }));
-    const res=rowSplit.parse(data);
-    assert.equal(res.cells[1],"");
+    const res = rowSplit.parse(data);
+    assert.equal(res.cells[1], "");
   })
-  it ("should allow blank quotes in quotes",()=>{
-    const data='a,"hello,this,"", test"';
-    
+  it("should allow blank quotes in quotes", () => {
+    const data = 'a,"hello,this,"", test"';
+
     const rowSplit = new RowSplit(new Converter({
       noheader: true
     }));
-    const res=rowSplit.parse(data);
-    assert.equal(res.cells[1],'hello,this,", test');
+    const res = rowSplit.parse(data);
+    assert.equal(res.cells[1], 'hello,this,", test');
+  })
+  it("should smart detect if an initial quote is only part of value ", () => {
+    const data = '"Weight" (kg),Error code,"Height" (m)';
+    const rowSplit = new RowSplit(new Converter({
+      noheader: true
+    }));
+    const res = rowSplit.parse(data);
+    console.log(res);
+    assert.equal(res.cells.length, 3);
+    assert(res.closed);
+    assert.equal(res.cells[0],'"Weight" (kg)');
+    assert.equal(res.cells[1],'Error code');
+    assert.equal(res.cells[2],'"Height" (m)');
+    
   })
-
 });

src/rowSplit.ts

@@ -2,7 +2,7 @@ import { CSVParseParam } from "./Parameters";
 import { Converter } from "./Converter";
 import { Fileline } from "./fileline";
 import getEol from "./getEol";
-import { filterArray,trimLeft,trimRight } from "./util";
+import { filterArray, trimLeft, trimRight } from "./util";
 
 const defaulDelimiters = [",", "|", "\t", ";", ":"];
 export class RowSplit {
@@ -24,7 +24,7 @@ export class RowSplit {
     this.escape = conv.parseParam.escape;
   }
   parse(fileline: Fileline): RowSplitResult {
-    if (fileline.length ===0 ||(this.conv.parseParam.ignoreEmpty && fileline.trim().length===0) ) {
+    if (fileline.length === 0 || (this.conv.parseParam.ignoreEmpty && fileline.trim().length === 0)) {
       return { cells: [], closed: true };
     }
     const quote = this.quote;
@@ -41,9 +41,9 @@ export class RowSplit {
     const delimiter = this.conv.parseRuntime.delimiter;
     const rowArr = fileline.split(delimiter);
     if (quote === "off") {
-      if (trim){
-        for (let i =0;i<rowArr.length;i++){
-          rowArr[i]=rowArr[i].trim();
+      if (trim) {
+        for (let i = 0; i < rowArr.length; i++) {
+          rowArr[i] = rowArr[i].trim();
         }
       }
       return { cells: rowArr, closed: true };
@@ -63,17 +63,24 @@ export class RowSplit {
       }
       const len = e.length;
       if (!inquote) {
-        if (len === 2 && e ===this.quote+this.quote){
+        if (len === 2 && e === this.quote + this.quote) {
           row.push("");
           continue;
-        }else if (this.isQuoteOpen(e)) { //quote open
+        } else if (this.isQuoteOpen(e)) { //quote open
           e = e.substr(1);
           if (this.isQuoteClose(e)) { //quote close
             e = e.substring(0, e.lastIndexOf(quote));
             e = this.escapeQuote(e);
             row.push(e);
             continue;
-          } else {
+          } else if (e.lastIndexOf(quote) !== 0) {
+            if (trim) {
+              e = trimRight(e);
+            }
+            row.push(quote+e);
+            continue;
+          }
+          else {
             inquote = true;
             quoteBuff += e;
             continue;
@@ -175,7 +182,7 @@ export class RowSplit {
     while (lines.length) {
       const line = left + lines.shift();
       const row = this.parse(line);
-      if (row.cells.length === 0 && this.conv.parseParam.ignoreEmpty){
+      if (row.cells.length === 0 && this.conv.parseParam.ignoreEmpty) {
         continue;
       }
       if (row.closed || this.conv.parseParam.alwaysSplitAtEOL) {

test/data/csvWithUnclosedHeader

@@ -0,0 +1,3 @@
+Date and time,Sample no.,Status,Sample type,Patient ID,Last name,First name,Sex of patient,Patient birth date,Patient note,Patient department,Accession number,Sample site,Physician,Operator,Department,Note,Test order code,Draw time,Approval status,Approval time,Report layout,Patient account number,,Errors detected during measurement,"Age" (years),Error code,"Weight" (kg),Error code,"Height" (m),Error code,"HCG_Beta(P)" (mIU/mL),Error code,"Troponin_I(P)" (ng/mL),Error code,"CK_MB(P)" (ng/mL),Error code,"D-Dimer(P)" (ng/mL),Error code,"hsCRP(P)" (mg/L),Error code,"Myoglobin(P)" (ng/mL),Error code,"NT-proBNP" (pg/mL),Error code,"CRP" (mg/dL),Error code,"BNP" (ng/L),Error code,"TnT" (ng/mL),Error code,"Demo(P)" (AQT),Error code,"PCT" (ng/mL),Error code,
+"06/14/2018  10:34 AM","12669","OK","","4206454447","","","Unknown","","","","4206454447","","","Anonymous","","","400",,"Not reviewed","","New","",,"",,,,,,,,,,,,,,,,,,,168,"",,,,,,,,,,
+"06/14/2018  10:19 AM","12668","OK","","4206454403","","","Unknown","","","","4206454403","","","Anonymous","","","101",,"Not reviewed","","New","",,"",,,,,,,,,<0.010,"",4.2,"",,,,,,,,,,,,,,,,,,

test/testCSVConverter3.ts

@@ -228,4 +228,16 @@ describe("testCSVConverter3", function () {
         assert.equal(jsonObj[0].field2, "");
       });
   })
+  it("should parse header with quotes correctly", function () {
+    var testData = __dirname + "/data/csvWithUnclosedHeader";
+    return csv({
+      headers:["exam_date","sample_no","status","sample_type","patient_id","last_name","first_name","gender_of_patient","patient_birth_date","patient_note","patient_department","accession_number","sample_site","physician","operator","department","note","test_order_code","draw_time","approval_status","approval_time","report_layout","patient_account_number","none_1","errors_detected_during_measurement","age","error_code_01","weight","error_code_02","height","error_code_03","hcg_beta_p","error_code_04","troponin_i_p","error_code_05","ck_mb_p","error_code_06","d_dimer_p","error_code_07","hscrp_p","error_code_08","myoglobin_p","error_code_09","nt_probnp","error_code_10","crp","error_code_11","bnp","error_code_12","tnt","error_code_13","demo_p","error_code_14","pct","error_code_15"]
+    })
+    .fromFile(testData)
+    .then((d)=>{
+      assert.equal(d.length,2);
+      assert.equal(d[0].sample_no,"12669");
+    })
+    
+  });
 });

v2/Converter.d.ts

@@ -3,7 +3,7 @@ import { Transform, TransformOptions, Readable } from "stream";
 import { CSVParseParam } from "./Parameters";
 import { ParseRuntime } from "./ParseRuntime";
 import CSVError from "./CSVError";
-export declare class Converter extends Transform {
+export declare class Converter extends Transform implements PromiseLike<Array<any>> {
     options: TransformOptions;
     preRawData(onRawData: PreRawDataCallback): Converter;
     preFileLine(onFileLine: PreFileLineCallback): Converter;