Bug fix: csv parsing

Author: runestonetest

runestone/spreadsheet/spreadsheet.py

@@ -13,28 +13,35 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-__author__ = 'bmiller'
+__author__ = "bmiller"
 
-import re, os
+import csv
+import io
+import os
+import re
 from docutils import nodes
 from docutils.parsers.rst import directives
-from runestone.common.runestonedirective import RunestoneNode, RunestoneIdDirective, get_node_line
+from runestone.common.runestonedirective import (
+    RunestoneNode,
+    RunestoneIdDirective,
+    get_node_line,
+)
 from runestone.server.componentdb import addQuestionToDB, addHTMLToDB
 
 #
 # setup is called when extensions are loaded. This registers the new directive and
 # logs any js or css files that should be loaded for this extension.
 #
 def setup(app):
-    app.add_directive('spreadsheet',SpreadSheet)
+    app.add_directive("spreadsheet", SpreadSheet)
 
-    app.add_autoversioned_javascript('spreadsheet.js')
-    app.add_javascript('jexcel.js')
-    app.add_javascript('japp.js')
+    app.add_autoversioned_javascript("spreadsheet.js")
+    app.add_javascript("jexcel.js")
+    app.add_javascript("japp.js")
 
-    app.add_autoversioned_stylesheet('spreadsheet.css')
-    app.add_stylesheet('jexcel.css')
-    app.add_stylesheet('japp.css')
+    app.add_autoversioned_stylesheet("spreadsheet.css")
+    app.add_stylesheet("jexcel.css")
+    app.add_stylesheet("japp.css")
 
     app.add_node(SpreadSheetNode, html=(visit_ss_node, depart_ss_node))
 
@@ -54,6 +61,7 @@ def __init__(self, content, **kwargs):
         super(SpreadSheetNode, self).__init__(**kwargs)
         self.ss_options = content
 
+
 #
 # The spreadsheet class implements the directive.
 # When the directive is processed the run method is called.
@@ -71,81 +79,104 @@ class SpreadSheet(RunestoneIdDirective):
         A1,B1,C1,D1...
         A2,B2,C2,D2...
     """
+
     required_arguments = 1
     optional_arguments = 5
     has_content = True
     option_spec = RunestoneIdDirective.option_spec.copy()
-    option_spec.update({
-        'fromcsv': directives.unchanged,
-        'colwidths': directives.unchanged,
-        'coltitles': directives.unchanged,
-        'mindimensions': directives.unchanged
-    })
-
+    option_spec.update(
+        {
+            "fromcsv": directives.unchanged,
+            "colwidths": directives.unchanged,
+            "coltitles": directives.unchanged,
+            "mindimensions": directives.unchanged,
+        }
+    )
 
     def run(self):
         super(SpreadSheet, self).run()
         env = self.state.document.settings.env
 
-        self.options['divid'] = self.arguments[0].strip()
+        self.options["divid"] = self.arguments[0].strip()
 
-        if '====' in self.content:
-            idx = self.content.index('====')
-            suffix = self.content[idx+1:]
-            self.options['asserts'] = suffix
-            self.options['autograde'] = 'data-autograde="true"'
+        if "====" in self.content:
+            idx = self.content.index("====")
+            suffix = self.content[idx + 1 :]
+            self.options["asserts"] = suffix
+            self.options["autograde"] = 'data-autograde="true"'
         else:
-            self.options['asserts'] = '""'
-            self.options['autograde'] = ''
+            self.options["asserts"] = '""'
+            self.options["autograde"] = ""
 
-        if 'fromcsv' in self.options:
-            self.content = self.body_from_csv(env, self.options['fromcsv'])
+        if "fromcsv" in self.options:
+            self.content = self.body_from_csv(env, self.options["fromcsv"])
         else:
             if self.content:
                 self.content = self.body_to_csv(self.content)
             else:
-                raise ValueError("You must specify either from csv or provide content in the body")
+                raise ValueError(
+                    "You must specify either from csv or provide content in the body"
+                )
 
-        self.options['data'] = self.content
+        self.options["data"] = self.content
 
-        if 'coltitles' not in  self.options:
-            self.options['coltitles'] = ""
+        if "coltitles" not in self.options:
+            self.options["coltitles"] = ""
         else:
-            self.options['coltitles'] = "data-coltitles=[{}]".format(self.options['coltitles'])
+            self.options["coltitles"] = "data-coltitles=[{}]".format(
+                ",".join([x.strip() for x in self.options["coltitles"].split(",")])
+            )
 
-        if 'mindimensions' not in self.options:
-            self.options['mindimensions'] = ""
+        if "mindimensions" not in self.options:
+            self.options["mindimensions"] = ""
         else:
-            self.options['mindimensions'] = "data-mindimensions=[{}]".format(self.options['mindimensions'])
+            self.options["mindimensions"] = "data-mindimensions=[{}]".format(
+                ",".join([x.strip() for x in self.options["mindimensions"].split(",")])
+            )
 
-        if 'colwidths' not in self.options:
-            self.options['colwidths'] = ""
+        if "colwidths" not in self.options:
+            self.options["colwidths"] = ""
         else:
-            self.options['colwidths'] = "data-colwidths=[{}]".format(self.options['colwidths'])
+            self.options["colwidths"] = "data-colwidths=[{}]".format(
+                ",".join([x.strip() for x in self.options["colwidths"].split(",")])
+            )
 
         ssnode = SpreadSheetNode(self.options, rawsource=self.block_text)
         ssnode.source, ssnode.line = self.state_machine.get_source_and_line(self.lineno)
-        self.add_name(ssnode)    # make this divid available as a target for :ref:
+        self.add_name(ssnode)  # make this divid available as a target for :ref:
 
         return [ssnode]
 
+    def body_to_csv(self, row_list):
+        """
+        Use the csv reader and writer functionality to better parse the body.
 
+        1. Convert the contents to a StringIO object
+        2. Then read and process using a csv reader
+        3. Formulas with ,'s in them should be in double quotes, if there are "'s in the
+           cell then they should be ""-ed
+        """
 
-    def body_to_csv(self, row_list):
         csvlist = []
+        body_list = []
         for row in row_list:
-            if re.match(r'^\s*====', row):
+            if re.match(r"^\s*====", row):
                 break
-            items = row.split(',')
+            body_list.append(row)
+        body_file = io.StringIO("\n".join(body_list))
+        body_reader = csv.reader(body_file)
+        for row in body_reader:
             ilist = []
-            for item in items:
+            for item in row:
                 item = item.strip()
+                if item and item[0] == '"' and item[-1] == '"':
+                    item = item[1:-1]
                 if is_float(item):
                     ilist.append(as_int_or_float(item))
                 elif item.startswith("="):
-                    ilist.append('{}'.format(item.upper()))
+                    ilist.append("{}".format(item.upper()))
                 else:
-                    ilist.append('{}'.format(item))
+                    ilist.append("{}".format(item))
             csvlist.append(ilist)
         return csvlist
 
@@ -155,18 +186,21 @@ def body_from_csv(self, env, csvfile):
         filename = os.path.join(env.srcdir, ffpath, csvfile)
 
         print("\n\nPATH=", self.srcpath)
-        with open(filename,'r') as csv:
+        with open(filename, "r") as csv:
             content = csv.readlines()
 
+        content = [line[:-1] for line in content]
         return self.body_to_csv(content)
 
+
 def is_float(s):
     try:
         x = float(s)
         return True
     except:
         return False
 
+
 def as_int_or_float(s):
     try:
         x = int(s)
@@ -187,9 +221,11 @@ def as_int_or_float(s):
 </div>
 """
 
-def visit_ss_node(self,node):
+
+def visit_ss_node(self, node):
     res = TEMPLATE.format(**node.ss_options)
     self.body.append(res)
 
-def depart_ss_node(self,node):
-    pass
+
+def depart_ss_node(self, node):
+    pass

runestone/spreadsheet/test/_sources/index.rst

@@ -37,3 +37,9 @@ Cells in the spreadsheet that are graded are initially colored light blue.  When
     ====
     assert A151 == 150
 
+
+.. spreadsheet:: ss3
+   :colwidths: 200,200
+
+   "=CONCATENATE(""abc"", ""xyz"")","=CONCATENATE(""abc"", ""xyz"")"
+   '=1+1, 2