Use pythonparser instead of ast for parsing Python.

Author: Dylan Trotter

.pylintrc

@@ -20,3 +20,8 @@ disable=design,similarities,no-self-use,attribute-defined-outside-init,locally-d
 [REPORTS]
 msg-template={path}:{line}: {msg} ({symbol})
 reports=no
+
+[TYPECHECK]
+# AST classes have dynamic members. Writer does not but for some reason pylint
+# barfs on some of its members.
+ignored-classes=pythonparser.ast.Module,grumpy.compiler.util.Writer

compiler/block.py

@@ -19,10 +19,11 @@
 from __future__ import unicode_literals
 
 import abc
-import ast
 import collections
 import re
 
+from pythonparser import algorithm
+from pythonparser import ast
 from pythonparser import source
 
 from grumpy.compiler import expr
@@ -226,7 +227,7 @@ class ModuleBlock(Block):
 
   def __init__(self, full_package_name, runtime, libroot, filename, src,
                future_features):
-    super(ModuleBlock, self).__init__(None, '<module>')
+    Block.__init__(self, None, '<module>')
     self._full_package_name = full_package_name
     self._runtime = runtime
     self._libroot = libroot
@@ -253,7 +254,7 @@ class ClassBlock(Block):
   """Python block for a class definition."""
 
   def __init__(self, parent_block, name, global_vars):
-    super(ClassBlock, self).__init__(parent_block, name)
+    Block.__init__(self, parent_block, name)
     self.global_vars = global_vars
 
   def bind_var(self, writer, name, value):
@@ -293,7 +294,7 @@ class FunctionBlock(Block):
   """Python block for a function definition."""
 
   def __init__(self, parent_block, name, block_vars, is_generator):
-    super(FunctionBlock, self).__init__(parent_block, name)
+    Block.__init__(self, parent_block, name)
     self.vars = block_vars
     self.parent_block = parent_block
     self.is_generator = is_generator
@@ -353,7 +354,7 @@ def __init__(self, name, var_type, arg_index=None):
       self.init_expr = None
 
 
-class BlockVisitor(ast.NodeVisitor):
+class BlockVisitor(algorithm.Visitor):
   """Visits nodes in a function or class to determine block variables."""
 
   # pylint: disable=invalid-name,missing-docstring
@@ -401,8 +402,9 @@ def visit_ImportFrom(self, node):
       self._register_local(alias.asname or alias.name)
 
   def visit_With(self, node):
-    if node.optional_vars:
-      self._assign_target(node.optional_vars)
+    for item in node.items:
+      if item.optional_vars:
+        self._assign_target(item.optional_vars)
     self.generic_visit(node)
 
   def _assign_target(self, target):
@@ -435,14 +437,14 @@ class FunctionBlockVisitor(BlockVisitor):
   # pylint: disable=invalid-name,missing-docstring
 
   def __init__(self, node):
-    super(FunctionBlockVisitor, self).__init__()
+    BlockVisitor.__init__(self)
     self.is_generator = False
     node_args = node.args
-    args = [a.id for a in node_args.args]
+    args = [a.arg for a in node_args.args]
     if node_args.vararg:
-      args.append(node_args.vararg)
+      args.append(node_args.vararg.arg)
     if node_args.kwarg:
-      args.append(node_args.kwarg)
+      args.append(node_args.kwarg.arg)
     for i, name in enumerate(args):
       if name in self.vars:
         msg = "duplicate argument '{}' in function definition".format(name)

compiler/block_test.py

@@ -18,10 +18,11 @@
 
 from __future__ import unicode_literals
 
-import ast
 import textwrap
 import unittest
 
+import pythonparser
+
 from grumpy.compiler import block
 from grumpy.compiler import stmt
 from grumpy.compiler import util
@@ -206,7 +207,7 @@ def testGlobalIsParam(self):
                             visitor.visit, _ParseStmt('global foo'))
 
   def testGlobalUsedPriorToDeclaration(self):
-    node = ast.parse('foo = 42\nglobal foo')
+    node = pythonparser.parse('foo = 42\nglobal foo')
     visitor = block.BlockVisitor()
     self.assertRaisesRegexp(util.ParseError, 'used prior to global declaration',
                             visitor.generic_visit, node)
@@ -250,7 +251,7 @@ def _MakeModuleBlock():
 
 
 def _ParseStmt(stmt_str):
-  return ast.parse(stmt_str).body[0]
+  return pythonparser.parse(stmt_str).body[0]
 
 
 if __name__ == '__main__':

compiler/expr_visitor.py

@@ -18,16 +18,18 @@
 
 from __future__ import unicode_literals
 
-import ast
 import contextlib
 import textwrap
 
+from pythonparser import algorithm
+from pythonparser import ast
+
 from grumpy.compiler import block
 from grumpy.compiler import expr
 from grumpy.compiler import util
 
 
-class ExprVisitor(ast.NodeVisitor):
+class ExprVisitor(algorithm.Visitor):
   """Builds and returns a Go expression representing the Python nodes."""
 
   # pylint: disable=invalid-name,missing-docstring
@@ -194,8 +196,10 @@ def visit_Set(self, node):
 
   def visit_DictComp(self, node):
     result = self.block.alloc_temp()
-    elt = ast.Tuple(elts=[node.key, node.value], context=ast.Load)
-    with self.visit(ast.GeneratorExp(elt, node.generators)) as gen:
+    elt = ast.Tuple(elts=[node.key, node.value])
+    gen_node = ast.GeneratorExp(
+        elt=elt, generators=node.generators, loc=node.loc)
+    with self.visit(gen_node) as gen:
       self.writer.write_checked_call2(
           result, 'πg.DictType.Call(πF, πg.Args{{{}}}, nil)', gen.expr)
     return result
@@ -218,12 +222,12 @@ def visit_ExtSlice(self, node):
     return result
 
   def visit_GeneratorExp(self, node):
-    body = ast.Expr(value=ast.Yield(node.elt), lineno=None)
+    body = ast.Expr(value=ast.Yield(value=node.elt), loc=node.loc)
     for comp_node in reversed(node.generators):
       for if_node in reversed(comp_node.ifs):
-        body = ast.If(test=if_node, body=[body], orelse=[], lineno=None)  # pylint: disable=redefined-variable-type
+        body = ast.If(test=if_node, body=[body], orelse=[], loc=node.loc)  # pylint: disable=redefined-variable-type
       body = ast.For(target=comp_node.target, iter=comp_node.iter,
-                     body=[body], orelse=[], lineno=None)
+                     body=[body], orelse=[], loc=node.loc)
 
     args = ast.arguments(args=[], vararg=None, kwarg=None, defaults=[])
     node = ast.FunctionDef(name='<generator>', args=args, body=[body])
@@ -259,7 +263,7 @@ def visit_Index(self, node):
     return result
 
   def visit_Lambda(self, node):
-    ret = ast.Return(node.body, lineno=node.lineno)
+    ret = ast.Return(value=node.body, loc=node.loc)
     func_node = ast.FunctionDef(
         name='<lambda>', args=node.args, body=[ret])
     return self.visit_function_inline(func_node)
@@ -273,7 +277,9 @@ def visit_List(self, node):
 
   def visit_ListComp(self, node):
     result = self.block.alloc_temp()
-    with self.visit(ast.GeneratorExp(node.elt, node.generators)) as gen:
+    gen_node = ast.GeneratorExp(
+        elt=node.elt, generators=node.generators, loc=node.loc)
+    with self.visit(gen_node) as gen:
       self.writer.write_checked_call2(
           result, 'πg.ListType.Call(πF, πg.Args{{{}}}, nil)', gen.expr)
     return result
@@ -438,7 +444,7 @@ def visit_function_inline(self, node):
         with self.visit(d) if d else expr.nil_expr as default:
           tmpl = '$args[$i] = πg.Param{Name: $name, Def: $default}'
           self.writer.write_tmpl(tmpl, args=func_args.expr, i=i,
-                                 name=util.go_str(a.id), default=default.expr)
+                                 name=util.go_str(a.arg), default=default.expr)
       flags = []
       if args.vararg:
         flags.append('πg.CodeFlagVarArg')

compiler/expr_visitor_test.py

@@ -18,11 +18,12 @@
 
 from __future__ import unicode_literals
 
-import ast
 import subprocess
 import textwrap
 import unittest
 
+import pythonparser
+
 from grumpy.compiler import block
 from grumpy.compiler import expr_visitor
 from grumpy.compiler import shard_test
@@ -231,7 +232,7 @@ def _MakeModuleBlock():
 
 
 def _ParseExpr(expr):
-  return ast.parse(expr).body[0].value
+  return pythonparser.parse(expr).body[0].value
 
 
 def _ParseAndVisitExpr(expr):