Use unicode_literals throughout compiler code.

Author: Dylan Trotter

Makefile

@@ -53,6 +53,8 @@ export GOPATH := $(ROOT_DIR)/build
 export PYTHONPATH := $(ROOT_DIR)/$(PY_DIR)
 export PATH := $(ROOT_DIR)/build/bin:$(PATH)
 
+PYTHONPARSER_SRCS := $(patsubst third_party/%,$(PY_DIR)/%,$(wildcard third_party/pythonparser/*.py))
+
 COMPILER_BIN := build/bin/grumpc
 COMPILER_SRCS := $(addprefix $(PY_DIR)/grumpy/compiler/,$(notdir $(shell find compiler -name '*.py' -not -name '*_test.py'))) $(PY_DIR)/grumpy/__init__.py
 COMPILER_TESTS := $(patsubst %.py,grumpy/%,$(filter-out compiler/expr_visitor_test.py compiler/stmt_test.py,$(wildcard compiler/*_test.py)))
@@ -62,7 +64,7 @@ COMPILER_PASS_FILES := $(patsubst %,$(PY_DIR)/%.pass,$(COMPILER_TESTS))
 COMPILER_EXPR_VISITOR_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/expr_visitor_test.%of32.pass,$(shell seq 32))
 COMPILER_STMT_PASS_FILES := $(patsubst %,$(PY_DIR)/grumpy/compiler/stmt_test.%of16.pass,$(shell seq 16))
 COMPILER_D_FILES := $(patsubst %,$(PY_DIR)/%.d,$(COMPILER_TESTS))
-COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS)
+COMPILER := $(COMPILER_BIN) $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
 
 RUNNER_BIN := build/bin/grumprun
 RUNTIME_SRCS := $(addprefix build/src/grumpy/,$(notdir $(wildcard runtime/*.go)))
@@ -73,7 +75,7 @@ RUNNER = $(RUNNER_BIN) $(COMPILER) $(RUNTIME) $(STDLIB)
 
 GRUMPY_STDLIB_SRCS := $(shell find lib -name '*.py')
 GRUMPY_STDLIB_PACKAGES := $(foreach x,$(GRUMPY_STDLIB_SRCS),$(patsubst lib/%.py,%,$(patsubst lib/%/__init__.py,%,$(x))))
-THIRD_PARTY_STDLIB_SRCS := $(shell find third_party -name '*.py')
+THIRD_PARTY_STDLIB_SRCS := $(shell find third_party/pypy -name '*.py') $(shell find third_party/stdlib -name '*.py')
 THIRD_PARTY_STDLIB_PACKAGES := $(foreach x,$(THIRD_PARTY_STDLIB_SRCS),$(patsubst third_party/stdlib/%.py,%,$(patsubst third_party/pypy/%.py,%,$(patsubst third_party/pypy/%/__init__.py,%,$(patsubst third_party/stdlib/%/__init__.py,%,$(x))))))
 STDLIB_SRCS := $(GRUMPY_STDLIB_SRCS) $(THIRD_PARTY_STDLIB_SRCS)
 STDLIB_PACKAGES := $(GRUMPY_STDLIB_PACKAGES) $(THIRD_PARTY_STDLIB_PACKAGES)
@@ -148,7 +150,7 @@ $(COMPILER_PASS_FILES): %.pass: %.py $(COMPILER)
 	@touch $@
 	@echo compiler/`basename $*` PASS
 
-$(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS)
+$(COMPILER_D_FILES): $(PY_DIR)/%.d: $(PY_DIR)/%.py $(COMPILER_SRCS) $(PYTHONPARSER_SRCS)
 	@$(PYTHON) -m modulefinder $< | awk '{if (match($$2, /^grumpy\>/)) { print "$(PY_DIR)/$*.pass: " substr($$3, length("$(ROOT_DIR)/") + 1) }}' > $@
 
 -include $(COMPILER_D_FILES)
@@ -268,6 +270,10 @@ $(eval $(foreach x,$(STDLIB_TESTS),$(call GRUMPY_STDLIB_TEST,$(x))))
 $(PY_DIR)/weetest.py: lib/weetest.py
 	@cp -f $< $@
 
+$(PYTHONPARSER_SRCS): $(PY_DIR)/%: third_party/%
+	@mkdir -p $(@D)
+	@cp -f $< $@
+
 $(patsubst %_test,build/%.go,$(ACCEPT_TESTS)): build/%.go: %_test.py $(COMPILER)
 	@mkdir -p $(@D)
 	@$(COMPILER_BIN) $< > $@

compiler/block.py

@@ -16,11 +16,15 @@
 
 """Classes for analyzing and storing the state of Python code blocks."""
 
+from __future__ import unicode_literals
+
 import abc
 import ast
 import collections
 import re
 
+from pythonparser import source
+
 from grumpy.compiler import expr
 from grumpy.compiler import util
 
@@ -56,7 +60,7 @@ class Block(object):
   _filename = None
   _full_package_name = None
   _libroot = None
-  _lines = None
+  _buffer = None
   _runtime = None
   _strings = None
   imports = None
@@ -96,8 +100,8 @@ def filename(self):
     return self._module_block._filename  # pylint: disable=protected-access
 
   @property
-  def lines(self):
-    return self._module_block._lines  # pylint: disable=protected-access
+  def buffer(self):
+    return self._module_block._buffer  # pylint: disable=protected-access
 
   @property
   def strings(self):
@@ -220,14 +224,14 @@ class ModuleBlock(Block):
     imports: A dict mapping fully qualified Go package names to Package objects.
   """
 
-  def __init__(self, full_package_name, runtime, libroot, filename, lines,
+  def __init__(self, full_package_name, runtime, libroot, filename, src,
                future_features):
     super(ModuleBlock, self).__init__(None, '<module>')
     self._full_package_name = full_package_name
     self._runtime = runtime
     self._libroot = libroot
     self._filename = filename
-    self._lines = lines
+    self._buffer = source.Buffer(src)
     self._strings = set()
     self.imports = {}
     self._future_features = future_features

compiler/block_test.py

@@ -16,6 +16,8 @@
 
 """Tests Package, Block, BlockVisitor and related classes."""
 
+from __future__ import unicode_literals
+
 import ast
 import textwrap
 import unittest
@@ -106,7 +108,7 @@ def testResolveName(self):
   def _ResolveName(self, b, name):
     writer = util.Writer()
     b.resolve_name(writer, name)
-    return writer.out.getvalue()
+    return writer.getvalue()
 
 
 class BlockVisitorTest(unittest.TestCase):
@@ -243,7 +245,7 @@ def testYieldExpr(self):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', '',
                            stmt.FutureFeatures())
 
 

compiler/expr.py

@@ -16,6 +16,8 @@
 
 """Classes representing generated expressions."""
 
+from __future__ import unicode_literals
+
 import abc
 
 from grumpy.compiler import util

compiler/expr_visitor.py

@@ -16,6 +16,8 @@
 
 """Visitor class for traversing Python expressions."""
 
+from __future__ import unicode_literals
+
 import ast
 import contextlib
 import textwrap
@@ -462,10 +464,10 @@ def visit_function_inline(self, node):
           self.writer.write('return πg.NewGenerator(πF, func(πSent *πg.Object) '
                             '(*πg.Object, *πg.BaseException) {')
           with self.writer.indent_block():
-            self.writer.write_block(func_block, visitor.writer.out.getvalue())
+            self.writer.write_block(func_block, visitor.writer.getvalue())
           self.writer.write('}).ToObject(), nil')
         else:
-          self.writer.write_block(func_block, visitor.writer.out.getvalue())
+          self.writer.write_block(func_block, visitor.writer.getvalue())
       self.writer.write('}), πF.Globals()).ToObject()')
     return result
 

compiler/expr_visitor_test.py

@@ -16,6 +16,8 @@
 
 """Tests for ExprVisitor."""
 
+from __future__ import unicode_literals
+
 import ast
 import subprocess
 import textwrap
@@ -35,11 +37,13 @@ def Test(self):
   return Test
 
 
-def _MakeLiteralTest(lit):
+def _MakeLiteralTest(lit, expected=None):
+  if expected is None:
+    expected = lit
   def Test(self):
     status, output = _GrumpRun('print repr({}),'.format(lit))
     self.assertEqual(0, status, output)
-    self.assertEqual(eval('repr({})'.format(lit)), output.strip())  # pylint: disable=eval-used
+    self.assertEqual(expected, output.strip())  # pylint: disable=eval-used
   return Test
 
 
@@ -130,9 +134,9 @@ def foo(a, b=2):
   testCompareNotInTuple = _MakeExprTest('10 < 12 not in (1, 2, 3)')
 
   testDictEmpty = _MakeLiteralTest('{}')
-  testDictNonEmpty = _MakeLiteralTest('{"foo": 42, "bar": 43}')
+  testDictNonEmpty = _MakeLiteralTest("{'foo': 42, 'bar': 43}")
 
-  testSetNoneEmpty = _MakeLiteralTest('{"foo", "bar"}')
+  testSetNonEmpty = _MakeLiteralTest("{'foo', 'bar'}", "set(['foo', 'bar'])")
 
   testDictCompFor = _MakeExprTest('{x: str(x) for x in range(3)}')
   testDictCompForIf = _MakeExprTest(
@@ -181,16 +185,15 @@ def foo():
 
   testNumInt = _MakeLiteralTest('42')
   testNumLong = _MakeLiteralTest('42L')
-  testNumIntLarge = _MakeLiteralTest('12345678901234567890')
+  testNumIntLarge = _MakeLiteralTest('12345678901234567890',
+                                     '12345678901234567890L')
   testNumFloat = _MakeLiteralTest('102.1')
-  testNumFloatOnlyDecimal = _MakeLiteralTest('.5')
-  # TODO: Current Grumpy's repr on float has different behavior than CPython.
-  # so skip these for now.
-  testNumFloatNoDecimal = unittest.expectedFailure(_MakeLiteralTest('5.'))
-  testNumFloatSci = unittest.expectedFailure(_MakeLiteralTest('1e6'))
-  testNumFloatSciCap = unittest.expectedFailure(_MakeLiteralTest('1E6'))
-  testNumFloatSciCapPlus = unittest.expectedFailure(_MakeLiteralTest('1E+6'))
-  testNumFloatSciMinus = _MakeLiteralTest('1e-6')
+  testNumFloatOnlyDecimal = _MakeLiteralTest('.5', '0.5')
+  testNumFloatNoDecimal = _MakeLiteralTest('5.', '5')
+  testNumFloatSci = _MakeLiteralTest('1e6', '1e+06')
+  testNumFloatSciCap = _MakeLiteralTest('1E6', '1e+06')
+  testNumFloatSciCapPlus = _MakeLiteralTest('1E+6', '1e+06')
+  testNumFloatSciMinus = _MakeLiteralTest('1e-06')
   testNumComplex = _MakeLiteralTest('3j')
 
   testSubscriptDictStr = _MakeExprTest('{"foo": 42}["foo"]')
@@ -205,14 +208,14 @@ def foo():
   testSubscriptMultiDimSlice = _MakeSliceTest(
       "'foo','bar':'baz':'qux'", "('foo', slice('bar', 'baz', 'qux'))")
 
-  testStrEmpty = _MakeLiteralTest('""')
-  testStrAscii = _MakeLiteralTest('"abc"')
-  testStrUtf8 = _MakeLiteralTest(r'"\tfoo\n\xcf\x80"')
-  testStrQuoted = _MakeLiteralTest('\'"foo"\'')
-  testStrUtf16 = _MakeLiteralTest(r'u"\u0432\u043e\u043b\u043d"')
+  testStrEmpty = _MakeLiteralTest("''")
+  testStrAscii = _MakeLiteralTest("'abc'")
+  testStrUtf8 = _MakeLiteralTest(r"'\tfoo\n\xcf\x80'")
+  testStrQuoted = _MakeLiteralTest('\'"foo"\'', '\'"foo"\'')
+  testStrUtf16 = _MakeLiteralTest("u'\\u0432\\u043e\\u043b\\u043d'")
 
-  testTupleEmpty = _MakeLiteralTest(())
-  testTupleNonEmpty = _MakeLiteralTest((1, 2, 3))
+  testTupleEmpty = _MakeLiteralTest('()')
+  testTupleNonEmpty = _MakeLiteralTest('(1, 2, 3)')
 
   testUnaryOpNot = _MakeExprTest('not True')
   testUnaryOpInvert = _MakeExprTest('~4')
@@ -223,7 +226,7 @@ def testUnaryOpNotImplemented(self):
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', '',
                            stmt.FutureFeatures())
 
 
@@ -235,7 +238,7 @@ def _ParseAndVisitExpr(expr):
   writer = util.Writer()
   visitor = expr_visitor.ExprVisitor(_MakeModuleBlock(), writer)
   visitor.visit(_ParseExpr(expr))
-  return writer.out.getvalue()
+  return writer.getvalue()
 
 
 def _GrumpRun(cmd):

compiler/shard_test.py

@@ -14,6 +14,8 @@
 
 """Wrapper for unit tests that loads a subset of all test methods."""
 
+from __future__ import unicode_literals
+
 import argparse
 import random
 import re

compiler/stmt.py

@@ -16,6 +16,8 @@
 
 """Visitor class for traversing Python statements."""
 
+from __future__ import unicode_literals
+
 import ast
 import string
 import textwrap
@@ -221,7 +223,7 @@ def visit_ClassDef(self, node):
       with self.writer.indent_block():
         self.writer.write_temp_decls(body_visitor.block)
         self.writer.write_block(body_visitor.block,
-                                body_visitor.writer.out.getvalue())
+                                body_visitor.writer.getvalue())
       tmpl = textwrap.dedent("""\
           }).Eval(πF, πF.Globals(), nil, nil)
           if πE != nil {
@@ -815,6 +817,6 @@ def _write_except_dispatcher(self, exc, tb, handlers):
 
   def _write_py_context(self, lineno):
     if lineno:
-      line = self.block.lines[lineno - 1].strip()
+      line = self.block.buffer.source_line(lineno).strip()
       self.writer.write('// line {}: {}'.format(lineno, line))
       self.writer.write('πF.SetLineno({})'.format(lineno))

compiler/stmt_test.py

@@ -16,6 +16,8 @@
 
 """Tests for StatementVisitor."""
 
+from __future__ import unicode_literals
+
 import ast
 import re
 import subprocess
@@ -573,7 +575,7 @@ def testWriteExceptDispatcherBareExcept(self):
         'exc', 'tb', handlers), [1, 2])
     expected = re.compile(r'ResolveGlobal\(.*foo.*\bIsInstance\(.*'
                           r'goto Label1.*goto Label2', re.DOTALL)
-    self.assertRegexpMatches(visitor.writer.out.getvalue(), expected)
+    self.assertRegexpMatches(visitor.writer.getvalue(), expected)
 
   def testWriteExceptDispatcherBareExceptionNotLast(self):
     visitor = stmt.StatementVisitor(_MakeModuleBlock())
@@ -593,19 +595,19 @@ def testWriteExceptDispatcherMultipleExcept(self):
         r'ResolveGlobal\(.*foo.*\bif .*\bIsInstance\(.*\{.*goto Label1.*'
         r'ResolveGlobal\(.*bar.*\bif .*\bIsInstance\(.*\{.*goto Label2.*'
         r'\bRaise\(exc\.ToObject\(\), nil, tb\.ToObject\(\)\)', re.DOTALL)
-    self.assertRegexpMatches(visitor.writer.out.getvalue(), expected)
+    self.assertRegexpMatches(visitor.writer.getvalue(), expected)
 
 
 def _MakeModuleBlock():
-  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', [],
+  return block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>', '',
                            stmt.FutureFeatures())
 
 
 def _ParseAndVisit(source):
   mod = ast.parse(source)
   future_features = stmt.visit_future(mod)
   b = block.ModuleBlock('__main__', 'grumpy', 'grumpy/lib', '<test>',
-                        source.split('\n'), future_features)
+                        source, future_features)
   visitor = stmt.StatementVisitor(b)
   visitor.visit(mod)
   return visitor

compiler/util.py

@@ -16,9 +16,13 @@
 
 """Utilities for generating Go code."""
 
+from __future__ import unicode_literals
+
+import codecs
 import contextlib
 import cStringIO
 import string
+import StringIO
 import textwrap
 
 
@@ -44,9 +48,12 @@ class Writer(object):
   """Utility class for writing blocks of Go code to a file-like object."""
 
   def __init__(self, out=None):
-    self.out = out or cStringIO.StringIO()
+    self.out = codecs.getwriter('utf8')(out or cStringIO.StringIO())
     self.indent_level = 0
 
+  def getvalue(self):
+    return self.out.getvalue().decode('utf8')
+
   @contextlib.contextmanager
   def indent_block(self, n=1):
     """A context manager that indents by n on entry and dedents on exit."""
@@ -127,7 +134,7 @@ def dedent(self, n=1):
 
 def go_str(value):
   """Returns value as a valid Go string literal."""
-  io = cStringIO.StringIO()
+  io = StringIO.StringIO()
   io.write('"')
   for c in value:
     if c in _ESCAPES: