Skip to content
This repository was archived by the owner on Mar 23, 2023. It is now read-only.

Commit dceae69

Browse files
trotterdylantrotterdylan
authored
Import module deps only in main, not packages (#356)
This implements option #2 in: #287 Basically, importing of packages corresponding to Python modules now does not happen in the generated code for the module. Instead, these imports only happen in the main Go package. This simplifies grumpc, since it does not need to track imports for the Python modules it imports. That work is now done in grumprun, which determines the transitive dependencies and imports all those packages in the generated main Go file.
1 parent dba35db commit dceae69

11 files changed

Lines changed: 242 additions & 212 deletions

File tree

‎Makefile‎

Lines changed: 5 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@ THIRD_PARTY_PYPY_SRCS := $(patsubst third_party/pypy/%,$(GOPATH_PY_ROOT)/%,$(she
8181
THIRD_PARTY_OUROBOROS_SRCS := $(patsubst third_party/ouroboros/%,$(GOPATH_PY_ROOT)/%,$(shell find third_party/ouroboros -name '*.py'))
8282
STDLIB_SRCS := $(LIB_SRCS) $(THIRD_PARTY_STDLIB_SRCS) $(THIRD_PARTY_PYPY_SRCS) $(THIRD_PARTY_OUROBOROS_SRCS)
8383

84-
8584
STDLIB_PACKAGES := $(patsubst $(GOPATH_PY_ROOT)/%.py,%,$(patsubst $(GOPATH_PY_ROOT)/%/__init__.py,%,$(STDLIB_SRCS)))
8685
STDLIB := $(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES))
8786
STDLIB_TESTS := \
@@ -264,23 +263,8 @@ $(patsubst %,build/src/__python__/%/module.d,$(STDLIB_PACKAGES)): build/bin/pyde
264263
$(patsubst %,$(PKG_DIR)/__python__/%.a,$(STDLIB_PACKAGES)): $(RUNTIME)
265264

266265
define GRUMPY_STDLIB_TEST
267-
build/testing/$(patsubst %_test,%_test_,$(notdir $(1))).go:
268-
@mkdir -p build/testing
269-
@echo 'package main' > $$@
270-
@echo 'import (' >> $$@
271-
@echo ' "os"' >> $$@
272-
@echo ' "grumpy"' >> $$@
273-
@echo ' mod "__python__/$(1)"' >> $$@
274-
@echo ')' >> $$@
275-
@echo 'func main() {' >> $$@
276-
@echo ' os.Exit(grumpy.RunMain(mod.Code))' >> $$@
277-
@echo '}' >> $$@
278-
279-
build/testing/$(notdir $(1)): build/testing/$(patsubst %_test,%_test_,$(notdir $(1))).go $(RUNTIME) $(PKG_DIR)/__python__/$(1).a
280-
@go build -o $$@ $$<
281-
282-
build/testing/$(notdir $(1)).pass: build/testing/$(notdir $(1))
283-
@$$<
266+
build/testing/$(notdir $(1)).pass: $(RUNTIME) $(PKG_DIR)/__python__/$(1).a $(RUNNER_BIN)
267+
@$(RUNNER_BIN) -m $(subst /,.,$(1))
284268
@touch $$@
285269
@echo 'lib/$(1) PASS'
286270

@@ -299,21 +283,14 @@ $(PYTHONPARSER_SRCS): $(PY_DIR)/grumpy/%: third_party/%
299283
@mkdir -p $(@D)
300284
@cp -f $< $@
301285

302-
$(patsubst %_test,build/%.go,$(ACCEPT_TESTS)): build/%.go: %_test.py $(COMPILER)
303-
@mkdir -p $(@D)
304-
@$(COMPILER_BIN) $< > $@
305-
306-
# TODO: These should not depend on stdlibs and should instead build a .d file.
307-
$(patsubst %,build/%,$(ACCEPT_TESTS)): build/%_test: build/%.go $(RUNTIME) $(STDLIB)
286+
$(ACCEPT_PASS_FILES): build/%_test.pass: %_test.py $(RUNTIME) $(STDLIB) $(RUNNER_BIN)
308287
@mkdir -p $(@D)
309-
@go build -o $@ $<
310-
311-
$(ACCEPT_PASS_FILES): build/%_test.pass: build/%_test
312-
@$<
288+
@$(RUNNER_BIN) < $<
313289
@touch $@
314290
@echo '$*_test PASS'
315291

316292
$(ACCEPT_PY_PASS_FILES): build/%_py.pass: %.py $(PY_DIR)/weetest.py
293+
@mkdir -p $(@D)
317294
@$(PYTHON) $<
318295
@touch $@
319296
@echo '$*_py PASS'

‎compiler/block.py‎

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -172,18 +172,7 @@ def del_var(self, writer, name):
172172
def resolve_name(self, writer, name):
173173
return self._resolve_global(writer, name)
174174

175-
def add_import(self, name):
176-
"""Register the named Go package for import.
177-
178-
Args:
179-
name: The fully qualified Go package name.
180-
Returns:
181-
A Package representing the import.
182-
"""
183-
return self.add_native_import('__python__/' + name)
184-
185-
def add_native_import(self, name):
186-
alias = None
175+
def add_native_import(self, name, alias=None):
187176
if name == 'grumpy':
188177
alias = 'πg'
189178
if name in self.imports:

‎compiler/block_test.py‎

Lines changed: 0 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -41,25 +41,6 @@ def testCreateGrump(self):
4141

4242
class BlockTest(unittest.TestCase):
4343

44-
def testAddImport(self):
45-
module_block = _MakeModuleBlock()
46-
func1_block = block.FunctionBlock(module_block, 'func1', {}, False)
47-
func2_block = block.FunctionBlock(func1_block, 'func2', {}, False)
48-
package = func2_block.root.add_import('foo/bar')
49-
self.assertEqual(package.name, '__python__/foo/bar')
50-
self.assertEqual(package.alias, 'π___python__ΓfooΓbar')
51-
self.assertEqual(module_block.imports, {'__python__/foo/bar': package})
52-
53-
def testAddImportRepeated(self):
54-
b = _MakeModuleBlock()
55-
package = b.root.add_import('foo')
56-
self.assertEqual(package.name, '__python__/foo')
57-
self.assertEqual(package.alias, 'π___python__Γfoo')
58-
self.assertEqual(b.imports, {'__python__/foo': package})
59-
package2 = b.root.add_import('foo')
60-
self.assertIs(package, package2)
61-
self.assertEqual(b.imports, {'__python__/foo': package})
62-
6344
def testLoop(self):
6445
b = _MakeModuleBlock()
6546
loop = b.push_loop()

‎compiler/imputil.py‎

Lines changed: 82 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@
2222
import collections
2323
import functools
2424
import os
25+
import os.path
2526

2627
from grumpy.compiler import util
28+
from grumpy import pythonparser
2729
from grumpy.pythonparser import algorithm
2830
from grumpy.pythonparser import ast
2931

@@ -44,8 +46,9 @@ class Import(object):
4446
MODULE = "<BindType 'module'>"
4547
MEMBER = "<BindType 'member'>"
4648

47-
def __init__(self, name, is_native=False):
49+
def __init__(self, name, script=None, is_native=False):
4850
self.name = name
51+
self.script = script
4952
self.is_native = is_native
5053
self.bindings = []
5154

@@ -145,11 +148,13 @@ def visit_ImportFrom(self, node):
145148

146149
def _resolve_import(self, node, modname):
147150
if not self.absolute_import and self.package_dir:
148-
if self._script_exists(self.package_dir, modname):
149-
return Import('{}.{}'.format(self.package_name, modname))
151+
script = find_script(self.package_dir, modname)
152+
if script:
153+
return Import('{}.{}'.format(self.package_name, modname), script)
150154
for dirname in self.pathdirs:
151-
if self._script_exists(dirname, modname):
152-
return Import(modname)
155+
script = find_script(dirname, modname)
156+
if script:
157+
return Import(modname, script)
153158
raise util.ImportError(node, 'no such module: {}'.format(modname))
154159

155160
def _resolve_relative_import(self, level, node, modname):
@@ -161,15 +166,81 @@ def _resolve_relative_import(self, level, node, modname):
161166
node, 'attempted relative import beyond toplevel package')
162167
dirname = os.path.normpath(os.path.join(
163168
self.package_dir, *(['..'] * uplevel)))
164-
if not self._script_exists(dirname, modname):
169+
script = find_script(dirname, modname)
170+
if not script:
165171
raise util.ImportError(node, 'no such module: {}'.format(modname))
166172
parts = self.package_name.split('.')
167-
return Import('.'.join(parts[:len(parts)-uplevel]) + '.' + modname)
173+
return Import('.'.join(parts[:len(parts)-uplevel]) + '.' + modname, script)
168174

169-
def _script_exists(self, dirname, name):
170-
prefix = os.path.join(dirname, name.replace('.', os.sep))
171-
return (os.path.isfile(prefix + '.py') or
172-
os.path.isfile(os.path.join(prefix, '__init__.py')))
175+
176+
class _ImportCollector(algorithm.Visitor):
177+
178+
# pylint: disable=invalid-name
179+
180+
def __init__(self, importer, future_node):
181+
self.importer = importer
182+
self.future_node = future_node
183+
self.imports = []
184+
185+
def visit_Import(self, node):
186+
self.imports.extend(self.importer.visit(node))
187+
188+
def visit_ImportFrom(self, node):
189+
if node.module == '__future__':
190+
if node != self.future_node:
191+
raise util.LateFutureError(node)
192+
return
193+
self.imports.extend(self.importer.visit(node))
194+
195+
196+
def collect_imports(modname, script, gopath):
197+
with open(script) as py_file:
198+
py_contents = py_file.read()
199+
mod = pythonparser.parse(py_contents)
200+
future_node, future_features = parse_future_features(mod)
201+
importer = Importer(gopath, modname, script, future_features.absolute_import)
202+
collector = _ImportCollector(importer, future_node)
203+
collector.visit(mod)
204+
return collector.imports
205+
206+
207+
def calculate_transitive_deps(modname, script, gopath):
208+
"""Determines all modules that script transitively depends upon."""
209+
deps = set()
210+
def calc(modname, script):
211+
if modname in deps:
212+
return
213+
deps.add(modname)
214+
for imp in collect_imports(modname, script, gopath):
215+
if imp.is_native:
216+
continue
217+
parts = imp.name.split('.')
218+
calc(imp.name, imp.script)
219+
if len(parts) == 1:
220+
continue
221+
# For submodules, the parent packages are also deps.
222+
package_dir, filename = os.path.split(imp.script)
223+
if filename == '__init__.py':
224+
package_dir = os.path.dirname(package_dir)
225+
for i in xrange(len(parts) - 1, 0, -1):
226+
modname = '.'.join(parts[:i])
227+
script = os.path.join(package_dir, '__init__.py')
228+
calc(modname, script)
229+
package_dir = os.path.dirname(package_dir)
230+
calc(modname, script)
231+
deps.remove(modname)
232+
return deps
233+
234+
235+
def find_script(dirname, name):
236+
prefix = os.path.join(dirname, name.replace('.', os.sep))
237+
script = prefix + '.py'
238+
if os.path.isfile(script):
239+
return script
240+
script = os.path.join(prefix, '__init__.py')
241+
if os.path.isfile(script):
242+
return script
243+
return None
173244

174245

175246
_FUTURE_FEATURES = (

0 commit comments

Comments
 (0)