Merge pull request #123 from Flamefire/ctrace-default

Make cProfile the default where available (Python 3) and add support for PyPy

Author: AndreasGocht

.github/workflows/unit_tests.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python: [2.7, 3.5, 3.6, 3.7, 3.8, 'pypy-2.7', 'pypy-3.6', 'pypy-3.7']
       fail-fast: false
 
     steps:
@@ -41,4 +41,4 @@ jobs:
       run: pip install .
     - name: Run tests
       working-directory: test
-      run: pytest
+      run: pytest

CMakeLists.txt

@@ -19,13 +19,16 @@ find_package(Python REQUIRED COMPONENTS Interpreter Development)
 Python_add_library(_bindings
   src/methods.cpp src/scorep_bindings.cpp src/scorepy/events.cpp
 )
-if(Python_VERSION_MAJOR GREATER_EQUAL 3)
+if(Python_VERSION_MAJOR GREATER_EQUAL 3 AND NOT Python_INTERPRETER_ID STREQUAL "PyPy")
   target_sources(_bindings PRIVATE
     src/classes.cpp
     src/scorepy/cInstrumenter.cpp
     src/scorepy/pathUtils.cpp
     src/scorepy/pythonHelpers.cpp
   )
+  target_compile_definitions(_bindings PRIVATE SCOREPY_ENABLE_CINSTRUMENTER=1)
+else()
+  target_compile_definitions(_bindings PRIVATE SCOREPY_ENABLE_CINSTRUMENTER=0)
 endif()
 target_link_libraries(_bindings PRIVATE Scorep::Plugin)
 target_compile_features(_bindings PRIVATE cxx_std_11)

scorep/__main__.py

@@ -24,7 +24,10 @@ def scorep_main(argv=None):
     no_default_threads = False
     no_default_compiler = False
     no_instrumenter = False
-    instrumenter_type = "profile"
+    if scorep.instrumenter.has_c_instrumenter():
+        instrumenter_type = "cProfile"
+    else:
+        instrumenter_type = "profile"
 
     for elem in argv[1:]:
         if parse_scorep_commands:

scorep/instrumenter.py

@@ -1,9 +1,18 @@
 import inspect
 import os
+import platform
+import sys
 
 global_instrumenter = None
 
 
+def has_c_instrumenter():
+    """Return true if the C instrumenter(s) are available"""
+    # We are using the UTF-8 string features from Python 3
+    # The C Instrumenter functions are not available on PyPy
+    return sys.version_info.major >= 3 and platform.python_implementation() != 'PyPy'
+
+
 def get_instrumenter(enable_instrumenter=False,
                      instrumenter_type="dummy"):
     """

setup.py

@@ -2,6 +2,7 @@
 import sys
 from distutils.core import setup, Extension
 import scorep.helper
+from scorep.instrumenter import has_c_instrumenter
 
 if scorep.helper.get_scorep_version() < 5.0:
     raise RuntimeError(
@@ -25,17 +26,23 @@
 src_folder = os.path.abspath('src')
 include += [src_folder]
 sources = ['src/methods.cpp', 'src/scorep_bindings.cpp', 'src/scorepy/events.cpp']
-if sys.version_info.major >= 3:
+define_macros = [('PY_SSIZE_T_CLEAN', '1')]
+# We are using the UTF-8 string features from Python 3
+# The C Instrumenter functions are not available on PyPy
+if has_c_instrumenter():
     sources.extend([
         'src/classes.cpp',
         'src/scorepy/cInstrumenter.cpp',
         'src/scorepy/pythonHelpers.cpp',
         'src/scorepy/pathUtils.cpp',
     ])
+    define_macros.append(('SCOREPY_ENABLE_CINSTRUMENTER', '1'))
+else:
+    define_macros.append(('SCOREPY_ENABLE_CINSTRUMENTER', '0'))
 
 cmodules.append(Extension('scorep._bindings',
                           include_dirs=include,
-                          define_macros=[('PY_SSIZE_T_CLEAN', '1')],
+                          define_macros=define_macros,
                           extra_compile_args=["-std=c++11"],
                           sources=sources))
 
@@ -70,6 +77,7 @@
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
         'Operating System :: POSIX',
         'Operating System :: Unix',
     ],

src/scorep_bindings.cpp

@@ -7,7 +7,7 @@ PyMODINIT_FUNC init_bindings(void)
 {
     (void)Py_InitModule("_bindings", scorepy::getMethodTable());
 }
-#else  /*python 3*/
+#else /*python 3*/
 static struct PyModuleDef scorepmodule = { PyModuleDef_HEAD_INIT, "_bindings", /* name of module */
                                            NULL, /* module documentation, may be NULL */
                                            -1,   /* size of per-interpreter state of the module,
@@ -16,23 +16,27 @@ static struct PyModuleDef scorepmodule = { PyModuleDef_HEAD_INIT, "_bindings", /
                                            scorepy::getMethodTable() };
 PyMODINIT_FUNC PyInit__bindings(void)
 {
+#if SCOREPY_ENABLE_CINSTRUMENTER
     auto* ctracerType = &scorepy::getCInstrumenterType();
     if (PyType_Ready(ctracerType) < 0)
         return nullptr;
+#endif
 
     auto* m = PyModule_Create(&scorepmodule);
     if (!m)
     {
         return nullptr;
     }
 
+#if SCOREPY_ENABLE_CINSTRUMENTER
     Py_INCREF(ctracerType);
     if (PyModule_AddObject(m, "CInstrumenter", (PyObject*)ctracerType) < 0)
     {
         Py_DECREF(ctracerType);
         Py_DECREF(m);
         return nullptr;
     }
+#endif
 
     return m;
 }

test/cases/io.py test/cases/file_io.pytest/cases/io.py renamed to test/cases/file_io.py

@@ -4,8 +4,12 @@
 import instrumentation2
 
 
+lock = threading.Lock()
+
+
 def worker(id, func):
-    print("Thread %s started" % id)
+    with lock:
+        print("Thread %s started" % id)
     # Use a random delay to add non-determinism to the output
     time.sleep(random.uniform(0.01, 0.9))
     func()

test/cases/use_threads.py

@@ -2,10 +2,11 @@
 
 import os
 import pkgutil
+import platform
+import pytest
 import re
 import subprocess
 import sys
-import pytest
 
 
 def call(arguments, expected_returncode=0, env=None):
@@ -50,7 +51,8 @@ def requires_package(name):
 
 
 cinstrumenter_skip_mark = pytest.mark.skipif(
-    sys.version_info.major < 3, reason="CInstrumenter only available in Python 3"
+    sys.version_info.major < 3 or platform.python_implementation() == 'PyPy',
+    reason="CInstrumenter only available in Python 3 and not in PyPy"
 )
 # All instrumenters (except dummy which isn't a real one)
 ALL_INSTRUMENTERS = [
@@ -85,6 +87,12 @@ def test_has_version():
     assert scorep.__version__ is not None
 
 
+@cinstrumenter_skip_mark
+def test_has_c_instrumenter():
+    from scorep.instrumenter import has_c_instrumenter
+    assert has_c_instrumenter()
+
+
 @foreach_instrumenter
 def test_user_regions(scorep_env, instrumenter):
     trace_path = get_trace_path(scorep_env)
@@ -448,7 +456,7 @@ def test_io(scorep_env, instrumenter):
 
     print("start")
     std_out, std_err = call_with_scorep(
-        "cases/io.py",
+        "cases/file_io.py",
         [
             "--nocompiler",
             "--instrumenter-type=" + instrumenter,