Merge branch 'master' of github.com:score-p/scorep_binding_python

Author: AndreasGocht

.github/workflows/unit_tests.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [2.7, 3.5, 3.6, 3.7, 3.8]
+        python: [2.7, 3.5, 3.6, 3.7, 3.8, 'pypy-2.7', 'pypy-3.6', 'pypy-3.7']
       fail-fast: false
 
     steps:
@@ -41,4 +41,4 @@ jobs:
       run: pip install .
     - name: Run tests
       working-directory: test
-      run: pytest
+      run: pytest

CMakeLists.txt

@@ -19,13 +19,16 @@ find_package(Python REQUIRED COMPONENTS Interpreter Development)
 Python_add_library(_bindings
   src/methods.cpp src/scorep_bindings.cpp src/scorepy/events.cpp
 )
-if(Python_VERSION_MAJOR GREATER_EQUAL 3)
+if(Python_VERSION_MAJOR GREATER_EQUAL 3 AND NOT Python_INTERPRETER_ID STREQUAL "PyPy")
   target_sources(_bindings PRIVATE
     src/classes.cpp
     src/scorepy/cInstrumenter.cpp
     src/scorepy/pathUtils.cpp
     src/scorepy/pythonHelpers.cpp
   )
+  target_compile_definitions(_bindings PRIVATE SCOREPY_ENABLE_CINSTRUMENTER=1)
+else()
+  target_compile_definitions(_bindings PRIVATE SCOREPY_ENABLE_CINSTRUMENTER=0)
 endif()
 target_link_libraries(_bindings PRIVATE Scorep::Plugin)
 target_compile_features(_bindings PRIVATE cxx_std_11)

scorep/__main__.py

@@ -3,10 +3,11 @@
 
 import scorep.instrumenter
 import scorep.subsystem
+from scorep.helper import print_err
 
 
 def _err_exit(msg):
-    sys.stderr.write("%s: %s\n" % ("scorep", msg))
+    print_err("scorep: " + msg)
     sys.exit(1)
 
 
@@ -19,17 +20,25 @@ def scorep_main(argv=None):
     parse_scorep_commands = True
 
     keep_files = False
+    verbose = False
     no_default_threads = False
     no_default_compiler = False
     no_instrumenter = False
-    instrumenter_type = "profile"
+    if scorep.instrumenter.has_c_instrumenter():
+        instrumenter_type = "cProfile"
+    else:
+        instrumenter_type = "profile"
 
     for elem in argv[1:]:
         if parse_scorep_commands:
-            if elem == "--mpi":
+            if elem == "--":
+                parse_scorep_commands = False
+            elif elem == "--mpi":
                 scorep_config.append("--mpp=mpi")
             elif elem == "--keep-files":
                 keep_files = True
+            elif elem == "--verbose" or elem == '-v':
+                verbose = True
             elif "--thread=" in elem:
                 scorep_config.append(elem)
                 no_default_threads = True
@@ -61,7 +70,7 @@ def scorep_main(argv=None):
         _err_exit("Did not find a script to run")
 
     if os.environ.get("SCOREP_PYTHON_BINDINGS_INITIALISED") != "true":
-        scorep.subsystem.init_environment(scorep_config, keep_files)
+        scorep.subsystem.init_environment(scorep_config, keep_files, verbose)
         os.environ["SCOREP_PYTHON_BINDINGS_INITIALISED"] = "true"
         """
         python -m starts the module as skript. i.e. sys.argv will loke like:

scorep/_version.py

@@ -1 +1 @@
-__version__ = "3.2.0"
+__version__ = "3.3.0"

scorep/helper.py

@@ -4,11 +4,15 @@
 import re
 
 
+def print_err(*args):
+    """Print to stderr"""
+    sys.stderr.write(' '.join(map(str, args)) + '\n')
+
+
 def call(arguments):
     """
     return a triple with (returncode, stdout, stderr) from the call to subprocess
     """
-    result = ()
     if sys.version_info > (3, 5):
         out = subprocess.run(
             arguments,
@@ -74,15 +78,15 @@ def add_to_ld_library_path(path):
         os.environ["LD_LIBRARY_PATH"] = ':'.join([path] + library_paths)
 
 
-def generate_compile_deps(config=[]):
+def generate_compile_deps(config):
     """
     Generates the data needed for compilation.
     """
 
-    scorep_config = ["scorep-config"] + config + ["--user"]
+    scorep_config = ["scorep-config"] + config
 
-    (retrun_code, _, _) = call(scorep_config)
-    if retrun_code != 0:
+    (return_code, _, _) = call(scorep_config)
+    if return_code != 0:
         raise ValueError(
             "given config {} is not supported".format(config))
 

scorep/instrumenter.py

@@ -1,9 +1,18 @@
 import inspect
 import os
+import platform
+import sys
 
 global_instrumenter = None
 
 
+def has_c_instrumenter():
+    """Return true if the C instrumenter(s) are available"""
+    # We are using the UTF-8 string features from Python 3
+    # The C Instrumenter functions are not available on PyPy
+    return sys.version_info.major >= 3 and platform.python_implementation() != 'PyPy'
+
+
 def get_instrumenter(enable_instrumenter=False,
                      instrumenter_type="dummy"):
     """

scorep/subsystem.py

@@ -1,10 +1,15 @@
 import os
-import sys
 import distutils.ccompiler
 import tempfile
 import shutil
 
 import scorep.helper
+from scorep.helper import print_err
+
+
+def _print_info(msg):
+    """Print an info message with a prefix"""
+    print_err("scorep: " + msg)
 
 
 def generate_subsystem_lib_name():
@@ -24,20 +29,19 @@ def generate_ld_preload(scorep_config):
     @return ld_preload string which needs to be passed to LD_PRELOAD
     """
 
-    (_, preload, _) = scorep.helper.call(
-        ["scorep-config"] + scorep_config + ["--user", "--preload-libs"])
-    return preload
+    (_, preload, _) = scorep.helper.call(["scorep-config"] + scorep_config + ["--preload-libs"])
+    return preload.strip()
 
 
-def generate_subsystem_code(config=[]):
+def generate_subsystem_code(config):
     """
     Generates the data needed to be preloaded.
     """
 
-    scorep_config = ["scorep-config"] + config + ["--user"]
+    scorep_config = ["scorep-config"] + config
 
-    (retrun_code, _, _) = scorep.helper.call(scorep_config)
-    if retrun_code != 0:
+    (return_code, _, _) = scorep.helper.call(scorep_config)
+    if return_code != 0:
         raise ValueError(
             "given config {} is not supported".format(scorep_config))
     (_, scorep_adapter_init, _) = scorep.helper.call(
@@ -75,9 +79,7 @@ def generate(scorep_config, keep_files=False):
 
     temp_dir = tempfile.mkdtemp(prefix="scorep.")
     if keep_files:
-        sys.stderr.write(
-            "Score-P files are keept at: {}\n".format(temp_dir))
-        sys.stderr.flush()
+        _print_info("Score-P files are kept at: " + temp_dir)
 
     with open(temp_dir + "/scorep_init.c", "w") as f:
         f.write(scorep_adapter_init)
@@ -99,35 +101,50 @@ def generate(scorep_config, keep_files=False):
     return(subsystem_lib_name, temp_dir)
 
 
-def init_environment(scorep_config=[], keep_files=False):
+def init_environment(scorep_config, keep_files=False, verbose=False):
     """
-    Set the inital needed environmet variables, to get everythin up an running.
-    As a few variables interact with LD env vars, the programms needs to be restarted after this.
+    Set the inital needed environment variables, to get everything up an running.
+    As a few variables interact with LD env vars, the program needs to be restarted after this.
 
     @param scorep_config configuration flags for score-p
     @param keep_files whether to keep the generated files, or not.
+    @param verbose Set to True to output information about config used and environment variables set.
     """
 
     if "libscorep" in os.environ.get("LD_PRELOAD", ""):
-        raise RuntimeError(
-            "Score-P is already loaded. This should not happen at this point")
+        raise RuntimeError("Score-P is already loaded. This should not happen at this point")
+
+    if "--user" not in scorep_config:
+        scorep_config.append("--user")
 
-    subsystem_lib_name, temp_dir = scorep.subsystem.generate(
-        scorep_config, keep_files)
+    if verbose:
+        _print_info("Score-P config: %s" % scorep_config)
+
+    old_env = os.environ.copy()
+
+    subsystem_lib_name, temp_dir = generate(scorep_config, keep_files)
     scorep_ld_preload = generate_ld_preload(scorep_config)
 
     scorep.helper.add_to_ld_library_path(temp_dir)
 
     preload_str = scorep_ld_preload + " " + subsystem_lib_name
-    if "LD_PRELOAD" in os.environ:
-        sys.stderr.write(
-            "LD_PRELOAD is already specified. If Score-P is already loaded this might lead to errors.")
+    if os.environ.get("LD_PRELOAD"):
+        print_err("LD_PRELOAD is already specified. If Score-P is already loaded this might lead to errors.")
         preload_str = os.environ["LD_PRELOAD"] + " " + preload_str
         os.environ["SCOREP_LD_PRELOAD_BACKUP"] = os.environ["LD_PRELOAD"]
     else:
         os.environ["SCOREP_LD_PRELOAD_BACKUP"] = ""
     os.environ["LD_PRELOAD"] = preload_str
 
+    if verbose:
+        for var in ("LD_LIBRARY_PATH", "LD_PRELOAD"):
+            # Shorten the setting to e.g.: FOO=new:$FOO
+            old_val = old_env.get(var)
+            new_val = os.environ[var]
+            if old_val:
+                new_val = new_val.replace(old_val, '$' + var)
+            _print_info('%s="%s"' % (var, new_val))
+
 
 def reset_preload():
     """

setup.py

@@ -2,6 +2,7 @@
 import sys
 from distutils.core import setup, Extension
 import scorep.helper
+from scorep.instrumenter import has_c_instrumenter
 
 if scorep.helper.get_scorep_version() < 5.0:
     raise RuntimeError(
@@ -16,33 +17,39 @@
 if "gcc" in check_compiler:
     gcc_plugin = scorep.helper.get_scorep_config("GCC plug-in support:")
     if not ("yes" in gcc_plugin):
-        raise RuntimeError(
-            "Score-P uses GCC but is not build with GCC Compiler Plugin. GCC plug-in support is:\n{}".format(gcc_plugin))
+        raise RuntimeError("Score-P uses GCC but is not build with GCC Compiler Plugin. "
+                           "GCC plug-in support is:\n{}".format(gcc_plugin))
 
 
 cmodules = []
-(include, _, _, _, _) = scorep.helper.generate_compile_deps()
+(include, _, _, _, _) = scorep.helper.generate_compile_deps([])
 src_folder = os.path.abspath('src')
 include += [src_folder]
 sources = ['src/methods.cpp', 'src/scorep_bindings.cpp', 'src/scorepy/events.cpp']
-if sys.version_info.major >= 3:
+define_macros = [('PY_SSIZE_T_CLEAN', '1')]
+# We are using the UTF-8 string features from Python 3
+# The C Instrumenter functions are not available on PyPy
+if has_c_instrumenter():
     sources.extend([
         'src/classes.cpp',
         'src/scorepy/cInstrumenter.cpp',
         'src/scorepy/pythonHelpers.cpp',
         'src/scorepy/pathUtils.cpp',
     ])
+    define_macros.append(('SCOREPY_ENABLE_CINSTRUMENTER', '1'))
+else:
+    define_macros.append(('SCOREPY_ENABLE_CINSTRUMENTER', '0'))
 
 cmodules.append(Extension('scorep._bindings',
                           include_dirs=include,
-                          define_macros=[('PY_SSIZE_T_CLEAN', '1')],
+                          define_macros=define_macros,
                           extra_compile_args=["-std=c++11"],
                           sources=sources))
 
 setup(
     name='scorep',
     version=scorep._version.__version__,
-    description='This is a scorep tracing package for python',
+    description='This is a Score-P tracing package for python',
     author='Andreas Gocht',
     author_email='andreas.gocht@tu-dresden.de',
     url='https://github.com/score-p/scorep_binding_python',
@@ -70,6 +77,7 @@
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: Implementation :: CPython',
+        'Programming Language :: Python :: Implementation :: PyPy',
         'Operating System :: POSIX',
         'Operating System :: Unix',
     ],

src/scorep_bindings.cpp

@@ -7,7 +7,7 @@ PyMODINIT_FUNC init_bindings(void)
 {
     (void)Py_InitModule("_bindings", scorepy::getMethodTable());
 }
-#else  /*python 3*/
+#else /*python 3*/
 static struct PyModuleDef scorepmodule = { PyModuleDef_HEAD_INIT, "_bindings", /* name of module */
                                            NULL, /* module documentation, may be NULL */
                                            -1,   /* size of per-interpreter state of the module,
@@ -16,23 +16,27 @@ static struct PyModuleDef scorepmodule = { PyModuleDef_HEAD_INIT, "_bindings", /
                                            scorepy::getMethodTable() };
 PyMODINIT_FUNC PyInit__bindings(void)
 {
+#if SCOREPY_ENABLE_CINSTRUMENTER
     auto* ctracerType = &scorepy::getCInstrumenterType();
     if (PyType_Ready(ctracerType) < 0)
         return nullptr;
+#endif
 
     auto* m = PyModule_Create(&scorepmodule);
     if (!m)
     {
         return nullptr;
     }
 
+#if SCOREPY_ENABLE_CINSTRUMENTER
     Py_INCREF(ctracerType);
     if (PyModule_AddObject(m, "CInstrumenter", (PyObject*)ctracerType) < 0)
     {
         Py_DECREF(ctracerType);
         Py_DECREF(m);
         return nullptr;
     }
+#endif
 
     return m;
 }