Merge pull request #92 from Flamefire/cTracing

Implement CInstrumenter

Author: AndreasGocht

CMakeLists.txt

@@ -10,19 +10,22 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 find_package(Scorep REQUIRED)
 find_package(Python REQUIRED COMPONENTS Interpreter Development)
 
-Python_add_library(scorep_bindings
+Python_add_library(_bindings
   src/methods.cpp src/scorep_bindings.cpp src/scorepy/events.cpp
 )
-target_link_libraries(scorep_bindings PRIVATE Scorep::Plugin)
-target_compile_features(scorep_bindings PRIVATE cxx_std_11)
-target_compile_definitions(scorep_bindings PRIVATE PY_SSIZE_T_CLEAN)
-target_include_directories(scorep_bindings PRIVATE src)
+if(Python_VERSION_MAJOR GREATER_EQUAL 3)
+  target_sources(_bindings PRIVATE src/classes.cpp src/scorepy/cInstrumenter.cpp src/scorepy/pythonHelpers.cpp)
+endif()
+target_link_libraries(_bindings PRIVATE Scorep::Plugin)
+target_compile_features(_bindings PRIVATE cxx_std_11)
+target_compile_definitions(_bindings PRIVATE PY_SSIZE_T_CLEAN)
+target_include_directories(_bindings PRIVATE src)
 
-set_target_properties(scorep_bindings PROPERTIES
+set_target_properties(_bindings PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/site-packages/scorep
 )
 add_custom_target(ScorepModule ALL
-  ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/scorep $<TARGET_FILE_DIR:scorep_bindings>
+  ${CMAKE_COMMAND} -E copy_directory ${CMAKE_CURRENT_LIST_DIR}/scorep $<TARGET_FILE_DIR:_bindings>
   COMMENT "Copying module files to build tree"
 )
 
@@ -40,4 +43,4 @@ set_tests_properties(ScorepPythonTests PROPERTIES ENVIRONMENT "PYTHONPATH=${pyth
 set(INSTALL_DIR "lib/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/site-packages")
 
 install(DIRECTORY scorep DESTINATION ${INSTALL_DIR})
-install(TARGETS scorep_bindings DESTINATION ${INSTALL_DIR}/scorep)
+install(TARGETS _bindings DESTINATION ${INSTALL_DIR}/scorep)

README.md

@@ -44,7 +44,8 @@ To trace the full script, you need to run
 python -m scorep <script.py>
 ```
 
-The usual Score-P environment Variables will be respected. Please have a look at:
+The usual Score-P environment Variables will be respected.
+Please have a look at:
 
 [www.vi-hps.org](http://www.vi-hps.org/projects/score-p/)
 
@@ -62,20 +63,24 @@ python -m scorep --mpp=mpi --thread=pthread <script.py>
 
 ## Instrumenter
 The instrumenter ist the key part of the bindings.
-He registers with the Python tracing interface, and cares about the fowarding of events to Score-P.
-There are currently three different instrumenter types available as described in the following section [Instrumenter Types](#instrumenter-types) .
+It registers with the Python tracing interface, and cares about the fowarding of events to Score-P.
+There are currently five different instrumenter types available as described in the following section [Instrumenter Types](#instrumenter-types) .
 A user interface, to dynamically enable and disable the automatic instrumentation, using the python hooks, is also available and described under [Instrumenter User Interface](#instrumenter-user-interface)
 
 ### Instrumenter Types
-With version 2.0 of the python bindings, the term "instrumenter" is introduced. The instrumenter describes the class that maps the Python `trace` or `profile` events to Score-P. Please be aware, that `trace` and `profile` does not refer to the traditional Score-P terms of tracing and profiling, but to the Python functions [sys.settrace](https://docs.python.org/3/library/sys.html#sys.settrace) and [sys.setprofile](https://docs.python.org/3/library/sys.html#sys.setprofile).
+With version 2.0 of the python bindings, the term "instrumenter" is introduced.
+The instrumenter describes the class that maps the Python `trace` or `profile` events to Score-P.
+Please be aware, that `trace` and `profile` does not refer to the traditional Score-P terms of tracing and profiling, but to the Python functions [sys.settrace](https://docs.python.org/3/library/sys.html#sys.settrace) and [sys.setprofile](https://docs.python.org/3/library/sys.html#sys.setprofile).
 
 The instrumenter that shall be used for tracing can be specified using `--instrumenter-type=<type>`.
-Currently there are the following tacers available:
+Currently there are the following instrumenters available:
  * `profile` (default) implements `call` and `return`  
  * `trace` implements `call` and `return`
+ * `cProfile` / `cTrace` are the same as the above but implemented in C++
  * `dummy` does nothing, can be used without `-m scorep` (as done by user instrumentation)
 
 The `profile` instrumenter should have a smaller overhead than `trace`.
+Using the instrumenters implemented in C++ additionally reduces the overhead but those are only available in Python 3.
 
 It is possible to disable the instrumenter passing  `--noinstrumenter`.
 However, the [Instrumenter User Interface](#instrumenter-user-interface) may override this flag.
@@ -196,22 +201,23 @@ scorep.user.parameter_uint(name, val)
 scorep.user.parameter_string(name, string)
 ```
 
-where `name` defines the name of the parameter or region, while `val` or `string` represents the value that is passed to Score-P. 
+where `name` defines the name of the parameter or region, while `val` or `string` represents the value that is passed to Score-P.
 
-Disabeling the recording with Score-P is still also possilbe:
+Disabling the recording with Score-P is still also possible:
 
 ```
 scorep.user.enable_recording()
 scorep.user.disable_recording()
 ```
 
-However, please be aware that the runtime impact of disabeling Score-P is rather small, as the instrumenter is still active. For details about the instrumenter, please see [Instrumenter](#Instrumenter).  
+However, please be aware that the runtime impact of disabling Score-P is rather small, as the instrumenter is still active.
+For details about the instrumenter, please see [Instrumenter](#Instrumenter).
 
 ## Overview about Flags
 
 The following flags are special to the python bindings:
 
- * `--noinstrumenter` disables the instrumentation of python code. Usefull for user instrumentation and to trace only specific code regions using `scorep.instrumenter.enable`.
+ * `--noinstrumenter` disables the instrumentation of python code. Useful for user instrumentation and to trace only specific code regions using `scorep.instrumenter.enable`.
  * `--instrumenter-type=<type>` choose an instrumenter. See  [Instrumenter](#Instrumenter).
  * `--keep-files` temporary files are kept.
 
@@ -235,7 +241,8 @@ To disable compiler instrumentation, please specify:
 python -m scorep --nocompiler <script.py>
 ```
 
-For other thread schemes just specify `--thread=<something>`. E.g. :
+For other thread schemes just specify `--thread=<something>`.
+E.g. :
 
 ```
 python -m scorep --thread=omp <script.py>
@@ -246,7 +253,7 @@ Please be aware the `--user` is always passed to Score-P, as this is needed for
 # Compatibility
 ## Working
 * python3 
-* python2.7
+* python2.7, but not all features are supported
 * mpi using mpi4py
 * threaded applications
 

benchmark/benchmark.py

@@ -3,6 +3,7 @@
 
 @author: gocht
 '''
+import sys
 import benchmark_helper
 import pickle
 
@@ -16,6 +17,9 @@
 
 for test in tests:
     results[test] = {"profile": {}, "trace": {}, "dummy": {}, "None": {}}
+    if sys.version_info.major >= 3:
+        results.update({"cProfile": {}, "cTrace": {}})
+
     for instrumenter in results[test]:
         if instrumenter == "None":
             enable_scorep = False

scorep/_instrumenters/scorep_cProfile.py

@@ -0,0 +1,8 @@
+from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter
+import scorep._bindings
+
+
+class ScorepCProfile(scorep._bindings.CInstrumenter, ScorepInstrumenter):
+    def __init__(self, enable_instrumenter):
+        scorep._bindings.CInstrumenter.__init__(self, interface='Profile')
+        ScorepInstrumenter.__init__(self, enable_instrumenter)

scorep/_instrumenters/scorep_cTrace.py

@@ -0,0 +1,8 @@
+from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter
+import scorep._bindings
+
+
+class ScorepCTrace(scorep._bindings.CInstrumenter, ScorepInstrumenter):
+    def __init__(self, enable_instrumenter):
+        scorep._bindings.CInstrumenter.__init__(self, interface='Trace')
+        ScorepInstrumenter.__init__(self, enable_instrumenter)

scorep/_instrumenters/scorep_instrumenter.py

@@ -2,7 +2,7 @@
 import inspect
 import os
 from scorep._instrumenters import base_instrumenter
-from scorep import scorep_bindings
+import scorep._bindings
 
 
 class ScorepInstrumenter(base_instrumenter.BaseInstrumenter):
@@ -58,12 +58,12 @@ def run(self, cmd, globals=None, locals=None):
 
     def region_begin(self, module_name, function_name, file_name, line_number):
         """Record a region begin event"""
-        scorep_bindings.region_begin(
+        scorep._bindings.region_begin(
             module_name, function_name, file_name, line_number)
 
     def region_end(self, module_name, function_name):
         """Record a region end event"""
-        scorep_bindings.region_end(module_name, function_name)
+        scorep._bindings.region_end(module_name, function_name)
 
     def rewind_begin(self, name, file_name=None, line_number=None):
         """
@@ -82,15 +82,15 @@ def rewind_begin(self, name, file_name=None, line_number=None):
         else:
             full_file_name = "None"
 
-        scorep_bindings.rewind_begin(name, full_file_name, line_number)
+        scorep._bindings.rewind_begin(name, full_file_name, line_number)
 
     def rewind_end(self, name, value):
         """
         End of an Rewind region.
         @param name name of the user region
         @param value True or False, whenether the region shall be rewinded or not.
         """
-        scorep_bindings.rewind_end(name, value)
+        scorep._bindings.rewind_end(name, value)
 
     def oa_region_begin(self, name, file_name=None, line_number=None):
         """
@@ -109,28 +109,28 @@ def oa_region_begin(self, name, file_name=None, line_number=None):
         else:
             full_file_name = "None"
 
-        scorep_bindings.oa_region_begin(name, full_file_name, line_number)
+        scorep._bindings.oa_region_begin(name, full_file_name, line_number)
 
     def oa_region_end(self, name):
         """End an Online Access region."""
-        scorep_bindings.oa_region_end(name)
+        scorep._bindings.oa_region_end(name)
 
     def user_enable_recording(self):
         """Enable writing of trace events in ScoreP"""
-        scorep_bindings.enable_recording()
+        scorep._bindings.enable_recording()
 
     def user_disable_recording(self):
         """Disable writing of trace events in ScoreP"""
-        scorep_bindings.disable_recording()
+        scorep._bindings.disable_recording()
 
     def user_parameter_int(self, name, val):
         """Record a parameter of type integer"""
-        scorep_bindings.parameter_int(name, val)
+        scorep._bindings.parameter_int(name, val)
 
     def user_parameter_uint(self, name, val):
         """Record a parameter of type unsigned integer"""
-        scorep_bindings.parameter_string(name, val)
+        scorep._bindings.parameter_string(name, val)
 
     def user_parameter_string(self, name, string):
         """Record a parameter of type string"""
-        scorep_bindings.parameter_string(name, string)
+        scorep._bindings.parameter_string(name, string)

scorep/_instrumenters/scorep_profile.py

@@ -3,7 +3,7 @@
 import sys
 from scorep._instrumenters.utils import get_module_name, get_file_name
 from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter
-from scorep import scorep_bindings
+import scorep._bindings
 
 try:
     import threading
@@ -42,9 +42,9 @@ def _globaltrace(self, frame, why, arg):
             if not code.co_name == "_unsetprofile" and not modulename[:6] == "scorep":
                 full_file_name = get_file_name(frame)
                 line_number = code.co_firstlineno
-                scorep_bindings.region_begin(modulename, code.co_name, full_file_name, line_number)
+                scorep._bindings.region_begin(modulename, code.co_name, full_file_name, line_number)
         elif why == 'return':
             code = frame.f_code
             modulename = get_module_name(frame)
             if not code.co_name == "_unsetprofile" and not modulename[:6] == "scorep":
-                scorep_bindings.region_end(modulename, code.co_name)
+                scorep._bindings.region_end(modulename, code.co_name)

scorep/_instrumenters/scorep_trace.py

@@ -3,7 +3,7 @@
 import sys
 from scorep._instrumenters.utils import get_module_name, get_file_name
 from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter
-from scorep import scorep_bindings
+import scorep._bindings
 
 try:
     import threading
@@ -40,13 +40,13 @@ def _globaltrace(self, frame, why, arg):
             if not code.co_name == "_unsettrace" and not modulename[:6] == "scorep":
                 full_file_name = get_file_name(frame)
                 line_number = code.co_firstlineno
-                scorep_bindings.region_begin(modulename, code.co_name, full_file_name, line_number)
+                scorep._bindings.region_begin(modulename, code.co_name, full_file_name, line_number)
                 return self._localtrace
         return None
 
     def _localtrace(self, frame, why, arg):
         if why == 'return':
             code = frame.f_code
             modulename = get_module_name(frame)
-            scorep_bindings.region_end(modulename, code.co_name)
+            scorep._bindings.region_end(modulename, code.co_name)
         return self._localtrace

scorep/instrumenter.py

@@ -24,6 +24,12 @@ def get_instrumenter(enable_instrumenter=False,
         elif instrumenter_type == "dummy":
             from scorep._instrumenters.dummy import ScorepDummy
             global_instrumenter = ScorepDummy(enable_instrumenter)
+        elif instrumenter_type == "cTrace":
+            from scorep._instrumenters.scorep_cTrace import ScorepCTrace
+            global_instrumenter = ScorepCTrace(enable_instrumenter)
+        elif instrumenter_type == "cProfile":
+            from scorep._instrumenters.scorep_cProfile import ScorepCProfile
+            global_instrumenter = ScorepCProfile(enable_instrumenter)
         else:
             raise RuntimeError('instrumenter_type "{}" unkown'.format(instrumenter_type))
 

setup.py

@@ -1,4 +1,5 @@
 import os
+import sys
 from distutils.core import setup, Extension
 import scorep.helper
 
@@ -24,8 +25,10 @@
 src_folder = os.path.abspath('src')
 include += [src_folder]
 sources = ['src/methods.cpp', 'src/scorep_bindings.cpp', 'src/scorepy/events.cpp']
+if sys.version_info.major >= 3:
+    sources.extend(['src/classes.cpp', 'src/scorepy/cInstrumenter.cpp', 'src/scorepy/pythonHelpers.cpp'])
 
-cmodules.append(Extension('scorep.scorep_bindings',
+cmodules.append(Extension('scorep._bindings',
                           include_dirs=include,
                           define_macros=[('PY_SSIZE_T_CLEAN', '1')],
                           extra_compile_args=["-std=c++11"],