Implement CInstrumenter

This is the equivalent of ScorepTrace/Profile but implemented in C/C++
TODO: Threading support

Author: Flamefire

CMakeLists.txt

@@ -13,6 +13,9 @@ find_package(Python REQUIRED COMPONENTS Interpreter Development)
 Python_add_library(scorep_bindings
   src/methods.cpp src/scorep_bindings.cpp src/scorepy/events.cpp
 )
+if(Python_VERSION_MAJOR GREATER_EQUAL 3)
+  target_sources(scorep_bindings PRIVATE src/classes.cpp src/scorepy/cInstrumenter.cpp src/scorepy/pythonHelpers.cpp)
+endif()
 target_link_libraries(scorep_bindings PRIVATE Scorep::Plugin)
 target_compile_features(scorep_bindings PRIVATE cxx_std_11)
 target_compile_definitions(scorep_bindings PRIVATE PY_SSIZE_T_CLEAN)

scorep/_instrumenters/scorep_cProfile.py

@@ -0,0 +1,8 @@
+from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter
+from scorep import scorep_bindings
+
+
+class ScorepCProfile(scorep_bindings.CInstrumenter, ScorepInstrumenter):
+    def __init__(self, enable_instrumenter):
+        scorep_bindings.CInstrumenter.__init__(self, tracingOrProfiling=False)
+        ScorepInstrumenter.__init__(self, enable_instrumenter)

scorep/_instrumenters/scorep_cTrace.py

@@ -0,0 +1,8 @@
+from scorep._instrumenters.scorep_instrumenter import ScorepInstrumenter
+from scorep import scorep_bindings
+
+
+class ScorepCTrace(scorep_bindings.CInstrumenter, ScorepInstrumenter):
+    def __init__(self, enable_instrumenter):
+        scorep_bindings.CInstrumenter.__init__(self, tracingOrProfiling=True)
+        ScorepInstrumenter.__init__(self, enable_instrumenter)

scorep/instrumenter.py

@@ -24,6 +24,12 @@ def get_instrumenter(enable_instrumenter=False,
         elif instrumenter_type == "dummy":
             from scorep._instrumenters.dummy import ScorepDummy
             global_instrumenter = ScorepDummy(enable_instrumenter)
+        elif instrumenter_type == "cTrace":
+            from scorep._instrumenters.scorep_cTrace import ScorepCTrace
+            global_instrumenter = ScorepCTrace(enable_instrumenter)
+        elif instrumenter_type == "cProfile":
+            from scorep._instrumenters.scorep_cProfile import ScorepCProfile
+            global_instrumenter = ScorepCProfile(enable_instrumenter)
         else:
             raise RuntimeError('instrumenter_type "{}" unkown'.format(instrumenter_type))
 

setup.py

@@ -1,4 +1,5 @@
 import os
+import sys
 from distutils.core import setup, Extension
 import scorep.helper
 
@@ -24,6 +25,8 @@
 src_folder = os.path.abspath('src')
 include += [src_folder]
 sources = ['src/methods.cpp', 'src/scorep_bindings.cpp', 'src/scorepy/events.cpp']
+if sys.version_info.major >= 3:
+    sources.extend(['src/classes.cpp', 'src/scorepy/cInstrumenter.cpp', 'src/scorepy/pythonHelpers.cpp'])
 
 cmodules.append(Extension('scorep.scorep_bindings',
                           include_dirs=include,

src/classes.cpp

@@ -0,0 +1,94 @@
+#include "classes.hpp"
+#include "scorepy/cInstrumenter.hpp"
+#include "scorepy/pythonHelpers.hpp"
+#include <Python.h>
+#include <type_traits>
+
+static_assert(std::is_trivial<scorepy::CInstrumenter>::value,
+              "Must be trivial or object creation by Python is UB");
+static_assert(std::is_standard_layout<scorepy::CInstrumenter>::value,
+              "Must be standard layout or object creation by Python is UB");
+
+extern "C"
+{
+
+    /// tp_new implementation that calls object.__new__ with not args to allow ABC classes to work
+    static PyObject* call_object_new(PyTypeObject* type, PyObject*, PyObject*)
+    {
+        scorepy::PyRefObject empty_tuple(PyTuple_New(0), scorepy::adopt_object);
+        if (!empty_tuple)
+            return nullptr;
+        scorepy::PyRefObject empty_dict(PyDict_New(), scorepy::adopt_object);
+        if (!empty_dict)
+            return nullptr;
+        return PyBaseObject_Type.tp_new(type, empty_tuple, empty_dict);
+    }
+
+    static int CInstrumenter_init(scorepy::CInstrumenter* self, PyObject* args, PyObject* kwds)
+    {
+        static const char* kwlist[] = { "tracingOrProfiling", nullptr };
+        int tracingOrProfiling;
+
+        if (!PyArg_ParseTupleAndKeywords(args, kwds, "p", const_cast<char**>(kwlist),
+                                         &tracingOrProfiling))
+            return -1;
+
+        self->init(tracingOrProfiling != 0);
+        return 0;
+    }
+
+    static PyObject* CInstrumenter_get_tracingOrProfiling(scorepy::CInstrumenter* self, void*)
+    {
+        scorepy::PyRefObject result(self->tracingOrProfiling ? Py_True : Py_False,
+                                    scorepy::retain_object);
+        return result;
+    }
+
+    static PyObject* CInstrumenter_enable_instrumenter(scorepy::CInstrumenter* self, PyObject*)
+    {
+        self->enable_instrumenter();
+        Py_RETURN_NONE;
+    }
+
+    static PyObject* CInstrumenter_disable_instrumenter(scorepy::CInstrumenter* self, PyObject*)
+    {
+        self->disable_instrumenter();
+        Py_RETURN_NONE;
+    }
+}
+
+namespace scorepy
+{
+
+PyTypeObject& getCInstrumenterType()
+{
+    static PyMethodDef methods[] = {
+        { "_enable_instrumenter", reinterpret_cast<PyCFunction>(CInstrumenter_enable_instrumenter),
+          METH_NOARGS, "Enable the instrumenter" },
+        { "_disable_instrumenter",
+          reinterpret_cast<PyCFunction>(CInstrumenter_disable_instrumenter), METH_NOARGS,
+          "Disable the instrumenter" },
+        { nullptr } /* Sentinel */
+    };
+    static PyGetSetDef getseters[] = {
+        { "tracingOrProfiling", reinterpret_cast<getter>(CInstrumenter_get_tracingOrProfiling),
+          nullptr, "Return whether the trace (True) or profile (False) instrumentation is used",
+          nullptr },
+        { nullptr } /* Sentinel */
+    };
+    // Sets the first few fields explicitely and remaining ones to zero
+    static PyTypeObject type = {
+        PyVarObject_HEAD_INIT(nullptr, 0)       /* header */
+        "scorep.scorep_bindings.CInstrumenter", /* tp_name */
+        sizeof(CInstrumenter),                  /* tp_basicsize */
+    };
+    type.tp_new = call_object_new;
+    type.tp_init = reinterpret_cast<initproc>(CInstrumenter_init);
+    type.tp_methods = methods;
+    type.tp_getset = getseters;
+    type.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE;
+    type.tp_doc = "Class for the C instrumenter interface of Score-P";
+    return type;
+}
+
+} // namespace scorepy

src/classes.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <Python.h>
+
+namespace scorepy
+{
+/// Return the type info to define for the python module
+PyTypeObject& getCInstrumenterType();
+} // namespace scorepy

src/scorep.hpp

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <Python.h>
+
+namespace scorepy
+{
+/// Return the methods to define for the python module
+PyMethodDef* getMethodTable();
+} // namespace scorepy

src/scorep_bindings.cpp

@@ -1,3 +1,4 @@
+#include "classes.hpp"
 #include "methods.hpp"
 #include <Python.h>
 
@@ -16,6 +17,22 @@ static struct PyModuleDef scorepmodule = { PyModuleDef_HEAD_INIT,
                                            scorepy::getMethodTable() };
 PyMODINIT_FUNC PyInit_scorep_bindings(void)
 {
-    return PyModule_Create(&scorepmodule);
+    auto* ctracerType = &scorepy::getCInstrumenterType();
+    if (PyType_Ready(ctracerType) < 0)
+        return nullptr;
+
+    auto* m = PyModule_Create(&scorepmodule);
+    if (!m)
+        return nullptr;
+
+    Py_INCREF(ctracerType);
+    if (PyModule_AddObject(m, "CInstrumenter", (PyObject*)ctracerType) < 0)
+    {
+        Py_DECREF(ctracerType);
+        Py_DECREF(m);
+        return nullptr;
+    }
+
+    return m;
 }
 #endif /*python 3*/

src/scorepy/cInstrumenter.cpp

@@ -0,0 +1,84 @@
+#include "cInstrumenter.hpp"
+#include "events.hpp"
+#include "pythonHelpers.hpp"
+#include <string>
+
+namespace scorepy
+{
+static const std::string& make_region_name(const char* moduleName, const char* name)
+{
+    static std::string region;
+    region = moduleName;
+    region += ":";
+    region += name;
+    return region;
+}
+
+void CInstrumenter::enable_instrumenter()
+{
+    // TODO: Known issue:  `sys.getprofile()` returns the user object (2nd arg)
+    // So `sys.setprofile(sys.getprofile())` will not round-trip as it will try to call the
+    // 2nd arg. If it is nullptr (here) it means it will be disabled completely
+    // See https://nedbatchelder.com/text/trace-function.html for details
+    const auto callback = [](PyObject* obj, PyFrameObject* frame, int what, PyObject* arg) -> int {
+        return fromPyObject(obj)->onEvent(*frame, what, arg) ? 0 : -1;
+    };
+    if (tracingOrProfiling)
+    {
+        PyEval_SetTrace(callback, toPyObject());
+    }
+    else
+    {
+        PyEval_SetProfile(callback, toPyObject());
+    }
+}
+
+void CInstrumenter::disable_instrumenter()
+{
+    if (tracingOrProfiling)
+        PyEval_SetTrace(nullptr, nullptr);
+    else
+        PyEval_SetProfile(nullptr, nullptr);
+}
+
+bool CInstrumenter::onEvent(PyFrameObject& frame, int what, PyObject*)
+{
+    switch (what)
+    {
+    case PyTrace_CALL:
+    {
+        const PyCodeObject& code = *frame.f_code;
+        const char* name = PyUnicode_AsUTF8(code.co_name);
+        const char* moduleName = get_module_name(frame);
+        assert(name);
+        assert(moduleName);
+        // TODO: Use string_view/CString comparison?
+        if (std::string(name) != "_unsetprofile" && std::string(moduleName, 0, 6) != "scorep")
+        {
+            const int lineNumber = code.co_firstlineno;
+            const auto& regionName = make_region_name(moduleName, name);
+            const auto fileName = get_file_name(frame);
+            region_begin(regionName, moduleName, fileName, lineNumber);
+        }
+        break;
+    }
+    case PyTrace_RETURN:
+    {
+        const PyCodeObject& code = *frame.f_code;
+        const char* name = PyUnicode_AsUTF8(code.co_name);
+        const char* moduleName = get_module_name(frame);
+        assert(name);
+        assert(moduleName);
+        // TODO: Use string_view/CString comparison?
+        if (std::string(name) != "_unsetprofile" && std::string(moduleName, 0, 6) != "scorep")
+        {
+            const auto& regionName = make_region_name(moduleName, name);
+            region_end(regionName);
+        }
+        break;
+    }
+    }
+    return true;
+}
+
+} // namespace scorepy