[Code] Add matrix debug markers and expose this feature into Python API

EgorOrachyov · EgorOrachyov · commit e97f951bb4bd · 2021-03-06T13:39:11.000+03:00
diff --git a/README.md b/README.md
@@ -29,25 +29,25 @@ prototyping algorithms on a local computer for later running on a powerful serve
 
 - [X] Library C interface
 - [X] Library setup
-- [X] CSR matrix
-- [X] CSR multiplication
-- [X] CSR element-wise addition
-- [X] CSR kronecker
-- [X] CSR transpose
-- [X] CSR submatrix
-- [X] CSR matrix reduce
-- [X] CSR slicing
-- [ ] Matrix cached filling
+- [X] Sparse matrix
+- [X] Sparse matrix multiplication
+- [X] Sparse matrix element-wise addition
+- [X] Sparse matrix kronecker
+- [X] Sparse matrix transpose
+- [X] Sparse matrix submatrix
+- [X] Sparse matrix reduce
+- [X] Sparse matrix slicing
+- [X] Matrix cached filling
 - [X] Sequential fallback backend for CPU
 - [X] Device capabilities query
 - [ ] IO matrix loading from mtx file
 - [ ] IO matrix saving into mtx file
 - [ ] IO matrix saving into gviz format
 - [X] IO user-defined file logging
 - [X] Wrapper for Python API
-- [ ] Wrapper syntax sugar
+- [X] Wrapper syntax sugar
 - [ ] Tests for Python wrapper
-- [ ] Pip package
+- [X] Pip package
 - [ ] Code examples
 - [ ] User guide
 - [X] Unit Tests collection
diff --git a/cubool/CMakeLists.txt b/cubool/CMakeLists.txt
@@ -39,6 +39,8 @@ set(CUBOOL_C_API_SOURCES
     sources/cuBool_Matrix_New.cpp
     sources/cuBool_Matrix_Build.cpp
     sources/cuBool_Matrix_SetElement.cpp
+    sources/cuBool_Matrix_SetMarker.cpp
+    sources/cuBool_Matrix_GetMarker.cpp
     sources/cuBool_Matrix_ExtractPairs.cpp
     sources/cuBool_Matrix_ExtractSubMatrix.cpp
     sources/cuBool_Matrix_Duplicate.cpp
diff --git a/cubool/include/cubool/cubool.h b/cubool/include/cubool/cubool.h
@@ -159,7 +159,7 @@ CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_GetVersion(
  * @note Pass `CUBOOL_HINT_LOG_WARNING` to include warning messages into log
  * @note Pass `CUBOOL_HINT_LOG_ALL` to include all messages into log
  *
- * @param logFileName UTF-8 encoded file name and path.
+ * @param logFileName UTF-8 encoded null-terminated file name and path string.
  * @param hints Logging hints to filter messages.
  *
  * @return Error code on this operation
@@ -264,6 +264,38 @@ CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Matrix_SetElement(
     cuBool_Index j
 );
 
+/**
+ * Sets to the matrix specific debug string marker.
+ * This marker will appear in the log messages as string identifier of the matrix.
+ *
+ * @param matrix Matrix handle to perform operation on
+ * @param marker UTF-8 encoded null-terminated string marker name.
+ *
+ * @return Error code on this operation
+ */
+CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Matrix_SetMarker(
+    cuBool_Matrix matrix,
+    const char* marker
+);
+
+/**
+ * Allows to get matrix debug string marker.
+ *
+ * @note Pass null marker if you want to retrieve only the required marker buffer size.
+ * @note After the function call the actual size of the marker is stored in the size variable.
+ *
+ * @param matrix Matrix handle to perform operation on
+ * @param[in,out] marker Where to store null-terminated UTF-8 encoded marker string.
+ * @param[in,out] size Size of the provided buffer in bytes to save marker string.
+ *
+ * @return Error code on this operation
+ */
+CUBOOL_EXPORT CUBOOL_API cuBool_Status cuBool_Matrix_GetMarker(
+    cuBool_Matrix matrix,
+    char* marker,
+    cuBool_Index* size
+);
+
 /**
  * Reads matrix data to the host visible CPU buffer as an array of values pair.
  *
diff --git a/cubool/sources/core/library.cpp b/cubool/sources/core/library.cpp
@@ -44,7 +44,7 @@
 
 namespace cubool {
 
-    std::unordered_set<class MatrixBase*> Library::mAllocated;
+    std::unordered_set<class Matrix*> Library::mAllocated;
     std::shared_ptr<class BackendBase> Library::mBackend = nullptr;
     std::shared_ptr<class Logger>  Library::mLogger = std::make_shared<DummyLogger>();
     bool Library::mRelaxedRelease = false;
@@ -92,11 +92,11 @@ namespace cubool {
         if (mBackend) {
             // Release all allocated resources implicitly
             if (mRelaxedRelease) {
-                for (auto m: mAllocated) {
-                    std::stringstream s;
-                    s << "Implicitly release matrix instance " << m;
+                LogStream stream(*getLogger());
+                stream << Logger::Level::Info << "Enabled relaxed library finalize" << LogStream::cmt;
 
-                    mLogger->logWarning(s.str());
+                for (auto m: mAllocated) {
+                    stream << Logger::Level::Warning << "Implicitly release matrix " << m->getDebugMarker() << LogStream::cmt;
                     delete m;
                 }
 
@@ -185,14 +185,25 @@ namespace cubool {
 
         auto m = new Matrix(nrows, ncols, *mBackend);
         mAllocated.emplace(m);
+
+        LogStream stream(*getLogger());
+        stream << Logger::Level::Info << "Create Matrix " << m->getDebugMarker()
+               << " (" << nrows << "," << ncols << ")" << LogStream::cmt;
+
         return m;
     }
 
     void Library::releaseMatrix(MatrixBase *matrixBase) {
         if (mRelaxedRelease && !mBackend) return;
 
-        mAllocated.erase(matrixBase);
-        delete matrixBase;
+        auto m = (Matrix*)(matrixBase);
+        CHECK_RAISE_ERROR(mAllocated.find(m) != mAllocated.end(), InvalidArgument, "No such matrix was allocated");
+
+        LogStream stream(*getLogger());
+        stream << Logger::Level::Info << "Release Matrix " << m->getDebugMarker() << LogStream::cmt;
+
+        mAllocated.erase(m);
+        delete m;
     }
 
     void Library::handleError(const std::exception& error) {
@@ -227,7 +238,7 @@ namespace cubool {
                << " warp size: " << caps.warp << std::endl
                << " globalMemoryKiBs: " << caps.globalMemoryKiBs << std::endl
                << " sharedMemoryPerMultiProcKiBs: " << caps.sharedMemoryPerMultiProcKiBs << std::endl
-               << " sharedMemoryPerBlockKiBs: " << caps.sharedMemoryPerBlockKiBs << std::endl;
+               << " sharedMemoryPerBlockKiBs: " << caps.sharedMemoryPerBlockKiBs;
         }
         else {
             ss << "Cuda device is not presented";
diff --git a/cubool/sources/core/library.hpp b/cubool/sources/core/library.hpp
@@ -47,7 +47,7 @@ namespace cubool {
         static class Logger* getLogger();
 
     private:
-        static std::unordered_set<class MatrixBase*> mAllocated;
+        static std::unordered_set<class Matrix*> mAllocated;
         static std::shared_ptr<class BackendBase> mBackend;
         static std::shared_ptr<class Logger> mLogger;
         static bool mRelaxedRelease;
diff --git a/cubool/sources/core/matrix.cpp b/cubool/sources/core/matrix.cpp
@@ -33,6 +33,11 @@ namespace cubool {
     Matrix::Matrix(size_t nrows, size_t ncols, BackendBase &backend) {
         mHnd = backend.createMatrix(nrows, ncols);
         mProvider = &backend;
+
+        // By default marker is the address of the matrix
+        std::stringstream s;
+        s << this;
+        mMarker = s.str();
     }
 
     Matrix::~Matrix() {
@@ -207,13 +212,21 @@ namespace cubool {
 
     void Matrix::setDebugMarker(const char *marker) {
         CHECK_RAISE_ERROR(marker, InvalidArgument, "Null pointer marker string");
-        mMarker = marker;
+
+        // Marker = "$marker (address)"
+        std::stringstream s;
+        s << marker << " (" << this << ")";
+        mMarker = s.str();
     }
 
     const char * Matrix::getDebugMarker() const {
         return mMarker.c_str();
     }
 
+    index Matrix::getDebugMarkerSizeWithNullT() const {
+        return mMarker.length() + 1;
+    }
+
     void Matrix::releaseCache() const {
         mCachedI.clear();
         mCachedJ.clear();
diff --git a/cubool/sources/core/matrix.hpp b/cubool/sources/core/matrix.hpp
@@ -32,6 +32,10 @@
 
 namespace cubool {
 
+    /**
+     * Proxy matrix for the actual backend matrix implementation.
+     * Behaves as validation/auxiliary layer.
+     */
     class Matrix final: public MatrixBase {
     public:
         Matrix(size_t nrows, size_t ncols, BackendBase& backend);
@@ -56,6 +60,7 @@ namespace cubool {
 
         void setDebugMarker(const char* marker);
         const char* getDebugMarker() const;
+        index getDebugMarkerSizeWithNullT() const;
 
     private:
 
diff --git a/cubool/sources/cuBool_Matrix_GetMarker.cpp b/cubool/sources/cuBool_Matrix_GetMarker.cpp
@@ -0,0 +1,49 @@
+/**********************************************************************************/
+/* MIT License                                                                    */
+/*                                                                                */
+/* Copyright (c) 2020, 2021 JetBrains-Research                                    */
+/*                                                                                */
+/* Permission is hereby granted, free of charge, to any person obtaining a copy   */
+/* of this software and associated documentation files (the "Software"), to deal  */
+/* in the Software without restriction, including without limitation the rights   */
+/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      */
+/* copies of the Software, and to permit persons to whom the Software is          */
+/* furnished to do so, subject to the following conditions:                       */
+/*                                                                                */
+/* The above copyright notice and this permission notice shall be included in all */
+/* copies or substantial portions of the Software.                                */
+/*                                                                                */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     */
+/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       */
+/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    */
+/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         */
+/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  */
+/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  */
+/* SOFTWARE.                                                                      */
+/**********************************************************************************/
+
+#include <cuBool_Common.hpp>
+#include <cstring>
+
+cuBool_Status cuBool_Matrix_GetMarker(
+        cuBool_Matrix matrix,
+        char* marker,
+        cuBool_Index* size
+) {
+    CUBOOL_BEGIN_BODY
+        CUBOOL_VALIDATE_LIBRARY
+        CUBOOL_ARG_NOT_NULL(matrix)
+        CUBOOL_ARG_NOT_NULL(size)
+
+        auto m = (cubool::Matrix*) matrix;
+        auto actualSize = m->getDebugMarkerSizeWithNullT();
+
+        if (marker != nullptr && *size > 0) {
+            const auto* text = m->getDebugMarker();
+            std::memcpy(marker, text, *size);
+            marker[*size - 1] = '\0';
+        }
+
+        *size = actualSize;
+    CUBOOL_END_BODY
+}
diff --git a/cubool/sources/cuBool_Matrix_SetMarker.cpp b/cubool/sources/cuBool_Matrix_SetMarker.cpp
@@ -0,0 +1,38 @@
+/**********************************************************************************/
+/* MIT License                                                                    */
+/*                                                                                */
+/* Copyright (c) 2020, 2021 JetBrains-Research                                    */
+/*                                                                                */
+/* Permission is hereby granted, free of charge, to any person obtaining a copy   */
+/* of this software and associated documentation files (the "Software"), to deal  */
+/* in the Software without restriction, including without limitation the rights   */
+/* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell      */
+/* copies of the Software, and to permit persons to whom the Software is          */
+/* furnished to do so, subject to the following conditions:                       */
+/*                                                                                */
+/* The above copyright notice and this permission notice shall be included in all */
+/* copies or substantial portions of the Software.                                */
+/*                                                                                */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR     */
+/* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,       */
+/* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE    */
+/* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER         */
+/* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,  */
+/* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE  */
+/* SOFTWARE.                                                                      */
+/**********************************************************************************/
+
+#include <cuBool_Common.hpp>
+
+cuBool_Status cuBool_Matrix_SetMarker(
+        cuBool_Matrix matrix,
+        const char* marker
+) {
+    CUBOOL_BEGIN_BODY
+        CUBOOL_VALIDATE_LIBRARY
+        CUBOOL_ARG_NOT_NULL(matrix)
+        CUBOOL_ARG_NOT_NULL(marker)
+        auto m = (cubool::Matrix*) matrix;
+        m->setDebugMarker(marker);
+    CUBOOL_END_BODY
+}
diff --git a/cubool/sources/io/logger.hpp b/cubool/sources/io/logger.hpp
@@ -28,6 +28,7 @@
 #include <string>
 #include <vector>
 #include <functional>
+#include <sstream>
 
 namespace cubool {
 
@@ -98,6 +99,47 @@ namespace cubool {
         size_t getMessagesCount() const override;
     };
 
+    /**
+     * Utility to build and log fancy messages.
+     */
+    class LogStream {
+    public:
+        struct Commit {};
+        constexpr static const Commit cmt = Commit{};
+
+        explicit LogStream(Logger& logger)
+            : mLogger(logger), mLevel(Logger::Level::Info) {
+        };
+        LogStream(LogStream&& other) noexcept = default;
+        ~LogStream() = default;
+
+        void commit() {
+            mLogger.log(mLevel, mStream.str());
+            mStream.str(std::string());
+        }
+
+        friend LogStream& operator<<(LogStream& stream, Logger::Level level) {
+            stream.mLevel = level;
+            return stream;
+        }
+
+        friend LogStream& operator<<(LogStream& stream, Commit commit) {
+            stream.commit();
+            return stream;
+        }
+
+        template<typename T>
+        friend LogStream& operator<<(LogStream& stream, T&& t) {
+            stream.mStream << std::forward<T>(t);
+            return stream;
+        }
+
+    private:
+        Logger& mLogger;
+        Logger::Level mLevel;
+        std::stringstream mStream;
+    };
+
 }
 
 #endif //CUBOOL_LOGGER_HPP
diff --git a/python/pycubool/bridge.py b/python/pycubool/bridge.py
@@ -108,6 +108,19 @@ def load_and_configure(cubool_lib_path: str):
         ctypes.c_uint
     ]
 
+    lib.cuBool_Matrix_SetMarker.restype = status_t
+    lib.cuBool_Matrix_SetMarker.argtypes = [
+        matrix_p,
+        ctypes.POINTER(ctypes.c_char)
+    ]
+
+    lib.cuBool_Matrix_GetMarker.restype = status_t
+    lib.cuBool_Matrix_GetMarker.argtypes = [
+        matrix_p,
+        ctypes.POINTER(ctypes.c_char),
+        ctypes.POINTER(ctypes.c_uint)
+    ]
+
     lib.cuBool_Matrix_ExtractPairs.restype = status_t
     lib.cuBool_Matrix_ExtractPairs.argtypes = [
         matrix_p,
diff --git a/python/pycubool/matrix.py b/python/pycubool/matrix.py
diff --git a/python/test_app.py b/python/test_app.py
