Merge pull request #1368 from PowerGridModel/pgm/feature/performance-prepare-scenario

Performance: only check independence for filled component scenarios

Author: TonyXiang8787

power_grid_model_c/power_grid_model/include/power_grid_model/auxiliary/dataset.hpp

@@ -10,18 +10,22 @@
 #include "meta_data.hpp"
 
 #include "../common/common.hpp"
+#include "../common/counting_iterator.hpp"
 #include "../common/exception.hpp"
 #include "../common/iterator_facade.hpp"
+#include "../common/typing.hpp"
 
 #include <algorithm>
 #include <cassert>
 #include <concepts>
 #include <cstddef>
+#include <iterator>
 #include <ranges>
 #include <span>
 #include <string>
 #include <string_view>
 #include <type_traits>
+#include <utility>
 #include <vector>
 
 namespace power_grid_model {
@@ -216,6 +220,8 @@ template <dataset_type_tag dataset_type_> class Dataset {
         std::span<Indptr> indptr{};
     };
 
+    template <class StructType> using SpanRange = std::span<StructType>;
+
     template <class StructType>
     using RangeObject = std::conditional_t<is_data_mutable_v<dataset_type>, mutable_range_object<StructType>,
                                            const_range_object<StructType>>;
@@ -452,7 +458,7 @@ template <dataset_type_tag dataset_type_> class Dataset {
     // get buffer by component type
     template <class type_getter, class ComponentType,
               class StructType = DataStruct<typename type_getter::template type<ComponentType>>>
-    std::span<StructType> get_buffer_span(Idx scenario = invalid_index) const {
+    SpanRange<StructType> get_buffer_span(Idx scenario = invalid_index) const {
         assert(scenario < batch_size());
 
         if (!is_batch() && scenario > 0) {
@@ -476,27 +482,17 @@ template <dataset_type_tag dataset_type_> class Dataset {
         return get_columnar_buffer_span_impl<StructType>(scenario, idx);
     }
 
-    // get buffer by component type for all scenarios in vector span
+    // get buffer by component type for all non-empty scenarios in vector span
     template <class type_getter, class ComponentType,
               class StructType = DataStruct<typename type_getter::template type<ComponentType>>>
-    std::vector<std::span<StructType>> get_buffer_span_all_scenarios() const {
-        Idx const idx = find_component(ComponentType::name, false);
-        std::vector<std::span<StructType>> result(batch_size());
-        for (Idx scenario{}; scenario != batch_size(); scenario++) {
-            result[scenario] = get_buffer_span_impl<StructType>(scenario, idx);
-        }
-        return result;
+    std::vector<SpanRange<StructType>> get_buffer_span_all_scenarios() const {
+        return get_buffer_span_all_scenarios_impl<type_getter, ComponentType, SpanRange>();
     }
 
     template <class type_getter, class ComponentType,
               class StructType = DataStruct<typename type_getter::template type<ComponentType>>>
     std::vector<RangeObject<StructType>> get_columnar_buffer_span_all_scenarios() const {
-        Idx const idx = find_component(ComponentType::name, false);
-        std::vector<RangeObject<StructType>> result(batch_size());
-        for (Idx scenario{}; scenario != batch_size(); scenario++) {
-            result[scenario] = get_columnar_buffer_span_impl<StructType>(scenario, idx);
-        }
-        return result;
+        return get_buffer_span_all_scenarios_impl<type_getter, ComponentType, RangeObject>();
     }
 
     // get individual dataset from batch
@@ -559,7 +555,7 @@ template <dataset_type_tag dataset_type_> class Dataset {
 
     template <class type_getter, class ComponentType, typename Func,
               class StructType = DataStruct<typename type_getter::template type<ComponentType>>>
-        requires std::invocable<Func, std::span<StructType>> && std::invocable<Func, RangeObject<StructType>>
+        requires std::invocable<Func, SpanRange<StructType>> && std::invocable<Func, RangeObject<StructType>>
     auto for_each_component(Func&& func, Idx scenario = invalid_index) const {
         if (is_columnar(ComponentType::name)) {
             return std::forward<Func>(func)(get_columnar_buffer_span<type_getter, ComponentType, StructType>(scenario));
@@ -659,31 +655,53 @@ template <dataset_type_tag dataset_type_> class Dataset {
                          std::begin(total_range) + info.elements_per_scenario * (scenario + 1)};
     }
 
-    // get non-empty row buffer
-    template <class StructType> std::span<StructType> get_buffer_span_impl(Idx scenario, Idx component_idx) const {
+    template <typename StructType, template <typename> typename RangeType>
+        requires std::ranges::view<RangeType<StructType>> &&
+                 (std::same_as<RangeType<StructType>, SpanRange<StructType>> ||
+                  std::same_as<RangeType<StructType>, RangeObject<StructType>>)
+    auto get_buffer_view_impl(Idx scenario, Idx component_idx) const {
+        using RangeType_ = RangeType<StructType>;
+
         // return empty span if the component does not exist
         if (component_idx < 0) {
-            return {};
+            return RangeType_{};
         }
         // return span based on uniform or non-uniform buffer
         ComponentInfo const& info = dataset_info_.component_info[component_idx];
         Buffer const& buffer = buffers_[component_idx];
-        auto const ptr = reinterpret_cast<StructType*>(buffer.data);
-        return get_span_impl(std::span<StructType>{ptr, ptr + info.total_elements}, scenario, buffer, info);
+
+        if constexpr (std::same_as<RangeType_, SpanRange<StructType>>) {
+            assert(is_row_based(buffer) || info.total_elements == 0);
+            auto const ptr = reinterpret_cast<StructType*>(buffer.data);
+            return get_span_impl(RangeType_{ptr, ptr + info.total_elements}, scenario, buffer, info);
+        } else if constexpr (std::same_as<RangeType_, RangeObject<StructType>>) {
+            assert(is_columnar(buffer));
+            return get_span_impl(RangeType_{info.total_elements, buffer.attributes}, scenario, buffer, info);
+        } else {
+            static_assert(false, "only span and RangeObject are supported");
+        }
+    }
+
+    // get non-empty row buffer
+    template <class StructType> SpanRange<StructType> get_buffer_span_impl(Idx scenario, Idx component_idx) const {
+        return get_buffer_view_impl<StructType, SpanRange>(scenario, component_idx);
     }
 
     // get non-empty columnar buffer
     template <class StructType>
     RangeObject<StructType> get_columnar_buffer_span_impl(Idx scenario, Idx component_idx) const {
-        // return empty span if the component does not exist
-        if (component_idx < 0) {
-            return {};
-        }
-        // return span based on uniform or non-uniform buffer
-        ComponentInfo const& info = dataset_info_.component_info[component_idx];
-        Buffer const& buffer = buffers_[component_idx];
-        assert(is_columnar(buffer));
-        return get_span_impl(RangeObject<StructType>{info.total_elements, buffer.attributes}, scenario, buffer, info);
+        return get_buffer_view_impl<StructType, RangeObject>(scenario, component_idx);
+    }
+
+    template <class type_getter, class ComponentType, template <typename> typename RangeType,
+              typename StructType = DataStruct<typename type_getter::template type<ComponentType>>>
+        requires std::ranges::view<RangeType<StructType>>
+    std::vector<RangeType<StructType>> get_buffer_span_all_scenarios_impl() const {
+        Idx const component_idx = find_component(ComponentType::name, false);
+        return IdxRange{batch_size()} | std::views::transform([this, component_idx](Idx scenario) {
+                   return get_buffer_view_impl<StructType, RangeType>(scenario, component_idx);
+               }) |
+               std::ranges::to<std::vector>();
     }
 };
 

power_grid_model_c/power_grid_model/include/power_grid_model/main_core/update.hpp

@@ -42,7 +42,7 @@ inline void iterate_component_sequence(Func func, Elements elements, std::span<I
     }
 }
 
-template <typename T> bool check_id_na(T const& obj) {
+template <typename T> inline bool check_id_na(T const& obj) {
     if constexpr (requires { obj.id; }) {
         return is_nan(obj.id);
     } else if constexpr (requires { obj.get().id; }) {
@@ -83,38 +83,38 @@ struct UpdateCompProperties {
     }
 };
 
-template <typename CompType> void process_buffer_span(auto const& all_spans, UpdateCompProperties& properties) {
-    properties.ids_all_na = std::ranges::all_of(all_spans, [](auto const& vec) {
+inline bool get_all_ids_na(auto const& all_spans) {
+    return std::ranges::all_of(all_spans, [](auto const& vec) {
         return std::ranges::all_of(vec, [](auto const& item) { return detail::check_id_na(item); });
     });
-    properties.ids_part_na = std::ranges::any_of(all_spans,
-                                                 [](auto const& vec) {
-                                                     return std::ranges::any_of(vec, [](auto const& item) {
-                                                         return detail::check_id_na(item);
-                                                     });
-                                                 }) &&
-                             !properties.ids_all_na;
+}
+
+inline bool get_any_ids_na(auto const& all_spans) {
+    return std::ranges::any_of(all_spans, [](auto const& vec) {
+        return std::ranges::any_of(vec, [](auto const& item) { return detail::check_id_na(item); });
+    });
+}
 
+template <typename CompType> inline bool get_update_ids_match(auto const& all_spans) {
     if (all_spans.empty()) {
-        properties.update_ids_match = true;
-        return;
+        return true; // if there are no elements, we can consider the ids to match
     }
+
     // Remember the begin iterator of the first scenario, then loop over the remaining scenarios and
     // check the ids
     auto const first_span = all_spans.front();
     // check the subsequent scenarios
     // only return true if ids of all scenarios match the ids of the first batch
 
-    properties.update_ids_match =
-        std::ranges::all_of(all_spans.cbegin() + 1, all_spans.cend(), [&first_span](auto const& current_span) {
-            return std::ranges::equal(current_span, first_span,
-                                      [](typename CompType::UpdateType const& obj,
-                                         typename CompType::UpdateType const& first) { return obj.id == first.id; });
-        });
+    return std::ranges::all_of(all_spans.cbegin() + 1, all_spans.cend(), [&first_span](auto const& current_span) {
+        return std::ranges::equal(current_span, first_span,
+                                  [](typename CompType::UpdateType const& obj,
+                                     typename CompType::UpdateType const& first) { return obj.id == first.id; });
+    });
 }
 
 template <class CompType>
-UpdateCompProperties check_component_independence(ConstDataset const& update_data, Idx n_component) {
+inline UpdateCompProperties check_component_independence(ConstDataset const& update_data, Idx n_component) {
     UpdateCompProperties properties;
     auto const component_idx = update_data.find_component(CompType::name, false);
     properties.is_columnar = update_data.is_columnar(CompType::name);
@@ -126,13 +126,25 @@ UpdateCompProperties check_component_independence(ConstDataset const& update_dat
         properties.uniform ? update_data.uniform_elements_per_scenario(CompType::name) : utils::invalid_index;
     properties.elements_in_base = n_component;
 
-    if (properties.is_columnar) {
-        process_buffer_span<CompType>(
-            update_data.template get_columnar_buffer_span_all_scenarios<meta_data::update_getter_s, CompType>(),
-            properties);
+    if (!properties.has_any_elements) {
+        properties.ids_all_na = true; // if there are no elements, we can consider all ids to be NA
+        properties.ids_part_na =
+            false; // if there are no elements, we cannot have some ids that are NA and some that are not
+        properties.update_ids_match = true; // if there are no elements, we can consider the
     } else {
-        process_buffer_span<CompType>(
-            update_data.template get_buffer_span_all_scenarios<meta_data::update_getter_s, CompType>(), properties);
+        auto const process_buffer_span = [&properties](auto const& all_spans) {
+            assert(!all_spans.empty());
+            properties.ids_all_na = get_all_ids_na(all_spans);
+            properties.ids_part_na = (!properties.ids_all_na) && get_any_ids_na(all_spans);
+            properties.update_ids_match = properties.uniform && get_update_ids_match<CompType>(all_spans);
+        };
+        if (properties.is_columnar) {
+            process_buffer_span(
+                update_data.template get_columnar_buffer_span_all_scenarios<meta_data::update_getter_s, CompType>());
+        } else {
+            process_buffer_span(
+                update_data.template get_buffer_span_all_scenarios<meta_data::update_getter_s, CompType>());
+        }
     }
 
     return properties;