Clean up pointers to middle_query_t in outputs to release memory

The outputs have a shared pointer to the middle_query_t to access the
data held by the middle. This in turn holds pointers to potentially
large datastructures and the flat node file. We need to release those
pointers after we don't need them any more so that if and when the
middle doesn't need those data structures any more they get cleaned up.

In effect this releases the memory used for the cache and flat node
store earlier.

Fixes #1563

Author: joto

src/osm2pgsql.cpp

@@ -24,6 +24,20 @@
 #include <memory>
 #include <utility>
 
+/**
+ * Output overall memory usage as debug message.
+ *
+ * This only works on Linux.
+ */
+static void show_memory_usage()
+{
+    osmium::MemoryUsage mem;
+    if (mem.peak() != 0) {
+        log_debug("Overall memory usage: peak={}MByte current={}MByte",
+                  mem.peak(), mem.current());
+    }
+}
+
 static void run(options_t const &options)
 {
     auto const files = prepare_input_files(
@@ -55,6 +69,8 @@ static void run(options_t const &options)
     process_files(files, &osmdata, options.append,
                   get_logger().show_progress());
 
+    show_memory_usage();
+
     // Process pending ways and relations. Cluster database tables and
     // create indexes.
     osmdata.stop();
@@ -76,13 +92,7 @@ int main(int argc, char *argv[])
 
         run(options);
 
-        // Output overall memory usage. This only works on Linux.
-        osmium::MemoryUsage mem;
-        if (mem.peak() != 0) {
-            log_debug("Overall memory usage: peak={}MByte current={}MByte",
-                      mem.peak(), mem.current());
-        }
-
+        show_memory_usage();
         log_info("osm2pgsql took {} overall.",
                  util::human_readable_duration(timer_overall.stop()));
     } catch (std::exception const &e) {

src/osmdata.cpp

@@ -372,6 +372,8 @@ void osmdata_t::reprocess_marked() const { m_output->reprocess_marked(); }
 
 void osmdata_t::postprocess_database() const
 {
+    m_output->free_middle_references();
+
     if (m_droptemp) {
         // When dropping middle tables, make sure they are gone before
         // indexing starts.

src/output.cpp

@@ -71,6 +71,11 @@ output_t::output_t(std::shared_ptr<middle_query_t> const &mid,
 
 output_t::~output_t() = default;
 
+void output_t::free_middle_references()
+{
+    m_mid.reset();
+}
+
 options_t const *output_t::get_options() const { return &m_options; }
 
 void output_t::merge_expire_trees(output_t *) {}

src/output.hpp

@@ -47,6 +47,13 @@ class output_t
     clone(std::shared_ptr<middle_query_t> const &mid,
           std::shared_ptr<db_copy_thread_t> const &copy_thread) const = 0;
 
+    /**
+     * Remove pointer to middle_query_t from output, so the middle_query_t
+     * is properly cleaned up and doesn't hold references to any datastructures
+     * any more.
+     */
+    void free_middle_references();
+
     virtual void start() = 0;
     virtual void stop() = 0;
     virtual void sync() = 0;