Refactor of expire code: Move tile writing code out of expire_tiles

In the future we want to have different ways of reporting expired
tiles. This splits out the code to write tiles to a file to make
it more straightforward to add other methods in the future.

The tests are also a bit more straightforward now and some now also test
the order the tiles are reported in.

This also adds a file tile-output.hpp which contains a streams
implementation for writing out tiles. Used for debugging when tests
fail.

Author: joto

src/expire-tiles.cpp

@@ -15,10 +15,13 @@
  * http://subversion.nexusuk.org/projects/openpistemap/trunk/scripts/expire_tiles.py
  */
 
+#include <algorithm>
 #include <cerrno>
 #include <cmath>
+#include <cstdio>
 #include <cstdlib>
 #include <cstring>
+#include <stdexcept>
 #include <string>
 
 #include "expire-tiles.hpp"
@@ -34,44 +37,12 @@
 // How many tiles worth of space to leave either side of a changed feature
 static constexpr double const tile_expiry_leeway = 0.1;
 
-tile_output_t::tile_output_t(char const *filename)
-: outfile(fopen(filename, "a"))
-{
-    if (outfile == nullptr) {
-        log_warn("Failed to open expired tiles file ({}).  Tile expiry "
-                 "list will not be written!",
-                 std::strerror(errno));
-    }
-}
-
-tile_output_t::~tile_output_t()
-{
-    if (outfile) {
-        fclose(outfile);
-    }
-}
-
-void tile_output_t::output_dirty_tile(tile_t const &tile)
-{
-    if (!outfile) {
-        return;
-    }
-
-    fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y());
-}
-
 expire_tiles::expire_tiles(uint32_t max_zoom, double max_bbox,
                            std::shared_ptr<reprojection> projection)
 : m_projection(std::move(projection)), m_max_bbox(max_bbox),
   m_maxzoom(max_zoom), m_map_width(1U << m_maxzoom)
 {}
 
-void expire_tiles::output_and_destroy(char const *filename, uint32_t minzoom)
-{
-    tile_output_t output_writer{filename};
-    output_and_destroy<tile_output_t>(output_writer, minzoom);
-}
-
 void expire_tiles::expire_tile(uint32_t x, uint32_t y)
 {
     // Only try to insert to tile into the set if the last inserted tile
@@ -278,6 +249,16 @@ int expire_tiles::from_result(pg_result_t const &result, osmid_t osm_id)
     return num_tuples;
 }
 
+std::vector<uint64_t> expire_tiles::get_tiles()
+{
+    std::vector<uint64_t> tiles;
+    tiles.reserve(m_dirty_tiles.size());
+    tiles.assign(m_dirty_tiles.begin(), m_dirty_tiles.end());
+    std::sort(tiles.begin(), tiles.end());
+    m_dirty_tiles.clear();
+    return tiles;
+}
+
 void expire_tiles::merge_and_destroy(expire_tiles *other)
 {
     if (m_map_width != other->m_map_width) {
@@ -294,3 +275,25 @@ void expire_tiles::merge_and_destroy(expire_tiles *other)
         other->m_dirty_tiles.clear();
     }
 }
+
+std::size_t output_tiles_to_file(std::vector<uint64_t> const &tiles_maxzoom,
+                                 char const *filename, uint32_t minzoom,
+                                 uint32_t maxzoom)
+{
+    FILE *outfile = std::fopen(filename, "a");
+    if (outfile == nullptr) {
+        log_warn("Failed to open expired tiles file ({}).  Tile expiry "
+                 "list will not be written!",
+                 std::strerror(errno));
+        return 0;
+    }
+
+    auto const count =
+        for_each_tile(tiles_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) {
+            fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y());
+        });
+
+    std::fclose(outfile);
+
+    return count;
+}

src/expire-tiles.hpp

@@ -12,6 +12,8 @@
 
 #include <memory>
 #include <unordered_set>
+#include <utility>
+#include <vector>
 
 #include "geom.hpp"
 #include "geom-box.hpp"
@@ -22,26 +24,6 @@
 
 class reprojection;
 
-/**
- * Implementation of the output of the tile expiry list to a file.
- */
-class tile_output_t
-{
-    FILE *outfile;
-
-public:
-    explicit tile_output_t(char const *filename);
-
-    ~tile_output_t();
-
-    /**
-     * Output dirty tile.
-     *
-     * \param tile The tile to write out.
-     */
-    void output_dirty_tile(tile_t const &tile);
-};
-
 class expire_tiles
 {
 public:
@@ -58,7 +40,7 @@ class expire_tiles
      * Expire tiles based on an osm id.
      *
      * \param result Result of a database query into some table returning the
-     *               geometries. (This is usally done using the "get_wkb"
+     *               geometries. (This is usually done using the "get_wkb"
      *               prepared statement.)
      * \param osm_id The OSM id to look for.
      * \return The number of elements that refer to the osm_id or -1 if
@@ -67,66 +49,15 @@ class expire_tiles
     int from_result(pg_result_t const &result, osmid_t osm_id);
 
     /**
-     * Write the list of expired tiles to a file.
-     *
-     * You will probably use tile_output_t as template argument for production code
-     * and another class which does not write to a file for unit tests.
-     *
-     * \param filename name of the file
-     * \param minzoom minimum zoom level
+     * Get tiles as a vector of quadkeys and remove them from the expire_tiles
+     * object.
      */
-    void output_and_destroy(char const *filename, uint32_t minzoom);
+    std::vector<uint64_t> get_tiles();
 
     /**
-     * Output expired tiles on all requested zoom levels.
-     *
-     * \tparam TILE_WRITER class which implements the method
-     * output_dirty_tile(tile_t const &tile) which usually writes the tile ID
-     * to a file (production code) or does something else (usually unit tests).
-     *
-     * \param minzoom minimum zoom level
+     * Merge the list of expired tiles in the other object into this
+     * object, destroying the list in the other object.
      */
-    template <class TILE_WRITER>
-    void output_and_destroy(TILE_WRITER &output_writer, uint32_t minzoom)
-    {
-        assert(minzoom <= m_maxzoom);
-        // build a sorted vector of all expired tiles
-        std::vector<uint64_t> tiles_maxzoom(m_dirty_tiles.begin(),
-                                            m_dirty_tiles.end());
-        std::sort(tiles_maxzoom.begin(), tiles_maxzoom.end());
-        /* Loop over all requested zoom levels (from maximum down to the minimum zoom level).
-         * Tile IDs of the tiles enclosing this tile at lower zoom levels are calculated using
-         * bit shifts.
-         *
-         * last_quadkey is initialized with a value which is not expected to exist
-         * (larger than largest possible quadkey). */
-        uint64_t last_quadkey = 1ULL << (2 * m_maxzoom);
-        std::size_t count = 0;
-        for (auto const quadkey : tiles_maxzoom) {
-            for (uint32_t dz = 0; dz <= m_maxzoom - minzoom; ++dz) {
-                // scale down to the current zoom level
-                uint64_t qt_current = quadkey >> (dz * 2);
-                /* If dz > 0, there are propably multiple elements whose quadkey
-                 * is equal because they are all sub-tiles of the same tile at the current
-                 * zoom level. We skip all of them after we have written the first sibling.
-                 */
-                if (qt_current == last_quadkey >> (dz * 2)) {
-                    continue;
-                }
-                auto const tile =
-                    tile_t::from_quadkey(qt_current, m_maxzoom - dz);
-                output_writer.output_dirty_tile(tile);
-                ++count;
-            }
-            last_quadkey = quadkey;
-        }
-        log_info("Wrote {} entries to expired tiles list", count);
-    }
-
-    /**
-    * merge the list of expired tiles in the other object into this
-    * object, destroying the list in the other object.
-    */
     void merge_and_destroy(expire_tiles *other);
 
 private:
@@ -161,4 +92,73 @@ class expire_tiles
     int m_map_width;
 };
 
+/**
+ * Iterate over tiles and call output function for each tile on all requested
+ * zoom levels.
+ *
+ * \tparam OUTPUT Class with operator() taking a tile_t argument
+ *
+ * \param tiles The list of tiles at maximum zoom level
+ * \param minzoom Minimum zoom level
+ * \param maxzoom Maximum zoom level
+ * \param output Output function
+ */
+template <class OUTPUT>
+std::size_t for_each_tile(std::vector<uint64_t> const &tiles, uint32_t minzoom,
+                          uint32_t maxzoom, OUTPUT &&output)
+{
+    assert(minzoom <= maxzoom);
+
+    if (minzoom == maxzoom) {
+        for (auto const quadkey : tiles) {
+            std::forward<OUTPUT>(output)(
+                tile_t::from_quadkey(quadkey, maxzoom));
+        }
+        return tiles.size();
+    }
+
+    /**
+     * Loop over all requested zoom levels (from maximum down to the minimum
+     * zoom level). Tile IDs of the tiles enclosing this tile at lower zoom
+     * levels are calculated using bit shifts.
+     *
+     * last_quadkey is initialized with a value which is not expected to exist
+     * (larger than largest possible quadkey).
+     */
+    uint64_t last_quadkey = 1ULL << (2 * maxzoom);
+    std::size_t count = 0;
+    for (auto const quadkey : tiles) {
+        for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) {
+            // scale down to the current zoom level
+            uint64_t const qt_current = quadkey >> (dz * 2);
+            /**
+             * If dz > 0, there are probably multiple elements whose quadkey
+             * is equal because they are all sub-tiles of the same tile at the
+             * current zoom level. We skip all of them after we have written
+             * the first sibling.
+             */
+            if (qt_current == last_quadkey >> (dz * 2)) {
+                continue;
+            }
+            auto const tile = tile_t::from_quadkey(qt_current, maxzoom - dz);
+            std::forward<OUTPUT>(output)(tile);
+            ++count;
+        }
+        last_quadkey = quadkey;
+    }
+    return count;
+}
+
+/**
+ * Write the list of tiles to a file.
+ *
+ * \param tiles The list of tiles at maximum zoom level
+ * \param filename Name of the file
+ * \param minzoom Minimum zoom level
+ * \param maxzoom Maximum zoom level
+ */
+std::size_t output_tiles_to_file(std::vector<uint64_t> const &tiles,
+                                 char const *filename, uint32_t minzoom,
+                                 uint32_t maxzoom);
+
 #endif // OSM2PGSQL_EXPIRE_TILES_HPP

src/output-flex.cpp

@@ -1857,9 +1857,11 @@ void output_flex_t::stop()
     }
 
     if (get_options()->expire_tiles_zoom_min > 0) {
-        m_expire.output_and_destroy(
-            get_options()->expire_tiles_filename.c_str(),
-            get_options()->expire_tiles_zoom_min);
+        auto const count = output_tiles_to_file(
+            m_expire.get_tiles(), get_options()->expire_tiles_filename.c_str(),
+            get_options()->expire_tiles_zoom_min,
+            get_options()->expire_tiles_zoom);
+        log_info("Wrote {} entries to expired tiles list", count);
     }
 }
 

src/output-pgsql.cpp

@@ -146,9 +146,11 @@ void output_pgsql_t::stop()
     }
 
     if (get_options()->expire_tiles_zoom_min > 0) {
-        m_expire.output_and_destroy(
-            get_options()->expire_tiles_filename.c_str(),
-            get_options()->expire_tiles_zoom_min);
+        auto const count = output_tiles_to_file(
+            m_expire.get_tiles(), get_options()->expire_tiles_filename.c_str(),
+            get_options()->expire_tiles_zoom_min,
+            get_options()->expire_tiles_zoom);
+        log_info("Wrote {} entries to expired tiles list", count);
     }
 }
 

src/tile-output.hpp

@@ -0,0 +1,25 @@
+#ifndef OSM2PGSQL_TILE_OUTPUT_HPP
+#define OSM2PGSQL_TILE_OUTPUT_HPP
+
+/**
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ *
+ * This file is part of osm2pgsql (https://osm2pgsql.org/).
+ *
+ * Copyright (C) 2006-2022 by the osm2pgsql developer community.
+ * For a full list of authors see the git log.
+ */
+
+#include "tile.hpp"
+
+#include <iosfwd>
+
+template <typename TChar, typename TTraits>
+std::basic_ostream<TChar, TTraits> &
+operator<<(std::basic_ostream<TChar, TTraits> &out, const tile_t &tile)
+{
+    return out << "TILE(" << tile.zoom() << ", " << tile.x() << ", " << tile.y()
+               << ')';
+}
+
+#endif // OSM2PGSQL_TILE_OUTPUT_HPP