Skip to content

Commit ac51627

Browse files
authored
Merge pull request #1802 from joto/refactor-expire
Various refactorings of expire code
2 parents d2392b2 + ae3f242 commit ac51627

9 files changed

Lines changed: 361 additions & 296 deletions

File tree

src/expire-tiles.cpp

Lines changed: 48 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,13 @@
1515
* http://subversion.nexusuk.org/projects/openpistemap/trunk/scripts/expire_tiles.py
1616
*/
1717

18+
#include <algorithm>
1819
#include <cerrno>
1920
#include <cmath>
21+
#include <cstdio>
2022
#include <cstdlib>
2123
#include <cstring>
24+
#include <stdexcept>
2225
#include <string>
2326

2427
#include "expire-tiles.hpp"
@@ -34,44 +37,12 @@
3437
// How many tiles worth of space to leave either side of a changed feature
3538
static constexpr double const tile_expiry_leeway = 0.1;
3639

37-
tile_output_t::tile_output_t(char const *filename)
38-
: outfile(fopen(filename, "a"))
39-
{
40-
if (outfile == nullptr) {
41-
log_warn("Failed to open expired tiles file ({}). Tile expiry "
42-
"list will not be written!",
43-
std::strerror(errno));
44-
}
45-
}
46-
47-
tile_output_t::~tile_output_t()
48-
{
49-
if (outfile) {
50-
fclose(outfile);
51-
}
52-
}
53-
54-
void tile_output_t::output_dirty_tile(tile_t const &tile)
55-
{
56-
if (!outfile) {
57-
return;
58-
}
59-
60-
fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y());
61-
}
62-
6340
expire_tiles::expire_tiles(uint32_t max_zoom, double max_bbox,
6441
std::shared_ptr<reprojection> projection)
6542
: m_projection(std::move(projection)), m_max_bbox(max_bbox),
6643
m_maxzoom(max_zoom), m_map_width(1U << m_maxzoom)
6744
{}
6845

69-
void expire_tiles::output_and_destroy(char const *filename, uint32_t minzoom)
70-
{
71-
tile_output_t output_writer{filename};
72-
output_and_destroy<tile_output_t>(output_writer, minzoom);
73-
}
74-
7546
void expire_tiles::expire_tile(uint32_t x, uint32_t y)
7647
{
7748
// Only try to insert to tile into the set if the last inserted tile
@@ -107,7 +78,7 @@ void expire_tiles::from_point_list(geom::point_list_t const &list)
10778
});
10879
}
10980

110-
void expire_tiles::from_geometry(geom::geometry_t const &geom, osmid_t osm_id)
81+
void expire_tiles::from_geometry(geom::geometry_t const &geom)
11182
{
11283
if (geom.srid() != 3857) {
11384
return;
@@ -125,10 +96,6 @@ void expire_tiles::from_geometry(geom::geometry_t const &geom, osmid_t osm_id)
12596
} else if (geom.is_polygon() || geom.is_multipolygon()) {
12697
auto const box = geom::envelope(geom);
12798
if (from_bbox(box)) {
128-
/* Bounding box too big - just expire tiles on the line */
129-
log_debug("Large polygon ({:.0f} x {:.0f} metres, OSM ID {})"
130-
" - only expiring perimeter",
131-
box.max_x() - box.min_x(), box.max_y() - box.min_y(), osm_id);
13299
if (geom.is_polygon()) {
133100
from_point_list(geom.get<geom::polygon_t>().outer());
134101
for (auto const &inner : geom.get<geom::polygon_t>().inners()) {
@@ -263,19 +230,14 @@ int expire_tiles::from_bbox(geom::box_t const &box)
263230
return 0;
264231
}
265232

266-
int expire_tiles::from_result(pg_result_t const &result, osmid_t osm_id)
233+
quadkey_list_t expire_tiles::get_tiles()
267234
{
268-
if (!enabled()) {
269-
return -1;
270-
}
271-
272-
auto const num_tuples = result.num_tuples();
273-
for (int i = 0; i < num_tuples; ++i) {
274-
char const *const wkb = result.get_value(i, 0);
275-
from_geometry(ewkb_to_geom(decode_hex(wkb)), osm_id);
276-
}
277-
278-
return num_tuples;
235+
quadkey_list_t tiles;
236+
tiles.reserve(m_dirty_tiles.size());
237+
tiles.assign(m_dirty_tiles.begin(), m_dirty_tiles.end());
238+
std::sort(tiles.begin(), tiles.end());
239+
m_dirty_tiles.clear();
240+
return tiles;
279241
}
280242

281243
void expire_tiles::merge_and_destroy(expire_tiles *other)
@@ -294,3 +256,40 @@ void expire_tiles::merge_and_destroy(expire_tiles *other)
294256
other->m_dirty_tiles.clear();
295257
}
296258
}
259+
260+
std::size_t output_tiles_to_file(quadkey_list_t const &tiles_maxzoom,
261+
char const *filename, uint32_t minzoom,
262+
uint32_t maxzoom)
263+
{
264+
FILE *outfile = std::fopen(filename, "a");
265+
if (outfile == nullptr) {
266+
log_warn("Failed to open expired tiles file ({}). Tile expiry "
267+
"list will not be written!",
268+
std::strerror(errno));
269+
return 0;
270+
}
271+
272+
auto const count =
273+
for_each_tile(tiles_maxzoom, minzoom, maxzoom, [&](tile_t const &tile) {
274+
fmt::print(outfile, "{}/{}/{}\n", tile.zoom(), tile.x(), tile.y());
275+
});
276+
277+
std::fclose(outfile);
278+
279+
return count;
280+
}
281+
282+
int expire_from_result(expire_tiles *expire, pg_result_t const &result)
283+
{
284+
if (!expire->enabled()) {
285+
return -1;
286+
}
287+
288+
auto const num_tuples = result.num_tuples();
289+
for (int i = 0; i < num_tuples; ++i) {
290+
char const *const wkb = result.get_value(i, 0);
291+
expire->from_geometry(ewkb_to_geom(decode_hex(wkb)));
292+
}
293+
294+
return num_tuples;
295+
}

src/expire-tiles.hpp

Lines changed: 85 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@
1212

1313
#include <memory>
1414
#include <unordered_set>
15+
#include <utility>
16+
#include <vector>
1517

1618
#include "geom.hpp"
1719
#include "geom-box.hpp"
@@ -22,26 +24,6 @@
2224

2325
class reprojection;
2426

25-
/**
26-
* Implementation of the output of the tile expiry list to a file.
27-
*/
28-
class tile_output_t
29-
{
30-
FILE *outfile;
31-
32-
public:
33-
explicit tile_output_t(char const *filename);
34-
35-
~tile_output_t();
36-
37-
/**
38-
* Output dirty tile.
39-
*
40-
* \param tile The tile to write out.
41-
*/
42-
void output_dirty_tile(tile_t const &tile);
43-
};
44-
4527
class expire_tiles
4628
{
4729
public:
@@ -50,83 +32,20 @@ class expire_tiles
5032

5133
bool enabled() const noexcept { return m_maxzoom != 0; }
5234

53-
void from_geometry(geom::geometry_t const &geom, osmid_t osm_id);
35+
void from_geometry(geom::geometry_t const &geom);
5436

5537
int from_bbox(geom::box_t const &box);
5638

5739
/**
58-
* Expire tiles based on an osm id.
59-
*
60-
* \param result Result of a database query into some table returning the
61-
* geometries. (This is usally done using the "get_wkb"
62-
* prepared statement.)
63-
* \param osm_id The OSM id to look for.
64-
* \return The number of elements that refer to the osm_id or -1 if
65-
* expire is disabled.
40+
* Get tiles as a vector of quadkeys and remove them from the expire_tiles
41+
* object.
6642
*/
67-
int from_result(pg_result_t const &result, osmid_t osm_id);
43+
quadkey_list_t get_tiles();
6844

6945
/**
70-
* Write the list of expired tiles to a file.
71-
*
72-
* You will probably use tile_output_t as template argument for production code
73-
* and another class which does not write to a file for unit tests.
74-
*
75-
* \param filename name of the file
76-
* \param minzoom minimum zoom level
77-
*/
78-
void output_and_destroy(char const *filename, uint32_t minzoom);
79-
80-
/**
81-
* Output expired tiles on all requested zoom levels.
82-
*
83-
* \tparam TILE_WRITER class which implements the method
84-
* output_dirty_tile(tile_t const &tile) which usually writes the tile ID
85-
* to a file (production code) or does something else (usually unit tests).
86-
*
87-
* \param minzoom minimum zoom level
46+
* Merge the list of expired tiles in the other object into this
47+
* object, destroying the list in the other object.
8848
*/
89-
template <class TILE_WRITER>
90-
void output_and_destroy(TILE_WRITER &output_writer, uint32_t minzoom)
91-
{
92-
assert(minzoom <= m_maxzoom);
93-
// build a sorted vector of all expired tiles
94-
std::vector<uint64_t> tiles_maxzoom(m_dirty_tiles.begin(),
95-
m_dirty_tiles.end());
96-
std::sort(tiles_maxzoom.begin(), tiles_maxzoom.end());
97-
/* Loop over all requested zoom levels (from maximum down to the minimum zoom level).
98-
* Tile IDs of the tiles enclosing this tile at lower zoom levels are calculated using
99-
* bit shifts.
100-
*
101-
* last_quadkey is initialized with a value which is not expected to exist
102-
* (larger than largest possible quadkey). */
103-
uint64_t last_quadkey = 1ULL << (2 * m_maxzoom);
104-
std::size_t count = 0;
105-
for (auto const quadkey : tiles_maxzoom) {
106-
for (uint32_t dz = 0; dz <= m_maxzoom - minzoom; ++dz) {
107-
// scale down to the current zoom level
108-
uint64_t qt_current = quadkey >> (dz * 2);
109-
/* If dz > 0, there are propably multiple elements whose quadkey
110-
* is equal because they are all sub-tiles of the same tile at the current
111-
* zoom level. We skip all of them after we have written the first sibling.
112-
*/
113-
if (qt_current == last_quadkey >> (dz * 2)) {
114-
continue;
115-
}
116-
auto const tile =
117-
tile_t::from_quadkey(qt_current, m_maxzoom - dz);
118-
output_writer.output_dirty_tile(tile);
119-
++count;
120-
}
121-
last_quadkey = quadkey;
122-
}
123-
log_info("Wrote {} entries to expired tiles list", count);
124-
}
125-
126-
/**
127-
* merge the list of expired tiles in the other object into this
128-
* object, destroying the list in the other object.
129-
*/
13049
void merge_and_destroy(expire_tiles *other);
13150

13251
private:
@@ -149,7 +68,7 @@ class expire_tiles
14968
void from_point_list(geom::point_list_t const &list);
15069

15170
/// This is where we collect all the expired tiles.
152-
std::unordered_set<uint64_t> m_dirty_tiles;
71+
std::unordered_set<quadkey_t> m_dirty_tiles;
15372

15473
/// The tile which has been added last to the unordered set.
15574
tile_t m_prev_tile;
@@ -159,6 +78,81 @@ class expire_tiles
15978
double m_max_bbox;
16079
uint32_t m_maxzoom;
16180
int m_map_width;
162-
};
81+
82+
}; // class expire_tiles
83+
84+
/**
85+
* Expire tiles based on an osm id.
86+
*
87+
* \param expire Where to mark expired tiles.
88+
* \param result Result of a database query into some table returning the
89+
* geometries. (This is usually done using the "get_wkb"
90+
* prepared statement.)
91+
* \return The number of tuples in the result or -1 if expire is disabled.
92+
*/
93+
int expire_from_result(expire_tiles *expire, pg_result_t const &result);
94+
95+
/**
96+
* Iterate over tiles and call output function for each tile on all requested
97+
* zoom levels.
98+
*
99+
* \tparam OUTPUT Class with operator() taking a tile_t argument
100+
*
101+
* \param tiles The list of tiles at maximum zoom level
102+
* \param minzoom Minimum zoom level
103+
* \param maxzoom Maximum zoom level
104+
* \param output Output function
105+
*/
106+
template <class OUTPUT>
107+
std::size_t for_each_tile(quadkey_list_t const &tiles, uint32_t minzoom,
108+
uint32_t maxzoom, OUTPUT &&output)
109+
{
110+
assert(minzoom <= maxzoom);
111+
112+
if (minzoom == maxzoom) {
113+
for (auto const quadkey : tiles) {
114+
std::forward<OUTPUT>(output)(
115+
tile_t::from_quadkey(quadkey, maxzoom));
116+
}
117+
return tiles.size();
118+
}
119+
120+
/**
121+
* Loop over all requested zoom levels (from maximum down to the minimum
122+
* zoom level).
123+
*/
124+
quadkey_t last_quadkey{};
125+
std::size_t count = 0;
126+
for (auto const quadkey : tiles) {
127+
for (uint32_t dz = 0; dz <= maxzoom - minzoom; ++dz) {
128+
auto const qt_current = quadkey.down(dz);
129+
/**
130+
* If dz > 0, there are probably multiple elements whose quadkey
131+
* is equal because they are all sub-tiles of the same tile at the
132+
* current zoom level. We skip all of them after we have written
133+
* the first sibling.
134+
*/
135+
if (qt_current != last_quadkey.down(dz)) {
136+
std::forward<OUTPUT>(output)(
137+
tile_t::from_quadkey(qt_current, maxzoom - dz));
138+
++count;
139+
}
140+
}
141+
last_quadkey = quadkey;
142+
}
143+
return count;
144+
}
145+
146+
/**
147+
* Write the list of tiles to a file.
148+
*
149+
* \param tiles The list of tiles at maximum zoom level
150+
* \param filename Name of the file
151+
* \param minzoom Minimum zoom level
152+
* \param maxzoom Maximum zoom level
153+
*/
154+
std::size_t output_tiles_to_file(quadkey_list_t const &tiles,
155+
char const *filename, uint32_t minzoom,
156+
uint32_t maxzoom);
163157

164158
#endif // OSM2PGSQL_EXPIRE_TILES_HPP

0 commit comments

Comments
 (0)