Skip to content

Commit ded35d5

Browse files
lonvialonvia
authored
Merge pull request #1310 from joto/option-to-disable-propagation
Add --middle-propagate-changes option
2 parents 639a7d1 + 59b57de commit ded35d5

16 files changed

Lines changed: 136 additions & 86 deletions

‎docs/osm2pgsql.1‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,14 +219,14 @@ The default is disabled.
219219
.RS
220220
.RE
221221
.TP
222-
.B \[en]middle\-schema=SCHEMA
222+
.B \-\-middle\-schema=SCHEMA
223223
Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in
224224
the middle (default is no schema, i.e.\ the \f[C]public\f[] schema is
225225
used).
226226
.RS
227227
.RE
228228
.TP
229-
.B \[en]middle\-way\-node\-index\-id\-shift=SHIFT
229+
.B \-\-middle\-way\-node\-index\-id\-shift=SHIFT
230230
Set ID shift for way node bucket index in middle.
231231
Experts only.
232232
See documentation for details.
@@ -369,7 +369,7 @@ Compute area column using spherical mercator coordinates.
369369
.RS
370370
.RE
371371
.TP
372-
.B \[en]output\-pgsql\-schema=SCHEMA
372+
.B \-\-output\-pgsql\-schema=SCHEMA
373373
Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in
374374
the pgsql and multi outputs (default is no schema, i.e.\ the
375375
\f[C]public\f[] schema is used).
@@ -404,6 +404,12 @@ index after the other.
404404
Specifies the number of parallel threads used for certain operations.
405405
.RS
406406
.RE
407+
.TP
408+
.B \-\-with\-forward\-dependencies=BOOL
409+
Propagate changes from nodes to ways and node/way members to relations
410+
(Default: \f[C]true\f[]).
411+
.RS
412+
.RE
407413
.SH SEE ALSO
408414
.IP \[bu] 2
409415
osm2pgsql website (https://osm2pgsql.org)

‎docs/osm2pgsql.md‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,11 +151,11 @@ Mandatory arguments to long options are mandatory for short options too.
151151
single large file. This mode is only recommended for full planet imports
152152
as it doesn't work well with small imports. The default is disabled.
153153

154-
--middle-schema=SCHEMA
154+
\--middle-schema=SCHEMA
155155
: Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in
156156
the middle (default is no schema, i.e. the `public` schema is used).
157157

158-
--middle-way-node-index-id-shift=SHIFT
158+
\--middle-way-node-index-id-shift=SHIFT
159159
: Set ID shift for way node bucket index in middle. Experts only. See
160160
documentation for details.
161161

@@ -245,7 +245,7 @@ Mandatory arguments to long options are mandatory for short options too.
245245
\--reproject-area
246246
: Compute area column using spherical mercator coordinates.
247247

248-
--output-pgsql-schema=SCHEMA
248+
\--output-pgsql-schema=SCHEMA
249249
: Use PostgreSQL schema SCHEMA for all tables, indexes, and functions in
250250
the pgsql and multi outputs (default is no schema, i.e. the `public` schema
251251
is used).
@@ -270,6 +270,10 @@ Mandatory arguments to long options are mandatory for short options too.
270270
\--number-processes=THREADS
271271
: Specifies the number of parallel threads used for certain operations.
272272

273+
\--with-forward-dependencies=BOOL
274+
: Propagate changes from nodes to ways and node/way members to relations
275+
(Default: `true`).
276+
273277
# SEE ALSO
274278

275279
* [osm2pgsql website](https://osm2pgsql.org)

‎src/middle-pgsql.cpp‎

Lines changed: 39 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -64,23 +64,18 @@ static std::string build_sql(options_t const &options, char const *templ)
6464

6565
middle_pgsql_t::table_desc::table_desc(options_t const &options,
6666
table_sql const &ts)
67-
: m_create(build_sql(options, ts.create_table)),
67+
: m_create_table(build_sql(options, ts.create_table)),
6868
m_prepare_query(build_sql(options, ts.prepare_query)),
6969
m_copy_target(std::make_shared<db_target_descr_t>())
7070
{
7171
m_copy_target->name = build_sql(options, ts.name);
7272
m_copy_target->schema = options.middle_dbschema;
7373
m_copy_target->id = "id"; // XXX hardcoded column name
7474

75-
// Gazetteer doesn't use mark-pending processing and consequently
76-
// needs no way-node index.
77-
// TODO Currently, set here to keep the impact on the code small.
78-
// We actually should have the output plugins report their needs
79-
// and pass that via the constructor to middle_t, so that middle_t
80-
// itself doesn't need to know about details of the output.
81-
if (options.output_backend != "gazetteer") {
82-
m_prepare_intarray = build_sql(options, ts.prepare_mark);
83-
m_array_indexes = build_sql(options, ts.create_index);
75+
if (options.with_forward_dependencies) {
76+
m_prepare_fw_dep_lookups =
77+
build_sql(options, ts.prepare_fw_dep_lookups);
78+
m_create_fw_dep_indexes = build_sql(options, ts.create_fw_dep_indexes);
8479
}
8580
}
8681

@@ -103,9 +98,9 @@ void middle_pgsql_t::table_desc::stop(std::string const &conninfo,
10398
auto const qual_name = qualified_name(
10499
m_copy_target->schema, m_copy_target->name);
105100
sql_conn.exec("DROP TABLE IF EXISTS {}"_format(qual_name));
106-
} else if (build_indexes && !m_array_indexes.empty()) {
101+
} else if (build_indexes && !m_create_fw_dep_indexes.empty()) {
107102
fmt::print(stderr, "Building index on table: {}\n", name());
108-
sql_conn.exec(m_array_indexes);
103+
sql_conn.exec(m_create_fw_dep_indexes);
109104
}
110105

111106
fmt::print(stderr, "Stopped table: {} in {}s\n", name(), timer.stop());
@@ -570,8 +565,8 @@ void middle_pgsql_t::start()
570565

571566
// Prepare queries for updating dependent objects
572567
for (auto &table : m_tables) {
573-
if (!table.m_prepare_intarray.empty()) {
574-
m_db_connection.exec(table.m_prepare_intarray);
568+
if (!table.m_prepare_fw_dep_lookups.empty()) {
569+
m_db_connection.exec(table.m_prepare_fw_dep_lookups);
575570
}
576571
}
577572
} else {
@@ -583,7 +578,7 @@ void middle_pgsql_t::start()
583578
table.m_copy_target->schema, table.m_copy_target->name);
584579
m_db_connection.exec(
585580
"DROP TABLE IF EXISTS {} CASCADE"_format(qual_name));
586-
m_db_connection.exec(table.m_create);
581+
m_db_connection.exec(table.m_create_table);
587582
}
588583

589584
// The extra query connection is only needed in append mode, so close.
@@ -666,33 +661,34 @@ static table_sql sql_for_ways(bool has_bucket_index,
666661
" WHERE id = ANY($1::int8[]);\n";
667662

668663
if (has_bucket_index) {
669-
sql.prepare_mark =
664+
sql.prepare_fw_dep_lookups =
670665
"PREPARE mark_ways_by_node(int8) AS"
671666
" SELECT id FROM {schema}{prefix}_ways w"
672667
" WHERE $1 = ANY(nodes)"
673668
" AND {schema}{prefix}_index_bucket(w.nodes)"
674669
" && {schema}{prefix}_index_bucket(ARRAY[$1]);\n";
675670
} else {
676-
sql.prepare_mark = "PREPARE mark_ways_by_node(int8) AS"
677-
" SELECT id FROM {schema}{prefix}_ways"
678-
" WHERE nodes && ARRAY[$1];\n";
671+
sql.prepare_fw_dep_lookups = "PREPARE mark_ways_by_node(int8) AS"
672+
" SELECT id FROM {schema}{prefix}_ways"
673+
" WHERE nodes && ARRAY[$1];\n";
679674
}
680675

681676
if (way_node_index_id_shift == 0) {
682-
sql.create_index =
677+
sql.create_fw_dep_indexes =
683678
"CREATE INDEX ON {schema}{prefix}_ways USING GIN (nodes)"
684679
" WITH (fastupdate = off) {index_tablespace};\n";
685680
} else {
686-
sql.create_index = "CREATE OR REPLACE FUNCTION"
687-
" {schema}{prefix}_index_bucket(int8[])"
688-
" RETURNS int8[] AS $$\n"
689-
" SELECT ARRAY(SELECT DISTINCT"
690-
" unnest($1) >> {way_node_index_id_shift})\n"
691-
"$$ LANGUAGE SQL IMMUTABLE;\n"
692-
"CREATE INDEX {schema}{prefix}_ways_nodes_bucket_idx"
693-
" ON {schema}{prefix}_ways"
694-
" USING GIN ({schema}{prefix}_index_bucket(nodes))"
695-
" WITH (fastupdate = off) {index_tablespace};\n";
681+
sql.create_fw_dep_indexes =
682+
"CREATE OR REPLACE FUNCTION"
683+
" {schema}{prefix}_index_bucket(int8[])"
684+
" RETURNS int8[] AS $$\n"
685+
" SELECT ARRAY(SELECT DISTINCT"
686+
" unnest($1) >> {way_node_index_id_shift})\n"
687+
"$$ LANGUAGE SQL IMMUTABLE;\n"
688+
"CREATE INDEX {schema}{prefix}_ways_nodes_bucket_idx"
689+
" ON {schema}{prefix}_ways"
690+
" USING GIN ({schema}{prefix}_index_bucket(nodes))"
691+
" WITH (fastupdate = off) {index_tablespace};\n";
696692
}
697693

698694
return sql;
@@ -717,17 +713,19 @@ static table_sql sql_for_relations() noexcept
717713
" SELECT members, tags"
718714
" FROM {schema}{prefix}_rels WHERE id = $1;\n";
719715

720-
sql.prepare_mark = "PREPARE mark_rels_by_node(int8) AS"
721-
" SELECT id FROM {schema}{prefix}_rels"
722-
" WHERE parts && ARRAY[$1]"
723-
" AND parts[1:way_off] && ARRAY[$1];\n"
724-
"PREPARE mark_rels_by_way(int8) AS"
725-
" SELECT id FROM {schema}{prefix}_rels"
726-
" WHERE parts && ARRAY[$1]"
727-
" AND parts[way_off+1:rel_off] && ARRAY[$1];\n";
728-
729-
sql.create_index = "CREATE INDEX ON {schema}{prefix}_rels USING GIN (parts)"
730-
" WITH (fastupdate = off) {index_tablespace};\n";
716+
sql.prepare_fw_dep_lookups =
717+
"PREPARE mark_rels_by_node(int8) AS"
718+
" SELECT id FROM {schema}{prefix}_rels"
719+
" WHERE parts && ARRAY[$1]"
720+
" AND parts[1:way_off] && ARRAY[$1];\n"
721+
"PREPARE mark_rels_by_way(int8) AS"
722+
" SELECT id FROM {schema}{prefix}_rels"
723+
" WHERE parts && ARRAY[$1]"
724+
" AND parts[way_off+1:rel_off] && ARRAY[$1];\n";
725+
726+
sql.create_fw_dep_indexes =
727+
"CREATE INDEX ON {schema}{prefix}_rels USING GIN (parts)"
728+
" WITH (fastupdate = off) {index_tablespace};\n";
731729

732730
return sql;
733731
}

‎src/middle-pgsql.hpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ struct table_sql {
4848
char const *name = "";
4949
char const *create_table = "";
5050
char const *prepare_query = "";
51-
char const *prepare_mark = "";
52-
char const *create_index = "";
51+
char const *prepare_fw_dep_lookups = "";
52+
char const *create_fw_dep_indexes = "";
5353
};
5454

5555
struct middle_pgsql_t : public slim_middle_t
@@ -87,10 +87,10 @@ struct middle_pgsql_t : public slim_middle_t
8787
void stop(std::string const &conninfo, bool droptemp,
8888
bool build_indexes);
8989

90-
std::string m_create;
90+
std::string m_create_table;
9191
std::string m_prepare_query;
92-
std::string m_prepare_intarray;
93-
std::string m_array_indexes;
92+
std::string m_prepare_fw_dep_lookups;
93+
std::string m_create_fw_dep_indexes;
9494

9595
std::shared_ptr<db_target_descr_t> m_copy_target;
9696
};

‎src/options.cpp‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ const struct option long_options[] = {
8080
{"username", required_argument, nullptr, 'U'},
8181
{"verbose", no_argument, nullptr, 'v'},
8282
{"version", no_argument, nullptr, 'V'},
83+
{"with-forward-dependencies", required_argument, nullptr, 217},
8384
{nullptr, 0, nullptr, 0}};
8485

8586
void short_usage(char *arg0)
@@ -126,6 +127,8 @@ void long_usage(char const *arg0, bool verbose)
126127
This file is a single > 40Gb large file. Only recommended\n\
127128
for full planet imports. Default is disabled.\n\
128129
--middle-schema Schema to use for middle tables (default: none)\n\
130+
--with-forward-dependencies true|false Propagate changes from nodes to ways\n\
131+
and node/way members to relations (Default: true).\n\
129132
\n\
130133
Database options:\n\
131134
-d|--database The name of the PostgreSQL database to connect to or\n\
@@ -187,8 +190,8 @@ void long_usage(char const *arg0, bool verbose)
187190
printf("%s", "\
188191
\n\
189192
Middle options (experts only):\n\
190-
--middle-way-node-index-id-shift shift Set ID shift for bucket\
191-
index. See documentation for details.\
193+
--middle-way-node-index-id-shift shift Set ID shift for bucket\n\
194+
index. See documentation for details.\n\
192195
\n\
193196
Expiry options:\n\
194197
-e|--expire-tiles [min_zoom-]max_zoom Create a tile expiry list.\n\
@@ -582,6 +585,17 @@ options_t::options_t(int argc, char *argv[]) : options_t()
582585
case 216:
583586
output_dbschema = optarg;
584587
break;
588+
case 217:
589+
if (std::strcmp(optarg, "false") == 0) {
590+
with_forward_dependencies = false;
591+
} else if (std::strcmp(optarg, "true") == 0) {
592+
with_forward_dependencies = true;
593+
} else {
594+
throw std::runtime_error{
595+
"Unknown value for --with-forward-dependencies option: {}\n"_format(
596+
optarg)};
597+
}
598+
break;
585599
case 300:
586600
way_node_index_id_shift = atoi(optarg);
587601
break;

‎src/options.hpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,12 @@ class options_t
105105
int num_procs;
106106
bool droptemp = false; ///< drop slim mode temp tables after act
107107

108+
/**
109+
* Should changes of objects be propagated forwards (from nodes to ways and
110+
* from node/way members to parent relations)?
111+
*/
112+
bool with_forward_dependencies = true;
113+
108114
/// only copy rows that match an explicitly listed key
109115
bool hstore_match_only = false;
110116

‎src/osm2pgsql.cpp‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,10 @@ int main(int argc, char *argv[])
6666
auto const outputs =
6767
output_t::create_outputs(middle->get_query_instance(), options);
6868

69-
bool const need_dependencies =
70-
std::any_of(outputs.cbegin(), outputs.cend(),
71-
[](std::shared_ptr<output_t> const &output) {
72-
return output->need_forward_dependencies();
73-
});
74-
7569
auto dependency_manager = std::unique_ptr<dependency_manager_t>(
76-
need_dependencies ? new full_dependency_manager_t{middle}
77-
: new dependency_manager_t{});
70+
options.with_forward_dependencies
71+
? new full_dependency_manager_t{middle}
72+
: new dependency_manager_t{});
7873

7974
osmdata_t osmdata{std::move(dependency_manager), middle, outputs,
8075
options};

‎src/osmdata.cpp‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ osmdata_t::osmdata_t(std::unique_ptr<dependency_manager_t> dependency_manager,
3030
m_outs(std::move(outs)), m_conninfo(options.database_options.conninfo()),
3131
m_num_procs(options.num_procs), m_append(options.append),
3232
m_droptemp(options.droptemp), m_parallel_indexing(options.parallel_indexing),
33-
m_with_extra_attrs(options.extra_attributes)
33+
m_with_extra_attrs(options.extra_attributes),
34+
m_with_forward_dependencies(options.with_forward_dependencies)
3435
{
3536
assert(m_dependency_manager);
3637
assert(m_mid);
@@ -193,11 +194,7 @@ class multithreaded_processor
193194
auto copy_thread = std::make_shared<db_copy_thread_t>(conninfo);
194195

195196
for (auto const &out : m_outputs) {
196-
if (out->need_forward_dependencies()) {
197-
m_clones[i].push_back(out->clone(midq, copy_thread));
198-
} else {
199-
m_clones[i].emplace_back(nullptr);
200-
}
197+
m_clones[i].push_back(out->clone(midq, copy_thread));
201198
}
202199
}
203200
}
@@ -246,9 +243,7 @@ class multithreaded_processor
246243
for (auto const &output : m_outputs) {
247244
for (auto const &clone : m_clones) {
248245
assert(n < clone.size());
249-
if (clone[n]) {
250-
output->merge_expire_trees(clone[n].get());
251-
}
246+
output->merge_expire_trees(clone[n].get());
252247
}
253248
++n;
254249
}
@@ -459,7 +454,7 @@ void osmdata_t::stop() const
459454
out->sync();
460455
}
461456

462-
if (m_append) {
457+
if (m_append && m_with_forward_dependencies) {
463458
process_dependents();
464459
}
465460

‎src/osmdata.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ class osmdata_t
9595
bool m_droptemp;
9696
bool m_parallel_indexing;
9797
bool m_with_extra_attrs;
98+
bool m_with_forward_dependencies;
9899

99100
};
100101

‎src/output-gazetteer.hpp‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,6 @@ class output_gazetteer_t : public output_t
4444
void stop(thread_pool_t *) noexcept override {}
4545
void sync() override;
4646

47-
bool need_forward_dependencies() const noexcept override { return false; }
48-
4947
void pending_way(osmid_t) noexcept override {}
5048
void pending_relation(osmid_t) noexcept override {}
5149

0 commit comments

Comments
 (0)