Skip to content

Commit 499300c

Browse files
jotojoto
authored
Merge pull request #1548 from geofabrik/info-command
add a 'status' command which prints the lag (& in JSON)
2 parents 717ee59 + 2d3e6f1 commit 499300c

2 files changed

Lines changed: 257 additions & 1 deletion

File tree

‎docs/osm2pgsql-replication.1‎

Lines changed: 118 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
osm2pgsql-replication
44
.SH SYNOPSIS
55
.B osm2pgsql-replication
6-
[-h] {init,update} ...
6+
[-h] {init,update,status} ...
77
.SH DESCRIPTION
88
Update an osm2pgsql database with changes from a OSM replication server.
99
.br
@@ -33,6 +33,9 @@ how to use osm2pgsql\-replication.
3333
.TP
3434
\fBosm2pgsql-replication\fR \fI\,update\/\fR
3535
Download newly available data and apply it to the database.
36+
.TP
37+
\fBosm2pgsql-replication\fR \fI\,status\/\fR
38+
Print information about the current replication status, optionally as JSON.
3639
.SH OPTIONS 'osm2pgsql-replication init'
3740
usage: osm2pgsql-replication init [-h] [-q] [-v] [-d DB] [-U NAME] [-H HOST]
3841
[-P PORT] [-p PREFIX]
@@ -205,6 +208,120 @@ Database server port
205208
\fB\-p\fR PREFIX, \fB\-\-prefix\fR PREFIX
206209
Prefix for table names (default 'planet_osm')
207210

211+
212+
.SH OPTIONS 'osm2pgsql-replication status'
213+
usage: osm2pgsql-replication status [-h] [-q] [-v] [-d DB] [-U NAME] [-H HOST]
214+
[-P PORT] [-p PREFIX] [--json]
215+
216+
Print information about the current replication status, optionally as JSON.
217+
.br
218+
219+
.br
220+
Sample output:
221+
.br
222+
223+
.br
224+
2021\-08\-17 15:20:28 [INFO]: Using replication service 'https://planet.openstreetmap.org/replication/minute', which is at sequence 4675115 ( 2021\-08\-17T13:19:43Z )
225+
.br
226+
2021\-08\-17 15:20:28 [INFO]: Replication server's most recent data is <1 minute old
227+
.br
228+
2021\-08\-17 15:20:28 [INFO]: Local database is 8288 sequences behind the server, i.e. 5 day(s) 20 hour(s) 58 minute(s)
229+
.br
230+
2021\-08\-17 15:20:28 [INFO]: Local database's most recent data is 5 day(s) 20 hour(s) 59 minute(s) old
231+
.br
232+
233+
.br
234+
235+
.br
236+
With the '\-\-json' option, the status is printed as a json object.
237+
.br
238+
239+
.br
240+
{
241+
.br
242+
"server": {
243+
.br
244+
"base_url": "https://planet.openstreetmap.org/replication/minute",
245+
.br
246+
"sequence": 4675116,
247+
.br
248+
"timestamp": "2021\-08\-17T13:20:43Z",
249+
.br
250+
"age_sec": 27
251+
.br
252+
},
253+
.br
254+
"local": {
255+
.br
256+
"sequence": 4666827,
257+
.br
258+
"timestamp": "2021\-08\-11T16:21:09Z",
259+
.br
260+
"age_sec": 507601
261+
.br
262+
},
263+
.br
264+
"status": 0
265+
.br
266+
}
267+
.br
268+
269+
.br
270+
271+
.br
272+
'status' is 0 if there were no problems getting the status. 1 & 2 for
273+
.br
274+
improperly set up replication. 3 for network issues. If status ≠ 0, then
275+
.br
276+
the 'error' key is an error message (as string). 'status' is used as the
277+
.br
278+
exit code.
279+
.br
280+
281+
.br
282+
'server' is the replication server's current status. 'sequence' is it's
283+
.br
284+
sequence number, 'timestamp' the time of that, and 'age_sec' the age of the
285+
.br
286+
data in seconds.
287+
.br
288+
289+
.br
290+
'local' is the status of your server.
291+
292+
293+
.TP
294+
\fB\-\-json\fR
295+
Output status as json.
296+
297+
.TP
298+
\fB\-q\fR, \fB\-\-quiet\fR
299+
Print only error messages
300+
301+
.TP
302+
\fB\-v\fR, \fB\-\-verbose\fR
303+
Increase verboseness of output
304+
305+
.TP
306+
\fB\-d\fR DB, \fB\-\-database\fR DB
307+
Name of PostgreSQL database to connect to or conninfo string
308+
309+
.TP
310+
\fB\-U\fR NAME, \fB\-\-username\fR NAME
311+
PostgreSQL user name
312+
313+
.TP
314+
\fB\-H\fR HOST, \fB\-\-host\fR HOST
315+
Database server host name or socket location
316+
317+
.TP
318+
\fB\-P\fR PORT, \fB\-\-port\fR PORT
319+
Database server port
320+
321+
.TP
322+
\fB\-p\fR PREFIX, \fB\-\-prefix\fR PREFIX
323+
Prefix for table names (default 'planet_osm')
324+
208325
.SH DISTRIBUTION
209326
The latest version of osm2pgsql\-replication may be downloaded from
210327
.UR osm2pgsql.org

‎scripts/osm2pgsql-replication‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,28 @@ from osmium import WriteHandler
3939

4040
LOG = logging.getLogger()
4141

42+
def pretty_format_timedelta(seconds):
43+
minutes = int(seconds/60)
44+
(hours, minutes) = divmod(minutes, 60)
45+
(days, hours) = divmod(hours, 24)
46+
(weeks, days) = divmod(days, 7)
47+
48+
if 0 < seconds < 60:
49+
output = "<1 minute"
50+
else:
51+
output = []
52+
# If weeks > 1 but hours == 0, we still want to show "0 hours"
53+
if weeks > 0:
54+
output.append("{} week(s)".format(weeks))
55+
if days > 0 or weeks > 0:
56+
output.append("{} day(s)".format(days))
57+
if hours > 0 or days > 0 or weeks > 0:
58+
output.append("{} hour(s)".format(hours))
59+
60+
output.append("{} minute(s)".format(minutes))
61+
output = " ".join(output)
62+
return output
63+
4264
def connect(args):
4365
""" Create a connection from the given command line arguments.
4466
"""
@@ -115,6 +137,113 @@ def update_replication_state(conn, table, seq, date):
115137

116138
conn.commit()
117139

140+
def status(conn, args):
141+
"""\
142+
Print information about the current replication status, optionally as JSON.
143+
144+
Sample output:
145+
146+
2021-08-17 15:20:28 [INFO]: Using replication service 'https://planet.openstreetmap.org/replication/minute', which is at sequence 4675115 ( 2021-08-17T13:19:43Z )
147+
2021-08-17 15:20:28 [INFO]: Replication server's most recent data is <1 minute old
148+
2021-08-17 15:20:28 [INFO]: Local database is 8288 sequences behind the server, i.e. 5 day(s) 20 hour(s) 58 minute(s)
149+
2021-08-17 15:20:28 [INFO]: Local database's most recent data is 5 day(s) 20 hour(s) 59 minute(s) old
150+
151+
152+
With the '--json' option, the status is printed as a json object.
153+
154+
{
155+
"server": {
156+
"base_url": "https://planet.openstreetmap.org/replication/minute",
157+
"sequence": 4675116,
158+
"timestamp": "2021-08-17T13:20:43Z",
159+
"age_sec": 27
160+
},
161+
"local": {
162+
"sequence": 4666827,
163+
"timestamp": "2021-08-11T16:21:09Z",
164+
"age_sec": 507601
165+
},
166+
"status": 0
167+
}
168+
169+
170+
'status' is 0 if there were no problems getting the status. 1 & 2 for
171+
improperly set up replication. 3 for network issues. If status ≠ 0, then
172+
the 'error' key is an error message (as string). 'status' is used as the
173+
exit code.
174+
175+
'server' is the replication server's current status. 'sequence' is it's
176+
sequence number, 'timestamp' the time of that, and 'age_sec' the age of the
177+
data in seconds.
178+
179+
'local' is the status of your server.
180+
"""
181+
182+
results = {}
183+
184+
with conn.cursor() as cur:
185+
cur.execute('SELECT * FROM pg_tables where tablename = %s', (args.table, ))
186+
if cur.rowcount < 1:
187+
results['status'] = 1
188+
results['error'] = "Cannot find replication status table. Run 'osm2pgsql-replication init' first."
189+
else:
190+
cur.execute('SELECT * FROM "{}"'.format(args.table))
191+
if cur.rowcount != 1:
192+
results['status'] = 2
193+
results['error'] = "Updates not set up correctly. Run 'osm2pgsql-updates init' first."
194+
else:
195+
196+
base_url, db_seq, db_ts = cur.fetchone()
197+
db_ts = db_ts.astimezone(dt.timezone.utc)
198+
results['server'] = {}
199+
results['local'] = {}
200+
results['server']['base_url'] = base_url
201+
results['local']['sequence'] = db_seq
202+
results['local']['timestamp'] = db_ts.strftime("%Y-%m-%dT%H:%M:%SZ")
203+
204+
205+
repl = ReplicationServer(base_url)
206+
state_info = repl.get_state_info()
207+
if state_info is None:
208+
# PyOsmium was unable to download the state information
209+
results['status'] = 3
210+
results['error'] = "Unable to download the state information from {}".format(base_url)
211+
else:
212+
results['status'] = 0
213+
now = dt.datetime.now(dt.timezone.utc)
214+
215+
server_seq, server_ts = state_info
216+
server_ts = server_ts.astimezone(dt.timezone.utc)
217+
218+
results['server']['sequence'] = server_seq
219+
results['server']['timestamp'] = server_ts.strftime("%Y-%m-%dT%H:%M:%SZ")
220+
results['server']['age_sec'] = int((now-server_ts).total_seconds())
221+
222+
results['local']['age_sec'] = int((now - db_ts).total_seconds())
223+
224+
if args.json:
225+
print(json.dumps(results))
226+
else:
227+
if results['status'] != 0:
228+
LOG.fatal(results['error'])
229+
else:
230+
print("Using replication service '{}', which is at sequence {} ( {} )".format(
231+
results['server']['base_url'], results['server']['sequence'], results['server']['timestamp']))
232+
print("Replication server's most recent data is {} old".format(pretty_format_timedelta(results['server']['age_sec'])))
233+
234+
if results['local']['sequence'] == results['server']['sequence']:
235+
print("Local database is up to date with server")
236+
else:
237+
print("Local database is {} sequences behind the server, i.e. {}".format(
238+
results['server']['sequence'] - results['local']['sequence'],
239+
pretty_format_timedelta(results['local']['age_sec'] - results['server']['age_sec'])
240+
))
241+
242+
print("Local database's most recent data is {} old".format(pretty_format_timedelta(results['local']['age_sec'])))
243+
244+
245+
return results['status']
246+
118247

119248
def init(conn, args):
120249
"""\
@@ -340,6 +469,16 @@ def get_parser():
340469
cmd.add_argument('--post-processing', metavar='SCRIPT',
341470
help='Post-processing script to run after each execution of osm2pgsql.')
342471

472+
# Arguments for status
473+
cmd = subs.add_parser('status', parents=[default_args],
474+
help=status.__doc__.split('\n', 1)[0],
475+
description=dedent(status.__doc__),
476+
formatter_class=RawDescriptionHelpFormatter,
477+
add_help=False)
478+
cmd.add_argument('--json', action="store_true", default=False, help="Output status as json.")
479+
cmd.set_defaults(handler=status)
480+
481+
343482
return parser
344483

345484
def main():

0 commit comments

Comments
 (0)