Skip to content

Commit 72a1faa

Browse files
sgallaghersgallagher
committed
Make string extraction a library function
Signed-off-by: Stephen Gallagher <sgallagh@redhat.com>
1 parent 09dde51 commit 72a1faa

2 files changed

Lines changed: 115 additions & 50 deletions

File tree

‎extract_translations.py‎

Lines changed: 49 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,55 @@
1212
# For more information on free software, see
1313
# <https://www.gnu.org/philosophy/free-sw.en.html>.
1414

15-
import os
15+
import koji
1616
import sys
1717
import subprocess
1818
import click
1919
import mmdzanata
20+
import requests
21+
2022
from babel.messages import pofile
2123

2224

25+
def get_fedora_rawhide_version(session, debug=False):
26+
# Koji sometimes disconnects for no apparent reason. Retry up to 5
27+
# times before failing.
28+
for attempt in range(5):
29+
try:
30+
build_targets = session.getBuildTargets('rawhide')
31+
except requests.exceptions.ConnectionError:
32+
if debug:
33+
print("Connection lost while retriving rawhide branch, "
34+
"retrying...",
35+
file=sys.stderr)
36+
else:
37+
# Succeeded this time, so break out of the loop
38+
break
39+
40+
return build_targets[0][
41+
'build_tag_name'].partition('-build')[0]
42+
43+
44+
def get_tags_for_fedora_branch(branch):
45+
return ['%s-modular' % branch,
46+
'%s-modular-override' % branch,
47+
'%s-modular-pending' % branch,
48+
'%s-modular-signing-pending' % branch,
49+
'%s-modular-updates' % branch,
50+
'%s-modular-updates-candidate' % branch,
51+
'%s-modular-updates-pending' % branch,
52+
'%s-modular-updates-testing' % branch,
53+
'%s-modular-updates-testing-pending' % branch]
54+
55+
2356
@click.command()
57+
@click.option('-k', '--koji-url',
58+
default='https://koji.fedoraproject.org/kojihub',
59+
type=str, help="""
60+
The URL of the Koji build system.
61+
(Default: https://koji.fedoraproject.org/kojihub)
62+
""",
63+
metavar="<URL>")
2464
@click.option('-b', '--branch', default="rawhide", type=str,
2565
help="The distribution release (Default: rawhide)",
2666
metavar="<branch_name>")
@@ -37,33 +77,24 @@
3777
(Default: fedora-modularity-translations)
3878
""",
3979
metavar="<zanata_project>")
40-
@click.option("-v", "--zanata-project-version", default="f29",
80+
@click.option("-v", "--zanata-project-version", default="rawhide",
4181
type=str, help="""
4282
The project version.
43-
(Default: f29)
83+
(Default: rawhide)
4484
""", metavar="[f28, f29, rawhide, ...]")
45-
def main(branch, zanata_url, zanata_project, zanata_project_version):
85+
def main(branch, koji_url, zanata_url, zanata_project,
86+
zanata_project_version):
4687
"""
4788
Extract translations from all modules included in a particular version of
4889
Fedora or EPEL.
4990
"""
50-
k = mmdzanata.get_koji_session()
91+
session = koji.ClientSession(koji_url)
5192

5293
if branch == "rawhide":
53-
branch = mmdzanata.get_rawhide_version(k)
54-
55-
tags = mmdzanata.get_tags_for_branch(branch)
56-
57-
tagged_builds = []
58-
for tag in tags:
59-
tagged_builds.extend(mmdzanata.get_latest_modules_in_tag(k, tag))
60-
61-
# Make the list unique since some modules may have multiple tags
62-
unique_builds = {}
63-
for build in tagged_builds:
64-
unique_builds[build['id']] = build
94+
branch = get_fedora_rawhide_version(session)
6595

66-
catalog = mmdzanata.get_module_catalog(k, unique_builds)
96+
catalog = mmdzanata.get_module_catalog_from_tags(
97+
session, get_tags_for_fedora_branch(branch), debug=True)
6798

6899
with open("fedora-modularity-translations.pot", mode="wb") as f:
69100
pofile.write_po(f, catalog, sort_by_file=True)

mmdzanata.py renamed to mmdzanata/__init__.py

Lines changed: 66 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -10,40 +10,43 @@
1010
# For more information on the license, see COPYING.
1111
# For more information on free software, see
1212
# <https://www.gnu.org/philosophy/free-sw.en.html>.
13+
#
14+
# This module provides utility functions for interacting with Zanata
15+
# translations of Fedora-style modules.
1316

17+
import sys
1418
import gi
15-
import koji
19+
import requests
1620

17-
from collections import defaultdict
1821
from babel.messages import Catalog
1922
from datetime import datetime
2023

2124
gi.require_version('Modulemd', '1.0')
2225
from gi.repository import Modulemd
2326

2427

25-
def get_koji_session():
26-
return koji.ClientSession('https://koji.fedoraproject.org/kojihub')
27-
28-
29-
def get_rawhide_version(session):
30-
return session.getBuildTargets('rawhide')[0]['build_tag_name'].partition('-build')[0]
31-
32-
33-
def get_tags_for_branch(branch):
34-
return ['%s-modular' % branch,
35-
'%s-modular-override' % branch,
36-
'%s-modular-pending' % branch,
37-
'%s-modular-signing-pending' % branch,
38-
'%s-modular-updates' % branch,
39-
'%s-modular-updates-candidate' % branch,
40-
'%s-modular-updates-pending' % branch,
41-
'%s-modular-updates-testing' % branch,
42-
'%s-modular-updates-testing-pending' % branch]
28+
def get_latest_modules_in_tag(session, tag, debug=False):
29+
"""
30+
Get the most-recently built versions of each (module,stream) pair from
31+
a Koji tag
32+
:param session: A Koji session
33+
:param tag: A koji tag
34+
:return: A list of the most recent build of all modules in the tag.
35+
"""
4336

44-
45-
def get_latest_modules_in_tag(session, tag):
46-
tagged = session.listTagged(tag, latest=False)
37+
# Koji sometimes disconnects for no apparent reason. Retry up to 5
38+
# times before failing.
39+
for attempt in range(5):
40+
try:
41+
tagged = session.listTagged(tag, latest=False)
42+
except requests.exceptions.ConnectionError:
43+
if debug:
44+
print("Connection lost while retrieving builds for tag %s, "
45+
"retrying..." % tag,
46+
file=sys.stderr)
47+
else:
48+
# Succeeded this time, so break out of the loop
49+
break
4750

4851
# Find the latest, in module terms. Pungi does this.
4952
# Collect all contexts that share the same NSV.
@@ -66,22 +69,53 @@ def get_latest_modules_in_tag(session, tag):
6669
return latest
6770

6871

69-
def get_module_catalog(session, builds):
70-
catalog = Catalog(project="fedora-modularity-translations")
72+
def get_module_catalog_from_tags(session, tags, debug=False):
73+
"""
74+
Construct a Babel translation source catalog from the contents of the
75+
provided tags.
76+
:param session: A Koji session
77+
:param tags: A set of Koji tags from which module metadata should be pulled
78+
:param debug: Whether to print debugging information to the console
79+
:return: A babel.messages.Catalog containing extracted translatable strings
80+
from any modules in the provided tags. Raises an exception if any of the
81+
retrieved modulemd is invalid.
82+
"""
7183

72-
for build_id in builds.keys():
73-
build = session.getBuild(build_id)
74-
print("Processing %s:%s" % (build['package_name'], build['nvr']))
84+
catalog = Catalog(project="fedora-modularity-translations")
7585

76-
module_stream = "%s:%s" % (
77-
build['extra']['typeinfo']['module']['name'],
78-
build['extra']['typeinfo']['module']['stream'])
86+
tagged_builds = []
87+
for tag in tags:
88+
tagged_builds.extend(get_latest_modules_in_tag(session, tag, debug))
89+
90+
# Make the list unique since some modules may have multiple tags
91+
unique_builds = {}
92+
for build in tagged_builds:
93+
unique_builds[build['id']] = build
94+
95+
for build_id in unique_builds.keys():
96+
# Koji sometimes disconnects for no apparent reason. Retry up to 5
97+
# times before failing.
98+
for attempt in range(5):
99+
try:
100+
build = session.getBuild(build_id)
101+
except requests.exceptions.ConnectionError:
102+
if debug:
103+
print("Connection lost while processing buildId %s, "
104+
"retrying..." % build_id,
105+
file=sys.stderr)
106+
else:
107+
# Succeeded this time, so break out of the loop
108+
break
109+
if debug:
110+
print("Processing %s:%s" % (build['package_name'], build['nvr']))
79111

80112
modulemds = Modulemd.objects_from_string(
81113
build['extra']['typeinfo']['module']['modulemd_str'])
82114

83115
# We should only get a single modulemd document from Koji
84-
assert len(modulemds) == 1
116+
if len(modulemds) != 1:
117+
raise ValueError("Koji build %s returned multiple modulemd YAML "
118+
"documents." % build['nvr'])
85119

86120
mmd = modulemds[0]
87121

0 commit comments

Comments
 (0)