[JUJU-2392] Cherrypick to fix wrong bases (#783)

* Cherrypicking to support bases.

Author: juanmanuel-tirado

juju/bundle.py

@@ -122,12 +122,13 @@ async def _handle_local_charms(self, bundle, bundle_dir):
                 pass
             except FileNotFoundError:
                 continue
-            series = (
-                app_dict.get('series') or
-                default_series or
-                await get_charm_series(charm_dir, self.model)
-            )
+            series = (app_dict.get('series') or default_series)
             if not series:
+                metadata = utils.get_local_charm_metadata(charm_dir)
+                series = await get_charm_series(metadata, self.model)
+            if not series:
+                metadata = utils.get_local_charm_metadata(charm_dir)
+                series = await get_charm_series(metadata, self.model)
                 raise JujuError(
                     "Couldn't determine series for charm at {}. "
                     "Add a 'series' key to the bundle.".format(charm_dir))
@@ -147,14 +148,18 @@ async def _handle_local_charms(self, bundle, bundle_dir):
 
             # Update the 'charm:' entry for each app with the new 'local:' url.
             for app_name, charm_url, (charm_dir, _) in zip(apps, charm_urls, args):
+                metadata = utils.get_local_charm_metadata(charm_dir)
                 resources = await self.model.add_local_resources(
                     app_name,
                     charm_url,
-                    utils.get_local_charm_metadata(charm_dir),
+                    metadata,
                     resources=bundle.get('applications', {app_name: {}})[app_name].get("resources", {}),
                 )
                 apps_dict[app_name]['charm'] = charm_url
                 apps_dict[app_name]["resources"] = resources
+                origin = client.CharmOrigin(source="local", risk="stable")
+
+                self.origins[charm_url] = {str(None): origin}
 
         return bundle
 
@@ -407,32 +412,17 @@ def is_local_charm(charm_url):
     return charm_url.startswith('.') or charm_url.startswith('local:') or os.path.isabs(charm_url)
 
 
-async def get_charm_series(path, model):
-    """Inspects the charm directory at ``path`` and returns a default
-    series from its metadata.yaml (the first item in the 'series' list).
+async def get_charm_series(metadata, model):
+    """Inspects the given metadata and returns a default series from its
+    metadata.yaml (the first item in the 'series' list).
 
-    Tries to extract the informiation from the given model if no
-    series is determined from the path.
+    Tries to extract the information from the given model if no
+    series is determined from the given metadata.
     Returns None if no series can be determined.
 
     """
-    path = Path(path)
-    try:
-        if path.suffix == '.charm':
-            md = "metadata.yaml in %s" % path
-            with zipfile.ZipFile(str(path), 'r') as charm_file:
-                data = yaml.safe_load(charm_file.read('metadata.yaml'))
-        else:
-            md = path / "metadata.yaml"
-            if not md.exists():
-                return None
-            data = yaml.safe_load(md.open())
-    except yaml.YAMLError as exc:
-        if hasattr(exc, "problem_mark"):
-            mark = exc.problem_mark
-            log.error("Error parsing YAML file {}, line {}, column: {}".format(md, mark.line, mark.column))
-        raise
-    _series = data.get('series')
+
+    _series = metadata.get('series')
     series = _series[0] if _series else None
 
     if series is None:

juju/model.py

@@ -1609,7 +1609,8 @@ async def deploy(
             raise JujuError('unknown charm or bundle {}'.format(entity_url))
         identifier = res.identifier
 
-        series = res.origin.series or series
+        # series = res.origin.series or series
+        charm_series = series
         if res.is_bundle:
             handler = BundleHandler(self, trusted=trust, forced=force)
             await handler.fetch_plan(url, res.origin, overlays=overlays)
@@ -1660,24 +1661,27 @@ async def deploy(
                                                                 identifier)
             else:
                 # We have a local charm dir that needs to be uploaded
-                charm_dir = os.path.abspath(
-                    os.path.expanduser(identifier))
+                charm_dir = os.path.abspath(os.path.expanduser(identifier))
                 charm_origin = res.origin
+                base = None
+
                 metadata = utils.get_local_charm_metadata(charm_dir)
+                charm_series = charm_series or await get_charm_series(metadata,
+                                                                      self)
+                base = utils.get_local_charm_base(
+                    charm_series, channel, metadata, charm_dir, client.Base)
+                charm_origin.base = base
+                if not application_name:
+                    application_name = metadata['name']
                 if not application_name:
                     application_name = metadata['name']
-                series = series or await get_charm_series(charm_dir, self)
-                if not series:
-                    model_config = await self.get_config()
-                    default_series = model_config.get("default-series")
-                    if default_series:
-                        series = default_series.value
-                if not series:
+                if base is None and charm_series is None:
                     raise JujuError(
-                        "Couldn't determine series for charm at {}. "
-                        "Pass a 'series' kwarg to Model.deploy().".format(
-                            charm_dir))
-                identifier = await self.add_local_charm_dir(charm_dir, series)
+                        "Either series or base is needed to deploy the "
+                        "charm at {}. ".format(charm_dir))
+
+                identifier = await self.add_local_charm_dir(charm_dir,
+                                                            charm_series)
                 resources = await self.add_local_resources(application_name,
                                                            identifier,
                                                            metadata,
@@ -1691,7 +1695,7 @@ async def deploy(
             return await self._deploy(
                 charm_url=identifier,
                 application=res.app_name,
-                series=series,
+                series=charm_series,
                 config=config,
                 constraints=constraints,
                 endpoint_bindings=bind,

juju/utils.py

@@ -9,7 +9,7 @@
 import yaml
 import zipfile
 
-from . import jasyncio
+from . import jasyncio, origin, errors
 
 
 async def execute_process(*cmd, log=None):
@@ -234,19 +234,172 @@ def generate_user_controller_access_token(username, controller_endpoints, secret
     return base64.urlsafe_b64encode(registration_string)
 
 
-def get_local_charm_metadata(path):
+def get_local_charm_data(path, yaml_file):
     """Retrieve Metadata of a Charm from its path
 
     :patam str path: Path of charm directory or .charm file
+    :patam str yaml_file: name of the yaml file, can be either
+    "metadata.yaml", or "manifest.yaml", or "charmcraft.yaml"
 
     :return: Object of charm metadata
     """
     if str(path).endswith('.charm'):
         with zipfile.ZipFile(str(path), 'r') as charm_file:
-            metadata = yaml.load(charm_file.read('metadata.yaml'), Loader=yaml.FullLoader)
+            metadata = yaml.load(charm_file.read(yaml_file), Loader=yaml.FullLoader)
     else:
         entity_path = Path(path)
-        metadata_path = entity_path / 'metadata.yaml'
+        metadata_path = entity_path / yaml_file
+        if not metadata_path.exists():
+            return {}
         metadata = yaml.load(metadata_path.read_text(), Loader=yaml.FullLoader)
 
     return metadata
+
+
+def get_local_charm_metadata(path):
+    return get_local_charm_data(path, 'metadata.yaml')
+
+
+def get_local_charm_manifest(path):
+    return get_local_charm_data(path, 'manifest.yaml')
+
+
+def get_local_charm_charmcraft_yaml(path):
+    return get_local_charm_data(path, 'charmcraft.yaml')
+
+
+PRECISE = "precise"
+QUANTAL = "quantal"
+RARING = "raring"
+SAUCY = "saucy"
+TRUSTY = "trusty"
+UTOPIC = "utopic"
+VIVID = "vivid"
+WILY = "wily"
+XENIAL = "xenial"
+YAKKETY = "yakkety"
+ZESTY = "zesty"
+ARTFUL = "artful"
+BIONIC = "bionic"
+COSMIC = "cosmic"
+DISCO = "disco"
+EOAN = "eoan"
+FOCAL = "focal"
+GROOVY = "groovy"
+HIRSUTE = "hirsute"
+IMPISH = "impish"
+JAMMY = "jammy"
+KINETIC = "kinetic"
+
+UBUNTU_SERIES = {
+    PRECISE: "12.04",
+    QUANTAL: "12.10",
+    RARING: "13.04",
+    SAUCY: "13.10",
+    TRUSTY: "14.04",
+    UTOPIC: "14.10",
+    VIVID: "15.04",
+    WILY: "15.10",
+    XENIAL: "16.04",
+    YAKKETY: "16.10",
+    ZESTY: "17.04",
+    ARTFUL: "17.10",
+    BIONIC: "18.04",
+    COSMIC: "18.10",
+    DISCO: "19.04",
+    EOAN: "19.10",
+    FOCAL: "20.04",
+    GROOVY: "20.10",
+    HIRSUTE: "21.04",
+    IMPISH: "21.10",
+    JAMMY: "22.04",
+    KINETIC: "22.10",
+}
+
+
+def get_series_version(series_name):
+    if series_name not in UBUNTU_SERIES:
+        raise errors.JujuError("Unknown series : %s", series_name)
+    return UBUNTU_SERIES[series_name]
+
+
+def get_version_series(version):
+    if version not in UBUNTU_SERIES.values():
+        raise errors.JujuError("Unknown version : %s", version)
+    return list(UBUNTU_SERIES.keys())[list(UBUNTU_SERIES.values()).index(version)]
+
+
+def get_local_charm_base(series, channel_from_arg, charm_metadata,
+                         charm_path, baseCls):
+    """Deduce the base [channel/osname] of a local charm based on what we
+    know already
+
+    :param str series: This may come from the argument or the metadata.yaml
+    :param str channel_from_arg: This is channel passed as argument, if any.
+    :param dict charm_metadata: metadata.yaml
+    :param str charm_path: Path of charm directory/.charm file
+    :param class baseCls:
+    :return: Instance of the baseCls with channel/osname informaiton
+    """
+
+    channel_for_base = ''
+    os_name_for_base = ''
+    # If user passed a channel_arg, then check the supported series against
+    # the channel's track
+    chnl_check = origin.Channel.parse(channel_from_arg) if channel_from_arg \
+        else None
+    if chnl_check:
+        not_supported_error = errors.JujuError(
+            "Given channel [track/risk] is not supported --"
+            "\n - Given channel : %s"
+            "\n - Series in Charm Metadata : %s" %
+            (channel_from_arg, charm_metadata['series']))
+        channel_for_base = chnl_check.track
+        intented_series = get_version_series(channel_for_base)
+        if intented_series not in charm_metadata['series']:
+            raise not_supported_error
+        # Also check the manifest if there's one
+        charm_manifest = get_local_charm_manifest(charm_path)
+        if 'bases' in charm_manifest:
+            for base in charm_manifest['bases']:
+                if channel_for_base == base['channel']:
+                    break
+            else:
+                raise not_supported_error
+
+    # If we know the series, use it to get a channel
+    if channel_for_base == '':
+        channel_for_base = get_series_version(series) if series else ''
+        if channel_for_base:
+            # we currently only support ubuntu series (statically)
+            # TODO (cderici) : go juju/core/series/supported.go and get the
+            #  others here too
+            os_name_for_base = 'ubuntu'
+
+    # Check the charm manifest
+    if channel_for_base == '':
+        charm_manifest = get_local_charm_manifest(charm_path)
+        if 'bases' in charm_manifest:
+            channel_for_base = charm_manifest['bases'][0]['channel']
+            os_name_for_base = charm_manifest['bases'][0]['name']
+        else:
+            # Also check the charmcraft.yaml
+            charmcraft_yaml = get_local_charm_charmcraft_yaml(charm_path)
+            if 'bases' in charmcraft_yaml:
+                channel_for_base = charmcraft_yaml['bases'][0]['run-on'][0]['channel']
+                os_name_for_base = charmcraft_yaml['bases'][0]['run-on'][0]['name']
+
+    if channel_for_base == '':
+        raise errors.JujuError("Unable to determine base for charm : %s" %
+                               charm_path)
+
+    return baseCls(channel_for_base, os_name_for_base)
+
+
+def base_channel_to_series(channel):
+    """Returns the series string using the track inside the base channel
+
+    :param str channel: is track/risk (e.g. 20.04/stable)
+    :return: str series (e.g. focal)
+    """
+    return get_version_series(origin.Channel.parse(channel).track)

tox.ini

@@ -26,6 +26,11 @@ passenv =
     TEST_AGENTS
     LXD_DIR
 deps =
+    macaroonbakery
+    theblues
+    toposort
+    typing-inspect
+    paramiko
     asynctest
     ipdb
     mock
@@ -35,9 +40,7 @@ deps =
     Twine
     websockets
     kubernetes
-    # use fork to pick up fix for https://github.com/aaugustin/websockets/pull/528
-    git+https://github.com/johnsca/websockets@bug/client-redirects#egg=websockets ; python_version<'3.6'
-    cffi <= 1.14.6 ; python_version<='3.5'
+    cffi
 
 [testenv:lint]
 commands =