Merge pull request #744 from cderici/backport-upgrade-charm-fixes-onto-2.9

#744

#### Description

This cherry-picks the `upgrade-charm` fixes from #729 and #742 to bring them onto `2.9` branch. 

Includes the fix for the `juju.errors.JujuAPIError: missing base name or channel not valid` error when upgrading local charms.

#### QA Steps

Same QA Steps for #729 and #742 should work here.

```
tox -e integration -- tests/integration/test_application.py::test_upgrade_local_charm
```
```
tox -e integration -- tests/integration/test_charmhub.py::test_list_resources
```
```
tox -e integration -- tests/integration/test_application.py::test_upgrade_charm_switch_channel
```

Author: jujubot

juju/application.py

@@ -18,12 +18,14 @@
 import pathlib
 
 from . import model, tag, utils, jasyncio
+from .url import URL, Schema
 from .status import derive_status
 from .annotationhelper import _get_annotations, _set_annotations
 from .client import client
 from .errors import JujuError, JujuApplicationConfigError
 from .bundle import get_charm_series
 from .placement import parse as parse_placement
+from .origin import Channel
 
 log = logging.getLogger(__name__)
 
@@ -609,98 +611,147 @@ async def refresh(
         :param str switch: Crossgrade charm url
 
         """
-        if path is not None:
-            await self.local_refresh(channel, force, force_series, force_units,
-                                     path, resources)
-            return
         if resources is not None:
             raise NotImplementedError("resources option is not implemented")
 
         if switch is not None and revision is not None:
             raise ValueError("switch and revision are mutually exclusive")
 
-        client_facade = client.ClientFacade.from_connection(self.connection)
-        resources_facade = client.ResourcesFacade.from_connection(
-            self.connection)
         app_facade = self._facade()
+        resources_facade = client.ResourcesFacade.from_connection(self.connection)
+        charms_facade = client.CharmsFacade.from_connection(self.connection)
 
-        charmstore = self.model.charmstore
-        charmstore_entity = None
+        # 1 - Figure out the destination origin and destination charm_url
+        # 2 - Then take care of the resources
+        # 3 - Finally execute the upgrade
 
-        if switch is not None:
-            charm_url = switch
-            if not charm_url.startswith('cs:'):
-                charm_url = 'cs:' + charm_url
-        else:
-            charm_url = self.data['charm-url']
-            charm_url = charm_url.rpartition('-')[0]
-            if revision is not None:
-                charm_url = "%s-%d" % (charm_url, revision)
-            else:
-                charmstore_entity = await charmstore.entity(charm_url,
-                                                            channel=channel)
-                charm_url = charmstore_entity['Id']
+        # Get the charm URL and charm origin of the given application is running at present.
+        charm_url_origin_result = await app_facade.GetCharmURLOrigin(application=self.name)
+        if charm_url_origin_result.error is not None:
+            err = charm_url_origin_result.error
+            raise JujuError("%s : %s" % (err.code, err.message))
+        charm_url = switch or charm_url_origin_result.url
+        origin = charm_url_origin_result.charm_origin
 
-        if charm_url == self.data['charm-url']:
-            raise JujuError('already running charm "%s"' % charm_url)
+        if path is not None:
+            await self.local_refresh(origin, force, force_series,
+                                     force_units, path, resources)
+            return
 
-        # Update charm
-        await client_facade.AddCharm(
-            url=charm_url,
-            force=force,
-            channel=channel
-        )
+        parsed_url = URL.parse(charm_url)
+        charm_name = parsed_url.name
+
+        if parsed_url.schema is None:
+            raise JujuError("A ch: or cs: schema is required for application "
+                            "refresh, given : %s " % str(parsed_url))
 
-        # Update resources
-        if not charmstore_entity:
-            charmstore_entity = await charmstore.entity(charm_url,
-                                                        channel=channel)
-        store_resources = charmstore_entity['Meta']['resources']
+        if revision is not None:
+            origin.revision = revision
 
+        # Make the source-specific changes to the origin/channel/url
+        # (and also get the resources necessary to deploy the (destination) charm -- for later)
+        if Schema.CHARM_HUB.matches(parsed_url.schema):
+            origin.source = 'charm-hub'
+            if channel:
+                ch = Channel.parse(channel).normalize()
+                origin.risk = ch.risk
+                origin.track = ch.track
+
+            charmhub = self.model.charmhub
+            charm_resources = await charmhub.list_resources(charm_name)
+        else:
+            charmstore = self.model.charmstore
+            charmstore_entity = None
+            if switch is None:
+                charm_url = charm_url.rpartition('-')[0]
+                if revision is not None:
+                    charm_url = "%s-%d" % (charm_url, revision)
+                else:
+                    charmstore_entity = await charmstore.entity(charm_url, channel=channel)
+                    charm_url = charmstore_entity['Id']
+            origin.source = 'charm-store'
+            if channel:
+                origin.risk = channel
+            if charmstore_entity is None:
+                charmstore_entity = await charmstore.entity(charm_url, channel=channel)
+            charm_resources = charmstore_entity['Meta']['resources']
+
+        # Resolve the given charm URLs with an optionally specified preferred channel.
+        # Channel provided via CharmOrigin.
+        resolved_charm_with_channel_results = await charms_facade.ResolveCharms(
+            resolve=[client.ResolveCharmWithChannel(
+                charm_origin=origin,
+                switch_charm=True if switch else False,  # rpc expects boolean type
+                reference=charm_url,
+            )])
+        resolved_charm = resolved_charm_with_channel_results.results[0]
+
+        # Get the destination origin and destination charm_url from the resolved charm
+        if resolved_charm.error is not None:
+            err = resolved_charm.error
+            raise JujuError("%s : %s" % (err.code, err.message))
+        dest_origin = resolved_charm.charm_origin
+        charm_url = resolved_charm.url
+
+        # Add the charm with the destination url and origin
+        charm_origin_result = await charms_facade.AddCharm(url=charm_url,
+                                                           force=force,
+                                                           charm_origin=dest_origin)
+        if charm_origin_result.error is not None:
+            err = charm_origin_result.error
+            raise JujuError("%s : %s" % (err.code, err.message))
+
+        # Now take care of the resources:
+
+        # Already prepped the charm_resources
+        # Now get the existing resources from the ResourcesFacade
         request_data = [client.Entity(self.tag)]
         response = await resources_facade.ListResources(entities=request_data)
         existing_resources = {
             resource.name: resource
             for resource in response.results[0].resources
         }
 
+        # Compute the difference btw resources needed and the existing resources
         resources_to_update = [
-            resource for resource in store_resources
-            if resource['Name'] not in existing_resources or
-            existing_resources[resource['Name']].origin != 'upload'
+            resource for resource in charm_resources
+            if resource.get('Name', resource.get('name')) not in existing_resources or
+            existing_resources[resource.get('Name', resource.get('name'))].origin != 'upload'
         ]
 
+        # Update the resources
         if resources_to_update:
-            request_data = [
-                client.CharmResource(
-                    description=resource.get('Description'),
-                    fingerprint=resource['Fingerprint'],
-                    name=resource['Name'],
-                    path=resource['Path'],
-                    revision=resource['Revision'],
-                    size=resource['Size'],
-                    type_=resource['Type'],
+            request_data = []
+            for resource in resources_to_update:
+                request_data.append(client.CharmResource(
+                    description=resource.get('Description', resource.get('description')),
+                    fingerprint=resource.get('Fingerprint', resource.get('fingerprint')),
+                    name=resource.get('Name', resource.get('name')),
+                    path=resource.get('Path', resource.get('filename')),
+                    revision=resource.get('Revision', resource.get('revision', -1)),
+                    size=resource.get('Size', resource.get('size')),
+                    type_=resource.get('Type', resource.get('type')),
                     origin='store',
-                ) for resource in resources_to_update
-            ]
+                ))
             response = await resources_facade.AddPendingResources(
                 application_tag=self.tag,
                 charm_url=charm_url,
-                resources=request_data
+                resources=request_data,
+                charm_origin=dest_origin,
             )
             pending_ids = response.pending_ids
             resource_ids = {
-                resource['Name']: id
+                resource.get('Name', resource.get('name')): id
                 for resource, id in zip(resources_to_update, pending_ids)
             }
         else:
             resource_ids = None
 
-        # Update application
+        # Update the application
         await app_facade.SetCharm(
             application=self.entity_id,
-            channel=channel,
             charm_url=charm_url,
+            charm_origin=dest_origin,
             config_settings=None,
             config_settings_yaml=None,
             force=force,
@@ -717,7 +768,8 @@ async def refresh(
     upgrade_charm = refresh
 
     async def local_refresh(
-            self, channel=None, force=False, force_series=False, force_units=False,
+            self, charm_origin=None, force=False, force_series=False,
+            force_units=False,
             path=None, resources=None):
         """Refresh the charm for this application with a local charm.
 
@@ -760,7 +812,7 @@ async def local_refresh(
         # Update application
         await app_facade.SetCharm(
             application=self.entity_id,
-            channel=channel,
+            charm_origin=charm_origin,
             charm_url=charm_url,
             config_settings=None,
             config_settings_yaml=None,

juju/charmhub.py

@@ -1,11 +1,56 @@
 from .client import client
 from .errors import JujuError
+from juju import jasyncio
+
+import requests
+import json
 
 
 class CharmHub:
     def __init__(self, model):
         self.model = model
 
+    def request_charmhub_with_retry(self, url, retries):
+        for attempt in range(retries):
+            _response = requests.get(url)
+            if _response.status_code == 200:
+                return _response
+            jasyncio.sleep(5)
+        raise JujuError("Got {} from {}".format(_response.status_code, url))
+
+    async def get_charm_id(self, charm_name):
+        conn, headers, path_prefix = self.model.connection().https_connection()
+
+        model_conf = await self.model.get_config()
+        charmhub_url = model_conf['charmhub-url']
+        url = "{}/v2/charms/info/{}".format(charmhub_url.value, charm_name)
+        _response = self.request_charmhub_with_retry(url, 5)
+        response = json.loads(_response.text)
+        return response['id'], response['name']
+
+    async def is_subordinate(self, charm_name):
+        conn, headers, path_prefix = self.model.connection().https_connection()
+
+        model_conf = await self.model.get_config()
+        charmhub_url = model_conf['charmhub-url']
+        url = "{}/v2/charms/info/{}?fields=default-release.revision.subordinate".format(charmhub_url.value, charm_name)
+        _response = self.request_charmhub_with_retry(url, 5)
+        response = json.loads(_response.text)
+        return 'subordinate' in response['default-release']['revision']
+
+    # TODO (caner) : we should be able to recreate the channel-map through the
+    #  api call without needing the CharmHub facade
+
+    async def list_resources(self, charm_name):
+        conn, headers, path_prefix = self.model.connection().https_connection()
+
+        model_conf = await self.model.get_config()
+        charmhub_url = model_conf['charmhub-url']
+        url = "{}/v2/charms/info/{}?fields=default-release.resources".format(charmhub_url.value, charm_name)
+        _response = self.request_charmhub_with_retry(url, 5)
+        response = json.loads(_response.text)
+        return response['default-release']['resources']
+
     async def info(self, name, channel=None):
         """info displays detailed information about a CharmHub charm. The charm
         can be specified by the exact name.

tests/charm-folder-symlink/metadata.yaml

@@ -1,4 +1,5 @@
 name: simple
 summary: A simple example charm with the new operator framework
+series: ["focal"]
 description: |
     Simple is an example charm