Merge pull request #526 from SimonRichardson/merge-master-into-2.9

#526

Merge master into 2.9, this should bring all the new changes in master into 2.9

From there we should be able to cut a new release with charmhub support.

Author: jujubot

VERSION

@@ -1 +1 @@
-2.8.6
+2.9.3

docs/changelog.rst

@@ -1,6 +1,40 @@
 Changelog
 ---------
 
+2.9.3
+^^^^^
+
+Monday August 12 2021
+
+* Bug fix - Fix 'Default to bundle series if the charm has no series field' #514
+
+2.9.2
+^^^^^
+
+Monday June 28 2021
+
+* Bug fix - Fix 'metadata referenced before assignment' error #509
+
+2.9.1
+^^^^^
+
+Wednesday June 16 2021
+
+* Bug fix - Bundle Exposed endpoints missing #502
+* Bug fix - Fix series requirement for local charms #504
+* Add local charm update support #507
+
+2.9.0
+^^^^^
+
+Thursday May 27 2021
+
+* Update facade methods for Juju 2.9.0
+* Update facade methods for Juju 2.9.1
+* Bug fix - Support for Juju client proxies (LP#1926595)
+* Bug fix - Honor charm channel in bundles #496
+* Remove machine workaround for Juju 2.2.3
+
 2.8.6
 ^^^^^
 

examples/deploy_local_resource.py

@@ -0,0 +1,46 @@
+"""
+This example:
+
+1. Connects to the current model
+2. Deploy a local charm with a oci-image resource and waits until it reports
+   itself active
+3. Destroys the unit and application
+
+"""
+from juju import loop
+from juju.model import Model
+from pathlib import Path
+
+
+async def main():
+    model = Model()
+    print('Connecting to model')
+    # connect to current model with current user, per Juju CLI
+    await model.connect()
+
+    try:
+        print('Deploying local-charm')
+        base_dir = Path(__file__).absolute().parent.parent
+        charm_path = '{}/tests/integration/oci-image-charm'.format(base_dir)
+        resources = {"oci-image": "ubuntu/latest"}
+        application = await model.deploy(
+            charm_path,
+            resources=resources,
+        )
+
+        print('Waiting for active')
+        await model.block_until(
+            lambda: all(unit.workload_status == 'active'
+                        for unit in application.units),
+            timeout=120,
+        )
+
+        print('Removing Charm')
+        await application.remove()
+    finally:
+        print('Disconnecting from model')
+        await model.disconnect()
+
+
+if __name__ == '__main__':
+    loop.run(main())

examples/local_refresh.py

@@ -0,0 +1,30 @@
+"""
+This example:
+
+1. Connects to the current model
+2. Upgrades previously deployed ubuntu charm
+
+"""
+from juju import loop
+from juju.model import Model
+
+
+async def main():
+    model = Model()
+    print('Connecting to model')
+    # connect to current model with current user, per Juju CLI
+    await model.connect()
+
+    try:
+        print('Get deployed application')
+        app = model.appplications["ubuntu"]
+
+        print('Refresh/Upgrade Ubuntu charm with local charm')
+        await app.refresh(path="path/to/local/ubuntu.charm")
+    finally:
+        print('Disconnecting from model')
+        await model.disconnect()
+
+
+if __name__ == '__main__':
+    loop.run(main())

juju/application.py

@@ -15,12 +15,14 @@
 import asyncio
 import json
 import logging
+import os
 
-from . import model, tag
+from . import model, tag, utils
 from .status import derive_status
 from .annotationhelper import _get_annotations, _set_annotations
 from .client import client
 from .errors import JujuError
+from .bundle import get_charm_series
 from .placement import parse as parse_placement
 
 log = logging.getLogger(__name__)
@@ -586,9 +588,10 @@ async def refresh(
         :param str switch: Crossgrade charm url
 
         """
-        # TODO: Support local upgrades
         if path is not None:
-            raise NotImplementedError("path option is not implemented")
+            await self.local_refresh(channel, force, force_series, force_units,
+                                     path, resources)
+            return
         if resources is not None:
             raise NotImplementedError("resources option is not implemented")
 
@@ -692,6 +695,61 @@ async def refresh(
 
     upgrade_charm = refresh
 
+    async def local_refresh(
+            self, channel=None, force=False, force_series=False, force_units=False,
+            path=None, resources=None):
+        """Refresh the charm for this application with a local charm.
+
+        :param str channel: Channel to use when getting the charm from the
+            charm store, e.g. 'development'
+        :param bool force_series: Refresh even if series of deployed
+            application is not supported by the new charm
+        :param bool force_units: Refresh all units immediately, even if in
+            error state
+        :param str path: Refresh to a charm located at path
+        :param dict resources: Dictionary of resource name/filepath pairs
+        :param int revision: Explicit refresh revision
+        :param str switch: Crossgrade charm url
+
+        """
+        app_facade = self._facade()
+
+        charm_dir = os.path.abspath(
+            os.path.expanduser(path))
+        model_config = await self.get_config()
+
+        series = get_charm_series(charm_dir)
+        if not series:
+            model_config = await self.get_config()
+            default_series = model_config.get("default-series")
+            if default_series:
+                series = default_series.value
+        charm_url = await self.model.add_local_charm_dir(charm_dir, series)
+        metadata = utils.get_local_charm_metadata(path)
+        if resources is not None:
+            resources = await self.model.add_local_resources(self.entity_id,
+                                                             charm_url,
+                                                             metadata,
+                                                             resources=resources)
+
+        # Update application
+        await app_facade.SetCharm(
+            application=self.entity_id,
+            channel=channel,
+            charm_url=charm_url,
+            config_settings=None,
+            config_settings_yaml=None,
+            force=force,
+            force_series=force_series,
+            force_units=force_units,
+            resource_ids=resources,
+            storage_constraints=None,
+        )
+
+        await self.model.block_until(
+            lambda: self.data['charm-url'] == charm_url
+        )
+
     async def get_metrics(self):
         """Get metrics for this application's units.
 

juju/bundle.py

@@ -456,6 +456,13 @@ async def run(self, context):
         :param context: is used for any methods or properties required to
             perform a change.
         """
+        # NB: this should really be handled by the controller when generating the
+        # bundle change plan, and this short-term workaround may be missing some
+        # aspects of the logic which the CLI client contains to handle edge cases.
+        if self.application in context.model.applications:
+            log.debug('Skipping %s; already in model', self.application)
+            return self.application
+
         # resolve indirect references
         charm = context.resolve(self.charm)
         options = {}
@@ -480,6 +487,9 @@ async def run(self, context):
         origin = context.origins.get(str(url), {}).get(str(channel), None)
         if origin is None:
             raise JujuError("expected origin to be valid for application {} and charm {} with channel {}".format(self.application, str(url), str(channel)))
+        if self.series is None or self.series == "":
+            self.series = context.bundle.get("bundle",
+                                             context.bundle.get("series", None))
 
         await context.model._deploy(
             charm_url=charm,
@@ -577,9 +587,9 @@ async def run(self, context):
             return self.charm
 
         if Schema.CHARM_STORE.matches(url.schema):
-            entity_id = await context.charmstore.entityId(self.charm)
+            entity_id = await context.charmstore.entityId(self.charm, channel=self.channel)
             log.debug('Adding %s', entity_id)
-            await context.client_facade.AddCharm(channel=None, url=entity_id, force=False)
+            await context.client_facade.AddCharm(channel=self.channel, url=entity_id, force=False)
             identifier = entity_id
             origin = client.CharmOrigin(source="charm-store", risk="stable")
 
@@ -754,6 +764,14 @@ async def run(self, context):
         ep1 = context.resolve_relation(self.endpoint1)
         ep2 = context.resolve_relation(self.endpoint2)
 
+        # NB: this should really be handled by the controller when generating the
+        # bundle change plan, and this short-term workaround may be missing some
+        # aspects of the logic which the CLI client contains to handle edge cases.
+        existing = [rel for rel in context.model.relations if rel.matches(ep1, ep2)]
+        if existing:
+            log.info('Skipping %s <-> %s; already related', ep1, ep2)
+            return existing[0]
+
         log.info('Relating %s <-> %s', ep1, ep2)
         return await context.model.add_relation(ep1, ep2)
 
@@ -960,6 +978,7 @@ class ExposeChange(ChangeInfo):
     def __init__(self, change_id, requires, params=None):
         super(ExposeChange, self).__init__(change_id, requires)
 
+        self.exposed_endpoints = None
         if isinstance(params, list):
             self.application = params[0]
         elif isinstance(params, dict):

juju/charm.py

@@ -13,7 +13,6 @@
 #     limitations under the License.
 
 import logging
-
 from . import model
 
 log = logging.getLogger(__name__)

juju/client/connection.py

@@ -163,8 +163,8 @@ class Monitor:
 
     def __init__(self, connection):
         self.connection = weakref.ref(connection)
-        self.reconnecting = asyncio.Lock(loop=connection.loop)
-        self.close_called = asyncio.Event(loop=connection.loop)
+        self.reconnecting = asyncio.Lock()
+        self.close_called = asyncio.Event()
 
     @property
     def status(self):
@@ -434,7 +434,7 @@ async def _pinger(self):
         async def _do_ping():
             try:
                 await pinger_facade.Ping()
-                await asyncio.sleep(10, loop=self.loop)
+                await asyncio.sleep(10)
             except CancelledError:
                 pass
 
@@ -636,7 +636,7 @@ async def _try_endpoint(endpoint, cacert, delay):
                                                      0.1 * i))
                  for i, (endpoint, cacert) in enumerate(endpoints)]
         for attempt in range(self._retries + 1):
-            for task in asyncio.as_completed(tasks, loop=self.loop):
+            for task in asyncio.as_completed(tasks):
                 try:
                     result = await task
                     break
@@ -805,7 +805,7 @@ async def login(self):
 
 class _Task:
     def __init__(self, task, loop):
-        self.stopped = asyncio.Event(loop=loop)
+        self.stopped = asyncio.Event()
         self.stopped.set()
         self.task = task
         self.loop = loop

juju/machine.py

@@ -140,7 +140,7 @@ async def _scp(self, source, destination, scp_opts):
             raise JujuError("command failed: %s" % cmd)
 
     async def ssh(
-            self, command, user=None, proxy=False, ssh_opts=None):
+            self, command, user='ubuntu', proxy=False, ssh_opts=None):
         """Execute a command over SSH on this machine.
 
         :param str command: Command to execute
@@ -164,10 +164,13 @@ async def ssh(
             cmd.extend(ssh_opts.split() if isinstance(ssh_opts, str) else ssh_opts)
         cmd.extend([command])
         loop = self.model.loop
-        process = await asyncio.create_subprocess_exec(*cmd, loop=loop)
-        await process.wait()
+        process = await asyncio.create_subprocess_exec(
+            *cmd, loop=loop, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
+        stdout, stderr = await process.communicate()
         if process.returncode != 0:
-            raise JujuError("command failed: %s" % cmd)
+            raise JujuError("command failed: %s with %s" % (cmd, stderr.decode()))
+        # stdout is a bytes-like object, returning a string might be more useful
+        return stdout.decode()
 
     def status_history(self, num=20, utc=False):
         """Get status history for this machine.
@@ -232,12 +235,11 @@ def dns_name(self):
 
         May return None if no suitable address is found.
         """
-        for scope in ['public', 'local-cloud']:
-            addresses = self.safe_data['addresses'] or []
-            addresses = [address for address in addresses
-                         if address['scope'] == scope]
-            if addresses:
-                return addresses[0]['value']
+        addresses = self.safe_data['addresses'] or []
+        for address in addresses:
+            scope = address['scope']
+            if scope == 'public' or scope == 'local-cloud':
+                return address['value']
         return None
 
     @property