Merge pull request #17 from JuliaParallel/revise

Revise support and worker state callbacks

Author: JamesWrigley

.github/dependabot.yml

@@ -5,3 +5,7 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "monthly"
+  - package-ecosystem: "julia"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -67,17 +67,7 @@ jobs:
           # GitHub will automatically expand to the correct value (`x86_64` or `aarch64`)
           # based on the architecture of the underlying GitHub Runner (virtual machine).
           arch: ${{ github.ref == '32' && 'x86' || runner.arch }}
-      - uses: actions/cache@v5
-        env:
-          cache-name: cache-artifacts
-        with:
-          path: ~/.julia/artifacts
-          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-test-${{ env.cache-name }}-
-            ${{ runner.os }}-test-${{ matrix.os }}
-            ${{ runner.os }}-
-      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/cache@v3
       - name: Decide what the value of JULIA_NUM_THREADS should be
         id: decide-numthreads-str
         run: |
@@ -115,6 +105,7 @@ jobs:
       - uses: julia-actions/setup-julia@latest
         with:
           version: '1'
+      - uses: julia-actions/cache@v3
       - name: Install dependencies
         run: julia --project=docs/ -e 'using Pkg; Pkg.instantiate()'
       - name: Build and deploy
@@ -130,8 +121,9 @@ jobs:
           persist-credentials: false
       - uses: julia-actions/setup-julia@latest
         with:
-          version: '1.12'
+          version: '1'
           # version: 'nightly'
+      - uses: julia-actions/cache@v3
       - run: julia --color=yes --project=ci/jet -e 'import Pkg; Pkg.instantiate()'
       - name: Run the JET tests
         run: julia --color=yes --project=ci/jet ci/jet/check.jl

.github/workflows/tagbot.yml

@@ -0,0 +1,18 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+    inputs:
+      lookback:
+        description: "[DEPRECATED] No longer has any effect"
+        default: "3"
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}

Project.toml

@@ -4,19 +4,28 @@ version = "1.1.1"
 
 [deps]
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+ScopedValues = "7e506255-f358-4e82-b7e4-beb19740aa63"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 
+[weakdeps]
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
+
+[extensions]
+ReviseExt = "Revise"
+
 [compat]
 Aqua = "0.8"
 Distributed = "1"
 LibSSH = "0.7"
 LinearAlgebra = "1"
 Random = "1"
+Revise = "3.7.0"
+ScopedValues = "1.6.0"
 Serialization = "1"
 Sockets = "1"
 Test = "1"
-julia = "1.9"
+julia = "1.10"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
@@ -26,4 +35,4 @@ LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "Distributed", "LibSSH", "LinearAlgebra", "Test"]
+test = ["Aqua", "Distributed", "LibSSH", "LinearAlgebra", "Revise", "Test"]

docs/src/_changelog.md

@@ -9,6 +9,12 @@ This documents notable changes in DistributedNext.jl. The format is based on
 
 ## Unreleased
 
+### Added
+- Implemented callback support for workers being added/removed etc ([#17]).
+- Added a package extension to support Revise.jl ([#17]).
+- Added support for setting worker statuses with [`setstatus`](@ref) and
+  [`getstatus`](@ref) ([#17]).
+
 ### Fixed
 - Modified the default implementations of methods like `take!` and `wait` on
   [`AbstractWorkerPool`](@ref) to be threadsafe and behave more consistently

docs/src/index.md

@@ -14,6 +14,10 @@ DistributedNext.rmprocs
 DistributedNext.interrupt
 DistributedNext.myid
 DistributedNext.pmap
+DistributedNext.getstatus
+DistributedNext.@getstatus
+DistributedNext.setstatus!
+DistributedNext.@setstatus!
 DistributedNext.RemoteException
 DistributedNext.ProcessExitedException
 DistributedNext.Future
@@ -53,6 +57,19 @@ DistributedNext.cluster_cookie()
 DistributedNext.cluster_cookie(::Any)
 ```
 
+## Callbacks
+
+```@docs
+DistributedNext.add_worker_starting_callback
+DistributedNext.remove_worker_starting_callback
+DistributedNext.add_worker_started_callback
+DistributedNext.remove_worker_started_callback
+DistributedNext.add_worker_exiting_callback
+DistributedNext.remove_worker_exiting_callback
+DistributedNext.add_worker_exited_callback
+DistributedNext.remove_worker_exited_callback
+```
+
 ## Cluster Manager Interface
 
 This interface provides a mechanism to launch and manage Julia workers on different cluster environments.

ext/ReviseExt.jl

@@ -0,0 +1,30 @@
+module ReviseExt
+
+import DistributedNext
+import DistributedNext: myid, workers, remotecall
+
+import Revise
+
+
+struct DistributedNextWorker <: Revise.AbstractWorker
+    id::Int
+end
+
+function get_workers()
+    map(DistributedNextWorker, workers())
+end
+
+function Revise.remotecall_impl(f, worker::DistributedNextWorker, args...; kwargs...)
+    remotecall(f, worker.id, args...; kwargs...)
+end
+
+Revise.is_master_worker(::typeof(get_workers)) = myid() == 1
+Revise.is_master_worker(worker::DistributedNextWorker) = worker.id == 1
+
+function __init__()
+    Revise.register_workers_function(get_workers)
+    DistributedNext.add_worker_started_callback(pid -> Revise.init_worker(DistributedNextWorker(pid));
+                                                key="DistributedNext-integration")
+end
+
+end

src/DistributedNext.jl

@@ -23,6 +23,12 @@ using Serialization, Sockets
 import Serialization: serialize, deserialize
 import Sockets: connect, wait_connected
 
+@static if VERSION < v"1.11"
+    using ScopedValues: ScopedValue, @with
+else
+    using Base.ScopedValues: ScopedValue, @with
+end
+
 # NOTE: clusterserialize.jl imports additional symbols from Serialization for use
 
 export