Fix StateChangedAt mismatch during container recovery from storage (#14482)

* Use Docker's FinishedAt timestamp in Transition() to fix StateChangedAt mismatch during container recovery

* Extract GetDockerFinishedAt() helper

* Address Feedback

* Fail if Docker event time is missing

* Address copilot feedback

* Use Docker stop event timestamp instead of InspectContainer()

* Apply copilot feedback

* PR Feedback

Author: beena352

src/windows/wslcsession/ContainerEventTracker.cpp

@@ -145,13 +145,18 @@ void ContainerEventTracker::OnEvent(const std::string_view& event)
         }
     }
 
+    auto timeEntry = parsed.find("time");
+    THROW_HR_IF_MSG(
+        E_INVALIDARG, timeEntry == parsed.end(), "Failed to parse time from event: %.*hs", static_cast<int>(event.size()), event.data());
+    std::uint64_t eventTime = timeEntry->get<std::uint64_t>();
+
     std::lock_guard lock{m_lock};
 
     for (const auto& e : m_callbacks)
     {
         if (e.ContainerId == containerId && (!e.ExecId.has_value() || e.ExecId == execId))
         {
-            e.Callback(it->second, exitCode);
+            e.Callback(it->second, exitCode, eventTime);
         }
     }
 }

src/windows/wslcsession/ContainerEventTracker.h

@@ -54,7 +54,7 @@ class ContainerEventTracker
         ContainerEventTracker* m_tracker = nullptr;
     };
 
-    using ContainerStateChangeCallback = std::function<void(ContainerEvent, std::optional<int>)>;
+    using ContainerStateChangeCallback = std::function<void(ContainerEvent, std::optional<int>, std::uint64_t)>;
 
     ContainerEventTracker(DockerHTTPClient& dockerClient, ULONG sessionId, IORelay& relay);
     ~ContainerEventTracker();

src/windows/wslcsession/WSLCContainer.cpp

@@ -355,7 +355,7 @@ WSLCContainerImpl::WSLCContainerImpl(
     m_eventTracker(EventTracker),
     m_ioRelay(Relay),
     m_containerEvents(EventTracker.RegisterContainerStateUpdates(
-        m_id, std::bind(&WSLCContainerImpl::OnEvent, this, std::placeholders::_1, std::placeholders::_2))),
+        m_id, std::bind(&WSLCContainerImpl::OnEvent, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3))),
     m_state(InitialState),
     m_initProcessFlags(InitProcessFlags),
     m_containerFlags(ContainerFlags)
@@ -547,8 +547,6 @@ void WSLCContainerImpl::Start(WSLCContainerStartFlags Flags, LPCSTR DetachKeys)
         m_initProcessControl = nullptr;
     });
 
-    m_stoppedNotifiedEvent.ResetEvent();
-
     auto volumeCleanup = MountVolumes(m_mountedVolumes, m_virtualMachine);
 
     auto portCleanup = wil::scope_exit_log(WI_DIAGNOSTICS_INFO, [this]() { UnmapPorts(); });
@@ -567,34 +565,34 @@ void WSLCContainerImpl::Start(WSLCContainerStartFlags Flags, LPCSTR DetachKeys)
     cleanup.release();
 }
 
-void WSLCContainerImpl::OnEvent(ContainerEvent event, std::optional<int> exitCode)
+void WSLCContainerImpl::OnEvent(ContainerEvent event, std::optional<int> exitCode, std::uint64_t eventTime)
 {
     if (event == ContainerEvent::Stop)
     {
-        m_stoppedNotifiedEvent.SetEvent();
-
         THROW_HR_IF(E_UNEXPECTED, !exitCode.has_value());
-        auto lock = m_lock.lock_exclusive();
-        auto previousState = m_state;
 
+        // If a Stop() call is in progress, provide the timestamp via the promise
+        // and let Stop() handle the state transition.
         {
-            std::lock_guard processesLock{m_processesLock};
-
-            // Notify all processes that the container has exited.
-            // N.B. The exec callback isn't always sent to execed processes, so do this to avoid 'stuck' processes.
-            for (auto& process : m_processes)
+            std::lock_guard stopLock{m_stopStateLock};
+            if (m_stopState.has_value())
             {
-                process->OnContainerReleased();
+                m_stopState->set_value(eventTime);
+                m_stopState.reset();
+                return;
             }
-
-            m_processes.clear();
         }
 
+        auto lock = m_lock.lock_exclusive();
+        auto previousState = m_state;
+
+        ReleaseProcesses();
+
         // Don't run the deletion logic if the container is already in a stopped / deleted state.
         // This can happen if Delete() is called by the user.
         if (previousState == WslcContainerStateRunning)
         {
-            Transition(WslcContainerStateExited);
+            Transition(WslcContainerStateExited, eventTime);
 
             ReleaseRuntimeResources();
 
@@ -639,6 +637,22 @@ void WSLCContainerImpl::Stop(WSLCSignal Signal, LONG TimeoutSeconds)
             m_state);
     }
 
+    // Set up a waitable stop state so OnEvent() can pass the Docker timestamp
+    // back to Stop() without needing to take m_lock.
+    std::future<std::uint64_t> stopFuture;
+    {
+        std::lock_guard stopLock{m_stopStateLock};
+        m_stopState.emplace();
+        stopFuture = m_stopState->get_future();
+    }
+
+    // Ensure m_stopState is cleared on all exit paths so OnEvent() doesn't
+    // take the promise path after a failed Stop().
+    auto resetStopState = wil::scope_exit([this]() {
+        std::lock_guard stopLock{m_stopStateLock};
+        m_stopState.reset();
+    });
+
     try
     {
         std::optional<WSLCSignal> SignalArg;
@@ -664,25 +678,24 @@ void WSLCContainerImpl::Stop(WSLCSignal Signal, LONG TimeoutSeconds)
         }
     }
 
-    Transition(WslcContainerStateExited);
+    // Wait for the stop event to get the Docker timestamp.
+    // Safe while holding m_lock since OnEvent() uses m_stopStateLock on this path.
+    std::optional<std::uint64_t> stopTimestamp;
+    if (stopFuture.wait_for(60s) == std::future_status::ready)
+    {
+        stopTimestamp = stopFuture.get();
+    }
+
+    Transition(WslcContainerStateExited, stopTimestamp);
+
+    ReleaseProcesses();
 
     ReleaseRuntimeResources();
 
     if (WI_IsFlagSet(m_containerFlags, WSLCContainerFlagsRm))
     {
         DeleteExclusiveLockHeld(WSLCDeleteFlagsForce);
     }
-    else
-    {
-        // Wait for the stop notification to arrive before returning.
-        // This is required so that a caller that Stops() and immediately calls Start() again doesn't see the container
-        // switch back to 'stopped' state due to the delayed event notification.
-
-        auto io = m_wslcSession.CreateIOContext();
-        io.AddHandle(std::make_unique<EventHandle>(m_stoppedNotifiedEvent.get()), MultiHandleWait::CancelOnCompleted);
-
-        io.Run({60s});
-    }
 }
 
 void WSLCContainerImpl::Delete(WSLCDeleteFlags Flags)
@@ -1427,6 +1440,20 @@ void WSLCContainerImpl::UnmapPorts()
     }
 }
 
+__requires_exclusive_lock_held(m_lock) void WSLCContainerImpl::ReleaseProcesses()
+{
+    std::lock_guard processesLock{m_processesLock};
+
+    // Notify all processes that the container has exited.
+    // The exec callback isn't always sent to execed processes, so do this to avoid 'stuck' processes.
+    for (auto& process : m_processes)
+    {
+        process->OnContainerReleased();
+    }
+
+    m_processes.clear();
+}
+
 __requires_exclusive_lock_held(m_lock) void WSLCContainerImpl::ReleaseRuntimeResources()
 {
     WSL_LOG("ReleaseRuntimeResources", TraceLoggingValue(m_id.c_str(), "ID"));
@@ -1468,7 +1495,7 @@ __requires_exclusive_lock_held(m_lock) void WSLCContainerImpl::DisconnectComWrap
     }
 }
 
-__requires_lock_held(m_lock) void WSLCContainerImpl::Transition(WSLCContainerState State) noexcept
+__requires_lock_held(m_lock) void WSLCContainerImpl::Transition(WSLCContainerState State, std::optional<std::uint64_t> stateChangedAt) noexcept
 {
     // N.B. A deleted container cannot transition back to any other state.
     WI_ASSERT(m_state != WslcContainerStateDeleted);
@@ -1480,7 +1507,7 @@ __requires_lock_held(m_lock) void WSLCContainerImpl::Transition(WSLCContainerSta
         TraceLoggingValue(m_id.c_str(), "ID"));
 
     m_state = State;
-    m_stateChangedAt = static_cast<std::uint64_t>(std::time(nullptr));
+    m_stateChangedAt = stateChangedAt.value_or(static_cast<std::uint64_t>(std::time(nullptr)));
 }
 
 WSLCContainer::WSLCContainer(WSLCContainerImpl* impl, std::function<void(const WSLCContainerImpl*)>&& OnDeleted) :

src/windows/wslcsession/WSLCContainer.h

@@ -93,7 +93,7 @@ class WSLCContainerImpl
     const std::string& Name() const noexcept;
     WSLCContainerState State() const noexcept;
 
-    __requires_lock_held(m_lock) void Transition(WSLCContainerState State) noexcept;
+    __requires_lock_held(m_lock) void Transition(WSLCContainerState State, std::optional<std::uint64_t> stateChangedAt = std::nullopt) noexcept;
 
     void OnProcessReleased(DockerExecProcessControl* process) noexcept;
 
@@ -130,10 +130,11 @@ class WSLCContainerImpl
     __requires_exclusive_lock_held(m_lock) void DeleteExclusiveLockHeld(WSLCDeleteFlags Flags);
 
     void AllocateBridgedModePorts();
-    void OnEvent(ContainerEvent event, std::optional<int> exitCode);
+    void OnEvent(ContainerEvent event, std::optional<int> exitCode, std::uint64_t eventTime);
     void WaitForContainerEvent();
     __requires_exclusive_lock_held(m_lock) void ReleaseResources();
     __requires_exclusive_lock_held(m_lock) void ReleaseRuntimeResources();
+    __requires_exclusive_lock_held(m_lock) void ReleaseProcesses();
     __requires_exclusive_lock_held(m_lock) void DisconnectComWrapper();
 
     std::unique_ptr<RelayedProcessIO> CreateRelayedProcessIO(wil::unique_handle&& stream, WSLCProcessFlags flags);
@@ -154,7 +155,8 @@ class WSLCContainerImpl
     __guarded_by(m_processesLock) Microsoft::WRL::ComPtr<IWSLCProcess> m_initProcess;
     __guarded_by(m_processesLock) DockerContainerProcessControl* m_initProcessControl = nullptr;
 
-    wil::unique_event m_stoppedNotifiedEvent{wil::EventOptions::ManualReset};
+    std::mutex m_stopStateLock;
+    std::optional<std::promise<std::uint64_t>> m_stopState;
     DockerHTTPClient& m_dockerClient;
     std::uint64_t m_stateChangedAt{static_cast<std::uint64_t>(std::time(nullptr))};
     std::uint64_t m_createdAt{static_cast<std::uint64_t>(std::time(nullptr))};

src/windows/wslcsession/WSLCProcessControl.cpp

@@ -45,7 +45,8 @@ DockerContainerProcessControl::DockerContainerProcessControl(WSLCContainerImpl&
     m_container(&Container),
     m_client(DockerClient),
     m_trackingReference(EventTracker.RegisterContainerStateUpdates(
-        Container.ID(), std::bind(&DockerContainerProcessControl::OnEvent, this, std::placeholders::_1, std::placeholders::_2)))
+        Container.ID(),
+        std::bind(&DockerContainerProcessControl::OnEvent, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)))
 {
 }
 
@@ -69,7 +70,7 @@ void DockerContainerProcessControl::ResizeTty(ULONG Rows, ULONG Columns)
     m_client.ResizeContainerTty(m_container->ID(), Rows, Columns);
 }
 
-void DockerContainerProcessControl::OnEvent(ContainerEvent Event, std::optional<int> ExitCode)
+void DockerContainerProcessControl::OnEvent(ContainerEvent Event, std::optional<int> ExitCode, std::uint64_t /*eventTime*/)
 {
     if (Event == ContainerEvent::Stop)
     {
@@ -118,7 +119,7 @@ DockerExecProcessControl::DockerExecProcessControl(
     m_id(Id),
     m_client(DockerClient),
     m_trackingReference(EventTracker.RegisterExecStateUpdates(
-        Container.ID(), Id, std::bind(&DockerExecProcessControl::OnEvent, this, std::placeholders::_1, std::placeholders::_2)))
+        Container.ID(), Id, std::bind(&DockerExecProcessControl::OnEvent, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)))
 {
 }
 
@@ -173,7 +174,7 @@ void DockerExecProcessControl::SetExitCode(int ExitCode)
     }
 }
 
-void DockerExecProcessControl::OnEvent(ContainerEvent Event, std::optional<int> ExitCode)
+void DockerExecProcessControl::OnEvent(ContainerEvent Event, std::optional<int> ExitCode, std::uint64_t /*eventTime*/)
 {
     if (Event == ContainerEvent::ExecDied && !m_exitEvent.is_signaled())
     {

src/windows/wslcsession/WSLCProcessControl.h

@@ -50,7 +50,7 @@ class DockerContainerProcessControl : public WSLCProcessControl
     void OnContainerReleased() noexcept;
 
 private:
-    void OnEvent(ContainerEvent Event, std::optional<int> ExitCode);
+    void OnEvent(ContainerEvent Event, std::optional<int> ExitCode, std::uint64_t eventTime);
 
     std::mutex m_lock;
     DockerHTTPClient& m_client;
@@ -72,7 +72,7 @@ class DockerExecProcessControl : public WSLCProcessControl
     void SetExitCode(int ExitCode);
 
 private:
-    void OnEvent(ContainerEvent Event, std::optional<int> ExitCode);
+    void OnEvent(ContainerEvent Event, std::optional<int> ExitCode, std::uint64_t eventTime);
 
     mutable std::mutex m_lock;
     std::string m_id;