Add VoiceActivityDetection options to realtime session abstractions (#7399)

* Add VoiceActivityDetection options to realtime session abstractions

Introduce VoiceActivityDetectionOptions with Enabled and AllowInterruption
properties to RealtimeSessionOptions. These represent the common VAD
options supported across multiple realtime AI models (OpenAI, Gemini,
Anthropic Claude, AWS Nova Sonic).

- Add VoiceActivityDetectionOptions class with Enabled (default true) and
  AllowInterruption (default true) properties
- Add VoiceActivityDetection property to RealtimeSessionOptions
- Map VAD options to OpenAI SDK TurnDetection in both conversation and
  transcription session paths
- Add concurrency guidance for SendAsync in IRealtimeClientSession docs
- Add unit tests for the new types

* Address PR review feedback

- Fix type name in SendAsync concurrency docs: FunctionInvokingRealtimeSession -> FunctionInvokingRealtimeClientSession
- Preserve existing TurnDetection from seed options (RawRepresentationFactory) by mutating InterruptResponseEnabled on existing RealtimeServerVadTurnDetection instead of replacing it, in both conversation and transcription paths

* Clarify relationship between Enabled and AllowInterruption in VAD docs

Document that AllowInterruption only takes effect when Enabled is true,
and that disabling VAD fully disables turn detection making interruption
not applicable.

---------

Co-authored-by: Tarek Mahmoud Sayed <tarekms@ntdev.microsoft.com>

Author: 2 people

src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/IRealtimeClientSession.cs

@@ -27,7 +27,17 @@ public interface IRealtimeClientSession : IAsyncDisposable
     /// <param name="cancellationToken">A token to cancel the operation.</param>
     /// <returns>A task that represents the asynchronous send operation.</returns>
     /// <remarks>
+    /// <para>
     /// This method allows for sending client messages to the session at any time, which can be used to influence the session's behavior or state.
+    /// </para>
+    /// <para>
+    /// <strong>Concurrency note for provider implementers:</strong> <see cref="SendAsync"/> may be called concurrently
+    /// from multiple sources. For example, a caller may stream audio via <see cref="SendAsync"/> on one thread while
+    /// middleware such as <c>FunctionInvokingRealtimeClientSession</c> calls <see cref="SendAsync"/> to return tool results
+    /// from within <see cref="GetStreamingResponseAsync"/> enumeration on another thread. If the underlying transport
+    /// (e.g., a WebSocket) does not support concurrent sends, provider implementations must serialize access — for
+    /// example by using a <see cref="System.Threading.SemaphoreSlim"/> — to prevent protocol violations.
+    /// </para>
     /// </remarks>
     Task SendAsync(RealtimeClientMessage message, CancellationToken cancellationToken = default);
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/RealtimeSessionOptions.cs

@@ -72,6 +72,15 @@ public class RealtimeSessionOptions
     /// </summary>
     public IReadOnlyList<AITool>? Tools { get; init; }
 
+    /// <summary>
+    /// Gets the voice activity detection (VAD) options for the session.
+    /// </summary>
+    /// <remarks>
+    /// When set, configures how the server detects user speech to manage turn-taking.
+    /// When <see langword="null"/>, the provider's default VAD behavior is used.
+    /// </remarks>
+    public VoiceActivityDetectionOptions? VoiceActivityDetection { get; init; }
+
     /// <summary>
     /// Gets a callback responsible for creating the raw representation of the session options from an underlying implementation.
     /// </summary>

src/Libraries/Microsoft.Extensions.AI.Abstractions/Realtime/VoiceActivityDetectionOptions.cs

@@ -0,0 +1,57 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Shared.DiagnosticIds;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Represents options for configuring voice activity detection (VAD) in a real-time session.
+/// </summary>
+/// <remarks>
+/// Voice activity detection automatically determines when a user starts and stops speaking,
+/// enabling natural turn-taking in conversational audio interactions.
+/// When <see cref="Enabled"/> is <see langword="true"/>, the server detects speech boundaries
+/// and manages turn transitions automatically.
+/// When <see cref="Enabled"/> is <see langword="false"/>, the client must explicitly signal
+/// activity boundaries (e.g., via audio buffer commit and response creation).
+/// </remarks>
+[Experimental(DiagnosticIds.Experiments.AIRealTime, UrlFormat = DiagnosticIds.UrlFormat)]
+public class VoiceActivityDetectionOptions
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="VoiceActivityDetectionOptions"/> class.
+    /// </summary>
+    public VoiceActivityDetectionOptions()
+    {
+    }
+
+    /// <summary>
+    /// Gets or sets a value indicating whether server-side voice activity detection is enabled.
+    /// </summary>
+    /// <remarks>
+    /// When <see langword="true"/>, the server automatically detects speech start and end,
+    /// and may automatically trigger responses when the user stops speaking.
+    /// When <see langword="false"/>, turn detection is fully disabled and the client controls
+    /// turn boundaries manually (e.g., via audio buffer commit and explicit response creation).
+    /// Other properties on this class, such as <see cref="AllowInterruption"/>, only take effect
+    /// when this property is <see langword="true"/>.
+    /// The default is <see langword="true"/>.
+    /// </remarks>
+    public bool Enabled { get; set; } = true;
+
+    /// <summary>
+    /// Gets or sets a value indicating whether the user's speech can interrupt the model's audio output.
+    /// </summary>
+    /// <remarks>
+    /// This property is only meaningful when <see cref="Enabled"/> is <see langword="true"/>.
+    /// When voice activity detection is disabled, the server does not detect speech, so interruption
+    /// does not apply.
+    /// When <see langword="true"/>, the model's response will be cut off when the user starts speaking (barge-in).
+    /// When <see langword="false"/>, the model's response will continue to completion regardless of user input.
+    /// The default is <see langword="true"/>.
+    /// Not all providers support this option; those that do not will ignore it.
+    /// </remarks>
+    public bool AllowInterruption { get; set; } = true;
+}

src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIRealtimeClientSession.cs

@@ -414,6 +414,26 @@ private static Sdk.RealtimeConversationSessionOptions BuildConversationSessionOp
             outputAudioOptions.Voice = new Sdk.RealtimeVoice(options.Voice);
         }
 
+        if (options.VoiceActivityDetection is { } vad)
+        {
+            if (!vad.Enabled)
+            {
+                inputAudioOptions.DisableTurnDetection();
+            }
+            else if (inputAudioOptions.TurnDetection is Sdk.RealtimeServerVadTurnDetection existingVad)
+            {
+                existingVad.InterruptResponseEnabled = vad.AllowInterruption;
+            }
+            else
+            {
+                inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection
+                {
+                    InterruptResponseEnabled = vad.AllowInterruption,
+                    CreateResponseEnabled = true,
+                };
+            }
+        }
+
         audioOptions.InputAudioOptions = inputAudioOptions;
         audioOptions.OutputAudioOptions = outputAudioOptions;
         convOptions.AudioOptions = audioOptions;
@@ -464,7 +484,7 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession
     {
         var transOptions = new Sdk.RealtimeTranscriptionSessionOptions();
 
-        if (options.InputAudioFormat is not null || options.TranscriptionOptions is not null)
+        if (options.InputAudioFormat is not null || options.TranscriptionOptions is not null || options.VoiceActivityDetection is not null)
         {
             var inputAudioOptions = new Sdk.RealtimeTranscriptionSessionInputAudioOptions();
 
@@ -483,6 +503,26 @@ private static Sdk.RealtimeTranscriptionSessionOptions BuildTranscriptionSession
                 };
             }
 
+            if (options.VoiceActivityDetection is { } vad)
+            {
+                if (!vad.Enabled)
+                {
+                    inputAudioOptions.DisableTurnDetection();
+                }
+                else if (inputAudioOptions.TurnDetection is Sdk.RealtimeServerVadTurnDetection existingVad)
+                {
+                    existingVad.InterruptResponseEnabled = vad.AllowInterruption;
+                }
+                else
+                {
+                    inputAudioOptions.TurnDetection = new Sdk.RealtimeServerVadTurnDetection
+                    {
+                        InterruptResponseEnabled = vad.AllowInterruption,
+                        CreateResponseEnabled = true,
+                    };
+                }
+            }
+
             transOptions.AudioOptions = new Sdk.RealtimeTranscriptionSessionAudioOptions
             {
                 InputAudioOptions = inputAudioOptions,

test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Realtime/RealtimeSessionOptionsTests.cs

@@ -26,6 +26,7 @@ public void Constructor_Parameterless_PropsDefaulted()
         Assert.Null(options.OutputModalities);
         Assert.Null(options.ToolMode);
         Assert.Null(options.Tools);
+        Assert.Null(options.VoiceActivityDetection);
     }
 
     [Fact]
@@ -90,4 +91,49 @@ public void TranscriptionOptions_PromptDefaultsToNull()
         Assert.Null(options.Prompt);
     }
 
+    [Fact]
+    public void VoiceActivityDetection_DefaultsToNull()
+    {
+        RealtimeSessionOptions options = new();
+        Assert.Null(options.VoiceActivityDetection);
+    }
+
+    [Fact]
+    public void VoiceActivityDetection_Roundtrip()
+    {
+        var vad = new VoiceActivityDetectionOptions();
+        RealtimeSessionOptions options = new()
+        {
+            VoiceActivityDetection = vad,
+        };
+
+        Assert.Same(vad, options.VoiceActivityDetection);
+    }
+
+    [Fact]
+    public void VoiceActivityDetectionOptions_DefaultValues()
+    {
+        var vad = new VoiceActivityDetectionOptions();
+        Assert.True(vad.Enabled);
+        Assert.True(vad.AllowInterruption);
+    }
+
+    [Fact]
+    public void VoiceActivityDetectionOptions_Properties_Roundtrip()
+    {
+        var vad = new VoiceActivityDetectionOptions
+        {
+            Enabled = false,
+            AllowInterruption = false,
+        };
+
+        Assert.False(vad.Enabled);
+        Assert.False(vad.AllowInterruption);
+
+        vad.Enabled = true;
+        vad.AllowInterruption = true;
+
+        Assert.True(vad.Enabled);
+        Assert.True(vad.AllowInterruption);
+    }
 }