Stage an MEAI 10.4.1 release (#7402)

Author: jeffhandley

eng/MSBuild/Packaging.targets

@@ -37,7 +37,7 @@
     <!-- https://learn.microsoft.com/dotnet/fundamentals/package-validation/overview -->
     <EnablePackageValidation>true</EnablePackageValidation>
 
-    <PackageValidationBaselineVersion Condition="'$(Stage)' == 'normal' and '$(PackageValidationBaselineVersion)' == ''">10.2.0</PackageValidationBaselineVersion>
+    <PackageValidationBaselineVersion Condition="'$(Stage)' == 'normal' and '$(PackageValidationBaselineVersion)' == ''">10.4.0</PackageValidationBaselineVersion>
   </PropertyGroup>
 
   <!-- Verify that the minimum supported TFM is actually used. Not applicable for project template packages. -->

eng/Versions.props

@@ -2,7 +2,7 @@
   <PropertyGroup Label="Version settings">
     <MajorVersion>10</MajorVersion>
     <MinorVersion>4</MinorVersion>
-    <PatchVersion>0</PatchVersion>
+    <PatchVersion>1</PatchVersion>
     <PreReleaseVersionLabel>preview</PreReleaseVersionLabel>
     <PreReleaseVersionIteration>1</PreReleaseVersionIteration>
     <VersionPrefix>$(MajorVersion).$(MinorVersion).$(PatchVersion)</VersionPrefix>

eng/packages/General.props

@@ -16,7 +16,7 @@
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp.Workspaces" Version="$(MicrosoftCodeAnalysisVersion)" />
     <PackageVersion Include="Microsoft.Extensions.VectorData.Abstractions" Version="$(MicrosoftExtensionsVectorDataAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.ML.Tokenizers" Version="$(MicrosoftMLTokenizersVersion)" />
-    <PackageVersion Include="ModelContextProtocol.Core" Version="0.4.0-preview.3" />
+    <PackageVersion Include="ModelContextProtocol.Core" Version="1.1.0" />
     <PackageVersion Include="OpenAI" Version="2.9.1" />
     <PackageVersion Include="Polly" Version="8.4.2" />
     <PackageVersion Include="Polly.Core" Version="8.4.2" />

src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## NOT YET RELEASED
 
+- Added `ITextToSpeechClient` abstraction for interacting with text-to-speech capabilities.
+
+## 10.4.0
+
 - `AddAIContentType` now automatically registers the content type against every base in the inheritance chain up to `AIContent`.
 - Added `IHostedFileClient` interface and related types for interacting with files hosted by the service.
 - Added `WebSearchToolCallContent` and `WebSearchToolResultContent` for representing web search tool calls and results.
@@ -10,6 +14,7 @@
 - Updated AI function parameter JSON schema generation to honor `[Required]` attributes.
 - Updated `AIFunctionFactory` to work better with `DynamicMethod`-based functions.
 - Removed the experimental `IToolReductionStrategy` type.
+- Fixed the serialization of `ResponseContinuationToken` / `AllowBackgroundResponses` properties.
 
 ## 10.3.0
 

src/Libraries/Microsoft.Extensions.AI.Abstractions/CompatibilitySuppressions.xml

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics;
+using System.Net.Mime;
 using System.Text.Json.Serialization;
 using Microsoft.Shared.Diagnostics;
 
@@ -18,6 +19,9 @@ namespace Microsoft.Extensions.AI;
 [DebuggerDisplay("{DebuggerDisplay,nq}")]
 public class UriContent : AIContent
 {
+    /// <summary>The default media type for unknown file extensions.</summary>
+    private const string DefaultMediaType = "application/octet-stream";
+
     /// <summary>The URI represented.</summary>
     private Uri _uri;
 
@@ -26,37 +30,35 @@ public class UriContent : AIContent
 
     /// <summary>Initializes a new instance of the <see cref="UriContent"/> class.</summary>
     /// <param name="uri">The URI to the represented content.</param>
-    /// <param name="mediaType">The media type (also known as MIME type) represented by the content.</param>
+    /// <param name="mediaType">
+    /// The media type (also known as MIME type) represented by the content. If not provided,
+    /// it will be inferred from the file extension of the URI. If it cannot be inferred,
+    /// "application/octet-stream" is used.
+    /// </param>
     /// <exception cref="ArgumentNullException"><paramref name="uri"/> is <see langword="null"/>.</exception>
-    /// <exception cref="ArgumentNullException"><paramref name="mediaType"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentException"><paramref name="mediaType"/> is an invalid media type.</exception>
     /// <exception cref="UriFormatException"><paramref name="uri"/> is an invalid URL.</exception>
-    /// <remarks>
-    /// A media type must be specified, so that consumers know what to do with the content.
-    /// If an exact media type is not known, but the category (e.g. image) is known, a wildcard
-    /// may be used (e.g. "image/*").
-    /// </remarks>
-    public UriContent(string uri, string mediaType)
+    public UriContent(string uri, string? mediaType = null)
         : this(new Uri(Throw.IfNull(uri)), mediaType)
     {
     }
 
     /// <summary>Initializes a new instance of the <see cref="UriContent"/> class.</summary>
     /// <param name="uri">The URI to the represented content.</param>
-    /// <param name="mediaType">The media type (also known as MIME type) represented by the content.</param>
+    /// <param name="mediaType">
+    /// The media type (also known as MIME type) represented by the content. If not provided,
+    /// it will be inferred from the file extension of the URI. If it cannot be inferred,
+    /// "application/octet-stream" is used.
+    /// </param>
     /// <exception cref="ArgumentNullException"><paramref name="uri"/> is <see langword="null"/>.</exception>
-    /// <exception cref="ArgumentNullException"><paramref name="mediaType"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentException"><paramref name="mediaType"/> is an invalid media type.</exception>
-    /// <remarks>
-    /// A media type must be specified, so that consumers know what to do with the content.
-    /// If an exact media type is not known, but the category (e.g. image) is known, a wildcard
-    /// may be used (e.g. "image/*").
-    /// </remarks>
     [JsonConstructor]
-    public UriContent(Uri uri, string mediaType)
+    public UriContent(Uri uri, string? mediaType = null)
     {
         _uri = Throw.IfNull(uri);
-        _mediaType = DataUriParser.ThrowIfInvalidMediaType(mediaType);
+        _mediaType = mediaType is not null
+            ? DataUriParser.ThrowIfInvalidMediaType(mediaType)
+            : InferMediaType(uri);
     }
 
     /// <summary>Gets or sets the <see cref="Uri"/> for this content.</summary>
@@ -90,4 +92,25 @@ public string MediaType
     /// <summary>Gets a string representing this instance to display in the debugger.</summary>
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]
     private string DebuggerDisplay => $"Uri = {_uri}";
+
+    /// <summary>Infers the media type from the URI's file extension.</summary>
+    private static string InferMediaType(Uri uri)
+    {
+        string path;
+        if (uri.IsAbsoluteUri)
+        {
+            path = uri.AbsolutePath;
+        }
+        else
+        {
+            path = uri.OriginalString;
+            int i = path.AsSpan().IndexOfAny('?', '#');
+            if (i >= 0)
+            {
+                path = path.Substring(0, i);
+            }
+        }
+
+        return MediaTypeMap.GetMediaType(path) ?? DefaultMediaType;
+    }
 }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/UriContent.cs

@@ -51,6 +51,40 @@ protected HostedFileDownloadStream()
     /// </remarks>
     public virtual string? FileName => null;
 
+    /// <inheritdoc />
+    public override bool CanWrite => false;
+
+    /// <inheritdoc />
+    public override void SetLength(long value) => throw new NotSupportedException();
+
+    /// <inheritdoc />
+    public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? callback, object? state) =>
+        throw new NotSupportedException();
+
+    /// <inheritdoc />
+    public override void EndWrite(IAsyncResult asyncResult) => throw new NotSupportedException();
+
+    /// <inheritdoc />
+    public override void WriteByte(byte value) => throw new NotSupportedException();
+
+    /// <inheritdoc />
+    public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+
+#if NET
+    /// <inheritdoc />
+    public override void Write(ReadOnlySpan<byte> buffer) => throw new NotSupportedException();
+#endif
+
+    /// <inheritdoc />
+    public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) =>
+        throw new NotSupportedException();
+
+#if NET
+    /// <inheritdoc />
+    public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default) =>
+        throw new NotSupportedException();
+#endif
+
     /// <summary>
     /// Reads the entire stream content from its current position and returns it as a <see cref="DataContent"/>.
     /// </summary>

src/Libraries/Microsoft.Extensions.AI.Abstractions/Files/HostedFileDownloadStream.cs

@@ -40,15 +40,29 @@ protected AIFunctionDeclaration()
     /// The metadata present in the schema document plays an important role in guiding AI function invocation.
     /// </para>
     /// <para>
+    /// When an <see cref="AIFunction"/> is created via <see cref="AIFunctionFactory"/>, this schema is automatically derived from the
+    /// method's parameters using the configured <see cref="JsonSerializerOptions"/> and <see cref="AIJsonSchemaCreateOptions"/>.
+    /// </para>
+    /// <para>
     /// When no schema is specified, consuming chat clients should assume the "{}" or "true" schema, indicating that any JSON input is admissible.
     /// </para>
     /// </remarks>
     public virtual JsonElement JsonSchema => AIJsonUtilities.DefaultJsonSchema;
 
     /// <summary>Gets a JSON Schema describing the function's return value.</summary>
     /// <remarks>
-    /// A <see langword="null"/> typically reflects a function that doesn't specify a return schema
-    /// or a function that returns <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>.
+    /// <para>
+    /// When an <see cref="AIFunction"/> is created via <see cref="AIFunctionFactory"/>, this schema is automatically derived from the
+    /// method's return type using the configured <see cref="JsonSerializerOptions"/> and <see cref="AIJsonSchemaCreateOptions"/>.
+    /// For methods returning <see cref="Task{TResult}"/> or <see cref="ValueTask{TResult}"/>, the schema is based on the
+    /// unwrapped result type. Return schema generation can be excluded by setting
+    /// <see cref="AIFunctionFactoryOptions.ExcludeResultSchema"/> to <see langword="true"/>.
+    /// </para>
+    /// <para>
+    /// A <see langword="null"/> value typically reflects a function that doesn't specify a return schema,
+    /// a function that returns <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>,
+    /// or a function for which <see cref="AIFunctionFactoryOptions.ExcludeResultSchema"/> was set to <see langword="true"/>.
+    /// </para>
     /// </remarks>
     public virtual JsonElement? ReturnJsonSchema => null;
 }

src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionDeclaration.cs

@@ -27,6 +27,19 @@
 namespace Microsoft.Extensions.AI;
 
 /// <summary>Provides factory methods for creating commonly-used implementations of <see cref="AIFunction"/>.</summary>
+/// <remarks>
+/// <para>
+/// The <see cref="AIFunctionFactory"/> class creates <see cref="AIFunction"/> instances that wrap .NET methods
+/// (specified as <see cref="Delegate"/> or <see cref="MethodInfo"/>). As part of this process, JSON schemas are
+/// automatically derived for both the function's input parameters (exposed via <see cref="AIFunctionDeclaration.JsonSchema"/>)
+/// and, by default, the function's return type (exposed via <see cref="AIFunctionDeclaration.ReturnJsonSchema"/>).
+/// These schemas are produced using the <see cref="AIFunctionFactoryOptions.SerializerOptions"/> and
+/// <see cref="AIFunctionFactoryOptions.JsonSchemaCreateOptions"/>, and enable AI services to understand and
+/// interact with the function. Return value serialization and schema derivation behavior can be customized
+/// via <see cref="AIFunctionFactoryOptions.MarshalResult"/> and <see cref="AIFunctionFactoryOptions.ExcludeResultSchema"/>,
+/// respectively.
+/// </para>
+/// </remarks>
 /// <related type="Article" href="https://learn.microsoft.com/dotnet/ai/quickstarts/use-function-calling">Invoke .NET functions using an AI model.</related>
 public static partial class AIFunctionFactory
 {
@@ -98,7 +111,14 @@ public static partial class AIFunctionFactory
     /// special-cased and are not serialized: the created function returns the original instance(s) directly to enable
     /// callers (such as an <c>IChatClient</c>) to perform type tests and implement specialized handling. If
     /// <see cref="AIFunctionFactoryOptions.MarshalResult"/> is supplied, that delegate governs the behavior instead.
-    /// Handling of return values can be overridden via <see cref="AIFunctionFactoryOptions.MarshalResult"/>.
+    /// </para>
+    /// <para>
+    /// In addition to the parameter schema, a JSON schema is also derived from the method's return type and exposed via the
+    /// returned <see cref="AIFunction"/>'s <see cref="AIFunctionDeclaration.ReturnJsonSchema"/>. For methods returning
+    /// <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>, no return schema is produced (the property is <see langword="null"/>).
+    /// For methods returning <see cref="Task{TResult}"/> or <see cref="ValueTask{TResult}"/>, the schema is derived from the
+    /// unwrapped result type. Return schema generation can be excluded via <see cref="AIFunctionFactoryOptions.ExcludeResultSchema"/>,
+    /// and its generation is governed by <paramref name="options"/>'s <see cref="AIFunctionFactoryOptions.JsonSchemaCreateOptions"/>.
     /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
@@ -169,6 +189,11 @@ public static AIFunction Create(Delegate method, AIFunctionFactoryOptions? optio
     /// derived type of <see cref="AIContent"/>, or any type assignable from <see cref="IEnumerable{AIContent}"/> are not serialized;
     /// they are returned as-is to facilitate specialized handling.
     /// </para>
+    /// <para>
+    /// A JSON schema is also derived from the method's return type and exposed via <see cref="AIFunctionDeclaration.ReturnJsonSchema"/>.
+    /// For methods returning <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>, no return schema is produced.
+    /// For methods returning <see cref="Task{TResult}"/> or <see cref="ValueTask{TResult}"/>, the schema is derived from the unwrapped result type.
+    /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
     /// <exception cref="JsonException">A parameter to <paramref name="method"/> is not serializable.</exception>
@@ -255,6 +280,14 @@ public static AIFunction Create(Delegate method, string? name = null, string? de
     /// any type assignable from <see cref="IEnumerable{AIContent}"/> are not serialized and are instead returned directly.
     /// Handling of return values can be overridden via <see cref="AIFunctionFactoryOptions.MarshalResult"/>.
     /// </para>
+    /// <para>
+    /// In addition to the parameter schema, a JSON schema is also derived from the method's return type and exposed via the
+    /// returned <see cref="AIFunction"/>'s <see cref="AIFunctionDeclaration.ReturnJsonSchema"/>. For methods returning
+    /// <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>, no return schema is produced (the property is <see langword="null"/>).
+    /// For methods returning <see cref="Task{TResult}"/> or <see cref="ValueTask{TResult}"/>, the schema is derived from the
+    /// unwrapped result type. Return schema generation can be excluded via <see cref="AIFunctionFactoryOptions.ExcludeResultSchema"/>,
+    /// and its generation is governed by <paramref name="options"/>'s <see cref="AIFunctionFactoryOptions.JsonSchemaCreateOptions"/>.
+    /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> represents an instance method but <paramref name="target"/> is null.</exception>
@@ -334,6 +367,11 @@ public static AIFunction Create(MethodInfo method, object? target, AIFunctionFac
     /// derived type of <see cref="AIContent"/>, or any type assignable from <see cref="IEnumerable{AIContent}"/> are returned
     /// without serialization to enable specialized handling.
     /// </para>
+    /// <para>
+    /// A JSON schema is also derived from the method's return type and exposed via <see cref="AIFunctionDeclaration.ReturnJsonSchema"/>.
+    /// For methods returning <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>, no return schema is produced.
+    /// For methods returning <see cref="Task{TResult}"/> or <see cref="ValueTask{TResult}"/>, the schema is derived from the unwrapped result type.
+    /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> represents an instance method but <paramref name="target"/> is null.</exception>
@@ -433,6 +471,14 @@ public static AIFunction Create(MethodInfo method, object? target, string? name
     /// assignable from <see cref="IEnumerable{AIContent}"/> are returned directly without serialization.
     /// Handling of return values can be overridden via <see cref="AIFunctionFactoryOptions.MarshalResult"/>.
     /// </para>
+    /// <para>
+    /// In addition to the parameter schema, a JSON schema is also derived from the method's return type and exposed via the
+    /// returned <see cref="AIFunction"/>'s <see cref="AIFunctionDeclaration.ReturnJsonSchema"/>. For methods returning
+    /// <see cref="void"/>, <see cref="Task"/>, or <see cref="ValueTask"/>, no return schema is produced (the property is <see langword="null"/>).
+    /// For methods returning <see cref="Task{TResult}"/> or <see cref="ValueTask{TResult}"/>, the schema is derived from the
+    /// unwrapped result type. Return schema generation can be excluded via <see cref="AIFunctionFactoryOptions.ExcludeResultSchema"/>,
+    /// and its generation is governed by <paramref name="options"/>'s <see cref="AIFunctionFactoryOptions.JsonSchemaCreateOptions"/>.
+    /// </para>
     /// </remarks>
     /// <exception cref="ArgumentNullException"><paramref name="method"/> is <see langword="null"/>.</exception>
     /// <exception cref="ArgumentNullException"><paramref name="createInstanceFunc"/> is <see langword="null"/>.</exception>