Add extensible chat client routing

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatRouting/RoutingChatClient.cs

@@ -0,0 +1,127 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Shared.DiagnosticIds;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>
+/// Provides a template for an <see cref="IChatClient"/> that selects and invokes another chat client.
+/// </summary>
+/// <remarks>
+/// Derived classes implement <see cref="SelectClientAsync"/> to supply one selection for each request. The selected
+/// client is invoked once with the selected options, and its response or failure is propagated to the caller.
+/// </remarks>
+[Experimental(DiagnosticIds.Experiments.AIRoutingChat, UrlFormat = DiagnosticIds.UrlFormat)]
+public abstract class RoutingChatClient : IChatClient
+{
+    /// <summary>Creates a routing client that selects one client and its options for each request.</summary>
+    /// <param name="selector">The callback that produces the selection to invoke.</param>
+    /// <returns>A routing client that uses <paramref name="selector"/>.</returns>
+    /// <remarks>The selected clients are caller-owned and are not disposed by the returned routing client.</remarks>
+    /// <exception cref="ArgumentNullException"><paramref name="selector"/> is <see langword="null"/>.</exception>
+    public static RoutingChatClient Create(
+        Func<RoutingContext, CancellationToken, ValueTask<RoutingSelection>> selector)
+    {
+        _ = Throw.IfNull(selector);
+
+        return new CallbackRoutingChatClient(selector);
+    }
+
+    /// <summary>Selects the client and complete options to use for the request.</summary>
+    /// <param name="context">The request-specific inputs.</param>
+    /// <param name="cancellationToken">The cancellation token supplied for the request.</param>
+    /// <returns>The selection to invoke.</returns>
+    /// <remarks>
+    /// The returned <see cref="RoutingSelection.ChatOptions"/> are complete attempt options, not an overlay.
+    /// Implementations may clone <see cref="RoutingContext.ChatOptions"/> and adjust the clone for the selected
+    /// attempt. Exceptions from this method propagate to the caller.
+    /// </remarks>
+    protected abstract ValueTask<RoutingSelection> SelectClientAsync(
+        RoutingContext context,
+        CancellationToken cancellationToken);
+
+    /// <inheritdoc/>
+    public virtual async Task<ChatResponse> GetResponseAsync(
+        IEnumerable<ChatMessage> messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
+    {
+        _ = Throw.IfNull(messages);
+
+        var context = new RoutingContext(messages, options);
+        RoutingSelection selection = await SelectClientAsync(context, cancellationToken).ConfigureAwait(false) ??
+            throw new InvalidOperationException($"{nameof(SelectClientAsync)} returned null.");
+
+        return await selection.Client.GetResponseAsync(
+            context.Messages,
+            selection.ChatOptions,
+            cancellationToken).ConfigureAwait(false);
+    }
+
+    /// <inheritdoc/>
+    public virtual async IAsyncEnumerable<ChatResponseUpdate> GetStreamingResponseAsync(
+        IEnumerable<ChatMessage> messages, ChatOptions? options = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default)
+    {
+        _ = Throw.IfNull(messages);
+
+        var context = new RoutingContext(messages, options);
+        RoutingSelection selection = await SelectClientAsync(context, cancellationToken).ConfigureAwait(false) ??
+            throw new InvalidOperationException($"{nameof(SelectClientAsync)} returned null.");
+
+        await foreach (ChatResponseUpdate update in
+            selection.Client.GetStreamingResponseAsync(
+                context.Messages,
+                selection.ChatOptions,
+                cancellationToken)
+                .WithCancellation(cancellationToken)
+                .ConfigureAwait(false))
+        {
+            yield return update;
+        }
+    }
+
+    /// <inheritdoc/>
+    public virtual object? GetService(Type serviceType, object? serviceKey = null)
+    {
+        _ = Throw.IfNull(serviceType);
+
+        return serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null;
+    }
+
+    /// <inheritdoc/>
+    public void Dispose()
+    {
+        Dispose(disposing: true);
+        GC.SuppressFinalize(this);
+    }
+
+    /// <summary>Provides a mechanism for releasing resources owned by the derived instance.</summary>
+    /// <param name="disposing"><see langword="true"/> when called from <see cref="Dispose()"/>.</param>
+    /// <remarks>The default implementation performs no operation.</remarks>
+    protected virtual void Dispose(bool disposing)
+    {
+    }
+
+    private sealed class CallbackRoutingChatClient : RoutingChatClient
+    {
+        private readonly Func<RoutingContext, CancellationToken, ValueTask<RoutingSelection>> _selector;
+
+        public CallbackRoutingChatClient(
+            Func<RoutingContext, CancellationToken, ValueTask<RoutingSelection>> selector)
+        {
+            _selector = selector;
+        }
+
+        protected override ValueTask<RoutingSelection> SelectClientAsync(
+            RoutingContext context,
+            CancellationToken cancellationToken) =>
+            _selector(context, cancellationToken);
+    }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatRouting/RoutingContext.cs

@@ -0,0 +1,49 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Shared.DiagnosticIds;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Provides request-specific inputs to a <see cref="RoutingChatClient"/>.</summary>
+/// <remarks>
+/// <para>
+/// One context is created for each call to <see cref="IChatClient.GetResponseAsync"/> and for each enumeration
+/// started from the sequence returned by <see cref="IChatClient.GetStreamingResponseAsync"/>.
+/// </para>
+/// <para>
+/// <see cref="ChatOptions"/> is cloned when the context is created and represents the request baseline. Selectors can
+/// clone and modify it to produce complete attempt options through <see cref="RoutingSelection"/>.
+/// </para>
+/// </remarks>
+[Experimental(DiagnosticIds.Experiments.AIRoutingChat, UrlFormat = DiagnosticIds.UrlFormat)]
+public class RoutingContext
+{
+    /// <summary>Initializes a new instance of the <see cref="RoutingContext"/> class.</summary>
+    /// <param name="messages">The messages to route.</param>
+    /// <param name="chatOptions">The options associated with this context.</param>
+    public RoutingContext(
+        IEnumerable<ChatMessage> messages,
+        ChatOptions? chatOptions)
+    {
+        _ = Throw.IfNull(messages);
+
+        Messages = messages;
+        ChatOptions = chatOptions?.Clone();
+    }
+
+    /// <summary>Gets the messages supplied to client selection and the selected client.</summary>
+    /// <remarks>
+    /// Selection and failover may enumerate this sequence multiple times. Callers should supply a repeatable sequence.
+    /// </remarks>
+    public IEnumerable<ChatMessage> Messages { get; }
+
+    /// <summary>Gets the request options supplied to client selection.</summary>
+    /// <remarks>
+    /// This is a clone of the caller's options. Selectors should clone it before applying attempt-specific changes.
+    /// </remarks>
+    public ChatOptions? ChatOptions { get; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatRouting/RoutingSelection.cs

@@ -0,0 +1,35 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using Microsoft.Shared.DiagnosticIds;
+using Microsoft.Shared.Diagnostics;
+
+namespace Microsoft.Extensions.AI;
+
+/// <summary>Represents the client and complete options selected for one routing attempt.</summary>
+/// <remarks>
+/// The exact <see cref="Client"/> instance represents the routing identity. <see cref="ChatOptions"/> are ephemeral
+/// attempt data and do not participate in that identity. The selected client may further transform the options.
+/// </remarks>
+[Experimental(DiagnosticIds.Experiments.AIRoutingChat, UrlFormat = DiagnosticIds.UrlFormat)]
+public sealed class RoutingSelection
+{
+    /// <summary>Initializes a new instance of the <see cref="RoutingSelection"/> class.</summary>
+    /// <param name="client">The client selected for the attempt.</param>
+    /// <param name="chatOptions">The complete options selected for the attempt, or <see langword="null"/>.</param>
+    /// <exception cref="ArgumentNullException"><paramref name="client"/> is <see langword="null"/>.</exception>
+    public RoutingSelection(IChatClient client, ChatOptions? chatOptions)
+    {
+        Client = Throw.IfNull(client);
+        ChatOptions = chatOptions?.Clone();
+    }
+
+    /// <summary>Gets the client selected for the attempt.</summary>
+    public IChatClient Client { get; }
+
+    /// <summary>Gets the complete options selected for the attempt.</summary>
+    /// <remarks>This is a clone of the options supplied to the constructor.</remarks>
+    public ChatOptions? ChatOptions { get; }
+}

src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json

@@ -3966,6 +3966,84 @@
         }
       ]
     },
+    {
+      "Type": "abstract class Microsoft.Extensions.AI.RoutingChatClient : Microsoft.Extensions.AI.IChatClient, System.IDisposable",
+      "Stage": "Experimental",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.RoutingChatClient.RoutingChatClient();",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "static Microsoft.Extensions.AI.RoutingChatClient Microsoft.Extensions.AI.RoutingChatClient.Create(System.Func<Microsoft.Extensions.AI.RoutingContext, System.Threading.CancellationToken, System.Threading.Tasks.ValueTask<Microsoft.Extensions.AI.RoutingSelection>> selector);",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "void Microsoft.Extensions.AI.RoutingChatClient.Dispose();",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "virtual void Microsoft.Extensions.AI.RoutingChatClient.Dispose(bool disposing);",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "virtual System.Threading.Tasks.Task<Microsoft.Extensions.AI.ChatResponse> Microsoft.Extensions.AI.RoutingChatClient.GetResponseAsync(System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.ChatMessage> messages, Microsoft.Extensions.AI.ChatOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "virtual object? Microsoft.Extensions.AI.RoutingChatClient.GetService(System.Type serviceType, object? serviceKey = null);",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "virtual System.Collections.Generic.IAsyncEnumerable<Microsoft.Extensions.AI.ChatResponseUpdate> Microsoft.Extensions.AI.RoutingChatClient.GetStreamingResponseAsync(System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.ChatMessage> messages, Microsoft.Extensions.AI.ChatOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "abstract System.Threading.Tasks.ValueTask<Microsoft.Extensions.AI.RoutingSelection> Microsoft.Extensions.AI.RoutingChatClient.SelectClientAsync(Microsoft.Extensions.AI.RoutingContext context, System.Threading.CancellationToken cancellationToken);",
+          "Stage": "Experimental"
+        }
+      ]
+    },
+    {
+      "Type": "class Microsoft.Extensions.AI.RoutingContext",
+      "Stage": "Experimental",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.RoutingContext.RoutingContext(System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.ChatMessage> messages, Microsoft.Extensions.AI.ChatOptions? chatOptions);",
+          "Stage": "Experimental"
+        }
+      ],
+      "Properties": [
+        {
+          "Member": "Microsoft.Extensions.AI.ChatOptions? Microsoft.Extensions.AI.RoutingContext.ChatOptions { get; }",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "System.Collections.Generic.IEnumerable<Microsoft.Extensions.AI.ChatMessage> Microsoft.Extensions.AI.RoutingContext.Messages { get; }",
+          "Stage": "Experimental"
+        }
+      ]
+    },
+    {
+      "Type": "sealed class Microsoft.Extensions.AI.RoutingSelection",
+      "Stage": "Experimental",
+      "Methods": [
+        {
+          "Member": "Microsoft.Extensions.AI.RoutingSelection.RoutingSelection(Microsoft.Extensions.AI.IChatClient client, Microsoft.Extensions.AI.ChatOptions? chatOptions);",
+          "Stage": "Experimental"
+        }
+      ],
+      "Properties": [
+        {
+          "Member": "Microsoft.Extensions.AI.ChatOptions? Microsoft.Extensions.AI.RoutingSelection.ChatOptions { get; }",
+          "Stage": "Experimental"
+        },
+        {
+          "Member": "Microsoft.Extensions.AI.IChatClient Microsoft.Extensions.AI.RoutingSelection.Client { get; }",
+          "Stage": "Experimental"
+        }
+      ]
+    },
     {
       "Type": "class Microsoft.Extensions.AI.SessionUpdateRealtimeClientMessage : Microsoft.Extensions.AI.RealtimeClientMessage",
       "Stage": "Experimental",