From af0af1a18f89f4710523f1ba812db37e504602f4 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Thu, 25 Jun 2026 19:28:19 -0400 Subject: [PATCH 01/13] Add IOcrClient OCR/document-extraction capability to Microsoft.Extensions.AI Introduces IOcrClient as a provider-neutral OCR/document-parsing capability in Microsoft.Extensions.AI.Abstractions, following the same abstraction + builder + middleware + DI shape as the existing capability family (IChatClient, ISpeechToTextClient, etc.). Abstractions (Microsoft.Extensions.AI.Abstractions): - IOcrClient, DelegatingOcrClient, OcrClientExtensions, OcrClientMetadata - OcrOptions, OcrResult, OcrPage, OcrTable, OcrTableCell, OcrBlock, OcrBoundingRegion, OcrUsage, OcrProgress Middleware + DI (Microsoft.Extensions.AI): - OcrClientBuilder, AsBuilder, AddOcrClient/AddKeyedOcrClient - LoggingOcrClient, OpenTelemetryOcrClient, ConfigureOptionsOcrClient and their builder extensions, mirroring the ISpeechToTextClient template All public surface is marked [Experimental] under the MEAI001 (AIOcr) diagnostic id. Includes unit tests for both libraries and updated API baselines. --- .../Microsoft.Extensions.AI.Abstractions.json | 358 +++++++++++++++++- .../Ocr/DelegatingOcrClient.cs | 72 ++++ .../Ocr/IOcrClient.cs | 67 ++++ .../Ocr/OcrBlock.cs | 33 ++ .../Ocr/OcrBoundingRegion.cs | 72 ++++ .../Ocr/OcrClientExtensions.cs | 60 +++ .../Ocr/OcrClientMetadata.cs | 38 ++ .../Ocr/OcrOptions.cs | 35 ++ .../Ocr/OcrPage.cs | 42 ++ .../Ocr/OcrProgress.cs | 25 ++ .../Ocr/OcrResult.cs | 55 +++ .../Ocr/OcrTable.cs | 51 +++ .../Ocr/OcrTableCell.cs | 43 +++ .../Ocr/OcrUsage.cs | 18 + .../Utilities/AIJsonUtilities.Defaults.cs | 6 + .../Microsoft.Extensions.AI.json | 150 ++++++++ .../Ocr/ConfigureOptionsOcrClient.cs | 58 +++ ...figureOptionsOcrClientBuilderExtensions.cs | 37 ++ .../Ocr/LoggingOcrClient.cs | 124 ++++++ .../Ocr/LoggingOcrClientBuilderExtensions.cs | 57 +++ .../Ocr/OcrClientBuilder.cs | 82 ++++ .../OcrClientBuilderOcrClientExtensions.cs | 27 ++ ...lientBuilderServiceCollectionExtensions.cs | 77 ++++ .../Ocr/OpenTelemetryOcrClient.cs | 241 ++++++++++++ ...OpenTelemetryOcrClientBuilderExtensions.cs | 43 +++ .../OpenTelemetryConsts.cs | 6 + src/Shared/DiagnosticIds/DiagnosticIds.cs | 1 + .../Ocr/DelegatingOcrClientTests.cs | 118 ++++++ .../Ocr/OcrBoundingRegionTests.cs | 38 ++ .../Ocr/OcrClientExtensionsTests.cs | 65 ++++ .../Ocr/OcrClientMetadataTests.cs | 29 ++ .../Ocr/OcrOptionsTests.cs | 42 ++ .../Ocr/OcrResultTests.cs | 31 ++ .../TestOcrClient.cs | 49 +++ .../Microsoft.Extensions.AI.Tests.csproj | 1 + .../Ocr/ConfigureOptionsOcrClientTests.cs | 71 ++++ .../Ocr/LoggingOcrClientTests.cs | 87 +++++ .../Ocr/OcrClientBuilderTests.cs | 104 +++++ .../OcrClientDependencyInjectionPatterns.cs | 107 ++++++ .../Ocr/OpenTelemetryOcrClientTests.cs | 101 +++++ 40 files changed, 2720 insertions(+), 1 deletion(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientMetadata.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClientBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClientBuilderExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilder.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderOcrClientExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClientBuilderExtensions.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientMetadataTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index 2d105bc2a32..c1be9fb1ecc 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -1835,6 +1835,38 @@ } ] }, + { + "Type": "class Microsoft.Extensions.AI.DelegatingOcrClient : Microsoft.Extensions.AI.IOcrClient, System.IDisposable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.DelegatingOcrClient.DelegatingOcrClient(Microsoft.Extensions.AI.IOcrClient innerClient);", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.AI.DelegatingOcrClient.Dispose();", + "Stage": "Experimental" + }, + { + "Member": "virtual void Microsoft.Extensions.AI.DelegatingOcrClient.Dispose(bool disposing);", + "Stage": "Experimental" + }, + { + "Member": "virtual object? Microsoft.Extensions.AI.DelegatingOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Stage": "Experimental" + }, + { + "Member": "virtual System.Threading.Tasks.Task Microsoft.Extensions.AI.DelegatingOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.IOcrClient Microsoft.Extensions.AI.DelegatingOcrClient.InnerClient { get; }", + "Stage": "Experimental" + } + ] + }, { "Type": "class Microsoft.Extensions.AI.DelegatingRealtimeClient : Microsoft.Extensions.AI.IRealtimeClient, System.IDisposable", "Stage": "Experimental", @@ -3164,6 +3196,20 @@ } ] }, + { + "Type": "interface Microsoft.Extensions.AI.IOcrClient : System.IDisposable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "object? Microsoft.Extensions.AI.IOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Stage": "Experimental" + }, + { + "Member": "System.Threading.Tasks.Task Microsoft.Extensions.AI.IOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ] + }, { "Type": "interface Microsoft.Extensions.AI.IRealtimeClient : System.IDisposable", "Stage": "Experimental", @@ -3296,6 +3342,316 @@ } ] }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrBlock", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrBlock.OcrBlock(string text);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrBlock.BoundingRegion { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "double? Microsoft.Extensions.AI.OcrBlock.Confidence { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrBlock.Kind { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrBlock.Text { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrBoundingRegion", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion.OcrBoundingRegion(int pageNumber, System.Collections.Generic.IReadOnlyList polygon);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrBoundingRegion Microsoft.Extensions.AI.OcrBoundingRegion.FromRectangle(int pageNumber, double left, double top, double right, double bottom);", + "Stage": "Experimental" + }, + { + "Member": "(float Left, float Top, float Right, float Bottom) Microsoft.Extensions.AI.OcrBoundingRegion.GetBounds();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "int Microsoft.Extensions.AI.OcrBoundingRegion.PageNumber { get; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrBoundingRegion.Polygon { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.AI.OcrClientExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static TService? Microsoft.Extensions.AI.OcrClientExtensions.GetService(this Microsoft.Extensions.AI.IOcrClient client, object? serviceKey = null);", + "Stage": "Experimental" + }, + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.GetTextAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.AI.OcrClientMetadata", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrClientMetadata.OcrClientMetadata(string? providerName = null, System.Uri? providerUri = null, string? defaultModelId = null);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "string? Microsoft.Extensions.AI.OcrClientMetadata.DefaultModelId { get; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrClientMetadata.ProviderName { get; }", + "Stage": "Experimental" + }, + { + "Member": "System.Uri? Microsoft.Extensions.AI.OcrClientMetadata.ProviderUri { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrOptions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrOptions.OcrOptions();", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrOptions Microsoft.Extensions.AI.OcrOptions.Clone();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrOptions.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrOptions.IncludeImages { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrOptions.ModelId { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrPage", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrPage.OcrPage(int index, string markdown);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrPage.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Blocks { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "double? Microsoft.Extensions.AI.OcrPage.Confidence { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.AI.OcrPage.Index { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrPage.Markdown { get; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Tables { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrProgress", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrProgress.OcrProgress();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "int? Microsoft.Extensions.AI.OcrProgress.PagesProcessed { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrProgress.Status { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.AI.OcrProgress.TotalPages { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrResult", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrResult.OcrResult(System.Collections.Generic.IReadOnlyList pages);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrResult.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrResult.Markdown { get; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrResult.ModelId { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrResult.OcrSource { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrResult.Pages { get; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.AI.OcrResult.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrUsage? Microsoft.Extensions.AI.OcrResult.Usage { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrTable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrTable.OcrTable(int rowCount, int columnCount, System.Collections.Generic.IReadOnlyList? cells = null, string? markdownRepresentation = null);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrTable.BoundingRegion { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IReadOnlyList? Microsoft.Extensions.AI.OcrTable.Cells { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.AI.OcrTable.ColumnCount { get; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrTable.MarkdownRepresentation { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.AI.OcrTable.RowCount { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrTableCell", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrTableCell.OcrTableCell(int rowIndex, int columnIndex, string content);", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "int Microsoft.Extensions.AI.OcrTableCell.ColumnIndex { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.AI.OcrTableCell.ColumnSpan { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrTableCell.Content { get; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrTableCell.Kind { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.AI.OcrTableCell.RowIndex { get; }", + "Stage": "Experimental" + }, + { + "Member": "int Microsoft.Extensions.AI.OcrTableCell.RowSpan { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrUsage", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrUsage.OcrUsage();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrUsage.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.AI.OcrUsage.PagesProcessed { get; set; }", + "Stage": "Experimental" + } + ] + }, { "Type": "class Microsoft.Extensions.AI.OutputTextAudioRealtimeServerMessage : Microsoft.Extensions.AI.RealtimeServerMessage", "Stage": "Experimental", @@ -4851,4 +5207,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs new file mode 100644 index 00000000000..fac27c2eef5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides an optional base class for an that passes through calls to another instance. +/// +/// This is recommended as a base type when building clients that can be chained in any order around an +/// underlying . The default implementation simply passes each call to the inner +/// client instance. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public class DelegatingOcrClient : IOcrClient +{ + /// Initializes a new instance of the class. + /// The wrapped client instance. + /// is . + protected DelegatingOcrClient(IOcrClient innerClient) + { + InnerClient = Throw.IfNull(innerClient); + } + + /// Gets the inner . + protected IOcrClient InnerClient { get; } + + /// + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// + public virtual Task GetTextAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + return InnerClient.GetTextAsync(document, mediaType, options, progress, cancellationToken); + } + + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + // If the key is non-null, we don't know what it means so pass through to the inner service. + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + InnerClient.GetService(serviceType, serviceKey); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + InnerClient.Dispose(); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs new file mode 100644 index 00000000000..e9d201dd6cf --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents an optical character recognition (OCR) / document-parsing client. +/// +/// +/// An transcribes a document or image into structured output: markdown, +/// per-page content, tables, layout blocks with bounding regions, and confidence. It is the +/// capability sibling to , IEmbeddingGenerator, and +/// ISpeechToTextClient for the document-extraction problem. +/// +/// +/// The contract is independent of . Most OCR / document-AI engines are not +/// chat models: they emit structured output (tables, bounding regions, confidence, reading order) +/// that does not map onto a chat response. An implementation may wrap a vision-capable +/// as the lowest-fidelity, transcription-only path, but the interface does +/// not require one. +/// +/// +/// Unless otherwise specified, all members of are thread-safe for concurrent +/// use. Implementations might mutate the supplied to ; +/// consumers should avoid sharing a single options instance across concurrent invocations when that is a +/// concern. The document stream passed to is not disposed by the implementation. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public interface IOcrClient : IDisposable +{ + /// Runs OCR / document parsing over a document stream and returns structured output. + /// The document or image content to parse. + /// The media type of , for example application/pdf or image/png. + /// The OCR options to configure the request. + /// + /// An optional progress reporter. Engines that poll a long-running operation (such as Azure Document + /// Intelligence) can report page-by-page progress; synchronous engines report once. Keeping the call + /// unary while exposing provides observability without modeling a batch + /// operation as a stream. + /// + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + Task GetTextAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be + /// provided by the , including itself or any services it might be wrapping. + /// + object? GetService(Type serviceType, object? serviceKey = null); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs new file mode 100644 index 00000000000..81689a8efff --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents a positioned layout block, such as a paragraph, heading, or figure. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrBlock +{ + /// Initializes a new instance of the class. + /// The text content of the block. + /// is . + public OcrBlock(string text) + { + Text = Throw.IfNull(text); + } + + /// Gets the text content of the block. + public string Text { get; } + + /// Gets or sets the kind of block, for example paragraph, title, or figure. + public string? Kind { get; set; } + + /// Gets or sets the region of the page the block occupies, when the engine provides geometry. + public OcrBoundingRegion? BoundingRegion { get; set; } + + /// Gets or sets the confidence for the block in the range [0, 1], when available. + public double? Confidence { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs new file mode 100644 index 00000000000..e6e49885bbd --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents a positioned region on a page. +/// +/// The region is a polygon (a flattened, clockwise sequence of [x1, y1, x2, y2, ...] vertices) +/// so it can faithfully carry a possibly rotation-skewed quadrilateral, such as Azure Document +/// Intelligence's BoundingRegion.Polygon, without loss. Engines that emit only an axis-aligned +/// rectangle (such as Mistral OCR) can convert via . The same type is reused +/// for layout-block geometry and for field grounding, providing one region primitive across providers. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrBoundingRegion +{ + /// Initializes a new instance of the class. + /// The one-based page number the region is on. + /// The flattened, clockwise polygon vertices. + /// is . + public OcrBoundingRegion(int pageNumber, IReadOnlyList polygon) + { + PageNumber = pageNumber; + Polygon = Throw.IfNull(polygon); + } + + /// Gets the one-based page number the region is on. + /// A region can reference a different page than its parent element. + public int PageNumber { get; } + + /// Gets the flattened polygon vertices [x1, y1, x2, y2, ...], in clockwise order. + /// An Azure Document Intelligence quadrilateral is eight floats. + public IReadOnlyList Polygon { get; } + + /// Builds a clockwise quadrilateral region from an axis-aligned rectangle. + /// The one-based page number the region is on. + /// The left coordinate. + /// The top coordinate. + /// The right coordinate. + /// The bottom coordinate. + /// A region whose polygon is the four corners of the rectangle. + public static OcrBoundingRegion FromRectangle(int pageNumber, double left, double top, double right, double bottom) + => new(pageNumber, + [ + (float)left, (float)top, + (float)right, (float)top, + (float)right, (float)bottom, + (float)left, (float)bottom, + ]); + + /// Computes the axis-aligned bounds of the polygon. + /// The minimum and maximum coordinates of the polygon. + public (float Left, float Top, float Right, float Bottom) GetBounds() + { + float minX = float.MaxValue, minY = float.MaxValue, maxX = float.MinValue, maxY = float.MinValue; + for (int i = 0; i + 1 < Polygon.Count; i += 2) + { + minX = Math.Min(minX, Polygon[i]); + maxX = Math.Max(maxX, Polygon[i]); + minY = Math.Min(minY, Polygon[i + 1]); + maxY = Math.Max(maxY, Polygon[i + 1]); + } + + return (minX, minY, maxX, maxY); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs new file mode 100644 index 00000000000..02f74cd8258 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extension methods for . +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OcrClientExtensions +{ + /// Asks the for an object of type . + /// The type of the object to be retrieved. + /// The client. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly typed services that might be + /// provided by the , including itself or any services it might be wrapping. + /// + public static TService? GetService(this IOcrClient client, object? serviceKey = null) + { + _ = Throw.IfNull(client); + + return (TService?)client.GetService(typeof(TService), serviceKey); + } + + /// Runs OCR over a single document provided as a . + /// The client. + /// The document content to parse. + /// The OCR options to configure the request. + /// An optional progress reporter. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + /// or is . + public static Task GetTextAsync( + this IOcrClient client, + DataContent document, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + var documentStream = MemoryMarshal.TryGetArray(document.Data, out var array) ? + new MemoryStream(array.Array!, array.Offset, array.Count) : + new MemoryStream(document.Data.ToArray()); + + return client.GetTextAsync(documentStream, document.MediaType, options, progress, cancellationToken); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientMetadata.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientMetadata.cs new file mode 100644 index 00000000000..800f4160d74 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientMetadata.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Provides metadata about an . +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public class OcrClientMetadata +{ + /// Initializes a new instance of the class. + /// The name of the OCR provider, if applicable. + /// The URL for accessing the OCR provider, if applicable. + /// The identifier of the model used by default, if applicable. + public OcrClientMetadata(string? providerName = null, Uri? providerUri = null, string? defaultModelId = null) + { + DefaultModelId = defaultModelId; + ProviderName = providerName; + ProviderUri = providerUri; + } + + /// Gets the name of the OCR provider. + public string? ProviderName { get; } + + /// Gets the URL for accessing the OCR provider. + public Uri? ProviderUri { get; } + + /// Gets the identifier of the default model used by this OCR client. + /// + /// This value can be if the name is unknown or if there are multiple possible + /// models associated with this instance. An individual request can override this value via + /// . + /// + public string? DefaultModelId { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs new file mode 100644 index 00000000000..ab2c8ba0f2f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents the options to configure an OCR request. +/// +/// Normalized options common to engines, plus an bag for +/// provider-specific settings, mirroring ChatOptions. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrOptions +{ + /// Gets or sets the model or deployment identifier to use for this request. + public string? ModelId { get; set; } + + /// Gets or sets a value indicating whether the engine should include rendered images inline, when supported. + public bool IncludeImages { get; set; } + + /// Gets or sets any additional provider-specific request settings. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } + + /// Produces a clone of the current instance. + /// A shallow clone of the options instance. + public OcrOptions Clone() => + new() + { + ModelId = ModelId, + IncludeImages = IncludeImages, + AdditionalProperties = AdditionalProperties?.Clone(), + }; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs new file mode 100644 index 00000000000..41aa69692d7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents one page of structured OCR output. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrPage +{ + /// Initializes a new instance of the class. + /// The zero-based page index. + /// The structured markdown for this page. + /// is . + public OcrPage(int index, string markdown) + { + Index = index; + Markdown = Throw.IfNull(markdown); + } + + /// Gets the zero-based page index. + public int Index { get; } + + /// Gets the structured markdown for this page, with headings, tables, and reading order preserved. + public string Markdown { get; } + + /// Gets or sets the tables extracted from this page. + public IReadOnlyList Tables { get; set; } = []; + + /// Gets or sets the layout blocks with bounding regions and confidence, when the engine provides them. + public IReadOnlyList Blocks { get; set; } = []; + + /// Gets or sets the page-level confidence in the range [0, 1], when available. + public double? Confidence { get; set; } + + /// Gets or sets any additional properties associated with the page. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs new file mode 100644 index 00000000000..c351295b035 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents progress reported during a long-running OCR request. +/// +/// Engines that poll a long-running operation (such as Azure Document Intelligence) report pages as +/// they complete; synchronous engines (such as Mistral OCR) report a single terminal update. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrProgress +{ + /// Gets or sets the number of pages processed so far, when known. + public int? PagesProcessed { get; set; } + + /// Gets or sets the total number of pages, when known. + public int? TotalPages { get; set; } + + /// Gets or sets a human-readable status for the operation, when available. + public string? Status { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs new file mode 100644 index 00000000000..1c52b722172 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs @@ -0,0 +1,55 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents the structured result of an OCR / document-parsing request. +/// +/// The result normalizes the content common to every engine (markdown, pages, tables, bounding +/// regions, confidence) while preserving everything provider-specific via +/// and , mirroring how +/// ChatResponse normalizes the common surface and preserves the raw. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrResult +{ + /// Initializes a new instance of the class. + /// The per-page structured content. + /// is . + public OcrResult(IReadOnlyList pages) + { + Pages = Throw.IfNull(pages); + } + + /// Gets the per-page structured content (markdown, tables, blocks, confidence). + public IReadOnlyList Pages { get; } + + /// Gets the full-document markdown, formed by joining the per-page markdown. + public string Markdown => string.Join("\n\n", Pages.Select(p => p.Markdown)); + + /// Gets or sets an identifier for the engine that produced this result. + /// This typically flows downstream as an ocr_source metadata value. + public string? OcrSource { get; set; } + + /// Gets or sets the model or deployment identifier that served the request. + public string? ModelId { get; set; } + + /// Gets or sets usage details associated with the request. + public OcrUsage? Usage { get; set; } + + /// Gets or sets the provider-native object underlying this result. + /// + /// The escape hatch for provider richness that does not map onto the normalized surface, mirroring + /// ChatResponse.RawRepresentation. Nothing is lost. + /// + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the result. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs new file mode 100644 index 00000000000..6de145996b6 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents a table extracted from a document. +/// +/// Cells are the primary, structured representation (row and column indices with spans, the Azure +/// Document Intelligence shape). is the fallback for engines that +/// only emit markdown or HTML (such as Mistral OCR). Consumers prefer when present +/// and fall back to otherwise. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrTable +{ + /// Initializes a new instance of the class. + /// The number of rows in the table. + /// The number of columns in the table. + /// The structured cells, or when only markdown is available. + /// The markdown or HTML representation, or when cells are available. + public OcrTable( + int rowCount, + int columnCount, + IReadOnlyList? cells = null, + string? markdownRepresentation = null) + { + RowCount = rowCount; + ColumnCount = columnCount; + Cells = cells; + MarkdownRepresentation = markdownRepresentation; + } + + /// Gets the number of rows in the table. + public int RowCount { get; } + + /// Gets the number of columns in the table. + public int ColumnCount { get; } + + /// Gets the structured cells, or when the engine only returned markdown. + public IReadOnlyList? Cells { get; } + + /// Gets the markdown or HTML table text, or when only cells were returned. + public string? MarkdownRepresentation { get; } + + /// Gets or sets the region of the page the table occupies, when the engine provides geometry. + public OcrBoundingRegion? BoundingRegion { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs new file mode 100644 index 00000000000..53165f1aeb4 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents a single cell within an . +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrTableCell +{ + /// Initializes a new instance of the class. + /// The zero-based row index of the cell. + /// The zero-based column index of the cell. + /// The text content of the cell. + /// is . + public OcrTableCell(int rowIndex, int columnIndex, string content) + { + RowIndex = rowIndex; + ColumnIndex = columnIndex; + Content = Throw.IfNull(content); + } + + /// Gets or sets the role of the cell, for example columnHeader or content. + public string? Kind { get; set; } + + /// Gets the zero-based row index of the cell. + public int RowIndex { get; } + + /// Gets the zero-based column index of the cell. + public int ColumnIndex { get; } + + /// Gets or sets the number of rows the cell spans. The default is 1. + public int RowSpan { get; set; } = 1; + + /// Gets or sets the number of columns the cell spans. The default is 1. + public int ColumnSpan { get; set; } = 1; + + /// Gets the text content of the cell. + public string Content { get; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs new file mode 100644 index 00000000000..bfea283448c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents usage details associated with an OCR request. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrUsage +{ + /// Gets or sets the number of pages processed by the request, when known. + public int? PagesProcessed { get; set; } + + /// Gets or sets any additional provider-specific usage details. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 70370c0170b..379a8c96674 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -139,6 +139,12 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(ImageGenerationOptions))] [JsonSerializable(typeof(ImageGenerationResponse))] + // IOcrClient + [JsonSerializable(typeof(OcrOptions))] + [JsonSerializable(typeof(OcrClientMetadata))] + [JsonSerializable(typeof(OcrResult))] + [JsonSerializable(typeof(OcrProgress))] + // IHostedFileClient [JsonSerializable(typeof(HostedFileClientOptions))] [JsonSerializable(typeof(HostedFileClientMetadata))] diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json index 9efb90882b9..3437a3ab9fe 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json @@ -273,6 +273,30 @@ } ] }, + { + "Type": "sealed class Microsoft.Extensions.AI.ConfigureOptionsOcrClient : Microsoft.Extensions.AI.DelegatingOcrClient", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ConfigureOptionsOcrClient(Microsoft.Extensions.AI.IOcrClient innerClient, System.Action configure);", + "Stage": "Experimental" + }, + { + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.ConfigureOptionsOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.AI.ConfigureOptionsOcrClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.AI.ConfigureOptionsOcrClientBuilderExtensions.ConfigureOptions(this Microsoft.Extensions.AI.OcrClientBuilder builder, System.Action configure);", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.ConfigureOptionsSpeechToTextClient : Microsoft.Extensions.AI.DelegatingSpeechToTextClient", "Stage": "Experimental", @@ -1019,6 +1043,36 @@ } ] }, + { + "Type": "class Microsoft.Extensions.AI.LoggingOcrClient : Microsoft.Extensions.AI.DelegatingOcrClient", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.LoggingOcrClient.LoggingOcrClient(Microsoft.Extensions.AI.IOcrClient innerClient, Microsoft.Extensions.Logging.ILogger logger);", + "Stage": "Experimental" + }, + { + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.LoggingOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "System.Text.Json.JsonSerializerOptions Microsoft.Extensions.AI.LoggingOcrClient.JsonSerializerOptions { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.AI.LoggingOcrClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.AI.LoggingOcrClientBuilderExtensions.UseLogging(this Microsoft.Extensions.AI.OcrClientBuilder builder, Microsoft.Extensions.Logging.ILoggerFactory? loggerFactory = null, System.Action? configure = null);", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.LoggingRealtimeClient : Microsoft.Extensions.AI.DelegatingRealtimeClient", "Stage": "Experimental", @@ -1131,6 +1185,64 @@ } ] }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrClientBuilder", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrClientBuilder.OcrClientBuilder(Microsoft.Extensions.AI.IOcrClient innerClient);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrClientBuilder.OcrClientBuilder(System.Func innerClientFactory);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.IOcrClient Microsoft.Extensions.AI.OcrClientBuilder.Build(System.IServiceProvider? services = null);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.AI.OcrClientBuilder.Use(System.Func clientFactory);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.AI.OcrClientBuilder.Use(System.Func clientFactory);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.AI.OcrClientBuilderOcrClientExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.AI.OcrClientBuilderOcrClientExtensions.AsBuilder(this Microsoft.Extensions.AI.IOcrClient innerClient);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.DependencyInjection.OcrClientBuilderServiceCollectionExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.DependencyInjection.OcrClientBuilderServiceCollectionExtensions.AddKeyedOcrClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, object? serviceKey, Microsoft.Extensions.AI.IOcrClient innerClient, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.DependencyInjection.OcrClientBuilderServiceCollectionExtensions.AddKeyedOcrClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, object? serviceKey, System.Func innerClientFactory, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.DependencyInjection.OcrClientBuilderServiceCollectionExtensions.AddOcrClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, Microsoft.Extensions.AI.IOcrClient innerClient, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.DependencyInjection.OcrClientBuilderServiceCollectionExtensions.AddOcrClient(this Microsoft.Extensions.DependencyInjection.IServiceCollection serviceCollection, System.Func innerClientFactory, Microsoft.Extensions.DependencyInjection.ServiceLifetime lifetime = Microsoft.Extensions.DependencyInjection.ServiceLifetime.Singleton);", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.OpenTelemetryChatClient : Microsoft.Extensions.AI.DelegatingChatClient", "Stage": "Stable", @@ -1307,6 +1419,44 @@ } ] }, + { + "Type": "sealed class Microsoft.Extensions.AI.OpenTelemetryOcrClient : Microsoft.Extensions.AI.DelegatingOcrClient", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OpenTelemetryOcrClient.OpenTelemetryOcrClient(Microsoft.Extensions.AI.IOcrClient innerClient, Microsoft.Extensions.Logging.ILogger? logger = null, string? sourceName = null);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.AI.OpenTelemetryOcrClient.Dispose(bool disposing);", + "Stage": "Experimental" + }, + { + "Member": "override object? Microsoft.Extensions.AI.OpenTelemetryOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Stage": "Experimental" + }, + { + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.OpenTelemetryOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "bool Microsoft.Extensions.AI.OpenTelemetryOcrClient.EnableSensitiveData { get; set; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "static class Microsoft.Extensions.AI.OpenTelemetryOcrClientBuilderExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.AI.OcrClientBuilder Microsoft.Extensions.AI.OpenTelemetryOcrClientBuilderExtensions.UseOpenTelemetry(this Microsoft.Extensions.AI.OcrClientBuilder builder, Microsoft.Extensions.Logging.ILoggerFactory? loggerFactory = null, string? sourceName = null, System.Action? configure = null);", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.OpenTelemetryRealtimeClient : Microsoft.Extensions.AI.DelegatingRealtimeClient", "Stage": "Experimental", diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs new file mode 100644 index 00000000000..45b6f0bf984 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents a delegating OCR client that configures an instance used by the remainder of the pipeline. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class ConfigureOptionsOcrClient : DelegatingOcrClient +{ + /// The callback delegate used to configure options. + private readonly Action _configureOptions; + + /// Initializes a new instance of the class with the specified callback. + /// The inner client. + /// + /// The delegate to invoke to configure the instance. It is passed a clone of the caller-supplied instance + /// (or a newly constructed instance if the caller-supplied instance is ). + /// + /// + /// The delegate is passed either a new instance of if + /// the caller didn't supply an instance, or a clone (via ) of the caller-supplied + /// instance if one was supplied. + /// + public ConfigureOptionsOcrClient(IOcrClient innerClient, Action configure) + : base(innerClient) + { + _configureOptions = Throw.IfNull(configure); + } + + /// + public override async Task GetTextAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + return await base.GetTextAsync(document, mediaType, Configure(options), progress, cancellationToken); + } + + /// Creates and configures the to pass along to the inner client. + private OcrOptions Configure(OcrOptions? options) + { + options = options?.Clone() ?? new(); + + _configureOptions(options); + + return options; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClientBuilderExtensions.cs new file mode 100644 index 00000000000..42e997d0d2c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClientBuilderExtensions.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class ConfigureOptionsOcrClientBuilderExtensions +{ + /// + /// Adds a callback that configures an to be passed to the next client in the pipeline. + /// + /// The . + /// + /// The delegate to invoke to configure the instance. + /// It is passed a clone of the caller-supplied instance (or a newly constructed instance if the caller-supplied instance is ). + /// + /// + /// This method can be used to set default options. The delegate is passed either a new instance of + /// if the caller didn't supply an instance, or a clone (via ) + /// of the caller-supplied instance if one was supplied. + /// + /// The . + public static OcrClientBuilder ConfigureOptions( + this OcrClientBuilder builder, Action configure) + { + _ = Throw.IfNull(builder); + _ = Throw.IfNull(configure); + + return builder.Use(innerClient => new ConfigureOptionsOcrClient(innerClient, configure)); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs new file mode 100644 index 00000000000..fd24cbb3de2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs @@ -0,0 +1,124 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A delegating OCR client that logs OCR operations to an . +/// +/// +/// The provided implementation of is thread-safe for concurrent use so long as the +/// employed is also thread-safe for concurrent use. +/// +/// +/// When the employed enables , the contents of +/// options and results are logged. These may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Options and results are not logged at other logging levels. +/// +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public partial class LoggingOcrClient : DelegatingOcrClient +{ + /// An instance used for all logging. + private readonly ILogger _logger; + + /// The to use for serialization of state written to the logger. + private JsonSerializerOptions _jsonSerializerOptions; + + /// Initializes a new instance of the class. + /// The underlying . + /// An instance that will be used for all logging. + public LoggingOcrClient(IOcrClient innerClient, ILogger logger) + : base(innerClient) + { + _logger = Throw.IfNull(logger); + _jsonSerializerOptions = AIJsonUtilities.DefaultOptions; + } + + /// Gets or sets JSON serialization options to use when serializing logging data. + public JsonSerializerOptions JsonSerializerOptions + { + get => _jsonSerializerOptions; + set => _jsonSerializerOptions = Throw.IfNull(value); + } + + /// + public override async Task GetTextAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokedSensitive(nameof(GetTextAsync), mediaType, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(GetTextAsync)); + } + } + + try + { + var result = await base.GetTextAsync(document, mediaType, options, progress, cancellationToken); + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogCompletedSensitive(nameof(GetTextAsync), AsJson(result)); + } + else + { + LogCompleted(nameof(GetTextAsync)); + } + } + + return result; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(GetTextAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(GetTextAsync), ex); + throw; + } + } + + private string AsJson(T value) => TelemetryHelpers.AsJson(value, _jsonSerializerOptions); + + [LoggerMessage(LogLevel.Debug, "{MethodName} invoked.")] + private partial void LogInvoked(string methodName); + + [LoggerMessage(LogLevel.Trace, "{MethodName} invoked: MediaType: {MediaType}. Options: {OcrOptions}. Metadata: {OcrClientMetadata}.")] + private partial void LogInvokedSensitive(string methodName, string mediaType, string ocrOptions, string ocrClientMetadata); + + [LoggerMessage(LogLevel.Debug, "{MethodName} completed.")] + private partial void LogCompleted(string methodName); + + [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {OcrResult}.")] + private partial void LogCompletedSensitive(string methodName, string ocrResult); + + [LoggerMessage(LogLevel.Debug, "{MethodName} canceled.")] + private partial void LogInvocationCanceled(string methodName); + + [LoggerMessage(LogLevel.Error, "{MethodName} failed.")] + private partial void LogInvocationFailed(string methodName, Exception error); +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClientBuilderExtensions.cs new file mode 100644 index 00000000000..85b53df5e35 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClientBuilderExtensions.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class LoggingOcrClientBuilderExtensions +{ + /// Adds logging to the OCR client pipeline. + /// The . + /// + /// An optional used to create a logger with which logging should be performed. + /// If not supplied, a required instance will be resolved from the service provider. + /// + /// An optional callback that can be used to configure the instance. + /// The . + /// + /// + /// When the employed enables , the contents of + /// options and results are logged. These may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Options and results are not logged at other logging levels. + /// + /// + public static OcrClientBuilder UseLogging( + this OcrClientBuilder builder, + ILoggerFactory? loggerFactory = null, + Action? configure = null) + { + _ = Throw.IfNull(builder); + + return builder.Use((innerClient, services) => + { + loggerFactory ??= services.GetRequiredService(); + + // If the factory we resolve is for the null logger, the LoggingOcrClient will end up + // being an expensive nop, so skip adding it and just return the inner client. + if (loggerFactory == NullLoggerFactory.Instance) + { + return innerClient; + } + + var ocrClient = new LoggingOcrClient(innerClient, loggerFactory.CreateLogger(typeof(LoggingOcrClient))); + configure?.Invoke(ocrClient); + return ocrClient; + }); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilder.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilder.cs new file mode 100644 index 00000000000..89c122fa353 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilder.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// A builder for creating pipelines of . +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrClientBuilder +{ + private readonly Func _innerClientFactory; + + /// The registered client factory instances. + private List>? _clientFactories; + + /// Initializes a new instance of the class. + /// The inner that represents the underlying backend. + public OcrClientBuilder(IOcrClient innerClient) + { + _ = Throw.IfNull(innerClient); + _innerClientFactory = _ => innerClient; + } + + /// Initializes a new instance of the class. + /// A callback that produces the inner that represents the underlying backend. + public OcrClientBuilder(Func innerClientFactory) + { + _innerClientFactory = Throw.IfNull(innerClientFactory); + } + + /// Builds an that represents the entire pipeline. Calls to this instance will pass through each of the pipeline stages in turn. + /// + /// The that should provide services to the instances. + /// If null, an empty will be used. + /// + /// An instance of that represents the entire pipeline. + public IOcrClient Build(IServiceProvider? services = null) + { + services ??= EmptyServiceProvider.Instance; + var ocrClient = _innerClientFactory(services); + + // To match intuitive expectations, apply the factories in reverse order, so that the first factory added is the outermost. + if (_clientFactories is not null) + { + for (var i = _clientFactories.Count - 1; i >= 0; i--) + { + ocrClient = _clientFactories[i](ocrClient, services) ?? + throw new InvalidOperationException( + $"The {nameof(OcrClientBuilder)} entry at index {i} returned null. " + + $"Ensure that the callbacks passed to {nameof(Use)} return non-null {nameof(IOcrClient)} instances."); + } + } + + return ocrClient; + } + + /// Adds a factory for an intermediate OCR client to the OCR client pipeline. + /// The client factory function. + /// The updated instance. + public OcrClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + return Use((innerClient, _) => clientFactory(innerClient)); + } + + /// Adds a factory for an intermediate OCR client to the OCR client pipeline. + /// The client factory function. + /// The updated instance. + public OcrClientBuilder Use(Func clientFactory) + { + _ = Throw.IfNull(clientFactory); + + (_clientFactories ??= []).Add(clientFactory); + return this; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderOcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderOcrClientExtensions.cs new file mode 100644 index 00000000000..2ddb4861c50 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderOcrClientExtensions.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extension methods for working with in the context of . +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OcrClientBuilderOcrClientExtensions +{ + /// Creates a new using as its inner client. + /// The client to use as the inner client. + /// The new instance. + /// + /// This method is equivalent to using the constructor directly, + /// specifying as the inner client. + /// + public static OcrClientBuilder AsBuilder(this IOcrClient innerClient) + { + _ = Throw.IfNull(innerClient); + + return new OcrClientBuilder(innerClient); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs new file mode 100644 index 00000000000..24d6e6a6e2f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DependencyInjection; + +/// Provides extension methods for registering with a . +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OcrClientBuilderServiceCollectionExtensions +{ + /// Registers a singleton in the . + /// The to which the client should be added. + /// The inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + public static OcrClientBuilder AddOcrClient( + this IServiceCollection serviceCollection, + IOcrClient innerClient, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddOcrClient(serviceCollection, _ => innerClient, lifetime); + + /// Registers a singleton in the . + /// The to which the client should be added. + /// A callback that produces the inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + public static OcrClientBuilder AddOcrClient( + this IServiceCollection serviceCollection, + Func innerClientFactory, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + _ = Throw.IfNull(serviceCollection); + _ = Throw.IfNull(innerClientFactory); + + var builder = new OcrClientBuilder(innerClientFactory); + serviceCollection.Add(new ServiceDescriptor(typeof(IOcrClient), builder.Build, lifetime)); + return builder; + } + + /// Registers a keyed singleton in the . + /// The to which the client should be added. + /// The key with which to associate the client. + /// The inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + public static OcrClientBuilder AddKeyedOcrClient( + this IServiceCollection serviceCollection, + object? serviceKey, + IOcrClient innerClient, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + => AddKeyedOcrClient(serviceCollection, serviceKey, _ => innerClient, lifetime); + + /// Registers a keyed singleton in the . + /// The to which the client should be added. + /// The key with which to associate the client. + /// A callback that produces the inner that represents the underlying backend. + /// The service lifetime for the client. Defaults to . + /// An that can be used to build a pipeline around the inner client. + public static OcrClientBuilder AddKeyedOcrClient( + this IServiceCollection serviceCollection, + object? serviceKey, + Func innerClientFactory, + ServiceLifetime lifetime = ServiceLifetime.Singleton) + { + _ = Throw.IfNull(serviceCollection); + _ = Throw.IfNull(innerClientFactory); + + var builder = new OcrClientBuilder(innerClientFactory); + serviceCollection.Add(new ServiceDescriptor(typeof(IOcrClient), serviceKey, factory: (services, serviceKey) => builder.Build(services), lifetime)); + return builder; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs new file mode 100644 index 00000000000..052a96d6e81 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs @@ -0,0 +1,241 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents a delegating OCR client that implements the OpenTelemetry Semantic Conventions for Generative AI systems. +/// +/// This class provides an implementation of the Semantic Conventions for Generative AI systems v1.41, defined at . +/// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OpenTelemetryOcrClient : DelegatingOcrClient +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + + private readonly Histogram _operationDurationHistogram; + + private readonly string? _defaultModelId; + private readonly string? _providerName; + private readonly string? _serverAddress; + private readonly int _serverPort; + + private readonly ILogger? _logger; + + /// Initializes a new instance of the class. + /// The underlying . + /// The to use for emitting any logging data from the client. + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryOcrClient(IOcrClient innerClient, ILogger? logger = null, string? sourceName = null) + : base(innerClient) + { + Debug.Assert(innerClient is not null, "Should have been validated by the base ctor"); + + _logger = logger; + + if (innerClient!.GetService() is OcrClientMetadata metadata) + { + _defaultModelId = metadata.DefaultModelId; + _providerName = metadata.ProviderName; + _serverAddress = metadata.ProviderUri?.Host; + _serverPort = metadata.ProviderUri?.Port ?? 0; + } + + string name = string.IsNullOrEmpty(sourceName) ? OpenTelemetryConsts.DefaultSourceName : sourceName!; + _activitySource = new(name); + _meter = new(name); + + _operationDurationHistogram = OtelMetricHelpers.CreateGenAIOperationDurationHistogram(_meter); + } + + /// + protected override void Dispose(bool disposing) + { + if (disposing) + { + _activitySource.Dispose(); + _meter.Dispose(); + } + + base.Dispose(disposing); + } + + /// + /// Gets or sets a value indicating whether potentially sensitive information should be included in telemetry. + /// + /// + /// if potentially sensitive information should be included in telemetry; + /// if telemetry shouldn't include raw inputs and outputs. + /// The default value is , unless the OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT + /// environment variable is set to "true" (case-insensitive). + /// + /// + /// By default, telemetry includes metadata, such as page counts, but not raw inputs + /// and outputs, such as document content. The default value can be overridden by setting the + /// OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT environment variable to "true". + /// Explicitly setting this property will override the environment variable. + /// + public bool EnableSensitiveData { get; set; } = TelemetryHelpers.EnableSensitiveDataDefault; + + /// + public override object? GetService(Type serviceType, object? serviceKey = null) => + serviceType == typeof(ActivitySource) ? _activitySource : + base.GetService(serviceType, serviceKey); + + /// + public override async Task GetTextAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(document); + + using Activity? activity = CreateAndConfigureActivity(options); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + string? requestModelId = options?.ModelId ?? _defaultModelId; + + OcrResult? response = null; + Exception? error = null; + try + { + response = await base.GetTextAsync(document, mediaType, options, progress, cancellationToken); + return response; + } + catch (Exception ex) + { + error = ex; + throw; + } + finally + { + TraceResponse(activity, requestModelId, response, error, stopwatch); + } + } + + /// Creates an activity for an OCR request, or returns if not enabled. + private Activity? CreateAndConfigureActivity(OcrOptions? options) + { + Activity? activity = null; + if (_activitySource.HasListeners()) + { + string? modelId = options?.ModelId ?? _defaultModelId; + + activity = _activitySource.StartActivity( + string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.GenerateContentName : $"{OpenTelemetryConsts.GenAI.GenerateContentName} {modelId}", + ActivityKind.Client); + + if (activity is { IsAllDataRequested: true }) + { + _ = activity + .AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName) + .AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId) + .AddTag(OpenTelemetryConsts.GenAI.Provider.Name, _providerName) + .AddTag(OpenTelemetryConsts.GenAI.Output.Type, OpenTelemetryConsts.TypeText); + + if (_serverAddress is not null) + { + _ = activity + .AddTag(OpenTelemetryConsts.Server.Address, _serverAddress) + .AddTag(OpenTelemetryConsts.Server.Port, _serverPort); + } + + if (EnableSensitiveData && options?.AdditionalProperties is { } props) + { + // Log all additional request options as raw values on the span. + // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag(prop.Key, prop.Value); + } + } + } + } + + return activity; + } + + /// Adds OCR response information to the activity. + private void TraceResponse( + Activity? activity, + string? requestModelId, + OcrResult? response, + Exception? error, + Stopwatch? stopwatch) + { + if (_operationDurationHistogram.Enabled && stopwatch is not null) + { + TagList tags = default; + + AddMetricTags(ref tags, requestModelId, response); + if (error is not null) + { + tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); + } + + _operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + + OpenTelemetryLog.RecordOperationError(activity, _logger, error); + + if (response is not null && activity is not null) + { + if (response.ModelId is not null) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Model, response.ModelId); + } + + if (response.Usage?.PagesProcessed is int pages) + { + _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.PagesProcessed, pages); + } + + // Log all additional response properties as raw values on the span. + // Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data. + if (EnableSensitiveData && response.AdditionalProperties is { } props) + { + foreach (KeyValuePair prop in props) + { + _ = activity.AddTag(prop.Key, prop.Value); + } + } + } + + void AddMetricTags(ref TagList tags, string? requestModelId, OcrResult? response) + { + tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName); + + if (requestModelId is not null) + { + tags.Add(OpenTelemetryConsts.GenAI.Request.Model, requestModelId); + } + + tags.Add(OpenTelemetryConsts.GenAI.Provider.Name, _providerName); + + if (_serverAddress is string endpointAddress) + { + tags.Add(OpenTelemetryConsts.Server.Address, endpointAddress); + tags.Add(OpenTelemetryConsts.Server.Port, _serverPort); + } + + if (response?.ModelId is string responseModel) + { + tags.Add(OpenTelemetryConsts.GenAI.Response.Model, responseModel); + } + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClientBuilderExtensions.cs new file mode 100644 index 00000000000..d3f47ac4d55 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClientBuilderExtensions.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extensions for configuring instances. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OpenTelemetryOcrClientBuilderExtensions +{ + /// + /// Adds OpenTelemetry support to the OCR client pipeline, following the OpenTelemetry Semantic Conventions for Generative AI systems. + /// + /// + /// The draft specification this follows is available at . + /// The specification is still experimental and subject to change; as such, the telemetry output by this client is also subject to change. + /// + /// The . + /// An optional to use to create a logger for logging events. + /// An optional source name that will be used on the telemetry data. + /// An optional callback that can be used to configure the instance. + /// The . + public static OcrClientBuilder UseOpenTelemetry( + this OcrClientBuilder builder, + ILoggerFactory? loggerFactory = null, + string? sourceName = null, + Action? configure = null) => + Throw.IfNull(builder).Use((innerClient, services) => + { + loggerFactory ??= services.GetService(); + + var client = new OpenTelemetryOcrClient(innerClient, loggerFactory?.CreateLogger(typeof(OpenTelemetryOcrClient)), sourceName); + configure?.Invoke(client); + + return client; + }); +} diff --git a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs index 804d0d9f684..a60952a5f9f 100644 --- a/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs +++ b/src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs @@ -173,6 +173,12 @@ public static class Usage public const string OutputAudioTokens = "gen_ai.usage.output_audio_tokens"; public const string OutputTextTokens = "gen_ai.usage.output_text_tokens"; public const string ReasoningOutputTokens = "gen_ai.usage.reasoning.output_tokens"; + + /// + /// Number of document pages processed by an OCR request. + /// This attribute is NOT part of the OpenTelemetry GenAI semantic conventions (as of v1.41). + /// + public const string PagesProcessed = "gen_ai.usage.pages_processed"; // Non-standard } /// diff --git a/src/Shared/DiagnosticIds/DiagnosticIds.cs b/src/Shared/DiagnosticIds/DiagnosticIds.cs index 4b4925850a4..50ca7f56a3a 100644 --- a/src/Shared/DiagnosticIds/DiagnosticIds.cs +++ b/src/Shared/DiagnosticIds/DiagnosticIds.cs @@ -58,6 +58,7 @@ internal static class Experiments internal const string AIChatReduction = AIExperiments; internal const string AIToolSearch = AIExperiments; internal const string AIRealTime = AIExperiments; + internal const string AIOcr = AIExperiments; internal const string AIFiles = AIExperiments; internal const string AIOpenAIRequestPolicies = AIExperiments; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs new file mode 100644 index 00000000000..54f58233707 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class DelegatingOcrClientTests +{ + [Fact] + public void RequiresInnerOcrClient() + { + Assert.Throws("innerClient", () => new NoOpDelegatingOcrClient(null!)); + } + + [Fact] + public async Task GetTextAsyncDefaultsToInnerClientAsync() + { + // Arrange + using var expectedDocument = new MemoryStream(); + var expectedMediaType = "application/pdf"; + var expectedOptions = new OcrOptions(); + var expectedCancellationToken = CancellationToken.None; + var expectedResult = new TaskCompletionSource(); + var expectedResponse = new OcrResult([]); + using var inner = new TestOcrClient + { + GetTextAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + { + Assert.Same(expectedDocument, document); + Assert.Same(expectedMediaType, mediaType); + Assert.Same(expectedOptions, options); + Assert.Equal(expectedCancellationToken, cancellationToken); + return expectedResult.Task; + } + }; + + using var delegating = new NoOpDelegatingOcrClient(inner); + + // Act + var resultTask = delegating.GetTextAsync(expectedDocument, expectedMediaType, expectedOptions, null, expectedCancellationToken); + + // Assert + Assert.False(resultTask.IsCompleted); + expectedResult.SetResult(expectedResponse); + Assert.True(resultTask.IsCompleted); + Assert.Same(expectedResponse, await resultTask); + } + + [Fact] + public void GetServiceThrowsForNullType() + { + using var inner = new TestOcrClient(); + using var delegating = new NoOpDelegatingOcrClient(inner); + Assert.Throws("serviceType", () => delegating.GetService(null!)); + } + + [Fact] + public void GetServiceReturnsSelfIfCompatibleWithRequestAndKeyIsNull() + { + // Arrange + using var inner = new TestOcrClient(); + using var delegating = new NoOpDelegatingOcrClient(inner); + + // Act + var client = delegating.GetService(); + + // Assert + Assert.Same(delegating, client); + } + + [Fact] + public void GetServiceDelegatesToInnerIfKeyIsNotNull() + { + // Arrange + var expectedKey = new object(); + using var expectedResult = new TestOcrClient(); + using var inner = new TestOcrClient + { + GetServiceCallback = (_, _) => expectedResult + }; + using var delegating = new NoOpDelegatingOcrClient(inner); + + // Act + var client = delegating.GetService(expectedKey); + + // Assert + Assert.Same(expectedResult, client); + } + + [Fact] + public void GetServiceDelegatesToInnerIfNotCompatibleWithRequest() + { + // Arrange + var expectedResult = TimeZoneInfo.Local; + var expectedKey = new object(); + using var inner = new TestOcrClient + { + GetServiceCallback = (type, key) => type == expectedResult.GetType() && key == expectedKey + ? expectedResult + : throw new InvalidOperationException("Unexpected call") + }; + using var delegating = new NoOpDelegatingOcrClient(inner); + + // Act + var tzi = delegating.GetService(expectedKey); + + // Assert + Assert.Same(expectedResult, tzi); + } + + private sealed class NoOpDelegatingOcrClient(IOcrClient innerClient) + : DelegatingOcrClient(innerClient); +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs new file mode 100644 index 00000000000..4a1d35efa27 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrBoundingRegionTests +{ + [Fact] + public void Constructor_NullPolygon_Throws() + { + Assert.Throws("polygon", () => new OcrBoundingRegion(1, null!)); + } + + [Fact] + public void FromRectangle_ProducesClockwiseQuadrilateral() + { + var region = OcrBoundingRegion.FromRectangle(2, left: 10, top: 20, right: 110, bottom: 220); + + Assert.Equal(2, region.PageNumber); + Assert.Equal(new float[] { 10, 20, 110, 20, 110, 220, 10, 220 }, region.Polygon); + } + + [Fact] + public void GetBounds_ReturnsAxisAlignedExtents() + { + var region = new OcrBoundingRegion(1, [30, 40, 100, 35, 110, 90, 25, 95]); + + var (left, top, right, bottom) = region.GetBounds(); + + Assert.Equal(25, left); + Assert.Equal(35, top); + Assert.Equal(110, right); + Assert.Equal(95, bottom); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs new file mode 100644 index 00000000000..4091972138b --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrClientExtensionsTests +{ + [Fact] + public void GetService_InvalidArgs_Throws() + { + Assert.Throws("client", () => + { + _ = OcrClientExtensions.GetService(null!); + }); + } + + [Fact] + public async Task GetTextAsync_InvalidArgs_Throws() + { + IOcrClient? client = null; + var content = new DataContent("data:application/pdf;base64,AQIDBA=="); + var ex1 = await Assert.ThrowsAsync(() => OcrClientExtensions.GetTextAsync(client!, content)); + Assert.Equal("client", ex1.ParamName); + + using var testClient = new TestOcrClient(); + DataContent? nullContent = null; + var ex2 = await Assert.ThrowsAsync(() => OcrClientExtensions.GetTextAsync(testClient, nullContent!)); + Assert.Equal("document", ex2.ParamName); + } + + [Fact] + public async Task GetTextAsync_DataContent_PassesStreamAndMediaTypeAsync() + { + // Arrange + var expectedResponse = new OcrResult([new OcrPage(0, "hello")]); + string? observedMediaType = null; + byte[]? observedBytes = null; + + using var client = new TestOcrClient + { + GetTextAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => + { + observedMediaType = mediaType; + using var ms = new MemoryStream(); + await document.CopyToAsync(ms, cancellationToken); + observedBytes = ms.ToArray(); + return expectedResponse; + } + }; + + // Act + var result = await OcrClientExtensions.GetTextAsync(client, new DataContent("data:application/pdf;base64,AQIDBA==")); + + // Assert + Assert.Same(expectedResponse, result); + Assert.Equal("application/pdf", observedMediaType); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, observedBytes); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientMetadataTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientMetadataTests.cs new file mode 100644 index 00000000000..7abb58d33b7 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientMetadataTests.cs @@ -0,0 +1,29 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrClientMetadataTests +{ + [Fact] + public void Constructor_NullValues_AllowedAndRoundtrip() + { + OcrClientMetadata metadata = new(null, null, null); + Assert.Null(metadata.ProviderName); + Assert.Null(metadata.ProviderUri); + Assert.Null(metadata.DefaultModelId); + } + + [Fact] + public void Constructor_Value_Roundtrips() + { + var uri = new Uri("https://example.com"); + OcrClientMetadata metadata = new("providerName", uri, "theModel"); + Assert.Equal("providerName", metadata.ProviderName); + Assert.Same(uri, metadata.ProviderUri); + Assert.Equal("theModel", metadata.DefaultModelId); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs new file mode 100644 index 00000000000..8d2ccfcf786 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrOptionsTests +{ + [Fact] + public void Clone_CopiesAllProperties() + { + var options = new OcrOptions + { + ModelId = "mistral-ocr-4-0", + IncludeImages = true, + AdditionalProperties = new() { ["custom"] = "value" }, + }; + + var clone = options.Clone(); + + Assert.NotSame(options, clone); + Assert.Equal("mistral-ocr-4-0", clone.ModelId); + Assert.True(clone.IncludeImages); + Assert.NotNull(clone.AdditionalProperties); + Assert.Equal("value", clone.AdditionalProperties!["custom"]); + } + + [Fact] + public void Clone_DeepCopiesAdditionalProperties() + { + var options = new OcrOptions + { + AdditionalProperties = new() { ["key"] = "original" }, + }; + + var clone = options.Clone(); + clone.AdditionalProperties!["key"] = "changed"; + + Assert.Equal("original", options.AdditionalProperties!["key"]); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs new file mode 100644 index 00000000000..efddd0357e0 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrResultTests +{ + [Fact] + public void Constructor_NullPages_Throws() + { + Assert.Throws("pages", () => new OcrResult(null!)); + } + + [Fact] + public void Markdown_JoinsPerPageMarkdown() + { + var result = new OcrResult([new OcrPage(0, "page one"), new OcrPage(1, "page two")]) + { + OcrSource = "test-engine", + ModelId = "model-1", + }; + + Assert.Equal("page one\n\npage two", result.Markdown); + Assert.Equal("test-engine", result.OcrSource); + Assert.Equal("model-1", result.ModelId); + Assert.Equal(2, result.Pages.Count); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs new file mode 100644 index 00000000000..c73da94dabe --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.AI; + +public sealed class TestOcrClient : IOcrClient +{ + public TestOcrClient() + { + GetServiceCallback = DefaultGetServiceCallback; + } + + public IServiceProvider? Services { get; set; } + + public Func?, + CancellationToken, + Task>? + GetTextAsyncCallback + { get; set; } + + public Func GetServiceCallback { get; set; } + + private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) + => serviceType is not null && serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; + + public Task GetTextAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + => GetTextAsyncCallback!.Invoke(document, mediaType, options, progress, cancellationToken); + + public object? GetService(Type serviceType, object? serviceKey = null) + => GetServiceCallback!.Invoke(serviceType, serviceKey); + + public void Dispose() + { + // Dispose of resources if any. + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj index 6c805e270c8..2089b8462e3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Microsoft.Extensions.AI.Tests.csproj @@ -23,6 +23,7 @@ + diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs new file mode 100644 index 00000000000..f7c8cfe060c --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class ConfigureOptionsOcrClientTests +{ + [Fact] + public void ConfigureOptionsOcrClient_InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new ConfigureOptionsOcrClient(null!, _ => { })); + Assert.Throws("configure", () => new ConfigureOptionsOcrClient(new TestOcrClient(), null!)); + } + + [Fact] + public void ConfigureOptions_InvalidArgs_Throws() + { + using var innerClient = new TestOcrClient(); + var builder = innerClient.AsBuilder(); + Assert.Throws("configure", () => builder.ConfigureOptions(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullProvidedOptions) + { + OcrOptions? providedOptions = nullProvidedOptions ? null : new() { ModelId = "test" }; + OcrOptions? returnedOptions = null; + OcrResult expectedResult = new([new OcrPage(0, "blue whale")]); + using CancellationTokenSource cts = new(); + + using IOcrClient innerClient = new TestOcrClient + { + GetTextAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + { + Assert.Same(returnedOptions, options); + Assert.Equal(cts.Token, cancellationToken); + return Task.FromResult(expectedResult); + }, + }; + + using var client = innerClient + .AsBuilder() + .ConfigureOptions(options => + { + Assert.NotSame(providedOptions, options); + if (nullProvidedOptions) + { + Assert.Null(options.ModelId); + } + else + { + Assert.Equal(providedOptions!.ModelId, options.ModelId); + } + + returnedOptions = options; + }) + .Build(); + + using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); + var result = await client.GetTextAsync(document, "application/pdf", providedOptions, null, cts.Token); + Assert.Same(expectedResult, result); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs new file mode 100644 index 00000000000..b9c66ac3caf --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Testing; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class LoggingOcrClientTests +{ + [Fact] + public void LoggingOcrClient_InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new LoggingOcrClient(null!, NullLogger.Instance)); + Assert.Throws("logger", () => new LoggingOcrClient(new TestOcrClient(), null!)); + } + + [Fact] + public void UseLogging_AvoidsInjectingNopClient() + { + using var innerClient = new TestOcrClient(); + + Assert.Null(innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(LoggingOcrClient))); + Assert.Same(innerClient, innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build().GetService(typeof(IOcrClient))); + + using var factory = LoggerFactory.Create(b => b.AddFakeLogging()); + Assert.NotNull(innerClient.AsBuilder().UseLogging(factory).Build().GetService(typeof(LoggingOcrClient))); + + ServiceCollection c = new(); + c.AddFakeLogging(); + var services = c.BuildServiceProvider(); + Assert.NotNull(innerClient.AsBuilder().UseLogging().Build(services).GetService(typeof(LoggingOcrClient))); + Assert.NotNull(innerClient.AsBuilder().UseLogging(null).Build(services).GetService(typeof(LoggingOcrClient))); + Assert.Null(innerClient.AsBuilder().UseLogging(NullLoggerFactory.Instance).Build(services).GetService(typeof(LoggingOcrClient))); + } + + [Theory] + [InlineData(LogLevel.Trace)] + [InlineData(LogLevel.Debug)] + [InlineData(LogLevel.Information)] + public async Task GetTextAsync_LogsInvocationAndCompletion(LogLevel level) + { + var collector = new FakeLogCollector(); + + ServiceCollection c = new(); + c.AddLogging(b => b.AddProvider(new FakeLoggerProvider(collector)).SetMinimumLevel(level)); + var services = c.BuildServiceProvider(); + + using IOcrClient innerClient = new TestOcrClient + { + GetTextAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + Task.FromResult(new OcrResult([new OcrPage(0, "blue whale")])), + }; + + using IOcrClient client = innerClient + .AsBuilder() + .UseLogging() + .Build(services); + + using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); + await client.GetTextAsync(document, "application/pdf", new OcrOptions { ModelId = "mistral-ocr-4-0" }); + + var logs = collector.GetSnapshot(); + if (level is LogLevel.Trace) + { + Assert.Collection(logs, + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} invoked:") && entry.Message.Contains("mistral-ocr-4-0")), + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} completed:") && entry.Message.Contains("blue whale"))); + } + else if (level is LogLevel.Debug) + { + Assert.Collection(logs, + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} invoked.") && !entry.Message.Contains("mistral-ocr-4-0")), + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} completed.") && !entry.Message.Contains("blue whale"))); + } + else + { + Assert.Empty(logs); + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs new file mode 100644 index 00000000000..63b07818a5d --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs @@ -0,0 +1,104 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrClientBuilderTests +{ + [Fact] + public void PassingNullInnerClientThrows() + { + Assert.Throws("innerClient", () => new OcrClientBuilder((IOcrClient)null!)); + Assert.Throws("innerClientFactory", () => new OcrClientBuilder((Func)null!)); + } + + [Fact] + public void BuildReturnsInnerClientWhenNoMiddleware() + { + using var inner = new TestOcrClient(); + var builder = inner.AsBuilder(); + + var built = builder.Build(); + + Assert.Same(inner, built); + } + + [Fact] + public void UseAppliesFactoriesInReverseOrderSoFirstAddedIsOutermost() + { + // Arrange + using var inner = new TestOcrClient(); + var order = new List(); + + var built = inner.AsBuilder() + .Use(c => + { + order.Add("outer-built"); + return new InspectorOcrClient(c, "outer", order); + }) + .Use(c => + { + order.Add("inner-built"); + return new InspectorOcrClient(c, "inner", order); + }) + .Build(); + + // The first factory added should be the outermost wrapper. + var outer = Assert.IsType(built); + Assert.Equal("outer", outer.Name); + var innerWrapper = Assert.IsType(outer.InnerClientPublic); + Assert.Equal("inner", innerWrapper.Name); + Assert.Same(inner, innerWrapper.InnerClientPublic); + + // Reverse-order application: inner factory runs before outer factory. + Assert.Equal(["inner-built", "outer-built"], order); + } + + [Fact] + public void BuildThrowsWhenFactoryReturnsNull() + { + using var inner = new TestOcrClient(); + var builder = inner.AsBuilder().Use(_ => null!); + + Assert.Throws(() => builder.Build()); + } + + [Fact] + public void UseNullFactoryThrows() + { + using var inner = new TestOcrClient(); + var builder = inner.AsBuilder(); + Assert.Throws("clientFactory", () => builder.Use((Func)null!)); + Assert.Throws("clientFactory", () => builder.Use((Func)null!)); + } + + [Fact] + public void ServicesAreFlowedThroughBuild() + { + using var inner = new TestOcrClient(); + IServiceProvider? observed = null; + + var services = new ServiceCollection().BuildServiceProvider(); + _ = inner.AsBuilder() + .Use((c, sp) => + { + observed = sp; + return c; + }) + .Build(services); + + Assert.Same(services, observed); + } + + private sealed class InspectorOcrClient(IOcrClient inner, string name, List order) : DelegatingOcrClient(inner) + { + public string Name => name; + public IOcrClient InnerClientPublic => base.InnerClient; + public List Order => order; + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs new file mode 100644 index 00000000000..ae8f5c9ea64 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrClientDependencyInjectionPatterns +{ + private IServiceCollection ServiceCollection { get; } = new ServiceCollection(); + + [Fact] + public void CanRegisterSingletonUsingFactory() + { + ServiceCollection.AddOcrClient(services => new TestOcrClient { Services = services }) + .Use((inner, services) => new SingletonMiddleware(inner, services)); + + var services = ServiceCollection.BuildServiceProvider(); + using var scope1 = services.CreateScope(); + using var scope2 = services.CreateScope(); + + var instance1 = scope1.ServiceProvider.GetRequiredService(); + var instance1Copy = scope1.ServiceProvider.GetRequiredService(); + var instance2 = scope2.ServiceProvider.GetRequiredService(); + + var instance = Assert.IsType(instance1); + Assert.Same(instance, instance1Copy); + Assert.Same(instance, instance2); + Assert.IsType(instance.InnerClientPublic); + } + + [Fact] + public void CanRegisterKeyedSingletonUsingSharedInstance() + { + using var singleton = new TestOcrClient(); + ServiceCollection.AddKeyedOcrClient("mykey", singleton) + .Use((inner, services) => new SingletonMiddleware(inner, services)); + + var services = ServiceCollection.BuildServiceProvider(); + using var scope1 = services.CreateScope(); + using var scope2 = services.CreateScope(); + + Assert.Null(services.GetService()); + + var instance1 = scope1.ServiceProvider.GetRequiredKeyedService("mykey"); + var instance1Copy = scope1.ServiceProvider.GetRequiredKeyedService("mykey"); + var instance2 = scope2.ServiceProvider.GetRequiredKeyedService("mykey"); + + var instance = Assert.IsType(instance1); + Assert.Same(instance, instance1Copy); + Assert.Same(instance, instance2); + Assert.IsType(instance.InnerClientPublic); + } + + [Theory] + [InlineData(null)] + [InlineData(ServiceLifetime.Singleton)] + [InlineData(ServiceLifetime.Scoped)] + [InlineData(ServiceLifetime.Transient)] + public void AddOcrClient_RegistersExpectedLifetime(ServiceLifetime? lifetime) + { + ServiceCollection sc = new(); + ServiceLifetime expectedLifetime = lifetime ?? ServiceLifetime.Singleton; + _ = lifetime.HasValue + ? sc.AddOcrClient(services => new TestOcrClient(), lifetime.Value) + : sc.AddOcrClient(services => new TestOcrClient()); + + ServiceDescriptor sd = Assert.Single(sc); + Assert.Equal(typeof(IOcrClient), sd.ServiceType); + Assert.False(sd.IsKeyedService); + Assert.Null(sd.ImplementationInstance); + Assert.NotNull(sd.ImplementationFactory); + Assert.IsType(sd.ImplementationFactory!(null!)); + Assert.Equal(expectedLifetime, sd.Lifetime); + } + + [Theory] + [InlineData(null)] + [InlineData(ServiceLifetime.Singleton)] + [InlineData(ServiceLifetime.Scoped)] + [InlineData(ServiceLifetime.Transient)] + public void AddKeyedOcrClient_RegistersExpectedLifetime(ServiceLifetime? lifetime) + { + ServiceCollection sc = new(); + ServiceLifetime expectedLifetime = lifetime ?? ServiceLifetime.Singleton; + _ = lifetime.HasValue + ? sc.AddKeyedOcrClient("key", services => new TestOcrClient(), lifetime.Value) + : sc.AddKeyedOcrClient("key", services => new TestOcrClient()); + + ServiceDescriptor sd = Assert.Single(sc); + Assert.Equal(typeof(IOcrClient), sd.ServiceType); + Assert.True(sd.IsKeyedService); + Assert.Equal("key", sd.ServiceKey); + Assert.Null(sd.KeyedImplementationInstance); + Assert.NotNull(sd.KeyedImplementationFactory); + Assert.IsType(sd.KeyedImplementationFactory!(null!, null!)); + Assert.Equal(expectedLifetime, sd.Lifetime); + } + + public class SingletonMiddleware(IOcrClient inner, IServiceProvider services) : DelegatingOcrClient(inner) + { + public IOcrClient InnerClientPublic => base.InnerClient; + public IServiceProvider Services => services; + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs new file mode 100644 index 00000000000..201aca196c5 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs @@ -0,0 +1,101 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using OpenTelemetry.Trace; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OpenTelemetryOcrClientTests +{ + [Fact] + public void InvalidArgs_Throws() + { + Assert.Throws("innerClient", () => new OpenTelemetryOcrClient(null!)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) + { + var sourceName = Guid.NewGuid().ToString(); + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var innerClient = new TestOcrClient + { + GetTextAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => + { + await Task.Yield(); + return new OcrResult([new OcrPage(0, "This is the recognized text.")]) + { + ModelId = "amazingmodel", + Usage = new() { PagesProcessed = 3 }, + }; + }, + + GetServiceCallback = (serviceType, serviceKey) => + serviceType == typeof(OcrClientMetadata) ? new OcrClientMetadata("testservice", new Uri("http://localhost:12345/something"), "amazingmodel") : + null, + }; + + using var client = innerClient + .AsBuilder() + .UseOpenTelemetry(null, sourceName, configure: instance => + { + instance.EnableSensitiveData = enableSensitiveData; + }) + .Build(); + + OcrOptions options = new() + { + ModelId = "mycoolocrmodel", + AdditionalProperties = new() + { + ["service_tier"] = "value1", + ["SomethingElse"] = "value2", + }, + }; + + _ = await client.GetTextAsync(Stream.Null, "application/pdf", options); + + var activity = Assert.Single(activities); + + Assert.NotNull(activity.Id); + Assert.NotEmpty(activity.Id); + + Assert.Equal("localhost", activity.GetTagItem("server.address")); + Assert.Equal(12345, (int)activity.GetTagItem("server.port")!); + + Assert.Equal("generate_content mycoolocrmodel", activity.DisplayName); + Assert.Equal("testservice", activity.GetTagItem("gen_ai.provider.name")); + + Assert.Equal("mycoolocrmodel", activity.GetTagItem("gen_ai.request.model")); + Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("service_tier")); + Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("SomethingElse")); + + Assert.Equal("amazingmodel", activity.GetTagItem("gen_ai.response.model")); + Assert.Equal(3, (int)activity.GetTagItem("gen_ai.usage.pages_processed")!); + + Assert.True(activity.Duration.TotalMilliseconds > 0); + } + + [Fact] + public void GetService_ReturnsActivitySource() + { + using var innerClient = new TestOcrClient(); + using var client = innerClient.AsBuilder().UseOpenTelemetry().Build(); + + Assert.NotNull(client.GetService()); + } +} From b89c7da6dd7485b57bdf1ef170b6a31364f2f04f Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 1 Jul 2026 13:58:33 -0400 Subject: [PATCH 02/13] Add OcrImage type and OcrPage.Images for extracted figures Adds an optional OcrImage sink so engines can return page images/figures when OcrOptions.IncludeImages is requested. Every member is optional: document-native engines (Mistral OCR inline images, Azure Document Intelligence figures) populate Content with rendered bytes; a vision-LLM transcriber may populate only Caption. One shape serves both provider archetypes. --- .../Ocr/OcrImage.cs | 31 +++++++++++++++++++ .../Ocr/OcrPage.cs | 3 ++ 2 files changed, 34 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs new file mode 100644 index 00000000000..2e522d95207 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents an image or figure extracted from a page during OCR. +/// +/// Populated when is requested and the engine supports it. Every +/// member is optional so each implementer fills what it can provide: document-native engines (for +/// example Mistral OCR inline images, or Azure Document Intelligence figures) populate +/// with the rendered image bytes, whereas a vision-LLM transcriber that cannot emit bytes may instead +/// populate only . This lets one shape serve both provider archetypes. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public sealed class OcrImage +{ + /// Gets or sets the rendered image bytes, when the engine returns them. + public DataContent? Content { get; set; } + + /// Gets or sets the region of the page the image occupies, when the engine provides geometry. + public OcrBoundingRegion? BoundingRegion { get; set; } + + /// Gets or sets a caption or description of the image, when available. + public string? Caption { get; set; } + + /// Gets or sets the confidence for the image in the range [0, 1], when available. + public double? Confidence { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs index 41aa69692d7..3b3f9b70945 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -34,6 +34,9 @@ public OcrPage(int index, string markdown) /// Gets or sets the layout blocks with bounding regions and confidence, when the engine provides them. public IReadOnlyList Blocks { get; set; } = []; + /// Gets or sets the images or figures extracted from this page, when requested and the engine provides them. + public IReadOnlyList Images { get; set; } = []; + /// Gets or sets the page-level confidence in the range [0, 1], when available. public double? Confidence { get; set; } From 996b4426f00067ffc6061f794719974b29265b84 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 1 Jul 2026 13:58:33 -0400 Subject: [PATCH 03/13] Add UriContent overload to OcrClientExtensions Symmetric with the existing DataContent overload. Handles self-contained data: URIs by delegating to the DataContent path and does no file/network IO. For file: and remote URIs it throws NotSupportedException, leaving native URL passthrough (Mistral document_url, Azure DI uriSource) as an explicit open design question. --- .../Ocr/OcrClientExtensions.cs | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs index 02f74cd8258..a8f17cd5868 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs @@ -57,4 +57,44 @@ public static Task GetTextAsync( return client.GetTextAsync(documentStream, document.MediaType, options, progress, cancellationToken); } + + /// Runs OCR over a single document referenced by a . + /// The client. + /// The document reference to parse. + /// The OCR options to configure the request. + /// An optional progress reporter. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + /// or is . + /// The references a remote URI, which this overload does not fetch. + /// + /// This overload handles self-contained data: URIs by delegating to the + /// overload. It intentionally does no file or network IO: for + /// file: and remote (http/https) URIs it throws, because whether to read/download + /// the bytes or hand the URL to the engine natively (for example Mistral document_url or Azure + /// Document Intelligence uriSource) is an open design question the abstraction does not decide. + /// For those, read the bytes yourself and pass a or , or + /// use an engine that accepts a URL directly. + /// + public static Task GetTextAsync( + this IOcrClient client, + UriContent document, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + Uri uri = document.Uri; + if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) + { + // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. + return client.GetTextAsync(new DataContent(uri), options, progress, cancellationToken); + } + + throw new NotSupportedException( + "This overload handles only self-contained data: URIs. For file or remote URIs, read the bytes " + + "and pass a stream or DataContent, or use an engine that accepts a URL natively."); + } } From ac530936905012e2164f18ce740c2cac38be250b Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 1 Jul 2026 15:24:15 -0400 Subject: [PATCH 04/13] Rename IOcrClient.GetTextAsync to ExtractAsync The method returns a rich OcrResult (markdown, tables, figures/images, blocks, confidence, language), not plain text, so GetTextAsync was a misnomer. ExtractAsync names the document-extraction operation accurately. Scoped to the Ocr/ types only; ISpeechToTextClient.GetTextAsync is unrelated and unchanged. Alternatives considered for maintainer discussion: GetDocumentAsync, AnalyzeAsync. --- .../Ocr/DelegatingOcrClient.cs | 4 ++-- .../Ocr/IOcrClient.cs | 6 +++--- .../Ocr/OcrClientExtensions.cs | 8 ++++---- .../Ocr/ConfigureOptionsOcrClient.cs | 4 ++-- .../Ocr/LoggingOcrClient.cs | 16 ++++++++-------- .../Ocr/OpenTelemetryOcrClient.cs | 4 ++-- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs index fac27c2eef5..bddc3befa58 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs @@ -39,14 +39,14 @@ public void Dispose() } /// - public virtual Task GetTextAsync( + public virtual Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default) { - return InnerClient.GetTextAsync(document, mediaType, options, progress, cancellationToken); + return InnerClient.ExtractAsync(document, mediaType, options, progress, cancellationToken); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs index e9d201dd6cf..c3be2d25d43 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs @@ -27,9 +27,9 @@ namespace Microsoft.Extensions.AI; /// /// /// Unless otherwise specified, all members of are thread-safe for concurrent -/// use. Implementations might mutate the supplied to ; +/// use. Implementations might mutate the supplied to ; /// consumers should avoid sharing a single options instance across concurrent invocations when that is a -/// concern. The document stream passed to is not disposed by the implementation. +/// concern. The document stream passed to is not disposed by the implementation. /// /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] @@ -47,7 +47,7 @@ public interface IOcrClient : IDisposable /// /// The to monitor for cancellation requests. The default is . /// The structured OCR result. - Task GetTextAsync( + Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs index a8f17cd5868..fe87a90e96f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs @@ -41,7 +41,7 @@ public static class OcrClientExtensions /// The to monitor for cancellation requests. The default is . /// The structured OCR result. /// or is . - public static Task GetTextAsync( + public static Task ExtractAsync( this IOcrClient client, DataContent document, OcrOptions? options = null, @@ -55,7 +55,7 @@ public static Task GetTextAsync( new MemoryStream(array.Array!, array.Offset, array.Count) : new MemoryStream(document.Data.ToArray()); - return client.GetTextAsync(documentStream, document.MediaType, options, progress, cancellationToken); + return client.ExtractAsync(documentStream, document.MediaType, options, progress, cancellationToken); } /// Runs OCR over a single document referenced by a . @@ -76,7 +76,7 @@ public static Task GetTextAsync( /// For those, read the bytes yourself and pass a or , or /// use an engine that accepts a URL directly. /// - public static Task GetTextAsync( + public static Task ExtractAsync( this IOcrClient client, UriContent document, OcrOptions? options = null, @@ -90,7 +90,7 @@ public static Task GetTextAsync( if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) { // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. - return client.GetTextAsync(new DataContent(uri), options, progress, cancellationToken); + return client.ExtractAsync(new DataContent(uri), options, progress, cancellationToken); } throw new NotSupportedException( diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs index 45b6f0bf984..239d767fdaa 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs @@ -36,14 +36,14 @@ public ConfigureOptionsOcrClient(IOcrClient innerClient, Action conf } /// - public override async Task GetTextAsync( + public override async Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default) { - return await base.GetTextAsync(document, mediaType, Configure(options), progress, cancellationToken); + return await base.ExtractAsync(document, mediaType, Configure(options), progress, cancellationToken); } /// Creates and configures the to pass along to the inner client. diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs index fd24cbb3de2..be9c2a36eb3 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs @@ -53,7 +53,7 @@ public JsonSerializerOptions JsonSerializerOptions } /// - public override async Task GetTextAsync( + public override async Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, @@ -64,27 +64,27 @@ public override async Task GetTextAsync( { if (_logger.IsEnabled(LogLevel.Trace)) { - LogInvokedSensitive(nameof(GetTextAsync), mediaType, AsJson(options), AsJson(this.GetService())); + LogInvokedSensitive(nameof(ExtractAsync), mediaType, AsJson(options), AsJson(this.GetService())); } else { - LogInvoked(nameof(GetTextAsync)); + LogInvoked(nameof(ExtractAsync)); } } try { - var result = await base.GetTextAsync(document, mediaType, options, progress, cancellationToken); + var result = await base.ExtractAsync(document, mediaType, options, progress, cancellationToken); if (_logger.IsEnabled(LogLevel.Debug)) { if (_logger.IsEnabled(LogLevel.Trace)) { - LogCompletedSensitive(nameof(GetTextAsync), AsJson(result)); + LogCompletedSensitive(nameof(ExtractAsync), AsJson(result)); } else { - LogCompleted(nameof(GetTextAsync)); + LogCompleted(nameof(ExtractAsync)); } } @@ -92,12 +92,12 @@ public override async Task GetTextAsync( } catch (OperationCanceledException) { - LogInvocationCanceled(nameof(GetTextAsync)); + LogInvocationCanceled(nameof(ExtractAsync)); throw; } catch (Exception ex) { - LogInvocationFailed(nameof(GetTextAsync), ex); + LogInvocationFailed(nameof(ExtractAsync), ex); throw; } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs index 052a96d6e81..eddfc78cfdb 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs @@ -96,7 +96,7 @@ protected override void Dispose(bool disposing) base.GetService(serviceType, serviceKey); /// - public override async Task GetTextAsync( + public override async Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, @@ -113,7 +113,7 @@ public override async Task GetTextAsync( Exception? error = null; try { - response = await base.GetTextAsync(document, mediaType, options, progress, cancellationToken); + response = await base.ExtractAsync(document, mediaType, options, progress, cancellationToken); return response; } catch (Exception ex) From e871445e9709c73c8465c36c5b55e8fa436c2691 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Thu, 2 Jul 2026 11:21:56 -0400 Subject: [PATCH 05/13] Add OcrImage to IngestionDocumentImage reader mapping --- .../OcrDocumentReader.cs | 109 ++++++++++++++++++ .../Readers/OcrDocumentReaderTests.cs | 78 +++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs create mode 100644 test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs new file mode 100644 index 00000000000..c62ab0a8389 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 // OCR abstractions are experimental. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.DataIngestion; + +/// +/// Reads documents by extracting structured OCR output using an . +/// +public sealed class OcrDocumentReader : IngestionDocumentReader +{ + private const string BoundingBoxMetadataKey = "bounding_box"; + private const string BoundingRegionMetadataKey = "bounding_region"; + + private readonly IOcrClient _ocrClient; + private readonly OcrOptions _options; + + /// + /// Initializes a new instance of the class. + /// + /// The OCR client to use for document extraction. + /// Optional OCR options. When not provided, image extraction is requested. + public OcrDocumentReader(IOcrClient ocrClient, OcrOptions? options = null) + { + _ocrClient = Throw.IfNull(ocrClient); + _options = options?.Clone() ?? new OcrOptions { IncludeImages = true }; + } + + /// + public override async Task ReadAsync(Stream source, string identifier, string mediaType, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(source); + _ = Throw.IfNullOrEmpty(identifier); + _ = Throw.IfNullOrEmpty(mediaType); + + OcrResult ocrResult = await _ocrClient + .ExtractAsync(source, mediaType, _options.Clone(), cancellationToken: cancellationToken) + .ConfigureAwait(false); + + return Map(ocrResult, identifier); + } + + private static IngestionDocument Map(OcrResult ocrResult, string identifier) + { + IngestionDocument document = new(identifier); + + foreach (OcrPage page in ocrResult.Pages) + { + IngestionDocumentSection section = new(); + int pageNumber = page.Index + 1; + + if (!string.IsNullOrWhiteSpace(page.Markdown)) + { + section.Elements.Add(new IngestionDocumentParagraph(page.Markdown) + { + Text = page.Markdown, + PageNumber = pageNumber + }); + } + + foreach (OcrImage image in page.Images) + { + section.Elements.Add(MapImage(image, pageNumber)); + } + + document.Sections.Add(section); + } + + return document; + } + + private static IngestionDocumentImage MapImage(OcrImage image, int pageNumber) + { + DataContent? content = image.Content; + IngestionDocumentImage element = new(CreateImageMarkdown(image)) + { + Content = content?.Data, + MediaType = content?.MediaType, + AlternativeText = image.Caption, + PageNumber = image.BoundingRegion?.PageNumber ?? pageNumber + }; + + if (image.BoundingRegion is not null) + { + (float left, float top, float right, float bottom) = image.BoundingRegion.GetBounds(); + element.Metadata[BoundingBoxMetadataKey] = new[] { left, top, right, bottom }; + element.Metadata[BoundingRegionMetadataKey] = image.BoundingRegion.Polygon.ToArray(); + } + + return element; + } + + private static string CreateImageMarkdown(OcrImage image) + { + string altText = image.Caption ?? string.Empty; + string uri = image.Content?.Uri ?? string.Empty; + + return $"![{altText}]({uri})"; + } +} diff --git a/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs new file mode 100644 index 00000000000..5dca4561ea5 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable MEAI001 // OCR abstractions are experimental. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Xunit; + +namespace Microsoft.Extensions.DataIngestion.Readers.Tests; + +public class OcrDocumentReaderTests +{ + [Fact] + public async Task MapsOcrImagesToIngestionDocumentImages() + { + byte[] imageBytes = [1, 2, 3, 4, 5]; + OcrResult ocrResult = new( + [ + new OcrPage(2, "Page text") + { + Images = + [ + new OcrImage + { + Content = new DataContent(imageBytes, "image/png"), + Caption = "Architecture diagram", + BoundingRegion = OcrBoundingRegion.FromRectangle(3, left: 1, top: 2, right: 10, bottom: 20) + } + ] + } + ]); + using TestOcrClient ocrClient = new(ocrResult); + OcrDocumentReader reader = new(ocrClient); + + using MemoryStream source = new([42]); + IngestionDocument document = await reader.ReadAsync(source, "doc-id", "application/pdf"); + + IngestionDocumentImage image = Assert.Single(document.EnumerateContent().OfType()); + Assert.Equal(imageBytes, image.Content?.ToArray()); + Assert.Equal("image/png", image.MediaType); + Assert.Equal("Architecture diagram", image.AlternativeText); + Assert.Equal(3, image.PageNumber); + Assert.Equal([1f, 2f, 10f, 20f], Assert.IsType(image.Metadata["bounding_box"])); + Assert.Equal([1f, 2f, 10f, 2f, 10f, 20f, 1f, 20f], Assert.IsType(image.Metadata["bounding_region"])); + Assert.Equal("application/pdf", ocrClient.MediaType); + Assert.True(ocrClient.Options?.IncludeImages); + } + + private sealed class TestOcrClient(OcrResult result) : IOcrClient + { + public string? MediaType { get; private set; } + + public OcrOptions? Options { get; private set; } + + public Task ExtractAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + MediaType = mediaType; + Options = options; + return Task.FromResult(result); + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } + } +} From ed72dc86881c023193ccb12b2fe4ea32f257a216 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 7 Jul 2026 14:51:07 -0400 Subject: [PATCH 06/13] Add ExtractFromUriAsync opt-in remote downloader to IOcrClient extensions --- .../Ocr/OcrClientExtensions.cs | 74 ++++++++++++++++++- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs index fe87a90e96f..8f7b2e2fdad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Net.Http; using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -73,8 +74,10 @@ public static Task ExtractAsync( /// file: and remote (http/https) URIs it throws, because whether to read/download /// the bytes or hand the URL to the engine natively (for example Mistral document_url or Azure /// Document Intelligence uriSource) is an open design question the abstraction does not decide. - /// For those, read the bytes yourself and pass a or , or - /// use an engine that accepts a URL directly. + /// To download a remote document explicitly, use + /// + /// with a caller-supplied ; or read the bytes yourself and pass a + /// or ; or use an engine that accepts a URL directly. /// public static Task ExtractAsync( this IOcrClient client, @@ -97,4 +100,71 @@ public static Task ExtractAsync( "This overload handles only self-contained data: URIs. For file or remote URIs, read the bytes " + "and pass a stream or DataContent, or use an engine that accepts a URL natively."); } + + /// + /// Runs OCR over a single document referenced by a , explicitly downloading the + /// bytes with a caller-supplied when the reference is remote. + /// + /// The client. + /// The document reference to download and parse. + /// The used to download remote (http/https) documents. + /// The OCR options to configure the request. + /// An optional progress reporter. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR result. + /// , , or is . + /// The uses a scheme other than data:, http, or https. + /// + /// This is the explicit, opt-in counterpart to + /// , + /// which never touches the network. Self-contained data: URIs are handled inline (no download); + /// http/https URIs are fetched with the supplied and passed + /// to the stream-based extraction path. The abstraction performs no ambient network IO: the caller owns + /// the and therefore its handlers, authentication, timeouts, and lifetime. + /// Engines that accept a URL natively (for example Azure Document Intelligence uriSource) should + /// expose that on the concrete client instead; this extension serves the bytes-only majority. + /// + public static async Task ExtractFromUriAsync( + this IOcrClient client, + UriContent document, + HttpClient httpClient, + OcrOptions? options = null, + IProgress? progress = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + _ = Throw.IfNull(httpClient); + + Uri uri = document.Uri; + + // Self-contained data: URIs carry their own bytes - no download needed. + if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) + { + return await client.ExtractAsync(document, options, progress, cancellationToken).ConfigureAwait(false); + } + + if (!uri.IsAbsoluteUri || + (!string.Equals(uri.Scheme, "http", StringComparison.OrdinalIgnoreCase) && + !string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase))) + { + throw new NotSupportedException( + "ExtractFromUriAsync downloads only http/https URIs (and inlines data: URIs). For other " + + "schemes, read the bytes and pass a stream or DataContent."); + } + + using HttpResponseMessage response = await httpClient + .GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken) + .ConfigureAwait(false); + _ = response.EnsureSuccessStatusCode(); + + string mediaType = response.Content.Headers.ContentType?.MediaType ?? document.MediaType; + +#if NET + using Stream contentStream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); +#else + using Stream contentStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); +#endif + return await client.ExtractAsync(contentStream, mediaType, options, progress, cancellationToken).ConfigureAwait(false); + } } From 4f7db5d6e9310de9ba1e710a9f7ccc497c645808 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Thu, 9 Jul 2026 11:53:02 -0400 Subject: [PATCH 07/13] Finish GetTextAsync to ExtractAsync rename in OCR tests and API baselines Complete the source rename across the OCR test suites in both Microsoft.Extensions.AI.Abstractions.Tests and Microsoft.Extensions.AI.Tests, and regenerate the two API baselines (which also reflect the round-2 members ExtractFromUriAsync, OcrImage, and OcrPage.Images). Also fix pre-existing StyleCop/Sonar violations in the OCR tests that were blocking the core test project from compiling. --- .../Microsoft.Extensions.AI.Abstractions.json | 52 ++++++++++++++++--- .../Microsoft.Extensions.AI.json | 8 +-- .../Ocr/DelegatingOcrClientTests.cs | 6 +-- .../Ocr/OcrClientExtensionsTests.cs | 13 +++-- .../TestOcrClient.cs | 6 +-- .../Ocr/ConfigureOptionsOcrClientTests.cs | 4 +- .../Ocr/LoggingOcrClientTests.cs | 14 ++--- .../Ocr/OcrClientBuilderTests.cs | 9 ++-- .../OcrClientDependencyInjectionPatterns.cs | 2 +- .../Ocr/OpenTelemetryOcrClientTests.cs | 5 +- 10 files changed, 78 insertions(+), 41 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index c1be9fb1ecc..4f18354d507 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -1852,11 +1852,11 @@ "Stage": "Experimental" }, { - "Member": "virtual object? Microsoft.Extensions.AI.DelegatingOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Member": "virtual System.Threading.Tasks.Task Microsoft.Extensions.AI.DelegatingOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "virtual System.Threading.Tasks.Task Microsoft.Extensions.AI.DelegatingOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "virtual object? Microsoft.Extensions.AI.DelegatingOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", "Stage": "Experimental" } ], @@ -3201,11 +3201,11 @@ "Stage": "Experimental", "Methods": [ { - "Member": "object? Microsoft.Extensions.AI.IOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Member": "System.Threading.Tasks.Task Microsoft.Extensions.AI.IOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "System.Threading.Tasks.Task Microsoft.Extensions.AI.IOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "object? Microsoft.Extensions.AI.IOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", "Stage": "Experimental" } ] @@ -3403,11 +3403,19 @@ "Stage": "Experimental", "Methods": [ { - "Member": "static TService? Microsoft.Extensions.AI.OcrClientExtensions.GetService(this Microsoft.Extensions.AI.IOcrClient client, object? serviceKey = null);", + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.GetTextAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractFromUriAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, System.Net.Http.HttpClient httpClient, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static TService? Microsoft.Extensions.AI.OcrClientExtensions.GetService(this Microsoft.Extensions.AI.IOcrClient client, object? serviceKey = null);", "Stage": "Experimental" } ] @@ -3436,6 +3444,34 @@ } ] }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrImage", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrImage.OcrImage();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrImage.BoundingRegion { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrImage.Caption { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "double? Microsoft.Extensions.AI.OcrImage.Confidence { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.DataContent? Microsoft.Extensions.AI.OcrImage.Content { get; set; }", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.OcrOptions", "Stage": "Experimental", @@ -3486,6 +3522,10 @@ "Member": "double? Microsoft.Extensions.AI.OcrPage.Confidence { get; set; }", "Stage": "Experimental" }, + { + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Images { get; set; }", + "Stage": "Experimental" + }, { "Member": "int Microsoft.Extensions.AI.OcrPage.Index { get; }", "Stage": "Experimental" diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json index 3437a3ab9fe..158a7000c62 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json @@ -282,7 +282,7 @@ "Stage": "Experimental" }, { - "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.ConfigureOptionsOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ] @@ -1052,7 +1052,7 @@ "Stage": "Experimental" }, { - "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.LoggingOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.LoggingOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ], @@ -1432,11 +1432,11 @@ "Stage": "Experimental" }, { - "Member": "override object? Microsoft.Extensions.AI.OpenTelemetryOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.OpenTelemetryOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.OpenTelemetryOcrClient.GetTextAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override object? Microsoft.Extensions.AI.OpenTelemetryOcrClient.GetService(System.Type serviceType, object? serviceKey = null);", "Stage": "Experimental" } ], diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs index 54f58233707..b9cb05eda2b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs @@ -18,7 +18,7 @@ public void RequiresInnerOcrClient() } [Fact] - public async Task GetTextAsyncDefaultsToInnerClientAsync() + public async Task ExtractAsyncDefaultsToInnerClientAsync() { // Arrange using var expectedDocument = new MemoryStream(); @@ -29,7 +29,7 @@ public async Task GetTextAsyncDefaultsToInnerClientAsync() var expectedResponse = new OcrResult([]); using var inner = new TestOcrClient { - GetTextAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => { Assert.Same(expectedDocument, document); Assert.Same(expectedMediaType, mediaType); @@ -42,7 +42,7 @@ public async Task GetTextAsyncDefaultsToInnerClientAsync() using var delegating = new NoOpDelegatingOcrClient(inner); // Act - var resultTask = delegating.GetTextAsync(expectedDocument, expectedMediaType, expectedOptions, null, expectedCancellationToken); + var resultTask = delegating.ExtractAsync(expectedDocument, expectedMediaType, expectedOptions, null, expectedCancellationToken); // Assert Assert.False(resultTask.IsCompleted); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs index 4091972138b..926bed63fa5 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using System.Threading; using System.Threading.Tasks; using Xunit; @@ -21,21 +20,21 @@ public void GetService_InvalidArgs_Throws() } [Fact] - public async Task GetTextAsync_InvalidArgs_Throws() + public async Task ExtractAsync_InvalidArgs_Throws() { IOcrClient? client = null; var content = new DataContent("data:application/pdf;base64,AQIDBA=="); - var ex1 = await Assert.ThrowsAsync(() => OcrClientExtensions.GetTextAsync(client!, content)); + var ex1 = await Assert.ThrowsAsync(() => OcrClientExtensions.ExtractAsync(client!, content)); Assert.Equal("client", ex1.ParamName); using var testClient = new TestOcrClient(); DataContent? nullContent = null; - var ex2 = await Assert.ThrowsAsync(() => OcrClientExtensions.GetTextAsync(testClient, nullContent!)); + var ex2 = await Assert.ThrowsAsync(() => OcrClientExtensions.ExtractAsync(testClient, nullContent!)); Assert.Equal("document", ex2.ParamName); } [Fact] - public async Task GetTextAsync_DataContent_PassesStreamAndMediaTypeAsync() + public async Task ExtractAsync_DataContent_PassesStreamAndMediaTypeAsync() { // Arrange var expectedResponse = new OcrResult([new OcrPage(0, "hello")]); @@ -44,7 +43,7 @@ public async Task GetTextAsync_DataContent_PassesStreamAndMediaTypeAsync() using var client = new TestOcrClient { - GetTextAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => { observedMediaType = mediaType; using var ms = new MemoryStream(); @@ -55,7 +54,7 @@ public async Task GetTextAsync_DataContent_PassesStreamAndMediaTypeAsync() }; // Act - var result = await OcrClientExtensions.GetTextAsync(client, new DataContent("data:application/pdf;base64,AQIDBA==")); + var result = await OcrClientExtensions.ExtractAsync(client, new DataContent("data:application/pdf;base64,AQIDBA==")); // Assert Assert.Same(expectedResponse, result); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs index c73da94dabe..d6a2fd8198a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs @@ -23,7 +23,7 @@ public TestOcrClient() IProgress?, CancellationToken, Task>? - GetTextAsyncCallback + ExtractAsyncCallback { get; set; } public Func GetServiceCallback { get; set; } @@ -31,13 +31,13 @@ public TestOcrClient() private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) => serviceType is not null && serviceKey is null && serviceType.IsInstanceOfType(this) ? this : null; - public Task GetTextAsync( + public Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, IProgress? progress = null, CancellationToken cancellationToken = default) - => GetTextAsyncCallback!.Invoke(document, mediaType, options, progress, cancellationToken); + => ExtractAsyncCallback!.Invoke(document, mediaType, options, progress, cancellationToken); public object? GetService(Type serviceType, object? serviceKey = null) => GetServiceCallback!.Invoke(serviceType, serviceKey); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs index f7c8cfe060c..1d23c719b81 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs @@ -38,7 +38,7 @@ public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullP using IOcrClient innerClient = new TestOcrClient { - GetTextAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => { Assert.Same(returnedOptions, options); Assert.Equal(cts.Token, cancellationToken); @@ -65,7 +65,7 @@ public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullP .Build(); using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); - var result = await client.GetTextAsync(document, "application/pdf", providedOptions, null, cts.Token); + var result = await client.ExtractAsync(document, "application/pdf", providedOptions, null, cts.Token); Assert.Same(expectedResult, result); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs index b9c66ac3caf..c505b0194da 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs @@ -44,7 +44,7 @@ public void UseLogging_AvoidsInjectingNopClient() [InlineData(LogLevel.Trace)] [InlineData(LogLevel.Debug)] [InlineData(LogLevel.Information)] - public async Task GetTextAsync_LogsInvocationAndCompletion(LogLevel level) + public async Task ExtractAsync_LogsInvocationAndCompletion(LogLevel level) { var collector = new FakeLogCollector(); @@ -54,7 +54,7 @@ public async Task GetTextAsync_LogsInvocationAndCompletion(LogLevel level) using IOcrClient innerClient = new TestOcrClient { - GetTextAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => Task.FromResult(new OcrResult([new OcrPage(0, "blue whale")])), }; @@ -64,20 +64,20 @@ public async Task GetTextAsync_LogsInvocationAndCompletion(LogLevel level) .Build(services); using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); - await client.GetTextAsync(document, "application/pdf", new OcrOptions { ModelId = "mistral-ocr-4-0" }); + await client.ExtractAsync(document, "application/pdf", new OcrOptions { ModelId = "mistral-ocr-4-0" }); var logs = collector.GetSnapshot(); if (level is LogLevel.Trace) { Assert.Collection(logs, - entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} invoked:") && entry.Message.Contains("mistral-ocr-4-0")), - entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} completed:") && entry.Message.Contains("blue whale"))); + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.ExtractAsync)} invoked:") && entry.Message.Contains("mistral-ocr-4-0")), + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.ExtractAsync)} completed:") && entry.Message.Contains("blue whale"))); } else if (level is LogLevel.Debug) { Assert.Collection(logs, - entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} invoked.") && !entry.Message.Contains("mistral-ocr-4-0")), - entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.GetTextAsync)} completed.") && !entry.Message.Contains("blue whale"))); + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.ExtractAsync)} invoked.") && !entry.Message.Contains("mistral-ocr-4-0")), + entry => Assert.True(entry.Message.Contains($"{nameof(IOcrClient.ExtractAsync)} completed.") && !entry.Message.Contains("blue whale"))); } else { diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs index 63b07818a5d..a0da3cf40bd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientBuilderTests.cs @@ -39,12 +39,12 @@ public void UseAppliesFactoriesInReverseOrderSoFirstAddedIsOutermost() .Use(c => { order.Add("outer-built"); - return new InspectorOcrClient(c, "outer", order); + return new InspectorOcrClient(c, "outer"); }) .Use(c => { order.Add("inner-built"); - return new InspectorOcrClient(c, "inner", order); + return new InspectorOcrClient(c, "inner"); }) .Build(); @@ -95,10 +95,9 @@ public void ServicesAreFlowedThroughBuild() Assert.Same(services, observed); } - private sealed class InspectorOcrClient(IOcrClient inner, string name, List order) : DelegatingOcrClient(inner) + private sealed class InspectorOcrClient(IOcrClient inner, string name) : DelegatingOcrClient(inner) { public string Name => name; - public IOcrClient InnerClientPublic => base.InnerClient; - public List Order => order; + public IOcrClient InnerClientPublic => InnerClient; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs index ae8f5c9ea64..284d43c1390 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OcrClientDependencyInjectionPatterns.cs @@ -101,7 +101,7 @@ public void AddKeyedOcrClient_RegistersExpectedLifetime(ServiceLifetime? lifetim public class SingletonMiddleware(IOcrClient inner, IServiceProvider services) : DelegatingOcrClient(inner) { - public IOcrClient InnerClientPublic => base.InnerClient; + public IOcrClient InnerClientPublic => InnerClient; public IServiceProvider Services => services; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs index 201aca196c5..f9376d64f98 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Threading; using System.Threading.Tasks; using OpenTelemetry.Trace; using Xunit; @@ -34,7 +33,7 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) using var innerClient = new TestOcrClient { - GetTextAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => { await Task.Yield(); return new OcrResult([new OcrPage(0, "This is the recognized text.")]) @@ -67,7 +66,7 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) }, }; - _ = await client.GetTextAsync(Stream.Null, "application/pdf", options); + _ = await client.ExtractAsync(Stream.Null, "application/pdf", options); var activity = Assert.Single(activities); From 49e55e227e56eb48442eb8523e058f8ebc1be340 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Mon, 13 Jul 2026 17:40:29 -0400 Subject: [PATCH 08/13] Fix net462 build: use buffered Stream.CopyToAsync overload in OCR test Stream.CopyToAsync(Stream, CancellationToken) does not exist on net462/ netstandard2.0, so the 2-arg call failed to compile (CS1503) on the net462 leg. Use the #if !NET conditional buffer-size overload, matching the existing DataContent.cs convention. --- .../Ocr/OcrClientExtensionsTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs index 926bed63fa5..681efbaa65e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs @@ -47,7 +47,12 @@ public async Task ExtractAsync_DataContent_PassesStreamAndMediaTypeAsync() { observedMediaType = mediaType; using var ms = new MemoryStream(); - await document.CopyToAsync(ms, cancellationToken); + await document.CopyToAsync( + ms, +#if !NET + 80 * 1024, // same as the default buffer size +#endif + cancellationToken); observedBytes = ms.ToArray(); return expectedResponse; } From fc6b00de3518499ca2ed18776916d6d03bdc9869 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Tue, 14 Jul 2026 13:53:28 -0400 Subject: [PATCH 09/13] Shape OCR bounding geometry as points and 1-base page numbers Introduce OcrPoint and OcrBoundingBox readonly record structs and retype OcrBoundingRegion.Polygon as IReadOnlyList with a typed OcrBoundingBox GetBounds() return. This makes odd or empty vertex lists unrepresentable (fixing the GetBounds inverted-bounds edge case) and reads self-documenting, while still carrying a rotation-skewed quadrilateral. Rename OcrPage.Index (zero-based) to OcrPage.PageNumber (one-based) so it correlates directly with OcrBoundingRegion.PageNumber, matching Azure Document Intelligence, PdfPig, and human page conventions, and removing the off-by-one adjustments providers had to make. Update the internal OcrDocumentReader and the OCR tests, and regenerate the Microsoft.Extensions.AI.Abstractions API baseline. --- .../Microsoft.Extensions.AI.Abstractions.json | 124 +++++++++++++++++- .../Ocr/OcrBoundingBox.cs | 15 +++ .../Ocr/OcrBoundingRegion.cs | 49 +++---- .../Ocr/OcrPage.cs | 10 +- .../Ocr/OcrPoint.cs | 13 ++ .../OcrDocumentReader.cs | 4 +- .../Ocr/OcrBoundingRegionTests.cs | 12 +- .../Ocr/OcrClientExtensionsTests.cs | 2 +- .../Ocr/OcrResultTests.cs | 2 +- .../Ocr/ConfigureOptionsOcrClientTests.cs | 2 +- .../Ocr/LoggingOcrClientTests.cs | 2 +- .../Ocr/OpenTelemetryOcrClientTests.cs | 2 +- 12 files changed, 195 insertions(+), 42 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingBox.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPoint.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index 4f18354d507..6ab7268126c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -3370,12 +3370,72 @@ } ] }, + { + "Type": "readonly class Microsoft.Extensions.AI.OcrBoundingBox(float Left, float Top, float Right, float Bottom)", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrBoundingBox.OcrBoundingBox(float Left, float Top, float Right, float Bottom);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrBoundingBox.OcrBoundingBox();", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.AI.OcrBoundingBox.Deconstruct(out float Left, out float Top, out float Right, out float Bottom);", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.AI.OcrBoundingBox.Equals(object obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrBoundingBox.Equals(Microsoft.Extensions.AI.OcrBoundingBox other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.AI.OcrBoundingBox.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrBoundingBox.operator ==(Microsoft.Extensions.AI.OcrBoundingBox left, Microsoft.Extensions.AI.OcrBoundingBox right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrBoundingBox.operator !=(Microsoft.Extensions.AI.OcrBoundingBox left, Microsoft.Extensions.AI.OcrBoundingBox right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.AI.OcrBoundingBox.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "float Microsoft.Extensions.AI.OcrBoundingBox.Bottom { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.AI.OcrBoundingBox.Left { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.AI.OcrBoundingBox.Right { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.AI.OcrBoundingBox.Top { get; init; }", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.OcrBoundingRegion", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrBoundingRegion.OcrBoundingRegion(int pageNumber, System.Collections.Generic.IReadOnlyList polygon);", + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion.OcrBoundingRegion(int pageNumber, System.Collections.Generic.IReadOnlyList polygon);", "Stage": "Experimental" }, { @@ -3383,7 +3443,7 @@ "Stage": "Experimental" }, { - "Member": "(float Left, float Top, float Right, float Bottom) Microsoft.Extensions.AI.OcrBoundingRegion.GetBounds();", + "Member": "Microsoft.Extensions.AI.OcrBoundingBox Microsoft.Extensions.AI.OcrBoundingRegion.GetBounds();", "Stage": "Experimental" } ], @@ -3393,7 +3453,7 @@ "Stage": "Experimental" }, { - "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrBoundingRegion.Polygon { get; }", + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrBoundingRegion.Polygon { get; }", "Stage": "Experimental" } ] @@ -3505,7 +3565,7 @@ "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrPage.OcrPage(int index, string markdown);", + "Member": "Microsoft.Extensions.AI.OcrPage.OcrPage(int pageNumber, string markdown);", "Stage": "Experimental" } ], @@ -3527,11 +3587,11 @@ "Stage": "Experimental" }, { - "Member": "int Microsoft.Extensions.AI.OcrPage.Index { get; }", + "Member": "string Microsoft.Extensions.AI.OcrPage.Markdown { get; }", "Stage": "Experimental" }, { - "Member": "string Microsoft.Extensions.AI.OcrPage.Markdown { get; }", + "Member": "int Microsoft.Extensions.AI.OcrPage.PageNumber { get; }", "Stage": "Experimental" }, { @@ -3540,6 +3600,58 @@ } ] }, + { + "Type": "readonly class Microsoft.Extensions.AI.OcrPoint(float X, float Y)", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrPoint.OcrPoint(float X, float Y);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrPoint.OcrPoint();", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.AI.OcrPoint.Deconstruct(out float X, out float Y);", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.AI.OcrPoint.Equals(object obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrPoint.Equals(Microsoft.Extensions.AI.OcrPoint other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.AI.OcrPoint.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrPoint.operator ==(Microsoft.Extensions.AI.OcrPoint left, Microsoft.Extensions.AI.OcrPoint right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrPoint.operator !=(Microsoft.Extensions.AI.OcrPoint left, Microsoft.Extensions.AI.OcrPoint right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.AI.OcrPoint.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "float Microsoft.Extensions.AI.OcrPoint.X { get; init; }", + "Stage": "Experimental" + }, + { + "Member": "float Microsoft.Extensions.AI.OcrPoint.Y { get; init; }", + "Stage": "Experimental" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.OcrProgress", "Stage": "Experimental", diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingBox.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingBox.cs new file mode 100644 index 00000000000..c8e58f0ff0b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingBox.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents the axis-aligned bounds of a bounding polygon, in the coordinate space defined by the OCR engine. +/// The minimum horizontal coordinate. +/// The minimum vertical coordinate. +/// The maximum horizontal coordinate. +/// The maximum vertical coordinate. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public readonly record struct OcrBoundingBox(float Left, float Top, float Right, float Bottom); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs index e6e49885bbd..f2fa7d8b46d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs @@ -11,20 +11,20 @@ namespace Microsoft.Extensions.AI; /// Represents a positioned region on a page. /// -/// The region is a polygon (a flattened, clockwise sequence of [x1, y1, x2, y2, ...] vertices) -/// so it can faithfully carry a possibly rotation-skewed quadrilateral, such as Azure Document -/// Intelligence's BoundingRegion.Polygon, without loss. Engines that emit only an axis-aligned -/// rectangle (such as Mistral OCR) can convert via . The same type is reused -/// for layout-block geometry and for field grounding, providing one region primitive across providers. +/// The region is a polygon (a clockwise sequence of vertices) so it can +/// faithfully carry a possibly rotation-skewed quadrilateral, such as Azure Document Intelligence's +/// BoundingRegion.Polygon, without loss. Engines that emit only an axis-aligned rectangle +/// (such as Mistral OCR) can convert via . The same type is reused for +/// layout-block geometry and for field grounding, providing one region primitive across providers. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] public sealed class OcrBoundingRegion { /// Initializes a new instance of the class. /// The one-based page number the region is on. - /// The flattened, clockwise polygon vertices. + /// The clockwise polygon vertices. /// is . - public OcrBoundingRegion(int pageNumber, IReadOnlyList polygon) + public OcrBoundingRegion(int pageNumber, IReadOnlyList polygon) { PageNumber = pageNumber; Polygon = Throw.IfNull(polygon); @@ -34,9 +34,9 @@ public OcrBoundingRegion(int pageNumber, IReadOnlyList polygon) /// A region can reference a different page than its parent element. public int PageNumber { get; } - /// Gets the flattened polygon vertices [x1, y1, x2, y2, ...], in clockwise order. - /// An Azure Document Intelligence quadrilateral is eight floats. - public IReadOnlyList Polygon { get; } + /// Gets the polygon vertices, in clockwise order. + /// An Azure Document Intelligence quadrilateral is four points. + public IReadOnlyList Polygon { get; } /// Builds a clockwise quadrilateral region from an axis-aligned rectangle. /// The one-based page number the region is on. @@ -48,25 +48,30 @@ public OcrBoundingRegion(int pageNumber, IReadOnlyList polygon) public static OcrBoundingRegion FromRectangle(int pageNumber, double left, double top, double right, double bottom) => new(pageNumber, [ - (float)left, (float)top, - (float)right, (float)top, - (float)right, (float)bottom, - (float)left, (float)bottom, + new OcrPoint((float)left, (float)top), + new OcrPoint((float)right, (float)top), + new OcrPoint((float)right, (float)bottom), + new OcrPoint((float)left, (float)bottom), ]); /// Computes the axis-aligned bounds of the polygon. - /// The minimum and maximum coordinates of the polygon. - public (float Left, float Top, float Right, float Bottom) GetBounds() + /// The axis-aligned bounds, or when the polygon has no vertices. + public OcrBoundingBox GetBounds() { + if (Polygon.Count == 0) + { + return default; + } + float minX = float.MaxValue, minY = float.MaxValue, maxX = float.MinValue, maxY = float.MinValue; - for (int i = 0; i + 1 < Polygon.Count; i += 2) + foreach (OcrPoint point in Polygon) { - minX = Math.Min(minX, Polygon[i]); - maxX = Math.Max(maxX, Polygon[i]); - minY = Math.Min(minY, Polygon[i + 1]); - maxY = Math.Max(maxY, Polygon[i + 1]); + minX = Math.Min(minX, point.X); + maxX = Math.Max(maxX, point.X); + minY = Math.Min(minY, point.Y); + maxY = Math.Max(maxY, point.Y); } - return (minX, minY, maxX, maxY); + return new OcrBoundingBox(minX, minY, maxX, maxY); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs index 3b3f9b70945..4c9567a9249 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -13,17 +13,17 @@ namespace Microsoft.Extensions.AI; public sealed class OcrPage { /// Initializes a new instance of the class. - /// The zero-based page index. + /// The one-based page number. /// The structured markdown for this page. /// is . - public OcrPage(int index, string markdown) + public OcrPage(int pageNumber, string markdown) { - Index = index; + PageNumber = pageNumber; Markdown = Throw.IfNull(markdown); } - /// Gets the zero-based page index. - public int Index { get; } + /// Gets the one-based page number. + public int PageNumber { get; } /// Gets the structured markdown for this page, with headings, tables, and reading order preserved. public string Markdown { get; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPoint.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPoint.cs new file mode 100644 index 00000000000..b252c017ba7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPoint.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents a single vertex of a bounding polygon, in the coordinate space defined by the OCR engine. +/// The horizontal coordinate. +/// The vertical coordinate. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public readonly record struct OcrPoint(float X, float Y); diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs index c62ab0a8389..9835f8b0975 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs @@ -56,7 +56,7 @@ private static IngestionDocument Map(OcrResult ocrResult, string identifier) foreach (OcrPage page in ocrResult.Pages) { IngestionDocumentSection section = new(); - int pageNumber = page.Index + 1; + int pageNumber = page.PageNumber; if (!string.IsNullOrWhiteSpace(page.Markdown)) { @@ -93,7 +93,7 @@ private static IngestionDocumentImage MapImage(OcrImage image, int pageNumber) { (float left, float top, float right, float bottom) = image.BoundingRegion.GetBounds(); element.Metadata[BoundingBoxMetadataKey] = new[] { left, top, right, bottom }; - element.Metadata[BoundingRegionMetadataKey] = image.BoundingRegion.Polygon.ToArray(); + element.Metadata[BoundingRegionMetadataKey] = image.BoundingRegion.Polygon.SelectMany(static p => new[] { p.X, p.Y }).ToArray(); } return element; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs index 4a1d35efa27..d3ec599c5c5 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs @@ -20,13 +20,13 @@ public void FromRectangle_ProducesClockwiseQuadrilateral() var region = OcrBoundingRegion.FromRectangle(2, left: 10, top: 20, right: 110, bottom: 220); Assert.Equal(2, region.PageNumber); - Assert.Equal(new float[] { 10, 20, 110, 20, 110, 220, 10, 220 }, region.Polygon); + Assert.Equal(new[] { new OcrPoint(10, 20), new OcrPoint(110, 20), new OcrPoint(110, 220), new OcrPoint(10, 220) }, region.Polygon); } [Fact] public void GetBounds_ReturnsAxisAlignedExtents() { - var region = new OcrBoundingRegion(1, [30, 40, 100, 35, 110, 90, 25, 95]); + var region = new OcrBoundingRegion(1, [new OcrPoint(30, 40), new OcrPoint(100, 35), new OcrPoint(110, 90), new OcrPoint(25, 95)]); var (left, top, right, bottom) = region.GetBounds(); @@ -35,4 +35,12 @@ public void GetBounds_ReturnsAxisAlignedExtents() Assert.Equal(110, right); Assert.Equal(95, bottom); } + + [Fact] + public void GetBounds_EmptyPolygon_ReturnsDefault() + { + var region = new OcrBoundingRegion(1, []); + + Assert.Equal(default, region.GetBounds()); + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs index 681efbaa65e..09f94bd10b8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs @@ -37,7 +37,7 @@ public async Task ExtractAsync_InvalidArgs_Throws() public async Task ExtractAsync_DataContent_PassesStreamAndMediaTypeAsync() { // Arrange - var expectedResponse = new OcrResult([new OcrPage(0, "hello")]); + var expectedResponse = new OcrResult([new OcrPage(1, "hello")]); string? observedMediaType = null; byte[]? observedBytes = null; diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs index efddd0357e0..86a1a3d350a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs @@ -17,7 +17,7 @@ public void Constructor_NullPages_Throws() [Fact] public void Markdown_JoinsPerPageMarkdown() { - var result = new OcrResult([new OcrPage(0, "page one"), new OcrPage(1, "page two")]) + var result = new OcrResult([new OcrPage(1, "page one"), new OcrPage(2, "page two")]) { OcrSource = "test-engine", ModelId = "model-1", diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs index 1d23c719b81..173364d3ca7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs @@ -33,7 +33,7 @@ public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullP { OcrOptions? providedOptions = nullProvidedOptions ? null : new() { ModelId = "test" }; OcrOptions? returnedOptions = null; - OcrResult expectedResult = new([new OcrPage(0, "blue whale")]); + OcrResult expectedResult = new([new OcrPage(1, "blue whale")]); using CancellationTokenSource cts = new(); using IOcrClient innerClient = new TestOcrClient diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs index c505b0194da..e05f7d331fb 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs @@ -55,7 +55,7 @@ public async Task ExtractAsync_LogsInvocationAndCompletion(LogLevel level) using IOcrClient innerClient = new TestOcrClient { ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => - Task.FromResult(new OcrResult([new OcrPage(0, "blue whale")])), + Task.FromResult(new OcrResult([new OcrPage(1, "blue whale")])), }; using IOcrClient client = innerClient diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs index f9376d64f98..e40efdcf8c7 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs @@ -36,7 +36,7 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) ExtractAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => { await Task.Yield(); - return new OcrResult([new OcrPage(0, "This is the recognized text.")]) + return new OcrResult([new OcrPage(1, "This is the recognized text.")]) { ModelId = "amazingmodel", Usage = new() { PagesProcessed = 3 }, From 6ee41c0c28b46b0a4d915b74010dcb6b57031408 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Wed, 15 Jul 2026 15:40:28 -0400 Subject: [PATCH 10/13] Align IOcrClient surface with the MEAI family ahead of API review Family-consistency changes surfaced by an API-review rehearsal against the sibling abstractions (IChatClient, ISpeechToTextClient): - Streaming: add IAsyncEnumerable via ExtractStreamingAsync (the unary/streaming/GetService triad, matching IChatClient) plus a ToOcrResult/ToOcrResultAsync reducer; retire IProgress and the OcrProgress type (progress folds onto OcrResponseUpdate). - Unseal the OCR DTOs (OcrOptions, OcrResult, OcrPage, OcrBlock, OcrTable, OcrTableCell, OcrImage, OcrBoundingRegion, OcrUsage) to mirror the unsealed ChatResponse/ChatOptions family shape. - Geometry units: add OcrPage.Width/Height and an OcrCoordinateUnit (ChatRole-style open struct: Pixel/Inch/Normalized) so bounding coordinates are interpretable across engines. - Model OcrBlock.Kind and OcrTableCell.Kind as ChatRole-style open structs (OcrBlockKind, OcrTableCellKind) instead of raw strings. - OcrBoundingRegion.GetBounds() now returns OcrBoundingBox? (null on empty polygon) instead of an ambiguous all-zero default. - Remove the leaky OcrResult.OcrSource; document OcrTable.Cells as authoritative when non-null. - Add docs to the AddOcrClient/AddKeyedOcrClient overloads. Updates the middleware (Logging/OpenTelemetry/ConfigureOptions), the MEDI OcrDocumentReader consumer, OCR tests, and both API baselines accordingly. --- .../Microsoft.Extensions.AI.Abstractions.json | 332 ++++++++++++++++-- .../Ocr/DelegatingOcrClient.cs | 14 +- .../Ocr/IOcrClient.cs | 34 +- .../Ocr/OcrBlock.cs | 6 +- .../Ocr/OcrBlockKind.cs | 93 +++++ .../Ocr/OcrBoundingRegion.cs | 12 +- .../Ocr/OcrClientExtensions.cs | 77 +++- .../Ocr/OcrCoordinateUnit.cs | 107 ++++++ .../Ocr/OcrImage.cs | 2 +- .../Ocr/OcrOptions.cs | 2 +- .../Ocr/OcrPage.cs | 24 +- .../Ocr/OcrProgress.cs | 25 -- .../Ocr/OcrResponseUpdate.cs | 73 ++++ .../Ocr/OcrResponseUpdateExtensions.cs | 99 ++++++ .../Ocr/OcrResult.cs | 6 +- .../Ocr/OcrTable.cs | 10 +- .../Ocr/OcrTableCell.cs | 6 +- .../Ocr/OcrTableCellKind.cs | 90 +++++ .../Ocr/OcrUsage.cs | 2 +- .../Utilities/AIJsonUtilities.Defaults.cs | 2 +- .../Microsoft.Extensions.AI.json | 18 +- .../Ocr/ConfigureOptionsOcrClient.cs | 14 +- .../Ocr/LoggingOcrClient.cs | 94 ++++- ...lientBuilderServiceCollectionExtensions.cs | 8 + .../Ocr/OpenTelemetryOcrClient.cs | 67 +++- .../OcrDocumentReader.cs | 8 +- .../Ocr/DelegatingOcrClientTests.cs | 53 ++- .../Ocr/OcrBoundingRegionTests.cs | 9 +- .../Ocr/OcrClientExtensionsTests.cs | 2 +- .../Ocr/OcrResponseUpdateExtensionsTests.cs | 84 +++++ .../Ocr/OcrResultTests.cs | 2 - .../TestOcrClient.cs | 20 +- .../Ocr/ConfigureOptionsOcrClientTests.cs | 4 +- .../Ocr/LoggingOcrClientTests.cs | 2 +- .../Ocr/OpenTelemetryOcrClientTests.cs | 2 +- .../Readers/OcrDocumentReaderTests.cs | 9 +- 36 files changed, 1286 insertions(+), 126 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlockKind.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs delete mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index 6ab7268126c..e4ad63d58de 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -1852,7 +1852,11 @@ "Stage": "Experimental" }, { - "Member": "virtual System.Threading.Tasks.Task Microsoft.Extensions.AI.DelegatingOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "virtual System.Threading.Tasks.Task Microsoft.Extensions.AI.DelegatingOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "virtual System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.DelegatingOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { @@ -3201,7 +3205,11 @@ "Stage": "Experimental", "Methods": [ { - "Member": "System.Threading.Tasks.Task Microsoft.Extensions.AI.IOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "System.Threading.Tasks.Task Microsoft.Extensions.AI.IOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.IOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { @@ -3343,7 +3351,7 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrBlock", + "Type": "class Microsoft.Extensions.AI.OcrBlock", "Stage": "Experimental", "Methods": [ { @@ -3361,7 +3369,7 @@ "Stage": "Experimental" }, { - "Member": "string? Microsoft.Extensions.AI.OcrBlock.Kind { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrBlockKind? Microsoft.Extensions.AI.OcrBlock.Kind { get; set; }", "Stage": "Experimental" }, { @@ -3370,6 +3378,80 @@ } ] }, + { + "Type": "readonly struct Microsoft.Extensions.AI.OcrBlockKind : System.IEquatable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrBlockKind.OcrBlockKind(string value);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrBlockKind.OcrBlockKind();", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.AI.OcrBlockKind.Equals(object? obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrBlockKind.Equals(Microsoft.Extensions.AI.OcrBlockKind other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.AI.OcrBlockKind.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrBlockKind.operator ==(Microsoft.Extensions.AI.OcrBlockKind left, Microsoft.Extensions.AI.OcrBlockKind right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrBlockKind.operator !=(Microsoft.Extensions.AI.OcrBlockKind left, Microsoft.Extensions.AI.OcrBlockKind right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.AI.OcrBlockKind.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "static Microsoft.Extensions.AI.OcrBlockKind Microsoft.Extensions.AI.OcrBlockKind.Figure { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrBlockKind Microsoft.Extensions.AI.OcrBlockKind.Paragraph { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrBlockKind Microsoft.Extensions.AI.OcrBlockKind.Title { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrBlockKind.Value { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrBlockKind.Converter", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrBlockKind.Converter.Converter();", + "Stage": "Experimental" + }, + { + "Member": "override Microsoft.Extensions.AI.OcrBlockKind Microsoft.Extensions.AI.OcrBlockKind.Converter.Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.AI.OcrBlockKind.Converter.Write(System.Text.Json.Utf8JsonWriter writer, Microsoft.Extensions.AI.OcrBlockKind value, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + } + ] + }, { "Type": "readonly class Microsoft.Extensions.AI.OcrBoundingBox(float Left, float Top, float Right, float Bottom)", "Stage": "Experimental", @@ -3431,7 +3513,7 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrBoundingRegion", + "Type": "class Microsoft.Extensions.AI.OcrBoundingRegion", "Stage": "Experimental", "Methods": [ { @@ -3443,7 +3525,7 @@ "Stage": "Experimental" }, { - "Member": "Microsoft.Extensions.AI.OcrBoundingBox Microsoft.Extensions.AI.OcrBoundingRegion.GetBounds();", + "Member": "Microsoft.Extensions.AI.OcrBoundingBox? Microsoft.Extensions.AI.OcrBoundingRegion.GetBounds();", "Stage": "Experimental" } ], @@ -3463,15 +3545,23 @@ "Stage": "Experimental", "Methods": [ { - "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractFromUriAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, System.Net.Http.HttpClient httpClient, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrClientExtensions.ExtractFromUriAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, System.Net.Http.HttpClient httpClient, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OcrClientExtensions.ExtractStreamingAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OcrClientExtensions.ExtractStreamingAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { @@ -3505,7 +3595,81 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrImage", + "Type": "readonly struct Microsoft.Extensions.AI.OcrCoordinateUnit : System.IEquatable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.OcrCoordinateUnit(string value);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.OcrCoordinateUnit();", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.AI.OcrCoordinateUnit.Equals(object? obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrCoordinateUnit.Equals(Microsoft.Extensions.AI.OcrCoordinateUnit other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.AI.OcrCoordinateUnit.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrCoordinateUnit.operator ==(Microsoft.Extensions.AI.OcrCoordinateUnit left, Microsoft.Extensions.AI.OcrCoordinateUnit right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrCoordinateUnit.operator !=(Microsoft.Extensions.AI.OcrCoordinateUnit left, Microsoft.Extensions.AI.OcrCoordinateUnit right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.AI.OcrCoordinateUnit.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "static Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Inch { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Normalized { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Pixel { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrCoordinateUnit.Value { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrCoordinateUnit.Converter", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.Converter.Converter();", + "Stage": "Experimental" + }, + { + "Member": "override Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Converter.Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.AI.OcrCoordinateUnit.Converter.Write(System.Text.Json.Utf8JsonWriter writer, Microsoft.Extensions.AI.OcrCoordinateUnit value, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.AI.OcrImage", "Stage": "Experimental", "Methods": [ { @@ -3533,7 +3697,7 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrOptions", + "Type": "class Microsoft.Extensions.AI.OcrOptions", "Stage": "Experimental", "Methods": [ { @@ -3561,7 +3725,7 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrPage", + "Type": "class Microsoft.Extensions.AI.OcrPage", "Stage": "Experimental", "Methods": [ { @@ -3582,6 +3746,14 @@ "Member": "double? Microsoft.Extensions.AI.OcrPage.Confidence { get; set; }", "Stage": "Experimental" }, + { + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrPage.CoordinateUnit { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "float? Microsoft.Extensions.AI.OcrPage.Height { get; set; }", + "Stage": "Experimental" + }, { "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Images { get; set; }", "Stage": "Experimental" @@ -3597,6 +3769,10 @@ { "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Tables { get; set; }", "Stage": "Experimental" + }, + { + "Member": "float? Microsoft.Extensions.AI.OcrPage.Width { get; set; }", + "Stage": "Experimental" } ] }, @@ -3653,31 +3829,69 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrProgress", + "Type": "class Microsoft.Extensions.AI.OcrResponseUpdate", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrProgress.OcrProgress();", + "Member": "Microsoft.Extensions.AI.OcrResponseUpdate.OcrResponseUpdate();", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrResponseUpdate.OcrResponseUpdate(Microsoft.Extensions.AI.OcrPage? page);", "Stage": "Experimental" } ], "Properties": [ { - "Member": "int? Microsoft.Extensions.AI.OcrProgress.PagesProcessed { get; set; }", + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrResponseUpdate.AdditionalProperties { get; set; }", "Stage": "Experimental" }, { - "Member": "string? Microsoft.Extensions.AI.OcrProgress.Status { get; set; }", + "Member": "string? Microsoft.Extensions.AI.OcrResponseUpdate.ModelId { get; set; }", "Stage": "Experimental" }, { - "Member": "int? Microsoft.Extensions.AI.OcrProgress.TotalPages { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrPage? Microsoft.Extensions.AI.OcrResponseUpdate.Page { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.AI.OcrResponseUpdate.PagesProcessed { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "object? Microsoft.Extensions.AI.OcrResponseUpdate.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "string? Microsoft.Extensions.AI.OcrResponseUpdate.Status { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.AI.OcrResponseUpdate.TotalPages { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrUsage? Microsoft.Extensions.AI.OcrResponseUpdate.Usage { get; set; }", "Stage": "Experimental" } ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrResult", + "Type": "static class Microsoft.Extensions.AI.OcrResponseUpdateExtensions", + "Stage": "Experimental", + "Methods": [ + { + "Member": "static Microsoft.Extensions.AI.OcrResult Microsoft.Extensions.AI.OcrResponseUpdateExtensions.ToOcrResult(this System.Collections.Generic.IEnumerable updates);", + "Stage": "Experimental" + }, + { + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrResponseUpdateExtensions.ToOcrResultAsync(this System.Collections.Generic.IAsyncEnumerable updates, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.AI.OcrResult", "Stage": "Experimental", "Methods": [ { @@ -3698,10 +3912,6 @@ "Member": "string? Microsoft.Extensions.AI.OcrResult.ModelId { get; set; }", "Stage": "Experimental" }, - { - "Member": "string? Microsoft.Extensions.AI.OcrResult.OcrSource { get; set; }", - "Stage": "Experimental" - }, { "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrResult.Pages { get; }", "Stage": "Experimental" @@ -3717,7 +3927,7 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrTable", + "Type": "class Microsoft.Extensions.AI.OcrTable", "Stage": "Experimental", "Methods": [ { @@ -3749,7 +3959,7 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrTableCell", + "Type": "class Microsoft.Extensions.AI.OcrTableCell", "Stage": "Experimental", "Methods": [ { @@ -3771,7 +3981,7 @@ "Stage": "Experimental" }, { - "Member": "string? Microsoft.Extensions.AI.OcrTableCell.Kind { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrTableCellKind? Microsoft.Extensions.AI.OcrTableCell.Kind { get; set; }", "Stage": "Experimental" }, { @@ -3785,7 +3995,77 @@ ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrUsage", + "Type": "readonly struct Microsoft.Extensions.AI.OcrTableCellKind : System.IEquatable", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrTableCellKind.OcrTableCellKind(string value);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrTableCellKind.OcrTableCellKind();", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.AI.OcrTableCellKind.Equals(object? obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrTableCellKind.Equals(Microsoft.Extensions.AI.OcrTableCellKind other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.AI.OcrTableCellKind.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrTableCellKind.operator ==(Microsoft.Extensions.AI.OcrTableCellKind left, Microsoft.Extensions.AI.OcrTableCellKind right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrTableCellKind.operator !=(Microsoft.Extensions.AI.OcrTableCellKind left, Microsoft.Extensions.AI.OcrTableCellKind right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.AI.OcrTableCellKind.ToString();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "static Microsoft.Extensions.AI.OcrTableCellKind Microsoft.Extensions.AI.OcrTableCellKind.ColumnHeader { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrTableCellKind Microsoft.Extensions.AI.OcrTableCellKind.Content { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrTableCellKind.Value { get; }", + "Stage": "Experimental" + } + ] + }, + { + "Type": "sealed class Microsoft.Extensions.AI.OcrTableCellKind.Converter", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrTableCellKind.Converter.Converter();", + "Stage": "Experimental" + }, + { + "Member": "override Microsoft.Extensions.AI.OcrTableCellKind Microsoft.Extensions.AI.OcrTableCellKind.Converter.Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + }, + { + "Member": "override void Microsoft.Extensions.AI.OcrTableCellKind.Converter.Write(System.Text.Json.Utf8JsonWriter writer, Microsoft.Extensions.AI.OcrTableCellKind value, System.Text.Json.JsonSerializerOptions options);", + "Stage": "Experimental" + } + ] + }, + { + "Type": "class Microsoft.Extensions.AI.OcrUsage", "Stage": "Experimental", "Methods": [ { diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs index bddc3befa58..7af7dc32ba6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading; @@ -43,10 +44,19 @@ public virtual Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { - return InnerClient.ExtractAsync(document, mediaType, options, progress, cancellationToken); + return InnerClient.ExtractAsync(document, mediaType, options, cancellationToken); + } + + /// + public virtual IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + CancellationToken cancellationToken = default) + { + return InnerClient.ExtractStreamingAsync(document, mediaType, options, cancellationToken); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs index c3be2d25d43..56bf2b0970a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading; @@ -27,9 +28,10 @@ namespace Microsoft.Extensions.AI; /// /// /// Unless otherwise specified, all members of are thread-safe for concurrent -/// use. Implementations might mutate the supplied to ; -/// consumers should avoid sharing a single options instance across concurrent invocations when that is a -/// concern. The document stream passed to is not disposed by the implementation. +/// use. Implementations might mutate the supplied to +/// and ; consumers should avoid sharing a single options instance across +/// concurrent invocations when that is a concern. The document stream passed to these methods is not +/// disposed by the implementation. /// /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] @@ -39,19 +41,31 @@ public interface IOcrClient : IDisposable /// The document or image content to parse. /// The media type of , for example application/pdf or image/png. /// The OCR options to configure the request. - /// - /// An optional progress reporter. Engines that poll a long-running operation (such as Azure Document - /// Intelligence) can report page-by-page progress; synchronous engines report once. Keeping the call - /// unary while exposing provides observability without modeling a batch - /// operation as a stream. - /// /// The to monitor for cancellation requests. The default is . /// The structured OCR result. Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, + CancellationToken cancellationToken = default); + + /// Runs OCR / document parsing over a document stream and streams back structured output as it is produced. + /// The document or image content to parse. + /// The media type of , for example application/pdf or image/png. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR updates representing the streamed output. + /// + /// Engines that produce pages incrementally (for example, while polling a long-running operation such as + /// Azure Document Intelligence) can yield each page as it completes, letting a consumer begin processing + /// early pages before later pages finish. Synchronous engines may yield a single terminal update. Use + /// to reassemble the stream into an + /// . + /// + IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, CancellationToken cancellationToken = default); /// Asks the for an object of the specified type . diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs index 81689a8efff..3504febddf6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs @@ -9,7 +9,7 @@ namespace Microsoft.Extensions.AI; /// Represents a positioned layout block, such as a paragraph, heading, or figure. [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrBlock +public class OcrBlock { /// Initializes a new instance of the class. /// The text content of the block. @@ -22,8 +22,8 @@ public OcrBlock(string text) /// Gets the text content of the block. public string Text { get; } - /// Gets or sets the kind of block, for example paragraph, title, or figure. - public string? Kind { get; set; } + /// Gets or sets the kind of block, for example , , or . + public OcrBlockKind? Kind { get; set; } /// Gets or sets the region of the page the block occupies, when the engine provides geometry. public OcrBoundingRegion? BoundingRegion { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlockKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlockKind.cs new file mode 100644 index 00000000000..18e4f880583 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlockKind.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Describes the kind of an , such as a paragraph, title, or figure. +/// +/// This is a small open set modeled on : the well-known values cover the common +/// layout categories, and a provider may introduce its own value when an engine reports a kind that is +/// not represented here. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct OcrBlockKind : IEquatable +{ + /// Gets the kind representing a paragraph of body text. + public static OcrBlockKind Paragraph { get; } = new("paragraph"); + + /// Gets the kind representing a title or heading. + public static OcrBlockKind Title { get; } = new("title"); + + /// Gets the kind representing a figure or image region. + public static OcrBlockKind Figure { get; } = new("figure"); + + /// Gets the value associated with this . + public string Value { get; } + + /// Initializes a new instance of the struct with the provided value. + /// The value to associate with this . + /// is . + /// is empty or composed entirely of whitespace. + [JsonConstructor] + public OcrBlockKind(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// Returns a value indicating whether two instances are equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(OcrBlockKind left, OcrBlockKind right) + { + return left.Equals(right); + } + + /// Returns a value indicating whether two instances are not equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(OcrBlockKind left, OcrBlockKind right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is OcrBlockKind otherKind && Equals(otherKind); + + /// + public bool Equals(OcrBlockKind other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override OcrBlockKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, OcrBlockKind value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs index f2fa7d8b46d..72d9473aff4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs @@ -18,7 +18,7 @@ namespace Microsoft.Extensions.AI; /// layout-block geometry and for field grounding, providing one region primitive across providers. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrBoundingRegion +public class OcrBoundingRegion { /// Initializes a new instance of the class. /// The one-based page number the region is on. @@ -55,12 +55,16 @@ public static OcrBoundingRegion FromRectangle(int pageNumber, double left, doubl ]); /// Computes the axis-aligned bounds of the polygon. - /// The axis-aligned bounds, or when the polygon has no vertices. - public OcrBoundingBox GetBounds() + /// The axis-aligned bounds, or when the polygon has no vertices. + /// + /// Returning for an empty polygon avoids conflating "no geometry" with a real + /// zero-size box located at the origin. + /// + public OcrBoundingBox? GetBounds() { if (Polygon.Count == 0) { - return default; + return null; } float minX = float.MaxValue, minY = float.MaxValue, maxX = float.MinValue, maxY = float.MinValue; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs index 8f7b2e2fdad..1fdea3f995e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Net.Http; @@ -38,7 +39,6 @@ public static class OcrClientExtensions /// The client. /// The document content to parse. /// The OCR options to configure the request. - /// An optional progress reporter. /// The to monitor for cancellation requests. The default is . /// The structured OCR result. /// or is . @@ -46,7 +46,6 @@ public static Task ExtractAsync( this IOcrClient client, DataContent document, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(client); @@ -56,14 +55,13 @@ public static Task ExtractAsync( new MemoryStream(array.Array!, array.Offset, array.Count) : new MemoryStream(document.Data.ToArray()); - return client.ExtractAsync(documentStream, document.MediaType, options, progress, cancellationToken); + return client.ExtractAsync(documentStream, document.MediaType, options, cancellationToken); } /// Runs OCR over a single document referenced by a . /// The client. /// The document reference to parse. /// The OCR options to configure the request. - /// An optional progress reporter. /// The to monitor for cancellation requests. The default is . /// The structured OCR result. /// or is . @@ -75,7 +73,7 @@ public static Task ExtractAsync( /// the bytes or hand the URL to the engine natively (for example Mistral document_url or Azure /// Document Intelligence uriSource) is an open design question the abstraction does not decide. /// To download a remote document explicitly, use - /// + /// /// with a caller-supplied ; or read the bytes yourself and pass a /// or ; or use an engine that accepts a URL directly. /// @@ -83,7 +81,6 @@ public static Task ExtractAsync( this IOcrClient client, UriContent document, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(client); @@ -93,7 +90,7 @@ public static Task ExtractAsync( if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) { // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. - return client.ExtractAsync(new DataContent(uri), options, progress, cancellationToken); + return client.ExtractAsync(new DataContent(uri), options, cancellationToken); } throw new NotSupportedException( @@ -109,14 +106,13 @@ public static Task ExtractAsync( /// The document reference to download and parse. /// The used to download remote (http/https) documents. /// The OCR options to configure the request. - /// An optional progress reporter. /// The to monitor for cancellation requests. The default is . /// The structured OCR result. /// , , or is . /// The uses a scheme other than data:, http, or https. /// /// This is the explicit, opt-in counterpart to - /// , + /// , /// which never touches the network. Self-contained data: URIs are handled inline (no download); /// http/https URIs are fetched with the supplied and passed /// to the stream-based extraction path. The abstraction performs no ambient network IO: the caller owns @@ -129,7 +125,6 @@ public static async Task ExtractFromUriAsync( UriContent document, HttpClient httpClient, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(client); @@ -141,7 +136,7 @@ public static async Task ExtractFromUriAsync( // Self-contained data: URIs carry their own bytes - no download needed. if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) { - return await client.ExtractAsync(document, options, progress, cancellationToken).ConfigureAwait(false); + return await client.ExtractAsync(document, options, cancellationToken).ConfigureAwait(false); } if (!uri.IsAbsoluteUri || @@ -165,6 +160,64 @@ public static async Task ExtractFromUriAsync( #else using Stream contentStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false); #endif - return await client.ExtractAsync(contentStream, mediaType, options, progress, cancellationToken).ConfigureAwait(false); + return await client.ExtractAsync(contentStream, mediaType, options, cancellationToken).ConfigureAwait(false); + } + + /// Runs streaming OCR over a single document provided as a . + /// The client. + /// The document content to parse. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR updates representing the streamed output. + /// or is . + public static IAsyncEnumerable ExtractStreamingAsync( + this IOcrClient client, + DataContent document, + OcrOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + var documentStream = MemoryMarshal.TryGetArray(document.Data, out var array) ? + new MemoryStream(array.Array!, array.Offset, array.Count) : + new MemoryStream(document.Data.ToArray()); + + return client.ExtractStreamingAsync(documentStream, document.MediaType, options, cancellationToken); + } + + /// Runs streaming OCR over a single document referenced by a . + /// The client. + /// The document reference to parse. + /// The OCR options to configure the request. + /// The to monitor for cancellation requests. The default is . + /// The structured OCR updates representing the streamed output. + /// or is . + /// The references a remote URI, which this overload does not fetch. + /// + /// This is the streaming counterpart to + /// ; it handles only + /// self-contained data: URIs and does no file or network IO. For file: and remote URIs it + /// throws, for the same reasons documented on the unary overload. + /// + public static IAsyncEnumerable ExtractStreamingAsync( + this IOcrClient client, + UriContent document, + OcrOptions? options = null, + CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(client); + _ = Throw.IfNull(document); + + Uri uri = document.Uri; + if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) + { + // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. + return client.ExtractStreamingAsync(new DataContent(uri), options, cancellationToken); + } + + throw new NotSupportedException( + "This overload handles only self-contained data: URIs. For file or remote URIs, read the bytes " + + "and pass a stream or DataContent, or use an engine that accepts a URL natively."); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs new file mode 100644 index 00000000000..1d7b7d8c6c2 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Describes the unit in which OCR geometry coordinates ( and +/// ) are expressed. +/// +/// +/// Coordinate conventions differ across OCR engines: some report pixels of the rendered page image, +/// some report a physical unit such as inches, and some normalize to the page. Pairing the geometry +/// with an and the page dimensions ( and +/// ) lets a consumer interpret or normalize regions with engine-agnostic +/// code. This type is a small open set modeled on : the well-known values cover +/// the common cases, and a provider may introduce its own value when needed. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct OcrCoordinateUnit : IEquatable +{ + /// Gets the unit for coordinates expressed in pixels of the rendered page image. + public static OcrCoordinateUnit Pixel { get; } = new("pixel"); + + /// Gets the unit for coordinates expressed in inches. + public static OcrCoordinateUnit Inch { get; } = new("inch"); + + /// Gets the unit for coordinates normalized to the range [0, 1] relative to the page width and height. + public static OcrCoordinateUnit Normalized { get; } = new("normalized"); + + /// Gets the value associated with this . + public string Value { get; } + + /// + /// Initializes a new instance of the struct with the provided value. + /// + /// The value to associate with this . + /// is . + /// is empty or composed entirely of whitespace. + [JsonConstructor] + public OcrCoordinateUnit(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// + /// Returns a value indicating whether two instances are equivalent, as + /// determined by a case-insensitive comparison of their values. + /// + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(OcrCoordinateUnit left, OcrCoordinateUnit right) + { + return left.Equals(right); + } + + /// + /// Returns a value indicating whether two instances are not equivalent, as + /// determined by a case-insensitive comparison of their values. + /// + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(OcrCoordinateUnit left, OcrCoordinateUnit right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is OcrCoordinateUnit otherUnit && Equals(otherUnit); + + /// + public bool Equals(OcrCoordinateUnit other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override OcrCoordinateUnit Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, OcrCoordinateUnit value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs index 2e522d95207..b23c8e0994f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs @@ -15,7 +15,7 @@ namespace Microsoft.Extensions.AI; /// populate only . This lets one shape serve both provider archetypes. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrImage +public class OcrImage { /// Gets or sets the rendered image bytes, when the engine returns them. public DataContent? Content { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs index ab2c8ba0f2f..87207fac1c9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs @@ -12,7 +12,7 @@ namespace Microsoft.Extensions.AI; /// provider-specific settings, mirroring ChatOptions. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrOptions +public class OcrOptions { /// Gets or sets the model or deployment identifier to use for this request. public string? ModelId { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs index 4c9567a9249..270af78c174 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI; /// Represents one page of structured OCR output. [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrPage +public class OcrPage { /// Initializes a new instance of the class. /// The one-based page number. @@ -40,6 +40,28 @@ public OcrPage(int pageNumber, string markdown) /// Gets or sets the page-level confidence in the range [0, 1], when available. public double? Confidence { get; set; } + /// Gets or sets the page width, expressed in , when the engine provides it. + /// + /// Together with and , this lets a consumer interpret or + /// normalize the geometry ( / ) on this page with + /// engine-agnostic code. For example, dividing a coordinate by the corresponding page dimension yields a + /// page-relative [0, 1] value regardless of the native unit. + /// + public float? Width { get; set; } + + /// Gets or sets the page height, expressed in , when the engine provides it. + /// See for how the page dimensions are used with . + public float? Height { get; set; } + + /// Gets or sets the unit in which this page's geometry coordinates are expressed, when known. + /// + /// OCR engines disagree on coordinate conventions (pixels, inches, or page-normalized values). When the + /// engine reports the unit, exposing it here alongside and makes + /// the region geometry interpretable without knowing which engine produced it. When , + /// the geometry should be treated as an opaque, provider-specific coordinate space. + /// + public OcrCoordinateUnit? CoordinateUnit { get; set; } + /// Gets or sets any additional properties associated with the page. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs deleted file mode 100644 index c351295b035..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrProgress.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; -using Microsoft.Shared.DiagnosticIds; - -namespace Microsoft.Extensions.AI; - -/// Represents progress reported during a long-running OCR request. -/// -/// Engines that poll a long-running operation (such as Azure Document Intelligence) report pages as -/// they complete; synchronous engines (such as Mistral OCR) report a single terminal update. -/// -[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrProgress -{ - /// Gets or sets the number of pages processed so far, when known. - public int? PagesProcessed { get; set; } - - /// Gets or sets the total number of pages, when known. - public int? TotalPages { get; set; } - - /// Gets or sets a human-readable status for the operation, when available. - public string? Status { get; set; } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs new file mode 100644 index 00000000000..f9558e5b5f5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents a single streaming update from an . +/// +/// +/// An request produces a sequence of +/// instances. A typical engine emits one update per page as that page +/// finishes (carrying the completed ), optionally interleaved with progress-only updates +/// (, , ) for long-running +/// operations such as Azure Document Intelligence. A synchronous engine may emit a single terminal update. +/// +/// +/// The relationship between and is codified in +/// , which reassembles a stream of updates into a +/// single . The conversion can be slightly lossy: for example, only one +/// slot is available on the assembled . +/// +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public class OcrResponseUpdate +{ + /// Initializes a new instance of the class. + [JsonConstructor] + public OcrResponseUpdate() + { + } + + /// Initializes a new instance of the class with the page completed in this update. + /// The page produced in this update. + public OcrResponseUpdate(OcrPage? page) + { + Page = page; + } + + /// Gets or sets the page produced in this update, when the update carries a completed page. + /// Progress-only updates leave this . + public OcrPage? Page { get; set; } + + /// Gets or sets the number of pages processed so far, when known. + public int? PagesProcessed { get; set; } + + /// Gets or sets the total number of pages, when known. + public int? TotalPages { get; set; } + + /// Gets or sets a human-readable status for the operation, when available. + public string? Status { get; set; } + + /// Gets or sets the model or deployment identifier that served the request. + public string? ModelId { get; set; } + + /// Gets or sets usage details associated with the request, when reported. + /// Usage is typically carried on a terminal update once the full document has been processed. + public OcrUsage? Usage { get; set; } + + /// Gets or sets the provider-native object underlying this update. + /// + /// If an is created to represent an underlying object from another object + /// model, this property can store that original object. This can be useful for debugging or for enabling + /// a consumer to access the underlying object model if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the update. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs new file mode 100644 index 00000000000..8524fdacdfe --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Provides extension methods for working with instances. +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public static class OcrResponseUpdateExtensions +{ + /// Combines instances into a single . + /// The updates to be combined. + /// The combined . + /// is . + public static OcrResult ToOcrResult(this IEnumerable updates) + { + _ = Throw.IfNull(updates); + + List pages = []; + OcrResult result = new(pages); + + foreach (var update in updates) + { + ProcessUpdate(update, pages, result); + } + + return result; + } + + /// Combines instances into a single . + /// The updates to be combined. + /// The to monitor for cancellation requests. The default is . + /// The combined . + /// is . + public static Task ToOcrResultAsync( + this IAsyncEnumerable updates, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(updates); + + return ToResultAsync(updates, cancellationToken); + + static async Task ToResultAsync( + IAsyncEnumerable updates, CancellationToken cancellationToken) + { + List pages = []; + OcrResult result = new(pages); + + await foreach (var update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) + { + ProcessUpdate(update, pages, result); + } + + return result; + } + } + + /// Incorporates one into the assembled . + /// The update to process. + /// The accumulating list of pages backing . + /// The being assembled. + private static void ProcessUpdate(OcrResponseUpdate update, List pages, OcrResult result) + { + if (update.Page is not null) + { + pages.Add(update.Page); + } + + if (update.ModelId is not null) + { + result.ModelId = update.ModelId; + } + + if (update.Usage is not null) + { + result.Usage = update.Usage; + } + + if (update.AdditionalProperties is not null) + { + if (result.AdditionalProperties is null) + { + result.AdditionalProperties = new(update.AdditionalProperties); + } + else + { + foreach (var entry in update.AdditionalProperties) + { + result.AdditionalProperties[entry.Key] = entry.Value; + } + } + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs index 1c52b722172..f74579e2790 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs @@ -17,7 +17,7 @@ namespace Microsoft.Extensions.AI; /// ChatResponse normalizes the common surface and preserves the raw. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrResult +public class OcrResult { /// Initializes a new instance of the class. /// The per-page structured content. @@ -33,10 +33,6 @@ public OcrResult(IReadOnlyList pages) /// Gets the full-document markdown, formed by joining the per-page markdown. public string Markdown => string.Join("\n\n", Pages.Select(p => p.Markdown)); - /// Gets or sets an identifier for the engine that produced this result. - /// This typically flows downstream as an ocr_source metadata value. - public string? OcrSource { get; set; } - /// Gets or sets the model or deployment identifier that served the request. public string? ModelId { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs index 6de145996b6..49d3ba12c94 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs @@ -10,12 +10,14 @@ namespace Microsoft.Extensions.AI; /// Represents a table extracted from a document. /// /// Cells are the primary, structured representation (row and column indices with spans, the Azure -/// Document Intelligence shape). is the fallback for engines that -/// only emit markdown or HTML (such as Mistral OCR). Consumers prefer when present -/// and fall back to otherwise. +/// Document Intelligence shape) and are authoritative when non-. +/// is the fallback for engines that only emit markdown or HTML +/// (such as Mistral OCR). Consumers prefer when present and fall back to +/// otherwise. On the markdown-only path and +/// may be 0 because the structure was not enumerated. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrTable +public class OcrTable { /// Initializes a new instance of the class. /// The number of rows in the table. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs index 53165f1aeb4..aae87b9fd09 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs @@ -9,7 +9,7 @@ namespace Microsoft.Extensions.AI; /// Represents a single cell within an . [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrTableCell +public class OcrTableCell { /// Initializes a new instance of the class. /// The zero-based row index of the cell. @@ -23,8 +23,8 @@ public OcrTableCell(int rowIndex, int columnIndex, string content) Content = Throw.IfNull(content); } - /// Gets or sets the role of the cell, for example columnHeader or content. - public string? Kind { get; set; } + /// Gets or sets the role of the cell, for example or . + public OcrTableCellKind? Kind { get; set; } /// Gets the zero-based row index of the cell. public int RowIndex { get; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs new file mode 100644 index 00000000000..4f1b20cb68b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Describes the role of an , such as a column header or a content cell. +/// +/// This is a small open set modeled on : the well-known values cover the common +/// table cell roles, and a provider may introduce its own value when an engine reports a role that is +/// not represented here. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonConverter(typeof(Converter))] +[DebuggerDisplay("{Value,nq}")] +public readonly struct OcrTableCellKind : IEquatable +{ + /// Gets the kind representing a column header cell. + public static OcrTableCellKind ColumnHeader { get; } = new("columnHeader"); + + /// Gets the kind representing a regular content cell. + public static OcrTableCellKind Content { get; } = new("content"); + + /// Gets the value associated with this . + public string Value { get; } + + /// Initializes a new instance of the struct with the provided value. + /// The value to associate with this . + /// is . + /// is empty or composed entirely of whitespace. + [JsonConstructor] + public OcrTableCellKind(string value) + { + Value = Throw.IfNullOrWhitespace(value); + } + + /// Returns a value indicating whether two instances are equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have equivalent values; otherwise, . + public static bool operator ==(OcrTableCellKind left, OcrTableCellKind right) + { + return left.Equals(right); + } + + /// Returns a value indicating whether two instances are not equivalent, using a case-insensitive comparison. + /// The first instance to compare. + /// The second instance to compare. + /// if left and right have different values; otherwise, . + public static bool operator !=(OcrTableCellKind left, OcrTableCellKind right) + { + return !(left == right); + } + + /// + public override bool Equals([NotNullWhen(true)] object? obj) + => obj is OcrTableCellKind otherKind && Equals(otherKind); + + /// + public bool Equals(OcrTableCellKind other) + => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); + + /// + public override int GetHashCode() + => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); + + /// + public override string ToString() => Value; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class Converter : JsonConverter + { + /// + public override OcrTableCellKind Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + new(reader.GetString()!); + + /// + public override void Write(Utf8JsonWriter writer, OcrTableCellKind value, JsonSerializerOptions options) => + Throw.IfNull(writer).WriteStringValue(value.Value); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs index bfea283448c..228345eafeb 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs @@ -8,7 +8,7 @@ namespace Microsoft.Extensions.AI; /// Represents usage details associated with an OCR request. [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public sealed class OcrUsage +public class OcrUsage { /// Gets or sets the number of pages processed by the request, when known. public int? PagesProcessed { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 379a8c96674..438ad3375e9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -143,7 +143,7 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(OcrOptions))] [JsonSerializable(typeof(OcrClientMetadata))] [JsonSerializable(typeof(OcrResult))] - [JsonSerializable(typeof(OcrProgress))] + [JsonSerializable(typeof(OcrResponseUpdate))] // IHostedFileClient [JsonSerializable(typeof(HostedFileClientOptions))] diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json index 158a7000c62..9ee5df97ae7 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json @@ -282,7 +282,11 @@ "Stage": "Experimental" }, { - "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ] @@ -1052,7 +1056,11 @@ "Stage": "Experimental" }, { - "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.LoggingOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.LoggingOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.LoggingOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ], @@ -1432,7 +1440,11 @@ "Stage": "Experimental" }, { - "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.OpenTelemetryOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.IProgress? progress = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Threading.Tasks.Task Microsoft.Extensions.AI.OpenTelemetryOcrClient.ExtractAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Stage": "Experimental" + }, + { + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OpenTelemetryOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs index 239d767fdaa..545c4ff53d7 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Threading; @@ -40,10 +41,19 @@ public override async Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { - return await base.ExtractAsync(document, mediaType, Configure(options), progress, cancellationToken); + return await base.ExtractAsync(document, mediaType, Configure(options), cancellationToken); + } + + /// + public override IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + CancellationToken cancellationToken = default) + { + return base.ExtractStreamingAsync(document, mediaType, Configure(options), cancellationToken); } /// Creates and configures the to pass along to the inner client. diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs index be9c2a36eb3..d7e6ccca2ca 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs @@ -2,8 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -57,7 +59,6 @@ public override async Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { if (_logger.IsEnabled(LogLevel.Debug)) @@ -74,7 +75,7 @@ public override async Task ExtractAsync( try { - var result = await base.ExtractAsync(document, mediaType, options, progress, cancellationToken); + var result = await base.ExtractAsync(document, mediaType, options, cancellationToken); if (_logger.IsEnabled(LogLevel.Debug)) { @@ -102,6 +103,89 @@ public override async Task ExtractAsync( } } + /// + public override async IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogInvokedSensitive(nameof(ExtractStreamingAsync), mediaType, AsJson(options), AsJson(this.GetService())); + } + else + { + LogInvoked(nameof(ExtractStreamingAsync)); + } + } + + IAsyncEnumerator e; + try + { + e = base.ExtractStreamingAsync(document, mediaType, options, cancellationToken).GetAsyncEnumerator(cancellationToken); + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ExtractStreamingAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ExtractStreamingAsync), ex); + throw; + } + + try + { + OcrResponseUpdate? update = null; + while (true) + { + try + { + if (!await e.MoveNextAsync()) + { + break; + } + + update = e.Current; + } + catch (OperationCanceledException) + { + LogInvocationCanceled(nameof(ExtractStreamingAsync)); + throw; + } + catch (Exception ex) + { + LogInvocationFailed(nameof(ExtractStreamingAsync), ex); + throw; + } + + if (_logger.IsEnabled(LogLevel.Debug)) + { + if (_logger.IsEnabled(LogLevel.Trace)) + { + LogStreamingUpdateSensitive(AsJson(update)); + } + else + { + LogStreamingUpdate(); + } + } + + yield return update; + } + + LogCompleted(nameof(ExtractStreamingAsync)); + } + finally + { + await e.DisposeAsync(); + } + } + private string AsJson(T value) => TelemetryHelpers.AsJson(value, _jsonSerializerOptions); [LoggerMessage(LogLevel.Debug, "{MethodName} invoked.")] @@ -116,6 +200,12 @@ public override async Task ExtractAsync( [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {OcrResult}.")] private partial void LogCompletedSensitive(string methodName, string ocrResult); + [LoggerMessage(LogLevel.Debug, "ExtractStreamingAsync received update.")] + private partial void LogStreamingUpdate(); + + [LoggerMessage(LogLevel.Trace, "ExtractStreamingAsync received update: {OcrResponseUpdate}")] + private partial void LogStreamingUpdateSensitive(string ocrResponseUpdate); + [LoggerMessage(LogLevel.Debug, "{MethodName} canceled.")] private partial void LogInvocationCanceled(string methodName); diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs index 24d6e6a6e2f..c74bd36d04c 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OcrClientBuilderServiceCollectionExtensions.cs @@ -18,6 +18,8 @@ public static class OcrClientBuilderServiceCollectionExtensions /// The inner that represents the underlying backend. /// The service lifetime for the client. Defaults to . /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . public static OcrClientBuilder AddOcrClient( this IServiceCollection serviceCollection, IOcrClient innerClient, @@ -29,6 +31,8 @@ public static OcrClientBuilder AddOcrClient( /// A callback that produces the inner that represents the underlying backend. /// The service lifetime for the client. Defaults to . /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . public static OcrClientBuilder AddOcrClient( this IServiceCollection serviceCollection, Func innerClientFactory, @@ -48,6 +52,8 @@ public static OcrClientBuilder AddOcrClient( /// The inner that represents the underlying backend. /// The service lifetime for the client. Defaults to . /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . public static OcrClientBuilder AddKeyedOcrClient( this IServiceCollection serviceCollection, object? serviceKey, @@ -61,6 +67,8 @@ public static OcrClientBuilder AddKeyedOcrClient( /// A callback that produces the inner that represents the underlying backend. /// The service lifetime for the client. Defaults to . /// An that can be used to build a pipeline around the inner client. + /// is . + /// is . public static OcrClientBuilder AddKeyedOcrClient( this IServiceCollection serviceCollection, object? serviceKey, diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs index eddfc78cfdb..67c9a9a1ea9 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs @@ -7,6 +7,7 @@ using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Metrics; using System.IO; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -100,7 +101,6 @@ public override async Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(document); @@ -113,7 +113,7 @@ public override async Task ExtractAsync( Exception? error = null; try { - response = await base.ExtractAsync(document, mediaType, options, progress, cancellationToken); + response = await base.ExtractAsync(document, mediaType, options, cancellationToken); return response; } catch (Exception ex) @@ -127,6 +127,69 @@ public override async Task ExtractAsync( } } + /// + public override async IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(document); + + using Activity? activity = CreateAndConfigureActivity(options); + Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + string? requestModelId = options?.ModelId ?? _defaultModelId; + + IAsyncEnumerable updates; + try + { + updates = base.ExtractStreamingAsync(document, mediaType, options, cancellationToken); + } + catch (Exception ex) + { + TraceResponse(activity, requestModelId, response: null, ex, stopwatch); + throw; + } + + var responseEnumerator = updates.GetAsyncEnumerator(cancellationToken); + List trackedUpdates = []; + Exception? error = null; + try + { + while (true) + { + OcrResponseUpdate update; + try + { + if (!await responseEnumerator.MoveNextAsync()) + { + break; + } + + update = responseEnumerator.Current; + } + catch (Exception ex) + { + error = ex; + throw; + } + + trackedUpdates.Add(update); + yield return update; + if (activity is not null) + { + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + } + finally + { + TraceResponse(activity, requestModelId, trackedUpdates.ToOcrResult(), error, stopwatch); + + await responseEnumerator.DisposeAsync(); + } + } + /// Creates an activity for an OCR request, or returns if not enabled. private Activity? CreateAndConfigureActivity(OcrOptions? options) { diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs index 9835f8b0975..059d621926b 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs @@ -91,8 +91,12 @@ private static IngestionDocumentImage MapImage(OcrImage image, int pageNumber) if (image.BoundingRegion is not null) { - (float left, float top, float right, float bottom) = image.BoundingRegion.GetBounds(); - element.Metadata[BoundingBoxMetadataKey] = new[] { left, top, right, bottom }; + if (image.BoundingRegion.GetBounds() is { } bounds) + { + (float left, float top, float right, float bottom) = bounds; + element.Metadata[BoundingBoxMetadataKey] = new[] { left, top, right, bottom }; + } + element.Metadata[BoundingRegionMetadataKey] = image.BoundingRegion.Polygon.SelectMany(static p => new[] { p.X, p.Y }).ToArray(); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs index b9cb05eda2b..181149aaf73 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -29,7 +30,7 @@ public async Task ExtractAsyncDefaultsToInnerClientAsync() var expectedResponse = new OcrResult([]); using var inner = new TestOcrClient { - ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = (document, mediaType, options, cancellationToken) => { Assert.Same(expectedDocument, document); Assert.Same(expectedMediaType, mediaType); @@ -42,7 +43,7 @@ public async Task ExtractAsyncDefaultsToInnerClientAsync() using var delegating = new NoOpDelegatingOcrClient(inner); // Act - var resultTask = delegating.ExtractAsync(expectedDocument, expectedMediaType, expectedOptions, null, expectedCancellationToken); + var resultTask = delegating.ExtractAsync(expectedDocument, expectedMediaType, expectedOptions, expectedCancellationToken); // Assert Assert.False(resultTask.IsCompleted); @@ -51,6 +52,54 @@ public async Task ExtractAsyncDefaultsToInnerClientAsync() Assert.Same(expectedResponse, await resultTask); } + [Fact] + public async Task ExtractStreamingAsyncDefaultsToInnerClientAsync() + { + // Arrange + using var expectedDocument = new MemoryStream(); + var expectedMediaType = "application/pdf"; + var expectedOptions = new OcrOptions(); + using var cts = new CancellationTokenSource(); + OcrResponseUpdate[] expectedUpdates = + [ + new(new OcrPage(1, "page one")), + new(new OcrPage(2, "page two")), + ]; + + using var inner = new TestOcrClient + { + ExtractStreamingAsyncCallback = (document, mediaType, options, cancellationToken) => + { + Assert.Same(expectedDocument, document); + Assert.Same(expectedMediaType, mediaType); + Assert.Same(expectedOptions, options); + Assert.Equal(cts.Token, cancellationToken); + return YieldAsync(expectedUpdates); + } + }; + + using var delegating = new NoOpDelegatingOcrClient(inner); + + // Act + List received = []; + await foreach (var update in delegating.ExtractStreamingAsync(expectedDocument, expectedMediaType, expectedOptions, cts.Token)) + { + received.Add(update); + } + + // Assert + Assert.Equal(expectedUpdates, received); + } + + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) + { + foreach (var update in updates) + { + await Task.Yield(); + yield return update; + } + } + [Fact] public void GetServiceThrowsForNullType() { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs index d3ec599c5c5..730b3bb751a 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrBoundingRegionTests.cs @@ -28,7 +28,10 @@ public void GetBounds_ReturnsAxisAlignedExtents() { var region = new OcrBoundingRegion(1, [new OcrPoint(30, 40), new OcrPoint(100, 35), new OcrPoint(110, 90), new OcrPoint(25, 95)]); - var (left, top, right, bottom) = region.GetBounds(); + var bounds = region.GetBounds(); + + Assert.NotNull(bounds); + var (left, top, right, bottom) = bounds.Value; Assert.Equal(25, left); Assert.Equal(35, top); @@ -37,10 +40,10 @@ public void GetBounds_ReturnsAxisAlignedExtents() } [Fact] - public void GetBounds_EmptyPolygon_ReturnsDefault() + public void GetBounds_EmptyPolygon_ReturnsNull() { var region = new OcrBoundingRegion(1, []); - Assert.Equal(default, region.GetBounds()); + Assert.Null(region.GetBounds()); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs index 09f94bd10b8..02958669e36 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrClientExtensionsTests.cs @@ -43,7 +43,7 @@ public async Task ExtractAsync_DataContent_PassesStreamAndMediaTypeAsync() using var client = new TestOcrClient { - ExtractAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = async (document, mediaType, options, cancellationToken) => { observedMediaType = mediaType; using var ms = new MemoryStream(); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs new file mode 100644 index 00000000000..381d030619c --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrResponseUpdateExtensionsTests +{ + [Fact] + public void ToOcrResult_NullUpdates_Throws() + { + Assert.Throws("updates", () => ((IEnumerable)null!).ToOcrResult()); + } + + [Fact] + public async Task ToOcrResultAsync_NullUpdates_ThrowsAsync() + { + await Assert.ThrowsAsync("updates", () => ((IAsyncEnumerable)null!).ToOcrResultAsync()); + } + + [Fact] + public void ToOcrResult_AssemblesPagesModelIdAndUsage() + { + OcrResponseUpdate[] updates = + [ + new(new OcrPage(1, "page one")), + new() { PagesProcessed = 1, TotalPages = 2, Status = "processing" }, + new(new OcrPage(2, "page two")) { ModelId = "model-x", Usage = new() { PagesProcessed = 2 } }, + ]; + + OcrResult result = updates.ToOcrResult(); + + Assert.Equal(2, result.Pages.Count); + Assert.Equal("page one\n\npage two", result.Markdown); + Assert.Equal("model-x", result.ModelId); + Assert.NotNull(result.Usage); + Assert.Equal(2, result.Usage!.PagesProcessed); + } + + [Fact] + public async Task ToOcrResultAsync_AssemblesPagesModelIdAndUsageAsync() + { + OcrResponseUpdate[] updates = + [ + new(new OcrPage(1, "page one")), + new(new OcrPage(2, "page two")) { ModelId = "model-y" }, + ]; + + OcrResult result = await YieldAsync(updates).ToOcrResultAsync(); + + Assert.Equal(2, result.Pages.Count); + Assert.Equal("page one\n\npage two", result.Markdown); + Assert.Equal("model-y", result.ModelId); + } + + [Fact] + public void ToOcrResult_MergesAdditionalProperties() + { + OcrResponseUpdate[] updates = + [ + new(new OcrPage(1, "page one")) { AdditionalProperties = new() { ["a"] = "1" } }, + new(new OcrPage(2, "page two")) { AdditionalProperties = new() { ["b"] = "2" } }, + ]; + + OcrResult result = updates.ToOcrResult(); + + Assert.NotNull(result.AdditionalProperties); + Assert.Equal("1", result.AdditionalProperties!["a"]); + Assert.Equal("2", result.AdditionalProperties!["b"]); + } + + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) + { + foreach (var update in updates) + { + await Task.Yield(); + yield return update; + } + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs index 86a1a3d350a..752d2b4e890 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs @@ -19,12 +19,10 @@ public void Markdown_JoinsPerPageMarkdown() { var result = new OcrResult([new OcrPage(1, "page one"), new OcrPage(2, "page two")]) { - OcrSource = "test-engine", ModelId = "model-1", }; Assert.Equal("page one\n\npage two", result.Markdown); - Assert.Equal("test-engine", result.OcrSource); Assert.Equal("model-1", result.ModelId); Assert.Equal(2, result.Pages.Count); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs index d6a2fd8198a..0df2626b12f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -20,12 +21,19 @@ public TestOcrClient() public Func?, CancellationToken, Task>? ExtractAsyncCallback { get; set; } + public Func>? + ExtractStreamingAsyncCallback + { get; set; } + public Func GetServiceCallback { get; set; } private object? DefaultGetServiceCallback(Type serviceType, object? serviceKey) @@ -35,9 +43,15 @@ public Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) - => ExtractAsyncCallback!.Invoke(document, mediaType, options, progress, cancellationToken); + => ExtractAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); + + public IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + CancellationToken cancellationToken = default) + => ExtractStreamingAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); public object? GetService(Type serviceType, object? serviceKey = null) => GetServiceCallback!.Invoke(serviceType, serviceKey); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs index 173364d3ca7..3da7ae0ea1d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/ConfigureOptionsOcrClientTests.cs @@ -38,7 +38,7 @@ public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullP using IOcrClient innerClient = new TestOcrClient { - ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = (document, mediaType, options, cancellationToken) => { Assert.Same(returnedOptions, options); Assert.Equal(cts.Token, cancellationToken); @@ -65,7 +65,7 @@ public async Task ConfigureOptions_ReturnedInstancePassedToNextClient(bool nullP .Build(); using var document = new MemoryStream(new byte[] { 1, 2, 3, 4 }); - var result = await client.ExtractAsync(document, "application/pdf", providedOptions, null, cts.Token); + var result = await client.ExtractAsync(document, "application/pdf", providedOptions, cts.Token); Assert.Same(expectedResult, result); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs index e05f7d331fb..9ef71dddc29 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/LoggingOcrClientTests.cs @@ -54,7 +54,7 @@ public async Task ExtractAsync_LogsInvocationAndCompletion(LogLevel level) using IOcrClient innerClient = new TestOcrClient { - ExtractAsyncCallback = (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = (document, mediaType, options, cancellationToken) => Task.FromResult(new OcrResult([new OcrPage(1, "blue whale")])), }; diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs index e40efdcf8c7..06baea533e6 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs @@ -33,7 +33,7 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) using var innerClient = new TestOcrClient { - ExtractAsyncCallback = async (document, mediaType, options, progress, cancellationToken) => + ExtractAsyncCallback = async (document, mediaType, options, cancellationToken) => { await Task.Yield(); return new OcrResult([new OcrPage(1, "This is the recognized text.")]) diff --git a/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs index 5dca4561ea5..f4b38b52b80 100644 --- a/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs +++ b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs @@ -4,6 +4,7 @@ #pragma warning disable MEAI001 // OCR abstractions are experimental. using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -61,7 +62,6 @@ public Task ExtractAsync( Stream document, string mediaType, OcrOptions? options = null, - IProgress? progress = null, CancellationToken cancellationToken = default) { MediaType = mediaType; @@ -69,6 +69,13 @@ public Task ExtractAsync( return Task.FromResult(result); } + public IAsyncEnumerable ExtractStreamingAsync( + Stream document, + string mediaType, + OcrOptions? options = null, + CancellationToken cancellationToken = default) + => throw new NotSupportedException(); + public object? GetService(Type serviceType, object? serviceKey = null) => null; public void Dispose() From 5e696788a507a598f3bc0067019c0c98aefc0627 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Thu, 23 Jul 2026 10:44:56 -0400 Subject: [PATCH 11/13] Reshape OCR result model: reading-order elements, doc-level coordinates, usage tokens Continue the IOcrClient reshape ahead of API review (#7587/#7588). - Replace parallel OcrPage.Blocks/Tables/Images with a single reading-order Elements list over a new polymorphic OcrElement base (OcrBlock/OcrTable/OcrImage derive); consumers project with OfType(). - Add optional nested OcrTableCell.Elements for structured cell content; keep the Content string as a flat-text convenience. - Convert OcrCoordinateUnit to a closed enum and add Point; add an OcrCoordinateOrigin enum; move both to document level (OcrResult/OcrPageResult) and remove the per-page CoordinateUnit. - Keep OcrBlockKind and OcrTableCellKind as open structs; add RowHeader and RowSection well-known cell kinds. - Add nullable Input/Output/TotalTokenCount to OcrUsage for the vision-LLM path. - Rename Markdown to Text on OcrPage/OcrResult; drop response ModelId, OcrPage.Confidence, and OcrOptions.IncludeImages; FromRectangle takes float. - Keep ToOcrResult/ToOcrResultAsync reducers and the convenience overloads as extension methods, matching the MEAI ChatResponseExtensions precedent. - Regenerate the ApiChief baselines for the reshaped surface. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 19ea13dd-f444-4942-b751-162711478580 --- .../Microsoft.Extensions.AI.Abstractions.json | 252 +++++++++--------- .../Ocr/DelegatingOcrClient.cs | 4 +- .../Ocr/IOcrClient.cs | 8 +- .../Ocr/OcrBlock.cs | 8 +- .../Ocr/OcrBoundingRegion.cs | 10 +- .../Ocr/OcrClientExtensions.cs | 8 +- .../Ocr/OcrCoordinateOrigin.cs | 28 ++ .../Ocr/OcrCoordinateUnit.cs | 102 ++----- .../Ocr/OcrElement.cs | 47 ++++ .../Ocr/OcrImage.cs | 10 +- .../Ocr/OcrOptions.cs | 4 - .../Ocr/OcrPage.cs | 55 ++-- ...{OcrResponseUpdate.cs => OcrPageResult.cs} | 50 ++-- ...tensions.cs => OcrPageResultExtensions.cs} | 28 +- .../Ocr/OcrResult.cs | 21 +- .../Ocr/OcrTable.cs | 5 +- .../Ocr/OcrTableCell.cs | 10 + .../Ocr/OcrTableCellKind.cs | 6 + .../Ocr/OcrUsage.cs | 10 + .../Utilities/AIJsonUtilities.Defaults.cs | 2 +- .../Microsoft.Extensions.AI.json | 6 +- .../Ocr/ConfigureOptionsOcrClient.cs | 4 +- .../Ocr/LoggingOcrClient.cs | 28 +- .../Ocr/OpenTelemetryOcrClient.cs | 24 +- .../OcrDocumentReader.cs | 12 +- .../Ocr/DelegatingOcrClientTests.cs | 12 +- .../Ocr/OcrElementTests.cs | 90 +++++++ .../Ocr/OcrOptionsTests.cs | 2 - ...sts.cs => OcrPageResultExtensionsTests.cs} | 47 ++-- .../Ocr/OcrResultTests.cs | 10 +- .../TestOcrClient.cs | 8 +- .../Ocr/OpenTelemetryOcrClientTests.cs | 2 - .../Readers/OcrDocumentReaderTests.cs | 6 +- 33 files changed, 503 insertions(+), 416 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrElement.cs rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/{OcrResponseUpdate.cs => OcrPageResult.cs} (54%) rename src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/{OcrResponseUpdateExtensions.cs => OcrPageResultExtensions.cs} (73%) create mode 100644 test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs rename test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/{OcrResponseUpdateExtensionsTests.cs => OcrPageResultExtensionsTests.cs} (53%) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index e4ad63d58de..fef135266c5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -1856,7 +1856,7 @@ "Stage": "Experimental" }, { - "Member": "virtual System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.DelegatingOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "virtual System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.DelegatingOcrClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { @@ -3209,7 +3209,7 @@ "Stage": "Experimental" }, { - "Member": "System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.IOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.IOcrClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { @@ -3351,7 +3351,7 @@ ] }, { - "Type": "class Microsoft.Extensions.AI.OcrBlock", + "Type": "class Microsoft.Extensions.AI.OcrBlock : Microsoft.Extensions.AI.OcrElement", "Stage": "Experimental", "Methods": [ { @@ -3360,14 +3360,6 @@ } ], "Properties": [ - { - "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrBlock.BoundingRegion { get; set; }", - "Stage": "Experimental" - }, - { - "Member": "double? Microsoft.Extensions.AI.OcrBlock.Confidence { get; set; }", - "Stage": "Experimental" - }, { "Member": "Microsoft.Extensions.AI.OcrBlockKind? Microsoft.Extensions.AI.OcrBlock.Kind { get; set; }", "Stage": "Experimental" @@ -3521,7 +3513,7 @@ "Stage": "Experimental" }, { - "Member": "static Microsoft.Extensions.AI.OcrBoundingRegion Microsoft.Extensions.AI.OcrBoundingRegion.FromRectangle(int pageNumber, double left, double top, double right, double bottom);", + "Member": "static Microsoft.Extensions.AI.OcrBoundingRegion Microsoft.Extensions.AI.OcrBoundingRegion.FromRectangle(int pageNumber, float left, float top, float right, float bottom);", "Stage": "Experimental" }, { @@ -3557,11 +3549,11 @@ "Stage": "Experimental" }, { - "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OcrClientExtensions.ExtractStreamingAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OcrClientExtensions.ExtractPagesAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.DataContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { - "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OcrClientExtensions.ExtractStreamingAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "static System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OcrClientExtensions.ExtractPagesAsync(this Microsoft.Extensions.AI.IOcrClient client, Microsoft.Extensions.AI.UriContent document, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { @@ -3595,81 +3587,89 @@ ] }, { - "Type": "readonly struct Microsoft.Extensions.AI.OcrCoordinateUnit : System.IEquatable", + "Type": "enum Microsoft.Extensions.AI.OcrCoordinateOrigin", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.OcrCoordinateUnit(string value);", - "Stage": "Experimental" - }, - { - "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.OcrCoordinateUnit();", - "Stage": "Experimental" - }, - { - "Member": "override bool Microsoft.Extensions.AI.OcrCoordinateUnit.Equals(object? obj);", - "Stage": "Experimental" - }, - { - "Member": "bool Microsoft.Extensions.AI.OcrCoordinateUnit.Equals(Microsoft.Extensions.AI.OcrCoordinateUnit other);", - "Stage": "Experimental" - }, - { - "Member": "override int Microsoft.Extensions.AI.OcrCoordinateUnit.GetHashCode();", + "Member": "Microsoft.Extensions.AI.OcrCoordinateOrigin.OcrCoordinateOrigin();", "Stage": "Experimental" - }, + } + ], + "Fields": [ { - "Member": "static bool Microsoft.Extensions.AI.OcrCoordinateUnit.operator ==(Microsoft.Extensions.AI.OcrCoordinateUnit left, Microsoft.Extensions.AI.OcrCoordinateUnit right);", - "Stage": "Experimental" + "Member": "const Microsoft.Extensions.AI.OcrCoordinateOrigin Microsoft.Extensions.AI.OcrCoordinateOrigin.BottomLeft", + "Stage": "Experimental", + "Value": "1" }, { - "Member": "static bool Microsoft.Extensions.AI.OcrCoordinateUnit.operator !=(Microsoft.Extensions.AI.OcrCoordinateUnit left, Microsoft.Extensions.AI.OcrCoordinateUnit right);", - "Stage": "Experimental" - }, + "Member": "const Microsoft.Extensions.AI.OcrCoordinateOrigin Microsoft.Extensions.AI.OcrCoordinateOrigin.TopLeft", + "Stage": "Experimental", + "Value": "0" + } + ] + }, + { + "Type": "enum Microsoft.Extensions.AI.OcrCoordinateUnit", + "Stage": "Experimental", + "Methods": [ { - "Member": "override string Microsoft.Extensions.AI.OcrCoordinateUnit.ToString();", + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.OcrCoordinateUnit();", "Stage": "Experimental" } ], - "Properties": [ + "Fields": [ { - "Member": "static Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Inch { get; }", - "Stage": "Experimental" + "Member": "const Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Inch", + "Stage": "Experimental", + "Value": "2" }, { - "Member": "static Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Normalized { get; }", - "Stage": "Experimental" + "Member": "const Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Normalized", + "Stage": "Experimental", + "Value": "3" }, { - "Member": "static Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Pixel { get; }", - "Stage": "Experimental" + "Member": "const Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Pixel", + "Stage": "Experimental", + "Value": "0" }, { - "Member": "string Microsoft.Extensions.AI.OcrCoordinateUnit.Value { get; }", - "Stage": "Experimental" + "Member": "const Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Point", + "Stage": "Experimental", + "Value": "1" } ] }, { - "Type": "sealed class Microsoft.Extensions.AI.OcrCoordinateUnit.Converter", + "Type": "abstract class Microsoft.Extensions.AI.OcrElement", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit.Converter.Converter();", + "Member": "Microsoft.Extensions.AI.OcrElement.OcrElement();", + "Stage": "Experimental" + } + ], + "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrElement.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrElement.BoundingRegion { get; set; }", "Stage": "Experimental" }, { - "Member": "override Microsoft.Extensions.AI.OcrCoordinateUnit Microsoft.Extensions.AI.OcrCoordinateUnit.Converter.Read(ref System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options);", + "Member": "double? Microsoft.Extensions.AI.OcrElement.Confidence { get; set; }", "Stage": "Experimental" }, { - "Member": "override void Microsoft.Extensions.AI.OcrCoordinateUnit.Converter.Write(System.Text.Json.Utf8JsonWriter writer, Microsoft.Extensions.AI.OcrCoordinateUnit value, System.Text.Json.JsonSerializerOptions options);", + "Member": "object? Microsoft.Extensions.AI.OcrElement.RawRepresentation { get; set; }", "Stage": "Experimental" } ] }, { - "Type": "class Microsoft.Extensions.AI.OcrImage", + "Type": "class Microsoft.Extensions.AI.OcrImage : Microsoft.Extensions.AI.OcrElement", "Stage": "Experimental", "Methods": [ { @@ -3678,18 +3678,10 @@ } ], "Properties": [ - { - "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrImage.BoundingRegion { get; set; }", - "Stage": "Experimental" - }, { "Member": "string? Microsoft.Extensions.AI.OcrImage.Caption { get; set; }", "Stage": "Experimental" }, - { - "Member": "double? Microsoft.Extensions.AI.OcrImage.Confidence { get; set; }", - "Stage": "Experimental" - }, { "Member": "Microsoft.Extensions.AI.DataContent? Microsoft.Extensions.AI.OcrImage.Content { get; set; }", "Stage": "Experimental" @@ -3714,10 +3706,6 @@ "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrOptions.AdditionalProperties { get; set; }", "Stage": "Experimental" }, - { - "Member": "bool Microsoft.Extensions.AI.OcrOptions.IncludeImages { get; set; }", - "Stage": "Experimental" - }, { "Member": "string? Microsoft.Extensions.AI.OcrOptions.ModelId { get; set; }", "Stage": "Experimental" @@ -3729,7 +3717,7 @@ "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrPage.OcrPage(int pageNumber, string markdown);", + "Member": "Microsoft.Extensions.AI.OcrPage.OcrPage(int pageNumber, string text);", "Stage": "Experimental" } ], @@ -3739,35 +3727,19 @@ "Stage": "Experimental" }, { - "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Blocks { get; set; }", - "Stage": "Experimental" - }, - { - "Member": "double? Microsoft.Extensions.AI.OcrPage.Confidence { get; set; }", - "Stage": "Experimental" - }, - { - "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrPage.CoordinateUnit { get; set; }", + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Elements { get; set; }", "Stage": "Experimental" }, { "Member": "float? Microsoft.Extensions.AI.OcrPage.Height { get; set; }", "Stage": "Experimental" }, - { - "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Images { get; set; }", - "Stage": "Experimental" - }, - { - "Member": "string Microsoft.Extensions.AI.OcrPage.Markdown { get; }", - "Stage": "Experimental" - }, { "Member": "int Microsoft.Extensions.AI.OcrPage.PageNumber { get; }", "Stage": "Experimental" }, { - "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Tables { get; set; }", + "Member": "string Microsoft.Extensions.AI.OcrPage.Text { get; }", "Stage": "Experimental" }, { @@ -3777,115 +3749,111 @@ ] }, { - "Type": "readonly class Microsoft.Extensions.AI.OcrPoint(float X, float Y)", + "Type": "class Microsoft.Extensions.AI.OcrPageResult", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrPoint.OcrPoint(float X, float Y);", + "Member": "Microsoft.Extensions.AI.OcrPageResult.OcrPageResult(Microsoft.Extensions.AI.OcrPage page);", "Stage": "Experimental" - }, + } + ], + "Properties": [ { - "Member": "Microsoft.Extensions.AI.OcrPoint.OcrPoint();", + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrPageResult.AdditionalProperties { get; set; }", "Stage": "Experimental" }, { - "Member": "void Microsoft.Extensions.AI.OcrPoint.Deconstruct(out float X, out float Y);", + "Member": "Microsoft.Extensions.AI.OcrCoordinateOrigin? Microsoft.Extensions.AI.OcrPageResult.CoordinateOrigin { get; set; }", "Stage": "Experimental" }, { - "Member": "override bool Microsoft.Extensions.AI.OcrPoint.Equals(object obj);", + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrPageResult.CoordinateUnit { get; set; }", "Stage": "Experimental" }, { - "Member": "bool Microsoft.Extensions.AI.OcrPoint.Equals(Microsoft.Extensions.AI.OcrPoint other);", + "Member": "Microsoft.Extensions.AI.OcrPage Microsoft.Extensions.AI.OcrPageResult.Page { get; }", "Stage": "Experimental" }, { - "Member": "override int Microsoft.Extensions.AI.OcrPoint.GetHashCode();", + "Member": "int? Microsoft.Extensions.AI.OcrPageResult.PagesProcessed { get; set; }", "Stage": "Experimental" }, { - "Member": "static bool Microsoft.Extensions.AI.OcrPoint.operator ==(Microsoft.Extensions.AI.OcrPoint left, Microsoft.Extensions.AI.OcrPoint right);", + "Member": "object? Microsoft.Extensions.AI.OcrPageResult.RawRepresentation { get; set; }", "Stage": "Experimental" }, { - "Member": "static bool Microsoft.Extensions.AI.OcrPoint.operator !=(Microsoft.Extensions.AI.OcrPoint left, Microsoft.Extensions.AI.OcrPoint right);", + "Member": "int? Microsoft.Extensions.AI.OcrPageResult.TotalPages { get; set; }", "Stage": "Experimental" }, { - "Member": "override string Microsoft.Extensions.AI.OcrPoint.ToString();", + "Member": "Microsoft.Extensions.AI.OcrUsage? Microsoft.Extensions.AI.OcrPageResult.Usage { get; set; }", "Stage": "Experimental" } - ], - "Properties": [ + ] + }, + { + "Type": "static class Microsoft.Extensions.AI.OcrPageResultExtensions", + "Stage": "Experimental", + "Methods": [ { - "Member": "float Microsoft.Extensions.AI.OcrPoint.X { get; init; }", + "Member": "static Microsoft.Extensions.AI.OcrResult Microsoft.Extensions.AI.OcrPageResultExtensions.ToOcrResult(this System.Collections.Generic.IEnumerable updates);", "Stage": "Experimental" }, { - "Member": "float Microsoft.Extensions.AI.OcrPoint.Y { get; init; }", + "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrPageResultExtensions.ToOcrResultAsync(this System.Collections.Generic.IAsyncEnumerable updates, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ] }, { - "Type": "class Microsoft.Extensions.AI.OcrResponseUpdate", + "Type": "readonly class Microsoft.Extensions.AI.OcrPoint(float X, float Y)", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrResponseUpdate.OcrResponseUpdate();", + "Member": "Microsoft.Extensions.AI.OcrPoint.OcrPoint(float X, float Y);", "Stage": "Experimental" }, { - "Member": "Microsoft.Extensions.AI.OcrResponseUpdate.OcrResponseUpdate(Microsoft.Extensions.AI.OcrPage? page);", - "Stage": "Experimental" - } - ], - "Properties": [ - { - "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrResponseUpdate.AdditionalProperties { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrPoint.OcrPoint();", "Stage": "Experimental" }, { - "Member": "string? Microsoft.Extensions.AI.OcrResponseUpdate.ModelId { get; set; }", + "Member": "void Microsoft.Extensions.AI.OcrPoint.Deconstruct(out float X, out float Y);", "Stage": "Experimental" }, { - "Member": "Microsoft.Extensions.AI.OcrPage? Microsoft.Extensions.AI.OcrResponseUpdate.Page { get; set; }", + "Member": "override bool Microsoft.Extensions.AI.OcrPoint.Equals(object obj);", "Stage": "Experimental" }, { - "Member": "int? Microsoft.Extensions.AI.OcrResponseUpdate.PagesProcessed { get; set; }", + "Member": "bool Microsoft.Extensions.AI.OcrPoint.Equals(Microsoft.Extensions.AI.OcrPoint other);", "Stage": "Experimental" }, { - "Member": "object? Microsoft.Extensions.AI.OcrResponseUpdate.RawRepresentation { get; set; }", + "Member": "override int Microsoft.Extensions.AI.OcrPoint.GetHashCode();", "Stage": "Experimental" }, { - "Member": "string? Microsoft.Extensions.AI.OcrResponseUpdate.Status { get; set; }", + "Member": "static bool Microsoft.Extensions.AI.OcrPoint.operator ==(Microsoft.Extensions.AI.OcrPoint left, Microsoft.Extensions.AI.OcrPoint right);", "Stage": "Experimental" }, { - "Member": "int? Microsoft.Extensions.AI.OcrResponseUpdate.TotalPages { get; set; }", + "Member": "static bool Microsoft.Extensions.AI.OcrPoint.operator !=(Microsoft.Extensions.AI.OcrPoint left, Microsoft.Extensions.AI.OcrPoint right);", "Stage": "Experimental" }, { - "Member": "Microsoft.Extensions.AI.OcrUsage? Microsoft.Extensions.AI.OcrResponseUpdate.Usage { get; set; }", + "Member": "override string Microsoft.Extensions.AI.OcrPoint.ToString();", "Stage": "Experimental" } - ] - }, - { - "Type": "static class Microsoft.Extensions.AI.OcrResponseUpdateExtensions", - "Stage": "Experimental", - "Methods": [ + ], + "Properties": [ { - "Member": "static Microsoft.Extensions.AI.OcrResult Microsoft.Extensions.AI.OcrResponseUpdateExtensions.ToOcrResult(this System.Collections.Generic.IEnumerable updates);", + "Member": "float Microsoft.Extensions.AI.OcrPoint.X { get; init; }", "Stage": "Experimental" }, { - "Member": "static System.Threading.Tasks.Task Microsoft.Extensions.AI.OcrResponseUpdateExtensions.ToOcrResultAsync(this System.Collections.Generic.IAsyncEnumerable updates, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "float Microsoft.Extensions.AI.OcrPoint.Y { get; init; }", "Stage": "Experimental" } ] @@ -3905,11 +3873,11 @@ "Stage": "Experimental" }, { - "Member": "string Microsoft.Extensions.AI.OcrResult.Markdown { get; }", + "Member": "Microsoft.Extensions.AI.OcrCoordinateOrigin? Microsoft.Extensions.AI.OcrResult.CoordinateOrigin { get; set; }", "Stage": "Experimental" }, { - "Member": "string? Microsoft.Extensions.AI.OcrResult.ModelId { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrResult.CoordinateUnit { get; set; }", "Stage": "Experimental" }, { @@ -3920,6 +3888,10 @@ "Member": "object? Microsoft.Extensions.AI.OcrResult.RawRepresentation { get; set; }", "Stage": "Experimental" }, + { + "Member": "string Microsoft.Extensions.AI.OcrResult.Text { get; }", + "Stage": "Experimental" + }, { "Member": "Microsoft.Extensions.AI.OcrUsage? Microsoft.Extensions.AI.OcrResult.Usage { get; set; }", "Stage": "Experimental" @@ -3927,7 +3899,7 @@ ] }, { - "Type": "class Microsoft.Extensions.AI.OcrTable", + "Type": "class Microsoft.Extensions.AI.OcrTable : Microsoft.Extensions.AI.OcrElement", "Stage": "Experimental", "Methods": [ { @@ -3936,10 +3908,6 @@ } ], "Properties": [ - { - "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrTable.BoundingRegion { get; set; }", - "Stage": "Experimental" - }, { "Member": "System.Collections.Generic.IReadOnlyList? Microsoft.Extensions.AI.OcrTable.Cells { get; }", "Stage": "Experimental" @@ -3980,6 +3948,10 @@ "Member": "string Microsoft.Extensions.AI.OcrTableCell.Content { get; }", "Stage": "Experimental" }, + { + "Member": "System.Collections.Generic.IReadOnlyList? Microsoft.Extensions.AI.OcrTableCell.Elements { get; set; }", + "Stage": "Experimental" + }, { "Member": "Microsoft.Extensions.AI.OcrTableCellKind? Microsoft.Extensions.AI.OcrTableCell.Kind { get; set; }", "Stage": "Experimental" @@ -4040,6 +4012,14 @@ "Member": "static Microsoft.Extensions.AI.OcrTableCellKind Microsoft.Extensions.AI.OcrTableCellKind.Content { get; }", "Stage": "Experimental" }, + { + "Member": "static Microsoft.Extensions.AI.OcrTableCellKind Microsoft.Extensions.AI.OcrTableCellKind.RowHeader { get; }", + "Stage": "Experimental" + }, + { + "Member": "static Microsoft.Extensions.AI.OcrTableCellKind Microsoft.Extensions.AI.OcrTableCellKind.RowSection { get; }", + "Stage": "Experimental" + }, { "Member": "string Microsoft.Extensions.AI.OcrTableCellKind.Value { get; }", "Stage": "Experimental" @@ -4078,9 +4058,21 @@ "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrUsage.AdditionalProperties { get; set; }", "Stage": "Experimental" }, + { + "Member": "int? Microsoft.Extensions.AI.OcrUsage.InputTokenCount { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.AI.OcrUsage.OutputTokenCount { get; set; }", + "Stage": "Experimental" + }, { "Member": "int? Microsoft.Extensions.AI.OcrUsage.PagesProcessed { get; set; }", "Stage": "Experimental" + }, + { + "Member": "int? Microsoft.Extensions.AI.OcrUsage.TotalTokenCount { get; set; }", + "Stage": "Experimental" } ] }, diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs index 7af7dc32ba6..b6fec1e2501 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/DelegatingOcrClient.cs @@ -50,13 +50,13 @@ public virtual Task ExtractAsync( } /// - public virtual IAsyncEnumerable ExtractStreamingAsync( + public virtual IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, CancellationToken cancellationToken = default) { - return InnerClient.ExtractStreamingAsync(document, mediaType, options, cancellationToken); + return InnerClient.ExtractPagesAsync(document, mediaType, options, cancellationToken); } /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs index 56bf2b0970a..08ece061437 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/IOcrClient.cs @@ -14,7 +14,7 @@ namespace Microsoft.Extensions.AI; /// Represents an optical character recognition (OCR) / document-parsing client. /// /// -/// An transcribes a document or image into structured output: markdown, +/// An transcribes a document or image into structured output: text, /// per-page content, tables, layout blocks with bounding regions, and confidence. It is the /// capability sibling to , IEmbeddingGenerator, and /// ISpeechToTextClient for the document-extraction problem. @@ -29,7 +29,7 @@ namespace Microsoft.Extensions.AI; /// /// Unless otherwise specified, all members of are thread-safe for concurrent /// use. Implementations might mutate the supplied to -/// and ; consumers should avoid sharing a single options instance across +/// and ; consumers should avoid sharing a single options instance across /// concurrent invocations when that is a concern. The document stream passed to these methods is not /// disposed by the implementation. /// @@ -59,10 +59,10 @@ Task ExtractAsync( /// Engines that produce pages incrementally (for example, while polling a long-running operation such as /// Azure Document Intelligence) can yield each page as it completes, letting a consumer begin processing /// early pages before later pages finish. Synchronous engines may yield a single terminal update. Use - /// to reassemble the stream into an + /// to reassemble the stream into an /// . /// - IAsyncEnumerable ExtractStreamingAsync( + IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs index 3504febddf6..8a8b5968e70 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBlock.cs @@ -9,7 +9,7 @@ namespace Microsoft.Extensions.AI; /// Represents a positioned layout block, such as a paragraph, heading, or figure. [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public class OcrBlock +public class OcrBlock : OcrElement { /// Initializes a new instance of the class. /// The text content of the block. @@ -24,10 +24,4 @@ public OcrBlock(string text) /// Gets or sets the kind of block, for example , , or . public OcrBlockKind? Kind { get; set; } - - /// Gets or sets the region of the page the block occupies, when the engine provides geometry. - public OcrBoundingRegion? BoundingRegion { get; set; } - - /// Gets or sets the confidence for the block in the range [0, 1], when available. - public double? Confidence { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs index 72d9473aff4..c9751fe04ac 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrBoundingRegion.cs @@ -45,13 +45,13 @@ public OcrBoundingRegion(int pageNumber, IReadOnlyList polygon) /// The right coordinate. /// The bottom coordinate. /// A region whose polygon is the four corners of the rectangle. - public static OcrBoundingRegion FromRectangle(int pageNumber, double left, double top, double right, double bottom) + public static OcrBoundingRegion FromRectangle(int pageNumber, float left, float top, float right, float bottom) => new(pageNumber, [ - new OcrPoint((float)left, (float)top), - new OcrPoint((float)right, (float)top), - new OcrPoint((float)right, (float)bottom), - new OcrPoint((float)left, (float)bottom), + new OcrPoint(left, top), + new OcrPoint(right, top), + new OcrPoint(right, bottom), + new OcrPoint(left, bottom), ]); /// Computes the axis-aligned bounds of the polygon. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs index 1fdea3f995e..d5cee6172c1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrClientExtensions.cs @@ -170,7 +170,7 @@ public static async Task ExtractFromUriAsync( /// The to monitor for cancellation requests. The default is . /// The structured OCR updates representing the streamed output. /// or is . - public static IAsyncEnumerable ExtractStreamingAsync( + public static IAsyncEnumerable ExtractPagesAsync( this IOcrClient client, DataContent document, OcrOptions? options = null, @@ -183,7 +183,7 @@ public static IAsyncEnumerable ExtractStreamingAsync( new MemoryStream(array.Array!, array.Offset, array.Count) : new MemoryStream(document.Data.ToArray()); - return client.ExtractStreamingAsync(documentStream, document.MediaType, options, cancellationToken); + return client.ExtractPagesAsync(documentStream, document.MediaType, options, cancellationToken); } /// Runs streaming OCR over a single document referenced by a . @@ -200,7 +200,7 @@ public static IAsyncEnumerable ExtractStreamingAsync( /// self-contained data: URIs and does no file or network IO. For file: and remote URIs it /// throws, for the same reasons documented on the unary overload. /// - public static IAsyncEnumerable ExtractStreamingAsync( + public static IAsyncEnumerable ExtractPagesAsync( this IOcrClient client, UriContent document, OcrOptions? options = null, @@ -213,7 +213,7 @@ public static IAsyncEnumerable ExtractStreamingAsync( if (uri.IsAbsoluteUri && string.Equals(uri.Scheme, "data", StringComparison.OrdinalIgnoreCase)) { // Reuse DataContent's data: URI parsing, then defer to the DataContent overload. - return client.ExtractStreamingAsync(new DataContent(uri), options, cancellationToken); + return client.ExtractPagesAsync(new DataContent(uri), options, cancellationToken); } throw new NotSupportedException( diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs new file mode 100644 index 00000000000..3ec028c527c --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// +/// Describes the origin corner and vertical axis direction of an OCR coordinate space. +/// +/// +/// Engines disagree on where a page's coordinate origin sits and which way the y axis grows: rasterized +/// page images place the origin at the top-left with y increasing downward, whereas PDF-native +/// (point) coordinates place it at the bottom-left with y increasing upward. The origin is reported once +/// at the document level on and , alongside +/// , so bounding regions from different engines can be compared and +/// normalized without guessing the convention. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public enum OcrCoordinateOrigin +{ + /// The origin sits at the top-left corner, with the y axis increasing downward. The convention for rasterized page images. + TopLeft, + + /// The origin sits at the bottom-left corner, with the y axis increasing upward. The convention for PDF-native point coordinates. + BottomLeft, +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs index 1d7b7d8c6c2..a6b273b6138 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs @@ -1,14 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.ComponentModel; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Text.Json; -using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; -using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; @@ -18,90 +12,26 @@ namespace Microsoft.Extensions.AI; /// /// /// Coordinate conventions differ across OCR engines: some report pixels of the rendered page image, -/// some report a physical unit such as inches, and some normalize to the page. Pairing the geometry -/// with an and the page dimensions ( and -/// ) lets a consumer interpret or normalize regions with engine-agnostic -/// code. This type is a small open set modeled on : the well-known values cover -/// the common cases, and a provider may introduce its own value when needed. +/// some report a physical unit such as points or inches, and some normalize to the page. The unit is +/// reported once at the document level on and , +/// paired with an and the page dimensions ( +/// and ), so a consumer can interpret or normalize regions with +/// engine-agnostic code. Unlike the taxonomy kinds (, +/// ), the set of coordinate units is physically bounded, so it is modeled +/// as a closed enumeration. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -[JsonConverter(typeof(Converter))] -[DebuggerDisplay("{Value,nq}")] -public readonly struct OcrCoordinateUnit : IEquatable +public enum OcrCoordinateUnit { - /// Gets the unit for coordinates expressed in pixels of the rendered page image. - public static OcrCoordinateUnit Pixel { get; } = new("pixel"); + /// Coordinates expressed in pixels of the rendered page image. + Pixel, - /// Gets the unit for coordinates expressed in inches. - public static OcrCoordinateUnit Inch { get; } = new("inch"); + /// Coordinates expressed in points (1/72 inch), the native unit of PDF content. + Point, - /// Gets the unit for coordinates normalized to the range [0, 1] relative to the page width and height. - public static OcrCoordinateUnit Normalized { get; } = new("normalized"); + /// Coordinates expressed in inches. + Inch, - /// Gets the value associated with this . - public string Value { get; } - - /// - /// Initializes a new instance of the struct with the provided value. - /// - /// The value to associate with this . - /// is . - /// is empty or composed entirely of whitespace. - [JsonConstructor] - public OcrCoordinateUnit(string value) - { - Value = Throw.IfNullOrWhitespace(value); - } - - /// - /// Returns a value indicating whether two instances are equivalent, as - /// determined by a case-insensitive comparison of their values. - /// - /// The first instance to compare. - /// The second instance to compare. - /// if left and right have equivalent values; otherwise, . - public static bool operator ==(OcrCoordinateUnit left, OcrCoordinateUnit right) - { - return left.Equals(right); - } - - /// - /// Returns a value indicating whether two instances are not equivalent, as - /// determined by a case-insensitive comparison of their values. - /// - /// The first instance to compare. - /// The second instance to compare. - /// if left and right have different values; otherwise, . - public static bool operator !=(OcrCoordinateUnit left, OcrCoordinateUnit right) - { - return !(left == right); - } - - /// - public override bool Equals([NotNullWhen(true)] object? obj) - => obj is OcrCoordinateUnit otherUnit && Equals(otherUnit); - - /// - public bool Equals(OcrCoordinateUnit other) - => string.Equals(Value, other.Value, StringComparison.OrdinalIgnoreCase); - - /// - public override int GetHashCode() - => StringComparer.OrdinalIgnoreCase.GetHashCode(Value); - - /// - public override string ToString() => Value; - - /// Provides a for serializing instances. - [EditorBrowsable(EditorBrowsableState.Never)] - public sealed class Converter : JsonConverter - { - /// - public override OcrCoordinateUnit Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => - new(reader.GetString()!); - - /// - public override void Write(Utf8JsonWriter writer, OcrCoordinateUnit value, JsonSerializerOptions options) => - Throw.IfNull(writer).WriteStringValue(value.Value); - } + /// Coordinates normalized to the range [0, 1] relative to the page width and height. + Normalized, } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrElement.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrElement.cs new file mode 100644 index 00000000000..44980423f6f --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrElement.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents a single positioned element within an , such as a block, table, or image. +/// +/// Elements appear in in reading order, so a consumer can walk a page as one +/// heterogeneous stream and project the kinds it cares about with +/// (for example +/// page.Elements.OfType<OcrTable>()). The full page text is available directly on +/// . The base is shaped to be promotable to a future shared document-element type. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(OcrBlock), typeDiscriminator: "block")] +[JsonDerivedType(typeof(OcrTable), typeDiscriminator: "table")] +[JsonDerivedType(typeof(OcrImage), typeDiscriminator: "image")] +public abstract class OcrElement +{ + /// Initializes a new instance of the class. + protected OcrElement() + { + } + + /// Gets or sets the region of the page the element occupies, when the engine provides geometry. + public OcrBoundingRegion? BoundingRegion { get; set; } + + /// Gets or sets the confidence for the element in the range [0, 1], when available. + public double? Confidence { get; set; } + + /// Gets or sets the provider-native object underlying this element. + /// + /// If an is created to represent an underlying object from another object model, + /// this property can store that original object. This can be useful for debugging or for enabling a + /// consumer to access the underlying object model if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the element. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs index b23c8e0994f..2d5afe3399e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrImage.cs @@ -8,24 +8,18 @@ namespace Microsoft.Extensions.AI; /// Represents an image or figure extracted from a page during OCR. /// -/// Populated when is requested and the engine supports it. Every +/// Populated when the engine supports it and images are present. Every /// member is optional so each implementer fills what it can provide: document-native engines (for /// example Mistral OCR inline images, or Azure Document Intelligence figures) populate /// with the rendered image bytes, whereas a vision-LLM transcriber that cannot emit bytes may instead /// populate only . This lets one shape serve both provider archetypes. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public class OcrImage +public class OcrImage : OcrElement { /// Gets or sets the rendered image bytes, when the engine returns them. public DataContent? Content { get; set; } - /// Gets or sets the region of the page the image occupies, when the engine provides geometry. - public OcrBoundingRegion? BoundingRegion { get; set; } - /// Gets or sets a caption or description of the image, when available. public string? Caption { get; set; } - - /// Gets or sets the confidence for the image in the range [0, 1], when available. - public double? Confidence { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs index 87207fac1c9..62e22cfec93 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrOptions.cs @@ -17,9 +17,6 @@ public class OcrOptions /// Gets or sets the model or deployment identifier to use for this request. public string? ModelId { get; set; } - /// Gets or sets a value indicating whether the engine should include rendered images inline, when supported. - public bool IncludeImages { get; set; } - /// Gets or sets any additional provider-specific request settings. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } @@ -29,7 +26,6 @@ public OcrOptions Clone() => new() { ModelId = ModelId, - IncludeImages = IncludeImages, AdditionalProperties = AdditionalProperties?.Clone(), }; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs index 270af78c174..0620e48705c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -14,54 +14,43 @@ public class OcrPage { /// Initializes a new instance of the class. /// The one-based page number. - /// The structured markdown for this page. - /// is . - public OcrPage(int pageNumber, string markdown) + /// The structured text for this page. + /// is . + public OcrPage(int pageNumber, string text) { PageNumber = pageNumber; - Markdown = Throw.IfNull(markdown); + Text = Throw.IfNull(text); } /// Gets the one-based page number. public int PageNumber { get; } - /// Gets the structured markdown for this page, with headings, tables, and reading order preserved. - public string Markdown { get; } + /// Gets the structured text for this page, with headings, tables, and reading order preserved. + public string Text { get; } - /// Gets or sets the tables extracted from this page. - public IReadOnlyList Tables { get; set; } = []; - - /// Gets or sets the layout blocks with bounding regions and confidence, when the engine provides them. - public IReadOnlyList Blocks { get; set; } = []; - - /// Gets or sets the images or figures extracted from this page, when requested and the engine provides them. - public IReadOnlyList Images { get; set; } = []; - - /// Gets or sets the page-level confidence in the range [0, 1], when available. - public double? Confidence { get; set; } + /// Gets or sets the elements extracted from this page, in reading order. + /// + /// A single heterogeneous stream of blocks, tables, and images in the order a human would read them. + /// Project a specific kind with , + /// for example Elements.OfType<OcrTable>(). The full page text is available directly on + /// , so reading-order consumers do not need geometry math. + /// + public IReadOnlyList Elements { get; set; } = []; - /// Gets or sets the page width, expressed in , when the engine provides it. + /// Gets or sets the page width, expressed in the document-level , when the engine provides it. /// - /// Together with and , this lets a consumer interpret or - /// normalize the geometry ( / ) on this page with - /// engine-agnostic code. For example, dividing a coordinate by the corresponding page dimension yields a - /// page-relative [0, 1] value regardless of the native unit. + /// Together with and the document-level and + /// , this lets a consumer interpret or normalize the geometry + /// ( / ) on this page with engine-agnostic code. For + /// example, dividing a coordinate by the corresponding page dimension yields a page-relative [0, 1] value + /// regardless of the native unit. /// public float? Width { get; set; } - /// Gets or sets the page height, expressed in , when the engine provides it. - /// See for how the page dimensions are used with . + /// Gets or sets the page height, expressed in the document-level , when the engine provides it. + /// See for how the page dimensions are used with the document-level coordinate unit. public float? Height { get; set; } - /// Gets or sets the unit in which this page's geometry coordinates are expressed, when known. - /// - /// OCR engines disagree on coordinate conventions (pixels, inches, or page-normalized values). When the - /// engine reports the unit, exposing it here alongside and makes - /// the region geometry interpretable without knowing which engine produced it. When , - /// the geometry should be treated as an opaque, provider-specific coordinate space. - /// - public OcrCoordinateUnit? CoordinateUnit { get; set; } - /// Gets or sets any additional properties associated with the page. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs similarity index 54% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs index f9558e5b5f5..4b65462de09 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdate.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs @@ -4,44 +4,40 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI; /// Represents a single streaming update from an . /// /// -/// An request produces a sequence of -/// instances. A typical engine emits one update per page as that page -/// finishes (carrying the completed ), optionally interleaved with progress-only updates -/// (, , ) for long-running -/// operations such as Azure Document Intelligence. A synchronous engine may emit a single terminal update. +/// An request produces a sequence of +/// instances, one per page as that page finishes (carrying the completed +/// ). Progress rides along on each result via and +/// for long-running operations such as Azure Document Intelligence. Completion is +/// signaled by the end of the sequence. /// /// -/// The relationship between and is codified in -/// , which reassembles a stream of updates into a +/// The relationship between and is codified in +/// , which reassembles a stream of updates into a /// single . The conversion can be slightly lossy: for example, only one /// slot is available on the assembled . /// /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public class OcrResponseUpdate +public class OcrPageResult { - /// Initializes a new instance of the class. - [JsonConstructor] - public OcrResponseUpdate() - { - } - - /// Initializes a new instance of the class with the page completed in this update. + /// Initializes a new instance of the class with the page completed in this update. /// The page produced in this update. - public OcrResponseUpdate(OcrPage? page) + /// is . + [JsonConstructor] + public OcrPageResult(OcrPage page) { - Page = page; + Page = Throw.IfNull(page); } - /// Gets or sets the page produced in this update, when the update carries a completed page. - /// Progress-only updates leave this . - public OcrPage? Page { get; set; } + /// Gets the page produced in this update. + public OcrPage Page { get; } /// Gets or sets the number of pages processed so far, when known. public int? PagesProcessed { get; set; } @@ -49,11 +45,15 @@ public OcrResponseUpdate(OcrPage? page) /// Gets or sets the total number of pages, when known. public int? TotalPages { get; set; } - /// Gets or sets a human-readable status for the operation, when available. - public string? Status { get; set; } + /// Gets or sets the unit in which this document's geometry coordinates are expressed, when known. + /// + /// Reported at the document level; the reducer () carries + /// the last non- value onto . + /// + public OcrCoordinateUnit? CoordinateUnit { get; set; } - /// Gets or sets the model or deployment identifier that served the request. - public string? ModelId { get; set; } + /// Gets or sets the origin corner and axis direction of this document's geometry coordinates, when known. + public OcrCoordinateOrigin? CoordinateOrigin { get; set; } /// Gets or sets usage details associated with the request, when reported. /// Usage is typically carried on a terminal update once the full document has been processed. @@ -61,7 +61,7 @@ public OcrResponseUpdate(OcrPage? page) /// Gets or sets the provider-native object underlying this update. /// - /// If an is created to represent an underlying object from another object + /// If an is created to represent an underlying object from another object /// model, this property can store that original object. This can be useful for debugging or for enabling /// a consumer to access the underlying object model if needed. /// diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs similarity index 73% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs index 8524fdacdfe..4e4efd905ba 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResponseUpdateExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs @@ -10,15 +10,15 @@ namespace Microsoft.Extensions.AI; -/// Provides extension methods for working with instances. +/// Provides extension methods for working with instances. [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public static class OcrResponseUpdateExtensions +public static class OcrPageResultExtensions { - /// Combines instances into a single . + /// Combines instances into a single . /// The updates to be combined. /// The combined . /// is . - public static OcrResult ToOcrResult(this IEnumerable updates) + public static OcrResult ToOcrResult(this IEnumerable updates) { _ = Throw.IfNull(updates); @@ -33,20 +33,20 @@ public static OcrResult ToOcrResult(this IEnumerable updates) return result; } - /// Combines instances into a single . + /// Combines instances into a single . /// The updates to be combined. /// The to monitor for cancellation requests. The default is . /// The combined . /// is . public static Task ToOcrResultAsync( - this IAsyncEnumerable updates, CancellationToken cancellationToken = default) + this IAsyncEnumerable updates, CancellationToken cancellationToken = default) { _ = Throw.IfNull(updates); return ToResultAsync(updates, cancellationToken); static async Task ToResultAsync( - IAsyncEnumerable updates, CancellationToken cancellationToken) + IAsyncEnumerable updates, CancellationToken cancellationToken) { List pages = []; OcrResult result = new(pages); @@ -60,20 +60,22 @@ static async Task ToResultAsync( } } - /// Incorporates one into the assembled . + /// Incorporates one into the assembled . /// The update to process. /// The accumulating list of pages backing . /// The being assembled. - private static void ProcessUpdate(OcrResponseUpdate update, List pages, OcrResult result) + private static void ProcessUpdate(OcrPageResult update, List pages, OcrResult result) { - if (update.Page is not null) + pages.Add(update.Page); + + if (update.CoordinateUnit is not null) { - pages.Add(update.Page); + result.CoordinateUnit = update.CoordinateUnit; } - if (update.ModelId is not null) + if (update.CoordinateOrigin is not null) { - result.ModelId = update.ModelId; + result.CoordinateOrigin = update.CoordinateOrigin; } if (update.Usage is not null) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs index f74579e2790..f37170da2e4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs @@ -11,8 +11,8 @@ namespace Microsoft.Extensions.AI; /// Represents the structured result of an OCR / document-parsing request. /// -/// The result normalizes the content common to every engine (markdown, pages, tables, bounding -/// regions, confidence) while preserving everything provider-specific via +/// The result normalizes the content common to every engine (text, pages, tables, bounding +/// regions) while preserving everything provider-specific via /// and , mirroring how /// ChatResponse normalizes the common surface and preserves the raw. /// @@ -27,14 +27,21 @@ public OcrResult(IReadOnlyList pages) Pages = Throw.IfNull(pages); } - /// Gets the per-page structured content (markdown, tables, blocks, confidence). + /// Gets the per-page structured content (text, tables, blocks). public IReadOnlyList Pages { get; } - /// Gets the full-document markdown, formed by joining the per-page markdown. - public string Markdown => string.Join("\n\n", Pages.Select(p => p.Markdown)); + /// Gets the full-document text, formed by joining the per-page text. + public string Text => string.Join("\n\n", Pages.Select(p => p.Text)); - /// Gets or sets the model or deployment identifier that served the request. - public string? ModelId { get; set; } + /// Gets or sets the unit in which this document's geometry coordinates are expressed, when known. + /// + /// Applies to every in the document. When , the + /// geometry should be treated as an opaque, provider-specific coordinate space. + /// + public OcrCoordinateUnit? CoordinateUnit { get; set; } + + /// Gets or sets the origin corner and axis direction of this document's geometry coordinates, when known. + public OcrCoordinateOrigin? CoordinateOrigin { get; set; } /// Gets or sets usage details associated with the request. public OcrUsage? Usage { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs index 49d3ba12c94..86c75d1815c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTable.cs @@ -17,7 +17,7 @@ namespace Microsoft.Extensions.AI; /// may be 0 because the structure was not enumerated. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] -public class OcrTable +public class OcrTable : OcrElement { /// Initializes a new instance of the class. /// The number of rows in the table. @@ -47,7 +47,4 @@ public OcrTable( /// Gets the markdown or HTML table text, or when only cells were returned. public string? MarkdownRepresentation { get; } - - /// Gets or sets the region of the page the table occupies, when the engine provides geometry. - public OcrBoundingRegion? BoundingRegion { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs index aae87b9fd09..1496e05652e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -40,4 +41,13 @@ public OcrTableCell(int rowIndex, int columnIndex, string content) /// Gets the text content of the cell. public string Content { get; } + + /// Gets or sets the nested content of the cell, in reading order, when the engine provides structured cell content. + /// + /// When , the cell is text-only and carries its text. When present, + /// the cell holds richer structured content (for example nested blocks or tables), and + /// remains a flat-text convenience. This mirrors the nested-content model used by engines such as Docling and + /// Google Document AI. + /// + public IReadOnlyList? Elements { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs index 4f1b20cb68b..0df6882c059 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCellKind.cs @@ -29,6 +29,12 @@ namespace Microsoft.Extensions.AI; /// Gets the kind representing a regular content cell. public static OcrTableCellKind Content { get; } = new("content"); + /// Gets the kind representing a row header cell (a header that labels the row it sits in). + public static OcrTableCellKind RowHeader { get; } = new("rowHeader"); + + /// Gets the kind representing a cell that introduces a labeled section spanning subsequent rows. + public static OcrTableCellKind RowSection { get; } = new("rowSection"); + /// Gets the value associated with this . public string Value { get; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs index 228345eafeb..398a7054634 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrUsage.cs @@ -13,6 +13,16 @@ public class OcrUsage /// Gets or sets the number of pages processed by the request, when known. public int? PagesProcessed { get; set; } + /// Gets or sets the number of input tokens consumed, when the engine reports token usage. + /// Typically reported only by vision-LLM OCR paths; classic OCR engines usually leave this . + public int? InputTokenCount { get; set; } + + /// Gets or sets the number of output tokens produced, when the engine reports token usage. + public int? OutputTokenCount { get; set; } + + /// Gets or sets the total number of tokens (input plus output), when the engine reports token usage. + public int? TotalTokenCount { get; set; } + /// Gets or sets any additional provider-specific usage details. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 438ad3375e9..238da0949ee 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -143,7 +143,7 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(OcrOptions))] [JsonSerializable(typeof(OcrClientMetadata))] [JsonSerializable(typeof(OcrResult))] - [JsonSerializable(typeof(OcrResponseUpdate))] + [JsonSerializable(typeof(OcrPageResult))] // IHostedFileClient [JsonSerializable(typeof(HostedFileClientOptions))] diff --git a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json index 9ee5df97ae7..5ad27e6e065 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json +++ b/src/Libraries/Microsoft.Extensions.AI/Microsoft.Extensions.AI.json @@ -286,7 +286,7 @@ "Stage": "Experimental" }, { - "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.ConfigureOptionsOcrClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ] @@ -1060,7 +1060,7 @@ "Stage": "Experimental" }, { - "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.LoggingOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.LoggingOcrClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" } ], @@ -1444,7 +1444,7 @@ "Stage": "Experimental" }, { - "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OpenTelemetryOcrClient.ExtractStreamingAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", + "Member": "override System.Collections.Generic.IAsyncEnumerable Microsoft.Extensions.AI.OpenTelemetryOcrClient.ExtractPagesAsync(System.IO.Stream document, string mediaType, Microsoft.Extensions.AI.OcrOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));", "Stage": "Experimental" }, { diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs index 545c4ff53d7..20947e32b4f 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/ConfigureOptionsOcrClient.cs @@ -47,13 +47,13 @@ public override async Task ExtractAsync( } /// - public override IAsyncEnumerable ExtractStreamingAsync( + public override IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, CancellationToken cancellationToken = default) { - return base.ExtractStreamingAsync(document, mediaType, Configure(options), cancellationToken); + return base.ExtractPagesAsync(document, mediaType, Configure(options), cancellationToken); } /// Creates and configures the to pass along to the inner client. diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs index d7e6ccca2ca..74a110f8358 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/LoggingOcrClient.cs @@ -104,7 +104,7 @@ public override async Task ExtractAsync( } /// - public override async IAsyncEnumerable ExtractStreamingAsync( + public override async IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, @@ -114,33 +114,33 @@ public override async IAsyncEnumerable ExtractStreamingAsync( { if (_logger.IsEnabled(LogLevel.Trace)) { - LogInvokedSensitive(nameof(ExtractStreamingAsync), mediaType, AsJson(options), AsJson(this.GetService())); + LogInvokedSensitive(nameof(ExtractPagesAsync), mediaType, AsJson(options), AsJson(this.GetService())); } else { - LogInvoked(nameof(ExtractStreamingAsync)); + LogInvoked(nameof(ExtractPagesAsync)); } } - IAsyncEnumerator e; + IAsyncEnumerator e; try { - e = base.ExtractStreamingAsync(document, mediaType, options, cancellationToken).GetAsyncEnumerator(cancellationToken); + e = base.ExtractPagesAsync(document, mediaType, options, cancellationToken).GetAsyncEnumerator(cancellationToken); } catch (OperationCanceledException) { - LogInvocationCanceled(nameof(ExtractStreamingAsync)); + LogInvocationCanceled(nameof(ExtractPagesAsync)); throw; } catch (Exception ex) { - LogInvocationFailed(nameof(ExtractStreamingAsync), ex); + LogInvocationFailed(nameof(ExtractPagesAsync), ex); throw; } try { - OcrResponseUpdate? update = null; + OcrPageResult? update = null; while (true) { try @@ -154,12 +154,12 @@ public override async IAsyncEnumerable ExtractStreamingAsync( } catch (OperationCanceledException) { - LogInvocationCanceled(nameof(ExtractStreamingAsync)); + LogInvocationCanceled(nameof(ExtractPagesAsync)); throw; } catch (Exception ex) { - LogInvocationFailed(nameof(ExtractStreamingAsync), ex); + LogInvocationFailed(nameof(ExtractPagesAsync), ex); throw; } @@ -178,7 +178,7 @@ public override async IAsyncEnumerable ExtractStreamingAsync( yield return update; } - LogCompleted(nameof(ExtractStreamingAsync)); + LogCompleted(nameof(ExtractPagesAsync)); } finally { @@ -200,11 +200,11 @@ public override async IAsyncEnumerable ExtractStreamingAsync( [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {OcrResult}.")] private partial void LogCompletedSensitive(string methodName, string ocrResult); - [LoggerMessage(LogLevel.Debug, "ExtractStreamingAsync received update.")] + [LoggerMessage(LogLevel.Debug, "ExtractPagesAsync received update.")] private partial void LogStreamingUpdate(); - [LoggerMessage(LogLevel.Trace, "ExtractStreamingAsync received update: {OcrResponseUpdate}")] - private partial void LogStreamingUpdateSensitive(string ocrResponseUpdate); + [LoggerMessage(LogLevel.Trace, "ExtractPagesAsync received update: {OcrPageResult}")] + private partial void LogStreamingUpdateSensitive(string ocrPageResult); [LoggerMessage(LogLevel.Debug, "{MethodName} canceled.")] private partial void LogInvocationCanceled(string methodName); diff --git a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs index 67c9a9a1ea9..1cb0c100a00 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Ocr/OpenTelemetryOcrClient.cs @@ -128,7 +128,7 @@ public override async Task ExtractAsync( } /// - public override async IAsyncEnumerable ExtractStreamingAsync( + public override async IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, @@ -140,10 +140,10 @@ public override async IAsyncEnumerable ExtractStreamingAsync( Stopwatch? stopwatch = _operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; string? requestModelId = options?.ModelId ?? _defaultModelId; - IAsyncEnumerable updates; + IAsyncEnumerable updates; try { - updates = base.ExtractStreamingAsync(document, mediaType, options, cancellationToken); + updates = base.ExtractPagesAsync(document, mediaType, options, cancellationToken); } catch (Exception ex) { @@ -152,13 +152,13 @@ public override async IAsyncEnumerable ExtractStreamingAsync( } var responseEnumerator = updates.GetAsyncEnumerator(cancellationToken); - List trackedUpdates = []; + List trackedUpdates = []; Exception? error = null; try { while (true) { - OcrResponseUpdate update; + OcrPageResult update; try { if (!await responseEnumerator.MoveNextAsync()) @@ -244,7 +244,7 @@ private void TraceResponse( { TagList tags = default; - AddMetricTags(ref tags, requestModelId, response); + AddMetricTags(ref tags, requestModelId); if (error is not null) { tags.Add(OpenTelemetryConsts.Error.Type, error.GetType().FullName); @@ -257,11 +257,6 @@ private void TraceResponse( if (response is not null && activity is not null) { - if (response.ModelId is not null) - { - _ = activity.AddTag(OpenTelemetryConsts.GenAI.Response.Model, response.ModelId); - } - if (response.Usage?.PagesProcessed is int pages) { _ = activity.AddTag(OpenTelemetryConsts.GenAI.Usage.PagesProcessed, pages); @@ -278,7 +273,7 @@ private void TraceResponse( } } - void AddMetricTags(ref TagList tags, string? requestModelId, OcrResult? response) + void AddMetricTags(ref TagList tags, string? requestModelId) { tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName); @@ -294,11 +289,6 @@ void AddMetricTags(ref TagList tags, string? requestModelId, OcrResult? response tags.Add(OpenTelemetryConsts.Server.Address, endpointAddress); tags.Add(OpenTelemetryConsts.Server.Port, _serverPort); } - - if (response?.ModelId is string responseModel) - { - tags.Add(OpenTelemetryConsts.GenAI.Response.Model, responseModel); - } } } } diff --git a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs index 059d621926b..d445b6a8737 100644 --- a/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs +++ b/src/Libraries/Microsoft.Extensions.DataIngestion/OcrDocumentReader.cs @@ -28,11 +28,11 @@ public sealed class OcrDocumentReader : IngestionDocumentReader /// Initializes a new instance of the class. /// /// The OCR client to use for document extraction. - /// Optional OCR options. When not provided, image extraction is requested. + /// Optional OCR options. public OcrDocumentReader(IOcrClient ocrClient, OcrOptions? options = null) { _ocrClient = Throw.IfNull(ocrClient); - _options = options?.Clone() ?? new OcrOptions { IncludeImages = true }; + _options = options?.Clone() ?? new OcrOptions(); } /// @@ -58,16 +58,16 @@ private static IngestionDocument Map(OcrResult ocrResult, string identifier) IngestionDocumentSection section = new(); int pageNumber = page.PageNumber; - if (!string.IsNullOrWhiteSpace(page.Markdown)) + if (!string.IsNullOrWhiteSpace(page.Text)) { - section.Elements.Add(new IngestionDocumentParagraph(page.Markdown) + section.Elements.Add(new IngestionDocumentParagraph(page.Text) { - Text = page.Markdown, + Text = page.Text, PageNumber = pageNumber }); } - foreach (OcrImage image in page.Images) + foreach (OcrImage image in page.Elements.OfType()) { section.Elements.Add(MapImage(image, pageNumber)); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs index 181149aaf73..5ce31072f45 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/DelegatingOcrClientTests.cs @@ -53,14 +53,14 @@ public async Task ExtractAsyncDefaultsToInnerClientAsync() } [Fact] - public async Task ExtractStreamingAsyncDefaultsToInnerClientAsync() + public async Task ExtractPagesAsyncDefaultsToInnerClientAsync() { // Arrange using var expectedDocument = new MemoryStream(); var expectedMediaType = "application/pdf"; var expectedOptions = new OcrOptions(); using var cts = new CancellationTokenSource(); - OcrResponseUpdate[] expectedUpdates = + OcrPageResult[] expectedUpdates = [ new(new OcrPage(1, "page one")), new(new OcrPage(2, "page two")), @@ -68,7 +68,7 @@ public async Task ExtractStreamingAsyncDefaultsToInnerClientAsync() using var inner = new TestOcrClient { - ExtractStreamingAsyncCallback = (document, mediaType, options, cancellationToken) => + ExtractPagesAsyncCallback = (document, mediaType, options, cancellationToken) => { Assert.Same(expectedDocument, document); Assert.Same(expectedMediaType, mediaType); @@ -81,8 +81,8 @@ public async Task ExtractStreamingAsyncDefaultsToInnerClientAsync() using var delegating = new NoOpDelegatingOcrClient(inner); // Act - List received = []; - await foreach (var update in delegating.ExtractStreamingAsync(expectedDocument, expectedMediaType, expectedOptions, cts.Token)) + List received = []; + await foreach (var update in delegating.ExtractPagesAsync(expectedDocument, expectedMediaType, expectedOptions, cts.Token)) { received.Add(update); } @@ -91,7 +91,7 @@ public async Task ExtractStreamingAsyncDefaultsToInnerClientAsync() Assert.Equal(expectedUpdates, received); } - private static async IAsyncEnumerable YieldAsync(IEnumerable updates) + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) { foreach (var update in updates) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs new file mode 100644 index 00000000000..cc601df3b98 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs @@ -0,0 +1,90 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class OcrElementTests +{ + [Fact] + public void Elements_OfType_ProjectsEachKindInReadingOrder() + { + OcrPage page = new(1, "page text") + { + Elements = + [ + new OcrBlock("intro"), + new OcrTable(1, 1), + new OcrImage { Caption = "figure" }, + new OcrBlock("outro"), + ], + }; + + Assert.Equal(4, page.Elements.Count); + Assert.Equal(["intro", "outro"], page.Elements.OfType().Select(b => b.Text)); + Assert.Single(page.Elements.OfType()); + Assert.Equal("figure", Assert.Single(page.Elements.OfType()).Caption); + } + + [Fact] + public void Elements_SerializePolymorphically_RoundTrip() + { + OcrResult result = new( + [ + new OcrPage(1, "page text") + { + Elements = + [ + new OcrBlock("title") { Kind = OcrBlockKind.Title, Confidence = 0.9 }, + new OcrTable(1, 2, [new OcrTableCell(0, 0, "a") { Kind = OcrTableCellKind.RowHeader }, new OcrTableCell(0, 1, "b")]), + new OcrImage { Caption = "figure", Confidence = 0.5 }, + ], + }, + ]) + { + CoordinateUnit = OcrCoordinateUnit.Point, + CoordinateOrigin = OcrCoordinateOrigin.BottomLeft, + }; + + string json = JsonSerializer.Serialize(result, AIJsonUtilities.DefaultOptions); + + Assert.Contains("$type", json); + Assert.Contains("block", json); + Assert.Contains("table", json); + Assert.Contains("image", json); + Assert.Contains("Point", json); + Assert.Contains("BottomLeft", json); + + OcrResult roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; + + Assert.Equal(OcrCoordinateUnit.Point, roundTripped.CoordinateUnit); + Assert.Equal(OcrCoordinateOrigin.BottomLeft, roundTripped.CoordinateOrigin); + + OcrPage page = Assert.Single(roundTripped.Pages); + Assert.Collection( + page.Elements, + e => Assert.Equal("title", Assert.IsType(e).Text), + e => Assert.Equal(2, Assert.IsType(e).ColumnCount), + e => Assert.Equal("figure", Assert.IsType(e).Caption)); + Assert.Equal(0.9, page.Elements.OfType().Single().Confidence); + } + + [Fact] + public void TableCell_NestedElements_RoundTrip() + { + OcrTableCell cell = new(0, 0, "flat text") + { + Elements = [new OcrBlock("nested paragraph")], + }; + + string json = JsonSerializer.Serialize(cell, AIJsonUtilities.DefaultOptions); + OcrTableCell roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; + + Assert.Equal("flat text", roundTripped.Content); + Assert.NotNull(roundTripped.Elements); + Assert.Equal("nested paragraph", Assert.IsType(Assert.Single(roundTripped.Elements!)).Text); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs index 8d2ccfcf786..ca76023404b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrOptionsTests.cs @@ -13,7 +13,6 @@ public void Clone_CopiesAllProperties() var options = new OcrOptions { ModelId = "mistral-ocr-4-0", - IncludeImages = true, AdditionalProperties = new() { ["custom"] = "value" }, }; @@ -21,7 +20,6 @@ public void Clone_CopiesAllProperties() Assert.NotSame(options, clone); Assert.Equal("mistral-ocr-4-0", clone.ModelId); - Assert.True(clone.IncludeImages); Assert.NotNull(clone.AdditionalProperties); Assert.Equal("value", clone.AdditionalProperties!["custom"]); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs similarity index 53% rename from test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs rename to test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs index 381d030619c..a188f2b97d9 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResponseUpdateExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs @@ -8,59 +8,56 @@ namespace Microsoft.Extensions.AI; -public class OcrResponseUpdateExtensionsTests +public class OcrPageResultExtensionsTests { [Fact] public void ToOcrResult_NullUpdates_Throws() { - Assert.Throws("updates", () => ((IEnumerable)null!).ToOcrResult()); + Assert.Throws("updates", () => ((IEnumerable)null!).ToOcrResult()); } [Fact] public async Task ToOcrResultAsync_NullUpdates_ThrowsAsync() { - await Assert.ThrowsAsync("updates", () => ((IAsyncEnumerable)null!).ToOcrResultAsync()); + await Assert.ThrowsAsync("updates", () => ((IAsyncEnumerable)null!).ToOcrResultAsync()); } [Fact] - public void ToOcrResult_AssemblesPagesModelIdAndUsage() + public void ToOcrResult_AssemblesPagesAndUsage() { - OcrResponseUpdate[] updates = + OcrPageResult[] updates = [ - new(new OcrPage(1, "page one")), - new() { PagesProcessed = 1, TotalPages = 2, Status = "processing" }, - new(new OcrPage(2, "page two")) { ModelId = "model-x", Usage = new() { PagesProcessed = 2 } }, + new(new OcrPage(1, "page one")) { PagesProcessed = 1, TotalPages = 2 }, + new(new OcrPage(2, "page two")) { Usage = new() { PagesProcessed = 2 } }, ]; OcrResult result = updates.ToOcrResult(); Assert.Equal(2, result.Pages.Count); - Assert.Equal("page one\n\npage two", result.Markdown); - Assert.Equal("model-x", result.ModelId); + Assert.Equal("page one\n\npage two", result.Text); Assert.NotNull(result.Usage); Assert.Equal(2, result.Usage!.PagesProcessed); } [Fact] - public async Task ToOcrResultAsync_AssemblesPagesModelIdAndUsageAsync() + public async Task ToOcrResultAsync_AssemblesPagesAndUsageAsync() { - OcrResponseUpdate[] updates = + OcrPageResult[] updates = [ new(new OcrPage(1, "page one")), - new(new OcrPage(2, "page two")) { ModelId = "model-y" }, + new(new OcrPage(2, "page two")), ]; OcrResult result = await YieldAsync(updates).ToOcrResultAsync(); Assert.Equal(2, result.Pages.Count); - Assert.Equal("page one\n\npage two", result.Markdown); - Assert.Equal("model-y", result.ModelId); + Assert.Equal("page one\n\npage two", result.Text); } [Fact] public void ToOcrResult_MergesAdditionalProperties() { - OcrResponseUpdate[] updates = + OcrPageResult[] updates = [ new(new OcrPage(1, "page one")) { AdditionalProperties = new() { ["a"] = "1" } }, new(new OcrPage(2, "page two")) { AdditionalProperties = new() { ["b"] = "2" } }, @@ -73,7 +70,23 @@ public void ToOcrResult_MergesAdditionalProperties() Assert.Equal("2", result.AdditionalProperties!["b"]); } - private static async IAsyncEnumerable YieldAsync(IEnumerable updates) + [Fact] + public void ToOcrResult_PropagatesCoordinateMetadata_LastNonNullWins() + { + OcrPageResult[] updates = + [ + new(new OcrPage(1, "page one")) { CoordinateUnit = OcrCoordinateUnit.Pixel, CoordinateOrigin = OcrCoordinateOrigin.TopLeft }, + new(new OcrPage(2, "page two")), + new(new OcrPage(3, "page three")) { CoordinateUnit = OcrCoordinateUnit.Point }, + ]; + + OcrResult result = updates.ToOcrResult(); + + Assert.Equal(OcrCoordinateUnit.Point, result.CoordinateUnit); + Assert.Equal(OcrCoordinateOrigin.TopLeft, result.CoordinateOrigin); + } + + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) { foreach (var update in updates) { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs index 752d2b4e890..4da31ea04ec 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrResultTests.cs @@ -15,15 +15,11 @@ public void Constructor_NullPages_Throws() } [Fact] - public void Markdown_JoinsPerPageMarkdown() + public void Text_JoinsPerPageText() { - var result = new OcrResult([new OcrPage(1, "page one"), new OcrPage(2, "page two")]) - { - ModelId = "model-1", - }; + var result = new OcrResult([new OcrPage(1, "page one"), new OcrPage(2, "page two")]); - Assert.Equal("page one\n\npage two", result.Markdown); - Assert.Equal("model-1", result.ModelId); + Assert.Equal("page one\n\npage two", result.Text); Assert.Equal(2, result.Pages.Count); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs index 0df2626b12f..b67bb4f59ac 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/TestOcrClient.cs @@ -30,8 +30,8 @@ public TestOcrClient() string, OcrOptions?, CancellationToken, - IAsyncEnumerable>? - ExtractStreamingAsyncCallback + IAsyncEnumerable>? + ExtractPagesAsyncCallback { get; set; } public Func GetServiceCallback { get; set; } @@ -46,12 +46,12 @@ public Task ExtractAsync( CancellationToken cancellationToken = default) => ExtractAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); - public IAsyncEnumerable ExtractStreamingAsync( + public IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, CancellationToken cancellationToken = default) - => ExtractStreamingAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); + => ExtractPagesAsyncCallback!.Invoke(document, mediaType, options, cancellationToken); public object? GetService(Type serviceType, object? serviceKey = null) => GetServiceCallback!.Invoke(serviceType, serviceKey); diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs index 06baea533e6..2b72455c259 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Ocr/OpenTelemetryOcrClientTests.cs @@ -38,7 +38,6 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) await Task.Yield(); return new OcrResult([new OcrPage(1, "This is the recognized text.")]) { - ModelId = "amazingmodel", Usage = new() { PagesProcessed = 3 }, }; }, @@ -83,7 +82,6 @@ public async Task ExpectedInformationLogged_Async(bool enableSensitiveData) Assert.Equal(enableSensitiveData ? "value1" : null, activity.GetTagItem("service_tier")); Assert.Equal(enableSensitiveData ? "value2" : null, activity.GetTagItem("SomethingElse")); - Assert.Equal("amazingmodel", activity.GetTagItem("gen_ai.response.model")); Assert.Equal(3, (int)activity.GetTagItem("gen_ai.usage.pages_processed")!); Assert.True(activity.Duration.TotalMilliseconds > 0); diff --git a/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs index f4b38b52b80..1759d95dfdb 100644 --- a/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs +++ b/test/Libraries/Microsoft.Extensions.DataIngestion.Tests/Readers/OcrDocumentReaderTests.cs @@ -24,7 +24,7 @@ public async Task MapsOcrImagesToIngestionDocumentImages() [ new OcrPage(2, "Page text") { - Images = + Elements = [ new OcrImage { @@ -49,7 +49,7 @@ public async Task MapsOcrImagesToIngestionDocumentImages() Assert.Equal([1f, 2f, 10f, 20f], Assert.IsType(image.Metadata["bounding_box"])); Assert.Equal([1f, 2f, 10f, 2f, 10f, 20f, 1f, 20f], Assert.IsType(image.Metadata["bounding_region"])); Assert.Equal("application/pdf", ocrClient.MediaType); - Assert.True(ocrClient.Options?.IncludeImages); + Assert.NotNull(ocrClient.Options); } private sealed class TestOcrClient(OcrResult result) : IOcrClient @@ -69,7 +69,7 @@ public Task ExtractAsync( return Task.FromResult(result); } - public IAsyncEnumerable ExtractStreamingAsync( + public IAsyncEnumerable ExtractPagesAsync( Stream document, string mediaType, OcrOptions? options = null, From 9c7a2057423a879736f7c2640df87d856c537db3 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Thu, 23 Jul 2026 13:55:16 -0400 Subject: [PATCH 12/13] Move OCR coordinate frame per-page; add OcrPageDimensions (SPIKE-07/08) SPIKE-07/08 primary-source verification refuted the round-1 assumption that a document uses one uniform coordinate unit. Google Document AI models Page.Dimension{width,height,unit} per page, and Azure DI's per-page DocumentPage.Unit is "pixel" for image inputs and "inch" for PDF, so one analyze call over a mixed batch returns different units on different pages. - Group OcrPage.Width/Height into a new OcrPageDimensions readonly record struct (the extent; an atomic pair several engines bundle). - Move CoordinateUnit + CoordinateOrigin from the document level (OcrResult, OcrPageResult) onto OcrPage; they are siblings to the dimensions extent, together describing the per-page coordinate system. - Drop the reducer's last-non-null coordinate carry-forward: pages already carry their own unit/origin. - Defer continuous page-rotation angle (Azure-family-only) to AdditionalProperties for v1; element rotation is already carried losslessly by the polygon OcrBoundingRegion. - Update per-page tests and regenerate the ApiChief baseline (32 public types). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 19ea13dd-f444-4942-b751-162711478580 --- .../Microsoft.Extensions.AI.Abstractions.json | 78 ++++++++++++++----- .../Ocr/OcrCoordinateOrigin.cs | 7 +- .../Ocr/OcrCoordinateUnit.cs | 12 +-- .../Ocr/OcrPage.cs | 27 ++++--- .../Ocr/OcrPageDimensions.cs | 19 +++++ .../Ocr/OcrPageResult.cs | 10 --- .../Ocr/OcrPageResultExtensions.cs | 10 --- .../Ocr/OcrResult.cs | 10 --- .../Ocr/OcrElementTests.cs | 13 ++-- .../Ocr/OcrPageResultExtensionsTests.cs | 25 ++++-- 10 files changed, 129 insertions(+), 82 deletions(-) create mode 100644 src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageDimensions.cs diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index fef135266c5..e1a4ed5ac26 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -3727,47 +3727,95 @@ "Stage": "Experimental" }, { - "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Elements { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrCoordinateOrigin? Microsoft.Extensions.AI.OcrPage.CoordinateOrigin { get; set; }", "Stage": "Experimental" }, { - "Member": "float? Microsoft.Extensions.AI.OcrPage.Height { get; set; }", + "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrPage.CoordinateUnit { get; set; }", "Stage": "Experimental" }, { - "Member": "int Microsoft.Extensions.AI.OcrPage.PageNumber { get; }", + "Member": "Microsoft.Extensions.AI.OcrPageDimensions? Microsoft.Extensions.AI.OcrPage.Dimensions { get; set; }", "Stage": "Experimental" }, { - "Member": "string Microsoft.Extensions.AI.OcrPage.Text { get; }", + "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrPage.Elements { get; set; }", "Stage": "Experimental" }, { - "Member": "float? Microsoft.Extensions.AI.OcrPage.Width { get; set; }", + "Member": "int Microsoft.Extensions.AI.OcrPage.PageNumber { get; }", + "Stage": "Experimental" + }, + { + "Member": "string Microsoft.Extensions.AI.OcrPage.Text { get; }", "Stage": "Experimental" } ] }, { - "Type": "class Microsoft.Extensions.AI.OcrPageResult", + "Type": "readonly class Microsoft.Extensions.AI.OcrPageDimensions(float Width, float Height)", "Stage": "Experimental", "Methods": [ { - "Member": "Microsoft.Extensions.AI.OcrPageResult.OcrPageResult(Microsoft.Extensions.AI.OcrPage page);", + "Member": "Microsoft.Extensions.AI.OcrPageDimensions.OcrPageDimensions(float Width, float Height);", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrPageDimensions.OcrPageDimensions();", + "Stage": "Experimental" + }, + { + "Member": "void Microsoft.Extensions.AI.OcrPageDimensions.Deconstruct(out float Width, out float Height);", + "Stage": "Experimental" + }, + { + "Member": "override bool Microsoft.Extensions.AI.OcrPageDimensions.Equals(object obj);", + "Stage": "Experimental" + }, + { + "Member": "bool Microsoft.Extensions.AI.OcrPageDimensions.Equals(Microsoft.Extensions.AI.OcrPageDimensions other);", + "Stage": "Experimental" + }, + { + "Member": "override int Microsoft.Extensions.AI.OcrPageDimensions.GetHashCode();", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrPageDimensions.operator ==(Microsoft.Extensions.AI.OcrPageDimensions left, Microsoft.Extensions.AI.OcrPageDimensions right);", + "Stage": "Experimental" + }, + { + "Member": "static bool Microsoft.Extensions.AI.OcrPageDimensions.operator !=(Microsoft.Extensions.AI.OcrPageDimensions left, Microsoft.Extensions.AI.OcrPageDimensions right);", + "Stage": "Experimental" + }, + { + "Member": "override string Microsoft.Extensions.AI.OcrPageDimensions.ToString();", "Stage": "Experimental" } ], "Properties": [ { - "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrPageResult.AdditionalProperties { get; set; }", + "Member": "float Microsoft.Extensions.AI.OcrPageDimensions.Height { get; init; }", "Stage": "Experimental" }, { - "Member": "Microsoft.Extensions.AI.OcrCoordinateOrigin? Microsoft.Extensions.AI.OcrPageResult.CoordinateOrigin { get; set; }", + "Member": "float Microsoft.Extensions.AI.OcrPageDimensions.Width { get; init; }", "Stage": "Experimental" - }, + } + ] + }, + { + "Type": "class Microsoft.Extensions.AI.OcrPageResult", + "Stage": "Experimental", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.OcrPageResult.OcrPageResult(Microsoft.Extensions.AI.OcrPage page);", + "Stage": "Experimental" + } + ], + "Properties": [ { - "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrPageResult.CoordinateUnit { get; set; }", + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrPageResult.AdditionalProperties { get; set; }", "Stage": "Experimental" }, { @@ -3872,14 +3920,6 @@ "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrResult.AdditionalProperties { get; set; }", "Stage": "Experimental" }, - { - "Member": "Microsoft.Extensions.AI.OcrCoordinateOrigin? Microsoft.Extensions.AI.OcrResult.CoordinateOrigin { get; set; }", - "Stage": "Experimental" - }, - { - "Member": "Microsoft.Extensions.AI.OcrCoordinateUnit? Microsoft.Extensions.AI.OcrResult.CoordinateUnit { get; set; }", - "Stage": "Experimental" - }, { "Member": "System.Collections.Generic.IReadOnlyList Microsoft.Extensions.AI.OcrResult.Pages { get; }", "Stage": "Experimental" diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs index 3ec028c527c..d5f040ab213 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateOrigin.cs @@ -12,10 +12,9 @@ namespace Microsoft.Extensions.AI; /// /// Engines disagree on where a page's coordinate origin sits and which way the y axis grows: rasterized /// page images place the origin at the top-left with y increasing downward, whereas PDF-native -/// (point) coordinates place it at the bottom-left with y increasing upward. The origin is reported once -/// at the document level on and , alongside -/// , so bounding regions from different engines can be compared and -/// normalized without guessing the convention. +/// (point) coordinates place it at the bottom-left with y increasing upward. The origin is reported per +/// page on , alongside , so bounding +/// regions from different engines can be compared and normalized without guessing the convention. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] public enum OcrCoordinateOrigin diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs index a6b273b6138..5086056f8ee 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrCoordinateUnit.cs @@ -13,12 +13,12 @@ namespace Microsoft.Extensions.AI; /// /// Coordinate conventions differ across OCR engines: some report pixels of the rendered page image, /// some report a physical unit such as points or inches, and some normalize to the page. The unit is -/// reported once at the document level on and , -/// paired with an and the page dimensions ( -/// and ), so a consumer can interpret or normalize regions with -/// engine-agnostic code. Unlike the taxonomy kinds (, -/// ), the set of coordinate units is physically bounded, so it is modeled -/// as a closed enumeration. +/// reported per page on , paired with an +/// and the page dimensions (), so a +/// consumer can interpret or normalize regions with engine-agnostic code. It is per page because engines +/// can emit different units for different pages of one document (for example, a batch mixing image and +/// PDF inputs). Unlike the taxonomy kinds (, ), +/// the set of coordinate units is physically bounded, so it is modeled as a closed enumeration. /// [Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] public enum OcrCoordinateUnit diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs index 0620e48705c..e7be99a43f4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -37,19 +37,26 @@ public OcrPage(int pageNumber, string text) /// public IReadOnlyList Elements { get; set; } = []; - /// Gets or sets the page width, expressed in the document-level , when the engine provides it. + /// Gets or sets the page dimensions (width and height), expressed in , when the engine provides them. /// - /// Together with and the document-level and - /// , this lets a consumer interpret or normalize the geometry - /// ( / ) on this page with engine-agnostic code. For - /// example, dividing a coordinate by the corresponding page dimension yields a page-relative [0, 1] value - /// regardless of the native unit. + /// Together with and , the dimensions let a consumer + /// interpret or normalize the geometry ( / ) on this page with + /// engine-agnostic code. For example, dividing a coordinate by the corresponding dimension yields a page-relative + /// [0, 1] value regardless of the native unit. /// - public float? Width { get; set; } + public OcrPageDimensions? Dimensions { get; set; } - /// Gets or sets the page height, expressed in the document-level , when the engine provides it. - /// See for how the page dimensions are used with the document-level coordinate unit. - public float? Height { get; set; } + /// Gets or sets the unit in which this page's geometry coordinates are expressed, when known. + /// + /// Reported per page: engines can emit different units for different pages of one document (for example, a batch + /// mixing image and PDF inputs). Applies to every on the page and to + /// . When , the geometry should be treated as an opaque, + /// provider-specific coordinate space. + /// + public OcrCoordinateUnit? CoordinateUnit { get; set; } + + /// Gets or sets the origin corner and axis direction of this page's geometry coordinates, when known. + public OcrCoordinateOrigin? CoordinateOrigin { get; set; } /// Gets or sets any additional properties associated with the page. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageDimensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageDimensions.cs new file mode 100644 index 00000000000..876525e5749 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageDimensions.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Extensions.AI; + +/// Represents the width and height of an , expressed in the page's . +/// The page width. +/// The page height. +/// +/// Together with the page's and , the +/// dimensions let a consumer interpret or normalize the geometry ( / ) +/// on the page with engine-agnostic code. For example, dividing a coordinate by the corresponding dimension yields a +/// page-relative [0, 1] value regardless of the native unit. +/// +[Experimental(DiagnosticIds.Experiments.AIOcr, UrlFormat = DiagnosticIds.UrlFormat)] +public readonly record struct OcrPageDimensions(float Width, float Height); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs index 4b65462de09..aa4070fbf4e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResult.cs @@ -45,16 +45,6 @@ public OcrPageResult(OcrPage page) /// Gets or sets the total number of pages, when known. public int? TotalPages { get; set; } - /// Gets or sets the unit in which this document's geometry coordinates are expressed, when known. - /// - /// Reported at the document level; the reducer () carries - /// the last non- value onto . - /// - public OcrCoordinateUnit? CoordinateUnit { get; set; } - - /// Gets or sets the origin corner and axis direction of this document's geometry coordinates, when known. - public OcrCoordinateOrigin? CoordinateOrigin { get; set; } - /// Gets or sets usage details associated with the request, when reported. /// Usage is typically carried on a terminal update once the full document has been processed. public OcrUsage? Usage { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs index 4e4efd905ba..2129b94b03a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPageResultExtensions.cs @@ -68,16 +68,6 @@ private static void ProcessUpdate(OcrPageResult update, List pages, Ocr { pages.Add(update.Page); - if (update.CoordinateUnit is not null) - { - result.CoordinateUnit = update.CoordinateUnit; - } - - if (update.CoordinateOrigin is not null) - { - result.CoordinateOrigin = update.CoordinateOrigin; - } - if (update.Usage is not null) { result.Usage = update.Usage; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs index f37170da2e4..e2bd8c06fc5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrResult.cs @@ -33,16 +33,6 @@ public OcrResult(IReadOnlyList pages) /// Gets the full-document text, formed by joining the per-page text. public string Text => string.Join("\n\n", Pages.Select(p => p.Text)); - /// Gets or sets the unit in which this document's geometry coordinates are expressed, when known. - /// - /// Applies to every in the document. When , the - /// geometry should be treated as an opaque, provider-specific coordinate space. - /// - public OcrCoordinateUnit? CoordinateUnit { get; set; } - - /// Gets or sets the origin corner and axis direction of this document's geometry coordinates, when known. - public OcrCoordinateOrigin? CoordinateOrigin { get; set; } - /// Gets or sets usage details associated with the request. public OcrUsage? Usage { get; set; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs index cc601df3b98..4b1cb6c5647 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs @@ -36,6 +36,8 @@ public void Elements_SerializePolymorphically_RoundTrip() [ new OcrPage(1, "page text") { + CoordinateUnit = OcrCoordinateUnit.Point, + CoordinateOrigin = OcrCoordinateOrigin.BottomLeft, Elements = [ new OcrBlock("title") { Kind = OcrBlockKind.Title, Confidence = 0.9 }, @@ -43,11 +45,7 @@ public void Elements_SerializePolymorphically_RoundTrip() new OcrImage { Caption = "figure", Confidence = 0.5 }, ], }, - ]) - { - CoordinateUnit = OcrCoordinateUnit.Point, - CoordinateOrigin = OcrCoordinateOrigin.BottomLeft, - }; + ]); string json = JsonSerializer.Serialize(result, AIJsonUtilities.DefaultOptions); @@ -60,10 +58,9 @@ public void Elements_SerializePolymorphically_RoundTrip() OcrResult roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; - Assert.Equal(OcrCoordinateUnit.Point, roundTripped.CoordinateUnit); - Assert.Equal(OcrCoordinateOrigin.BottomLeft, roundTripped.CoordinateOrigin); - OcrPage page = Assert.Single(roundTripped.Pages); + Assert.Equal(OcrCoordinateUnit.Point, page.CoordinateUnit); + Assert.Equal(OcrCoordinateOrigin.BottomLeft, page.CoordinateOrigin); Assert.Collection( page.Elements, e => Assert.Equal("title", Assert.IsType(e).Text), diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs index a188f2b97d9..a0648ec3f11 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs @@ -71,19 +71,34 @@ public void ToOcrResult_MergesAdditionalProperties() } [Fact] - public void ToOcrResult_PropagatesCoordinateMetadata_LastNonNullWins() + public void ToOcrResult_PreservesPerPageCoordinateMetadata() { OcrPageResult[] updates = [ - new(new OcrPage(1, "page one")) { CoordinateUnit = OcrCoordinateUnit.Pixel, CoordinateOrigin = OcrCoordinateOrigin.TopLeft }, + new(new OcrPage(1, "page one") { CoordinateUnit = OcrCoordinateUnit.Pixel, CoordinateOrigin = OcrCoordinateOrigin.TopLeft }), new(new OcrPage(2, "page two")), - new(new OcrPage(3, "page three")) { CoordinateUnit = OcrCoordinateUnit.Point }, + new(new OcrPage(3, "page three") { CoordinateUnit = OcrCoordinateUnit.Point, CoordinateOrigin = OcrCoordinateOrigin.BottomLeft }), ]; OcrResult result = updates.ToOcrResult(); - Assert.Equal(OcrCoordinateUnit.Point, result.CoordinateUnit); - Assert.Equal(OcrCoordinateOrigin.TopLeft, result.CoordinateOrigin); + Assert.Collection( + result.Pages, + p => + { + Assert.Equal(OcrCoordinateUnit.Pixel, p.CoordinateUnit); + Assert.Equal(OcrCoordinateOrigin.TopLeft, p.CoordinateOrigin); + }, + p => + { + Assert.Null(p.CoordinateUnit); + Assert.Null(p.CoordinateOrigin); + }, + p => + { + Assert.Equal(OcrCoordinateUnit.Point, p.CoordinateUnit); + Assert.Equal(OcrCoordinateOrigin.BottomLeft, p.CoordinateOrigin); + }); } private static async IAsyncEnumerable YieldAsync(IEnumerable updates) From 108134e81728a9003f15d3c92a644bc52727a3f0 Mon Sep 17 00:00:00 2001 From: luisquintanilla Date: Thu, 23 Jul 2026 16:14:26 -0400 Subject: [PATCH 13/13] Add per-cell geometry and OcrPage.RawRepresentation (SPIKE-06) SPIKE-06 built an exhaustive 14-engine raw-output inventory to justify the RawRepresentation/AdditionalProperties escape hatches, then a promote interview resolved which recurring un-homed fields become first-class. - Add BoundingRegion, Confidence, RawRepresentation, and AdditionalProperties to OcrTableCell, mirrored member-for-member from OcrElement. Five engines (Textract, Google Document AI, Azure DI, Adobe, Docling) emit per-cell geometry, and a cell has its own rectangle even when empty or padded. OcrTableCell deliberately does not derive from OcrElement (a cell is not a page-reading-order element) and no shared base type is extracted yet; mirroring the members keeps that a reversible way-station under [Experimental]. - Add [JsonIgnore] object? RawRepresentation to OcrPage. Every other node carries it; the page node was the lone exception. Because the page rides through ToOcrResult reduction, provider-native page data now survives into OcrResult.Pages, closing a silent per-page raw data-loss. - Detected language is deferred (not promoted) to a standardized AdditionalProperties key rather than a lossy typed field. - Add OCR tests for cell geometry round-trip and per-page raw survival; regenerate the ApiChief baseline (five additive members, no new types). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 19ea13dd-f444-4942-b751-162711478580 --- .../Microsoft.Extensions.AI.Abstractions.json | 20 ++++++++++++++++ .../Ocr/OcrPage.cs | 12 ++++++++++ .../Ocr/OcrTableCell.cs | 24 +++++++++++++++++++ .../Ocr/OcrElementTests.cs | 22 +++++++++++++++++ .../Ocr/OcrPageResultExtensionsTests.cs | 20 ++++++++++++++++ 5 files changed, 98 insertions(+) diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index e1a4ed5ac26..9a9a2202ea0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -3746,6 +3746,10 @@ "Member": "int Microsoft.Extensions.AI.OcrPage.PageNumber { get; }", "Stage": "Experimental" }, + { + "Member": "object? Microsoft.Extensions.AI.OcrPage.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, { "Member": "string Microsoft.Extensions.AI.OcrPage.Text { get; }", "Stage": "Experimental" @@ -3976,6 +3980,14 @@ } ], "Properties": [ + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.OcrTableCell.AdditionalProperties { get; set; }", + "Stage": "Experimental" + }, + { + "Member": "Microsoft.Extensions.AI.OcrBoundingRegion? Microsoft.Extensions.AI.OcrTableCell.BoundingRegion { get; set; }", + "Stage": "Experimental" + }, { "Member": "int Microsoft.Extensions.AI.OcrTableCell.ColumnIndex { get; }", "Stage": "Experimental" @@ -3984,6 +3996,10 @@ "Member": "int Microsoft.Extensions.AI.OcrTableCell.ColumnSpan { get; set; }", "Stage": "Experimental" }, + { + "Member": "double? Microsoft.Extensions.AI.OcrTableCell.Confidence { get; set; }", + "Stage": "Experimental" + }, { "Member": "string Microsoft.Extensions.AI.OcrTableCell.Content { get; }", "Stage": "Experimental" @@ -3996,6 +4012,10 @@ "Member": "Microsoft.Extensions.AI.OcrTableCellKind? Microsoft.Extensions.AI.OcrTableCell.Kind { get; set; }", "Stage": "Experimental" }, + { + "Member": "object? Microsoft.Extensions.AI.OcrTableCell.RawRepresentation { get; set; }", + "Stage": "Experimental" + }, { "Member": "int Microsoft.Extensions.AI.OcrTableCell.RowIndex { get; }", "Stage": "Experimental" diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs index e7be99a43f4..bb3679b3325 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrPage.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -58,6 +59,17 @@ public OcrPage(int pageNumber, string text) /// Gets or sets the origin corner and axis direction of this page's geometry coordinates, when known. public OcrCoordinateOrigin? CoordinateOrigin { get; set; } + /// Gets or sets the provider-native object underlying this page. + /// + /// If an is created to represent an underlying object from another object model, this + /// property can store that original object. This can be useful for debugging or for enabling a consumer to + /// access the underlying object model if needed. Because the page node rides through + /// reduction, provider-native page data set here survives + /// into . + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + /// Gets or sets any additional properties associated with the page. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs index 1496e05652e..19b379c5290 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Ocr/OcrTableCell.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Text.Json.Serialization; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; @@ -50,4 +51,27 @@ public OcrTableCell(int rowIndex, int columnIndex, string content) /// Google Document AI. /// public IReadOnlyList? Elements { get; set; } + + /// Gets or sets the region of the page the cell occupies, when the engine provides geometry. + /// + /// A cell can carry its own rectangle even when it is empty or padded, so this is not always derivable from + /// the geometry of its nested . These members mirror so a cell + /// can be promoted to a shared positioned-node type later without a reshape. + /// + public OcrBoundingRegion? BoundingRegion { get; set; } + + /// Gets or sets the confidence for the cell in the range [0, 1], when available. + public double? Confidence { get; set; } + + /// Gets or sets the provider-native object underlying this cell. + /// + /// If an is created to represent an underlying object from another object model, + /// this property can store that original object. This can be useful for debugging or for enabling a + /// consumer to access the underlying object model if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// Gets or sets any additional properties associated with the cell. + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs index 4b1cb6c5647..bf6b7f3b55e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrElementTests.cs @@ -84,4 +84,26 @@ public void TableCell_NestedElements_RoundTrip() Assert.NotNull(roundTripped.Elements); Assert.Equal("nested paragraph", Assert.IsType(Assert.Single(roundTripped.Elements!)).Text); } + + [Fact] + public void TableCell_GeometryConfidenceAndProperties_RoundTrip() + { + OcrTableCell cell = new(0, 0, "flat text") + { + BoundingRegion = OcrBoundingRegion.FromRectangle(1, left: 10, top: 20, right: 110, bottom: 220), + Confidence = 0.75, + RawRepresentation = new { ignored = true }, + AdditionalProperties = new() { ["detectedLanguages"] = "en" }, + }; + + string json = JsonSerializer.Serialize(cell, AIJsonUtilities.DefaultOptions); + OcrTableCell roundTripped = JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions)!; + + Assert.NotNull(roundTripped.BoundingRegion); + Assert.Equal(1, roundTripped.BoundingRegion!.PageNumber); + Assert.Equal(0.75, roundTripped.Confidence); + Assert.NotNull(roundTripped.AdditionalProperties); + Assert.True(roundTripped.AdditionalProperties!.ContainsKey("detectedLanguages")); + Assert.Null(roundTripped.RawRepresentation); + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs index a0648ec3f11..c7eb59daf2f 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Ocr/OcrPageResultExtensionsTests.cs @@ -101,6 +101,26 @@ public void ToOcrResult_PreservesPerPageCoordinateMetadata() }); } + [Fact] + public void ToOcrResult_PreservesPerPageRawRepresentation() + { + object rawPageOne = new { page = 1 }; + object rawPageTwo = new { page = 2 }; + + OcrPageResult[] updates = + [ + new(new OcrPage(1, "page one") { RawRepresentation = rawPageOne }), + new(new OcrPage(2, "page two") { RawRepresentation = rawPageTwo }), + ]; + + OcrResult result = updates.ToOcrResult(); + + Assert.Collection( + result.Pages, + p => Assert.Same(rawPageOne, p.RawRepresentation), + p => Assert.Same(rawPageTwo, p.RawRepresentation)); + } + private static async IAsyncEnumerable YieldAsync(IEnumerable updates) { foreach (var update in updates)