Surface + document the loopback Local API for on-device agents (Settings toggle + docs)

Fluid.xcodeproj/project.pbxproj

@@ -28,6 +28,9 @@
 		803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 803000000000000000000001 /* MediaPlaybackServiceTests.swift */; };
 		803000000000000000000003 /* MediaRemoteAdapter in Frameworks */ = {isa = PBXBuildFile; productRef = 803000000000000000000004 /* MediaRemoteAdapter */; };
 		7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
+		0GG000000000000000000001 /* LocalAPIAudioDecoderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0GG000000000000000000002 /* LocalAPIAudioDecoderTests.swift */; };
+		0GG000000000000000000003 /* dictation_fixture.ogg in Resources */ = {isa = PBXBuildFile; fileRef = 0GG000000000000000000004 /* dictation_fixture.ogg */; };
+		0GG000000000000000000005 /* Opus.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0GG000000000000000000006 /* Opus.xcframework */; };
 		7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
 		7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };
 		86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
@@ -75,6 +78,9 @@
 		803000000000000000000001 /* MediaPlaybackServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlaybackServiceTests.swift; sourceTree = "<group>"; };
 		7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };
 		7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };
+		0GG000000000000000000002 /* LocalAPIAudioDecoderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalAPIAudioDecoderTests.swift; sourceTree = "<group>"; };
+		0GG000000000000000000004 /* dictation_fixture.ogg */ = {isa = PBXFileReference; lastKnownFileType = file; path = dictation_fixture.ogg; sourceTree = "<group>"; };
+		0GG000000000000000000006 /* Opus.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Vendor/Opus.xcframework; sourceTree = SOURCE_ROOT; };
 		7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
 /* End PBXFileReference section */
 
@@ -91,6 +97,7 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				0GG000000000000000000005 /* Opus.xcframework in Frameworks */,
 				7C3697892ED70F9C005874CE /* DynamicNotchKit in Frameworks */,
 				7C5AF14B2F15041600DE21B0 /* MediaRemoteAdapter in Frameworks */,
 				7C9A71022F58B00000FB7CAF /* TranscribeCpp in Frameworks */,
@@ -155,6 +162,7 @@
 				DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
 				7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */,
 				803000000000000000000001 /* MediaPlaybackServiceTests.swift */,
+				0GG000000000000000000002 /* LocalAPIAudioDecoderTests.swift */,
 			);
 			path = FluidDictationIntegrationTests;
 			sourceTree = "<group>";
@@ -171,13 +179,15 @@
 			isa = PBXGroup;
 			children = (
 				7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */,
+				0GG000000000000000000004 /* dictation_fixture.ogg */,
 			);
 			path = Resources;
 			sourceTree = "<group>";
 		};
 		7CDB0A282F3C4D5600FB7CAD /* Frameworks */ = {
 			isa = PBXGroup;
 			children = (
+				0GG000000000000000000006 /* Opus.xcframework */,
 				7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */,
 			);
 			name = Frameworks;
@@ -293,6 +303,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */,
+				0GG000000000000000000003 /* dictation_fixture.ogg in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -324,6 +335,7 @@
 				DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
 				7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */,
 				803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */,
+				0GG000000000000000000001 /* LocalAPIAudioDecoderTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

README.md

@@ -173,6 +173,29 @@ Whisper supports up to 99 languages, depending on the model size you choose.
 
 ---
 
+## Local API (for on-device agents)
+
+FluidVoice can expose a **loopback-only** HTTP API so other apps and AI agents running **on the same Mac** can reuse its transcription and post-processing — instead of bundling and loading their own ASR model. One model stays warm in FluidVoice; agents get FluidVoice-quality transcription with no extra memory footprint.
+
+**Enable it:** `Settings → Local API (on-device agents)`. It is **off by default** and **refuses all non-loopback connections** — any non-local peer is rejected at accept time, before it can send a request — so the API is usable only from processes on this Mac. Every route requires the per-user bearer token from `defaults read com.FluidApp.app LocalAPIAuthToken`.
+
+- Base URL: `http://127.0.0.1:47733` (port configurable via the `LocalAPIPort` preference)
+- `GET  /v1/health` — liveness check
+- `POST /v1/transcribe` — transcribe an audio file → `{ text, confidence, sampleCount, provider }`
+- `POST /v1/postprocess` — run text post-processing/enhancement
+
+```bash
+curl -s http://127.0.0.1:47733/v1/transcribe \
+  -H "Authorization: Bearer $(defaults read com.FluidApp.app LocalAPIAuthToken)" \
+  -H 'Content-Type: application/json' \
+  -d '{"path": "/absolute/path/to/audio.ogg"}'
+# → {"text":"…","confidence":0.97,"sampleCount":…,"provider":"Parakeet TDT v3 (Multilingual)"}
+```
+
+Audio can also be sent inline as `{"audioBase64": "…", "filename": "clip.wav"}`. Limits: **25 MB** request body, **25 MB** referenced OGG/Opus file, and **300 s (5 min)** of audio per transcription (independent caps). File-path input and inline OGG/Opus input longer than 300 s are rejected; other inline formats are truncated to the first 300 s. Note: `/v1/transcribe` runs on-device, but `/v1/postprocess` uses your configured AI provider (Settings → AI Enhancement), which may be remote. Full reference: [docs/local-api.md](docs/local-api.md).
+
+---
+
 ## Requirements
 
 - macOS 15.0 (Sequoia) or later

Sources/Fluid/Persistence/SettingsStore.swift

@@ -1817,6 +1817,15 @@ final class SettingsStore: ObservableObject {
         set { self.defaults.set(newValue, forKey: Keys.copyTranscriptionToClipboard) }
     }
 
+    /// When enabled, the loopback-only Local API server (`LocalAPIServer`) is started so
+    /// on-device agents can reuse FluidVoice's transcription (POST /v1/transcribe) instead
+    /// of shipping their own ASR model. Off by default. Uses the same "LocalAPIEnabled"
+    /// key that `LocalAPI.Configuration` reads.
+    var localAPIEnabled: Bool {
+        get { self.defaults.bool(forKey: Keys.localAPIEnabled) }
+        set { self.defaults.set(newValue, forKey: Keys.localAPIEnabled) }
+    }
+
     var preferredInputDeviceUID: String? {
         get { self.defaults.string(forKey: Keys.preferredInputDeviceUID) }
         set { self.defaults.set(newValue, forKey: Keys.preferredInputDeviceUID) }
@@ -5262,6 +5271,7 @@ private extension SettingsStore {
         static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled"
         static let enableAIStreaming = "EnableAIStreaming"
         static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
+        static let localAPIEnabled = "LocalAPIEnabled"
         static let textInsertionMode = "TextInsertionMode"
         static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled"
         static let betaReleasesEnabled = "BetaReleasesEnabled"

Sources/Fluid/Services/ASRService.swift

@@ -11,7 +11,7 @@ import AudioToolbox
 import CoreAudio
 
 /// Serializes transcription operations and lets teardown cancel the real queued work.
-private actor TranscriptionExecutor {
+actor TranscriptionExecutor {
     private var lastTask: Task<Void, Never>?
     private var operationCancellations: [UUID: () -> Void] = [:]
 
@@ -26,7 +26,11 @@ private actor TranscriptionExecutor {
         self.operationCancellations[operationID] = { task.cancel() }
         self.lastTask = Task { _ = try? await task.value }
         defer { self.operationCancellations.removeValue(forKey: operationID) }
-        return try await task.value
+        return try await withTaskCancellationHandler {
+            try await task.value
+        } onCancel: {
+            task.cancel()
+        }
     }
 
     func cancelAndAwaitPending() async {
@@ -2417,7 +2421,7 @@ final class ASRService: ObservableObject {
             )
         }
 
-        let estimatedSamples = try LocalAPIAudioDecoder.validateDurationWithinLimit(for: fileURL)
+        let preparedAudio = try LocalAPIAudioDecoder.prepareForTranscription(fileURL: fileURL)
 
         try await self.ensureAsrReady()
         let provider = self.transcriptionProvider
@@ -2429,7 +2433,7 @@ final class ASRService: ObservableObject {
             )
         }
 
-        guard provider.prefersNativeFileTranscription else {
+        guard provider.prefersNativeFileTranscription, !preparedAudio.requiresBufferedTranscription else {
             let samples = try LocalAPIAudioDecoder.samples(from: fileURL)
             let result = try await self.transcribeSamplesForAPI(samples)
             return (result, samples.count)
@@ -2449,7 +2453,10 @@ final class ASRService: ObservableObject {
             ASRService.applyCustomDictionary(ASRService.removeFillerWords(result.text))
         )
         self.recordWordBoostHitIfAny(transcribedText: cleanedText)
-        return (ASRTranscriptionResult(text: cleanedText, confidence: result.confidence), estimatedSamples)
+        return (
+            ASRTranscriptionResult(text: cleanedText, confidence: result.confidence),
+            preparedAudio.estimatedSamples
+        )
     }
 
     func stopWithoutTranscription() async {

Sources/Fluid/Services/LocalAPI/InferenceAPIController.swift

@@ -2,6 +2,17 @@ import Foundation
 
 @MainActor
 final class InferenceAPIController: LocalAPIRouteHandler {
+    typealias SampleTranscriber = ([Float]) async throws -> (text: String, confidence: Float, provider: String)
+
+    private let sampleTranscriber: SampleTranscriber
+
+    init(sampleTranscriber: @escaping SampleTranscriber = { samples in
+        let result = try await AppServices.shared.asr.transcribeSamplesForAPI(samples)
+        return (result.text, result.confidence, SettingsStore.shared.selectedSpeechModel.displayName)
+    }) {
+        self.sampleTranscriber = sampleTranscriber
+    }
+
     struct TranscribeJSONRequest: Decodable {
         let path: String?
         let audioBase64: String?
@@ -55,13 +66,13 @@ final class InferenceAPIController: LocalAPIRouteHandler {
             }
 
             let samples = try self.decodeAudioSamples(from: request)
-            let result = try await AppServices.shared.asr.transcribeSamplesForAPI(samples)
+            let result = try await self.sampleTranscriber(samples)
             return LocalAPI.json(
                 TranscribeResponse(
                     text: result.text,
                     confidence: result.confidence,
                     sampleCount: samples.count,
-                    provider: SettingsStore.shared.selectedSpeechModel.displayName
+                    provider: result.provider
                 )
             )
         } catch {

Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift

@@ -2,10 +2,50 @@ import AVFoundation
 import Foundation
 
 enum LocalAPIAudioDecoder {
+    struct PreparedAudio {
+        let estimatedSamples: Int
+        let requiresBufferedTranscription: Bool
+    }
+
     static let sampleRate: Double = 16_000
     static let maxDurationSeconds: Double = 300
+    private static let oggSniffBytes = 64 * 1024
+
+    static func requiresBufferedTranscription(for fileURL: URL) throws -> Bool {
+        try self.isOggOpusFile(at: fileURL)
+    }
+
+    static func prepareForTranscription(fileURL: URL) throws -> PreparedAudio {
+        if try self.isOggOpusFile(at: fileURL) {
+            try self.validateOggFileSize(at: fileURL)
+            return PreparedAudio(
+                estimatedSamples: try OggOpusDecoder.sampleCount(fromFileAt: fileURL),
+                requiresBufferedTranscription: true
+            )
+        }
+        return PreparedAudio(
+            estimatedSamples: try self.validateDurationWithinLimitUsingAVFoundation(for: fileURL),
+            requiresBufferedTranscription: false
+        )
+    }
 
     static func samples(from fileURL: URL) throws -> [Float] {
+        if try self.isOggOpusFile(at: fileURL) {
+            try self.validateOggFileSize(at: fileURL)
+            return try self.oggOpusSamples(from: Data(contentsOf: fileURL, options: .mappedIfSafe))
+        }
+        do {
+            return try self.samplesUsingAVFoundation(from: fileURL)
+        } catch {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -7,
+                userInfo: [NSLocalizedDescriptionKey: "Unsupported or invalid audio file: \(error.localizedDescription)"]
+            )
+        }
+    }
+
+    private static func samplesUsingAVFoundation(from fileURL: URL) throws -> [Float] {
         let file = try AVAudioFile(forReading: fileURL)
         let sourceFormat = file.processingFormat
         let maxFrames = AVAudioFramePosition(sourceFormat.sampleRate * self.maxDurationSeconds)
@@ -39,7 +79,53 @@ enum LocalAPIAudioDecoder {
     }
 
     static func validateDurationWithinLimit(for fileURL: URL) throws -> Int {
-        let file = try AVAudioFile(forReading: fileURL)
+        if try self.isOggOpusFile(at: fileURL) {
+            try self.validateOggFileSize(at: fileURL)
+            return try OggOpusDecoder.sampleCount(fromFileAt: fileURL)
+        }
+        return try self.validateDurationWithinLimitUsingAVFoundation(for: fileURL)
+    }
+
+    private static func validateDurationWithinLimitUsingAVFoundation(for fileURL: URL) throws -> Int {
+        do {
+            let file = try AVAudioFile(forReading: fileURL)
+            return try self.validateDurationWithinLimit(for: file)
+        } catch {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -7,
+                userInfo: [NSLocalizedDescriptionKey: "Unsupported or invalid audio file: \(error.localizedDescription)"]
+            )
+        }
+    }
+
+    private static func isOggOpusFile(at fileURL: URL) throws -> Bool {
+        let values = try fileURL.resourceValues(forKeys: [.isRegularFileKey])
+        guard values.isRegularFile == true else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -10,
+                userInfo: [NSLocalizedDescriptionKey: "Audio path must reference a regular file."]
+            )
+        }
+
+        let handle = try FileHandle(forReadingFrom: fileURL)
+        defer { try? handle.close() }
+        return OggOpusDecoder.isOggOpus(try handle.read(upToCount: self.oggSniffBytes) ?? Data())
+    }
+
+    private static func validateOggFileSize(at fileURL: URL) throws {
+        let values = try fileURL.resourceValues(forKeys: [.fileSizeKey])
+        guard let fileSize = values.fileSize, fileSize <= LocalAPI.maxRequestBytes else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -8,
+                userInfo: [NSLocalizedDescriptionKey: "Audio input exceeds the 25 MB API limit."]
+            )
+        }
+    }
+
+    private static func validateDurationWithinLimit(for file: AVAudioFile) throws -> Int {
         let sourceFormat = file.processingFormat
         guard sourceFormat.sampleRate > 0 else {
             throw NSError(domain: "LocalAPIAudioDecoder", code: -6, userInfo: [NSLocalizedDescriptionKey: "Audio file has an invalid sample rate."])
@@ -56,4 +142,54 @@ enum LocalAPIAudioDecoder {
 
         return Int((Double(file.length) * self.sampleRate / sourceFormat.sampleRate).rounded())
     }
+
+    static func resampleOpusTo16k(_ samples: [Float]) throws -> [Float] {
+        guard !samples.isEmpty else { return [] }
+        guard let sourceFormat = AVAudioFormat(
+            commonFormat: .pcmFormatFloat32,
+            sampleRate: 48_000,
+            channels: 1,
+            interleaved: false
+        ), let sourceBuffer = AVAudioPCMBuffer(
+            pcmFormat: sourceFormat,
+            frameCapacity: AVAudioFrameCount(samples.count)
+        ), let channelData = sourceBuffer.floatChannelData else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -11,
+                userInfo: [NSLocalizedDescriptionKey: "Unable to allocate an OGG/Opus audio buffer."]
+            )
+        }
+
+        sourceBuffer.frameLength = AVAudioFrameCount(samples.count)
+        samples.withUnsafeBufferPointer { pointer in
+            channelData[0].update(from: pointer.baseAddress!, count: samples.count)
+        }
+        return try AudioBufferConverter.monoSamples(
+            from: sourceBuffer,
+            targetSampleRate: self.sampleRate
+        )
+    }
+
+    static func oggOpusSamples(from data: Data) throws -> [Float] {
+        try self.validateRequestSize(data)
+        guard OggOpusDecoder.isOggOpus(data) else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -9,
+                userInfo: [NSLocalizedDescriptionKey: "Audio input is not an OGG/Opus stream."]
+            )
+        }
+        return try self.resampleOpusTo16k(OggOpusDecoder.samples(from: data))
+    }
+
+    private static func validateRequestSize(_ data: Data) throws {
+        guard data.count <= LocalAPI.maxRequestBytes else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -8,
+                userInfo: [NSLocalizedDescriptionKey: "Audio input exceeds the 25 MB API limit."]
+            )
+        }
+    }
 }

Sources/Fluid/Services/LocalAPI/LocalAPIModels.swift

@@ -7,6 +7,7 @@ enum LocalAPI {
     struct Configuration {
         let enabled: Bool
         let port: UInt16
+        let authToken: String
 
         static var current: Configuration {
             let defaults = UserDefaults.standard
@@ -19,7 +20,14 @@ enum LocalAPI {
 
             let rawPort = defaults.integer(forKey: "LocalAPIPort")
             let port = rawPort > 0 && rawPort <= Int(UInt16.max) ? UInt16(rawPort) : LocalAPI.defaultPort
-            return Configuration(enabled: enabled, port: port)
+            let authToken: String
+            if let storedToken = defaults.string(forKey: "LocalAPIAuthToken"), !storedToken.isEmpty {
+                authToken = storedToken
+            } else {
+                authToken = UUID().uuidString + UUID().uuidString
+                defaults.set(authToken, forKey: "LocalAPIAuthToken")
+            }
+            return Configuration(enabled: enabled, port: port, authToken: authToken)
         }
     }