From 422272d974ffda87f9754830c633f1fcda59166b Mon Sep 17 00:00:00 2001 From: altic-dev Date: Wed, 29 Jul 2026 03:49:13 -0700 Subject: [PATCH 01/15] add spoken send --- Fluid.xcodeproj/project.pbxproj | 4 + Sources/Fluid/ContentView.swift | 255 +++++++++++++- Sources/Fluid/Persistence/BackupService.swift | 4 + Sources/Fluid/Persistence/SettingsStore.swift | 91 +++++ Sources/Fluid/Services/ASRService.swift | 40 +++ .../Fluid/Services/GlobalHotkeyManager.swift | 8 + Sources/Fluid/Services/MenuBarManager.swift | 12 + .../Fluid/Services/NotchOverlayManager.swift | 1 + Sources/Fluid/Services/SpokenSendParser.swift | 95 ++++++ Sources/Fluid/Services/TypingService.swift | 230 +++++++++++-- Sources/Fluid/UI/SettingsView.swift | 79 +++++ Sources/Fluid/Views/BottomOverlayView.swift | 56 +++- Sources/Fluid/Views/NotchContentViews.swift | 138 +++++++- .../SpokenSendTests.swift | 316 ++++++++++++++++++ 14 files changed, 1278 insertions(+), 51 deletions(-) create mode 100644 Sources/Fluid/Services/SpokenSendParser.swift create mode 100644 Tests/FluidDictationIntegrationTests/SpokenSendTests.swift diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj index f281dd4c..407c9464 100644 --- a/Fluid.xcodeproj/project.pbxproj +++ b/Fluid.xcodeproj/project.pbxproj @@ -22,6 +22,7 @@ C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; }; DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; }; 7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */; }; + B51800000000000000000002 /* SpokenSendTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51800000000000000000001 /* SpokenSendTests.swift */; }; 7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; }; 7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; }; 7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; }; @@ -59,6 +60,7 @@ D1A600000000000000000201 /* SpeakerTurnMergingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpeakerTurnMergingTests.swift; sourceTree = ""; }; C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = ""; }; DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = ""; }; + B51800000000000000000001 /* SpokenSendTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpokenSendTests.swift; sourceTree = ""; }; 7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = ""; }; 7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -141,6 +143,7 @@ C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */, DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */, 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */, + B51800000000000000000001 /* SpokenSendTests.swift */, ); path = FluidDictationIntegrationTests; sourceTree = ""; @@ -303,6 +306,7 @@ C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */, DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */, 7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */, + B51800000000000000000002 /* SpokenSendTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index a3cacb21..91721d8e 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -243,6 +243,7 @@ struct ContentView: View { @State private var playgroundUsed: Bool = SettingsStore.shared.playgroundUsed @State private var recordingAppInfo: (name: String, bundleId: String, windowTitle: String)? = nil @State private var recordingPrecedingText: String = "" + @State private var recordingFocusTarget: TypingService.CapturedFocusTarget? = nil // Command Mode State // @State private var showCommandMode: Bool = false @@ -280,6 +281,11 @@ struct ContentView: View { @State private var accessibilityGuideRequestID: UUID? @State private var prewarmDictationTask: Task? @State private var overlayLifecycleID: UInt64 = 0 + @State private var spokenSendAutoStopTask: Task? + @State private var spokenSendAutoStopTriggered = false + @State private var spokenSendPartialRevision: UInt64 = 0 + @State private var spokenSendCountdownStartedAt: TimeInterval? + @State private var spokenSendLastVoiceActivityAt: TimeInterval = 0 private var isRecordingAnyShortcutCapture: Bool { self.activeShortcutRecordingTarget != nil @@ -353,6 +359,12 @@ struct ContentView: View { .onReceive(NotificationCenter.default.publisher(for: .settingsBackupDidRestore)) { _ in self.reloadSettingsStateAfterBackupRestore() } + .onReceive(self.asr.$partialTranscription) { text in + self.handleSpokenSendPartialTranscription(text) + } + .onReceive(self.asr.audioLevelPublisher) { level in + self.handleSpokenSendAudioLevel(level) + } .toolbar { if !self.settings.shouldShowOnboarding { ToolbarItemGroup(placement: .primaryAction) { @@ -1610,6 +1622,48 @@ struct ContentView: View { return (name: "Unknown", bundleId: "unknown", windowTitle: "") } + private func isSpokenSendBlockedApp( + _ appInfo: (name: String, bundleId: String, windowTitle: String) + ) -> Bool { + let identity = "\(appInfo.name) \(appInfo.bundleId)".lowercased() + return identity.contains("terminal") + || identity.contains("iterm") + || identity.contains("warp") + || identity.contains("ghostty") + } + + private func deliverSpokenSend( + _ outputPlan: DictationLiteralOutputPlan, + targetPID: pid_t?, + textReadyAt: TimeInterval + ) async -> TypingService.DeliveryOutcome { + let sendsExistingDraft = outputPlan.plainText.isEmpty + let outcome = await self.asr.typeOutputPlanToActiveFieldAndWait( + outputPlan, + preferredTargetPID: targetPID, + textReadyAt: textReadyAt, + postInsertionKey: self.settings.spokenSendKey, + requiredFocusTarget: self.recordingFocusTarget + ) + if outcome.didDispatchAction { + NotchContentState.shared.setSpokenSendIndicatorState(.sent) + try? await Task.sleep(nanoseconds: 350_000_000) + return outcome + } + + NotchContentState.shared.setSpokenSendIndicatorState(.failed) + DebugLogger.shared.warning( + "Spoken Send skipped because delivery safety checks did not pass", + source: "ContentView" + ) + let message = outcome.didInsert + ? "Text inserted — send skipped" + : sendsExistingDraft ? "Couldn't send" : "Couldn't insert or send" + NotchOverlayManager.shared.updateTranscriptionText(message) + try? await Task.sleep(nanoseconds: 650_000_000) + return outcome + } + /// Best-effort frontmost window title lookup for the current app private func getFrontmostWindowTitle(ownerPid: pid_t) -> String? { let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements] @@ -1628,7 +1682,9 @@ struct ContentView: View { private func captureRecordingTargetContext() { // Capture the focused target PID BEFORE any overlay/UI changes. // Used to restore focus when the user interacts with overlay dropdowns. - let focusedPID = TypingService.captureSystemFocusedPID() + let focusTarget = TypingService.captureSystemFocusTarget() + self.recordingFocusTarget = focusTarget + let focusedPID = focusTarget?.pid ?? NSWorkspace.shared.frontmostApplication?.processIdentifier NotchContentState.shared.recordingTargetPID = focusedPID @@ -2050,7 +2106,8 @@ struct ContentView: View { !wasRewriteMode && !wasCommandMode && !promptTest.isActive && - !shouldUseAIOnStop + !shouldUseAIOnStop && + !self.settings.spokenSendEnabled var didRequestOverlayHideOnStop = false DebugLogger.shared.info( "Routing decision snapshot | activeMode=\(modeAtStop.rawValue) | rewrite=\(wasRewriteMode) | command=\(wasCommandMode) | overlay=\(NotchContentState.shared.mode.rawValue)", @@ -2180,16 +2237,25 @@ struct ContentView: View { var aiFallbackReason: String? var postProcessingModel: String? let appInfo = self.recordingAppInfo ?? self.getCurrentAppInfo() - let normalizedTranscribedText = ASRService.applySpokenPunctuationFormatting( + let punctuationFormattedText = ASRService.applySpokenPunctuationFormatting( transcribedText, appName: appInfo.name, bundleID: appInfo.bundleId, windowTitle: appInfo.windowTitle ) + let spokenSendParse = SpokenSendParser.parse( + punctuationFormattedText, + phrase: self.settings.spokenSendPhrase, + enabled: route == .normal && self.settings.spokenSendEnabled + ) + self.updateSpokenSendIndicatorForFinalParse(shouldSend: spokenSendParse.shouldSend) + let normalizedTranscribedText = spokenSendParse.text + let sendsExistingDraft = spokenSendParse.shouldSend && + normalizedTranscribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - let shouldUseAI = activeDictationSlot.map { + let shouldUseAI = !sendsExistingDraft && (activeDictationSlot.map { DictationAIPostProcessingGate.isConfigured(for: $0, appBundleID: appInfo.bundleId) - } ?? DictationAIPostProcessingGate.isConfigured(for: .primary, appBundleID: appInfo.bundleId) + } ?? DictationAIPostProcessingGate.isConfigured(for: .primary, appBundleID: appInfo.bundleId)) let transcriptionModelInfo = self.currentTranscriptionModelInfo() if shouldUseAI { @@ -2322,7 +2388,7 @@ struct ContentView: View { "mode": AnalyticsMode.dictation.rawValue, "words_bucket": AnalyticsBuckets.bucketWords(AnalyticsBuckets.wordCount(in: finalText)), "ai_used": shouldUseAI, - "ai_changed_text": transcribedText != finalText, + "ai_changed_text": normalizedTranscribedText != finalText, "transcription_provider": transcriptionModelInfo.provider, "transcription_model": transcriptionModelInfo.model, ] @@ -2350,13 +2416,13 @@ struct ContentView: View { let isFluidFrontmost = frontmostApp?.bundleIdentifier == Bundle.main.bundleIdentifier // Save to transcription history (transcription mode only, if enabled) - if shouldPersistOutputs, SettingsStore.shared.saveTranscriptionHistory { + if shouldPersistOutputs, !sendsExistingDraft, SettingsStore.shared.saveTranscriptionHistory { let historyEntryID = UUID() let historyTimestamp = Date() TranscriptionHistoryStore.shared.addEntry( id: historyEntryID, timestamp: historyTimestamp, - rawText: transcribedText, + rawText: spokenSendParse.shouldSend ? normalizedTranscribedText : transcribedText, processedText: finalText, appName: appInfo.name, windowTitle: appInfo.windowTitle, @@ -2374,6 +2440,7 @@ struct ContentView: View { // When FluidVoice itself is frontmost, the bound editor already receives `finalText`. // Avoid re-inserting or overwriting the clipboard in that self-target case. let shouldCopyToClipboard = shouldPersistOutputs && + !sendsExistingDraft && SettingsStore.shared.copyTranscriptionToClipboard && !isFluidFrontmost @@ -2398,21 +2465,55 @@ struct ContentView: View { if shouldTypeExternally { let typingTarget = self.resolveTypingTargetPID() + let spokenSendRequested = spokenSendParse.shouldSend + let targetMatchesRecordingFocus = typingTarget.pid != nil + && typingTarget.pid == self.recordingFocusTarget?.pid + let spokenSendAllowed = spokenSendRequested + && aiFallbackReason == nil + && (sendsExistingDraft || !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + && targetMatchesRecordingFocus + && !self.isSpokenSendBlockedApp(appInfo) // Dispatch insertion as soon as the destination app is ready; the // overlay hides asynchronously after output so it cannot delay paste. if typingTarget.shouldRestoreOriginalFocus { await self.restoreFocusToRecordingTarget() } + if spokenSendAllowed { + NotchContentState.shared.setSpokenSendIndicatorState(.sending) + NotchOverlayManager.shared.updateTranscriptionText("Sending") + } self.appBench( "text_ready_to_type_request elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - finalTextReadyAt) * 1000).rounded()))" ) - self.asr.typeOutputPlanToActiveField( - finalOutputPlan, - preferredTargetPID: typingTarget.pid, - textReadyAt: finalTextReadyAt, - tracksDictionaryCorrections: true - ) - didTypeExternally = true + if spokenSendAllowed { + let deliveryOutcome = await self.deliverSpokenSend( + finalOutputPlan, + targetPID: typingTarget.pid, + textReadyAt: finalTextReadyAt + ) + didTypeExternally = deliveryOutcome.didInsert + } else { + self.asr.typeOutputPlanToActiveField( + finalOutputPlan, + preferredTargetPID: typingTarget.pid, + textReadyAt: finalTextReadyAt, + tracksDictionaryCorrections: true + ) + didTypeExternally = true + } + if spokenSendRequested, !spokenSendAllowed { + NotchContentState.shared.setSpokenSendIndicatorState(.failed) + DebugLogger.shared.warning( + "Spoken Send skipped because delivery safety checks did not pass", + source: "ContentView" + ) + if aiFallbackReason == nil { + NotchOverlayManager.shared.updateTranscriptionText("Text inserted — send skipped") + try? await Task.sleep(nanoseconds: 650_000_000) + } + } + NotchOverlayManager.shared.updateTranscriptionText("") + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) if !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop { self.hideOverlayAfterOutput() } @@ -2500,11 +2601,123 @@ struct ContentView: View { return true } + private func updateSpokenSendIndicatorForFinalParse(shouldSend: Bool) { + if shouldSend, NotchContentState.shared.spokenSendIndicatorState == .sending { + return + } + NotchContentState.shared.setSpokenSendIndicatorState(shouldSend ? .detected : .hidden) + } + private func advanceOverlayLifecycle() { + self.spokenSendAutoStopTask?.cancel() + self.spokenSendAutoStopTask = nil + self.spokenSendAutoStopTriggered = false + self.spokenSendPartialRevision = 0 + self.spokenSendCountdownStartedAt = nil + self.spokenSendLastVoiceActivityAt = ProcessInfo.processInfo.systemUptime self.overlayLifecycleID &+= 1 NotchContentState.shared.clearAIProcessingFailure() } + private func handleSpokenSendPartialTranscription(_ text: String) { + self.spokenSendPartialRevision &+= 1 + let isDictationMode = self.activeRecordingMode == .dictate || self.activeRecordingMode == .promptMode + let shouldStop = isDictationMode && + self.currentDictationOutputRouteForHotkeyStop() == .normal && + self.asr.isRunning && + !self.spokenSendAutoStopTriggered && + SpokenSendParser.shouldStopImmediately( + text, + phrase: self.settings.spokenSendPhrase, + spokenSendEnabled: self.settings.spokenSendEnabled, + sendImmediatelyEnabled: self.settings.spokenSendImmediatelyEnabled + ) + + guard shouldStop else { + self.spokenSendAutoStopTask?.cancel() + self.spokenSendAutoStopTask = nil + self.spokenSendCountdownStartedAt = nil + if !self.spokenSendAutoStopTriggered, + NotchContentState.shared.spokenSendIndicatorState == .countingDown + { + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + return + } + + guard self.spokenSendAutoStopTask == nil else { + return + } + + let expectedOverlayLifecycleID = self.overlayLifecycleID + let expectedPartialRevision = self.spokenSendPartialRevision + let countdownStartedAt = ProcessInfo.processInfo.systemUptime + self.spokenSendCountdownStartedAt = countdownStartedAt + let expectedCountdownID = NotchContentState.shared.beginSpokenSendCountdown() + self.spokenSendAutoStopTask = Task { @MainActor in + // Keep one countdown across harmless streaming refinements such as punctuation or casing. + try? await Task.sleep(nanoseconds: SpokenSendParser.immediateStopSettleNanoseconds) + let quietDuration = ProcessInfo.processInfo.systemUptime - self.spokenSendLastVoiceActivityAt + guard !Task.isCancelled, + self.overlayLifecycleID == expectedOverlayLifecycleID, + NotchContentState.shared.spokenSendCountdownID == expectedCountdownID, + self.asr.isRunning, + self.activeRecordingMode == .dictate || self.activeRecordingMode == .promptMode, + self.currentDictationOutputRouteForHotkeyStop() == .normal, + !self.spokenSendAutoStopTriggered, + SpokenSendParser.canCompleteImmediateStop( + self.asr.partialTranscription, + phrase: self.settings.spokenSendPhrase, + spokenSendEnabled: self.settings.spokenSendEnabled, + sendImmediatelyEnabled: self.settings.spokenSendImmediatelyEnabled, + receivedFreshTranscript: self.spokenSendPartialRevision > expectedPartialRevision, + quietDuration: quietDuration + ) + else { + if self.overlayLifecycleID == expectedOverlayLifecycleID, + NotchContentState.shared.spokenSendCountdownID == expectedCountdownID + { + self.spokenSendAutoStopTask = nil + self.spokenSendCountdownStartedAt = nil + if NotchContentState.shared.spokenSendIndicatorState == .countingDown { + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + } + return + } + + self.spokenSendAutoStopTask = nil + self.spokenSendCountdownStartedAt = nil + self.spokenSendAutoStopTriggered = true + NotchContentState.shared.setSpokenSendIndicatorState(.sending) + DebugLogger.shared.info("Spoken Send countdown completed; stopping dictation", source: "ContentView") + await self.stopAndProcessTranscription(route: .normal) + } + } + + private func handleSpokenSendAudioLevel(_ level: CGFloat) { + guard level > 0 else { return } + + let activityAt = ProcessInfo.processInfo.systemUptime + self.spokenSendLastVoiceActivityAt = activityAt + guard let countdownStartedAt = self.spokenSendCountdownStartedAt, + SpokenSendParser.shouldCancelCountdownForVoiceActivity( + countdownStartedAt: countdownStartedAt, + voiceActivityAt: activityAt + ), + self.spokenSendAutoStopTask != nil + else { + return + } + + self.spokenSendAutoStopTask?.cancel() + self.spokenSendAutoStopTask = nil + self.spokenSendCountdownStartedAt = nil + if NotchContentState.shared.spokenSendIndicatorState == .countingDown { + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + } + private func hideOverlayAsync(reason: String) { let expectedOverlayLifecycleID = self.overlayLifecycleID self.appBench("overlay_hide_request reason=\(reason) lifecycle=\(expectedOverlayLifecycleID)") @@ -3185,6 +3398,18 @@ struct ContentView: View { guard let pid = NotchContentState.shared.recordingTargetPID else { return } let startedAt = ProcessInfo.processInfo.systemUptime self.appBench("focus_restore_start targetPID=\(pid)") + if let focusTarget = self.recordingFocusTarget, focusTarget.pid == pid { + if TypingService.isExactFocusTargetActive(focusTarget) { + self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") + return + } + let activated = TypingService.activateApp(pid: pid) + let focusedElementRestored = TypingService.restoreFocusTarget(focusTarget) + self.appBench( + "focus_restore_result activated=\(activated) element=\(focusedElementRestored) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))" + ) + return + } if TypingService.isCapturedFocusStillActive(for: pid) { self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") DebugLogger.shared.debug( diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift index 4953a598..0d104176 100644 --- a/Sources/Fluid/Persistence/BackupService.swift +++ b/Sources/Fluid/Persistence/BackupService.swift @@ -62,6 +62,10 @@ struct SettingsBackupPayload: Codable, Equatable { let enableAIStreaming: Bool let copyTranscriptionToClipboard: Bool let textInsertionMode: SettingsStore.TextInsertionMode + let spokenSendEnabled: Bool? + let spokenSendImmediatelyEnabled: Bool? + let spokenSendPhrase: String? + let spokenSendKey: SettingsStore.SpokenSendKey? let preferredInputDeviceUID: String? // Optional so backups created before microphone priority ordering still decode. // swiftlint:disable:next discouraged_optional_collection diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 50b378d4..8a04e4b2 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -3195,6 +3195,10 @@ final class SettingsStore: ObservableObject { enableAIStreaming: self.enableAIStreaming, copyTranscriptionToClipboard: self.copyTranscriptionToClipboard, textInsertionMode: self.textInsertionMode, + spokenSendEnabled: self.spokenSendEnabled, + spokenSendImmediatelyEnabled: self.spokenSendImmediatelyEnabled, + spokenSendPhrase: self.spokenSendPhrase, + spokenSendKey: self.spokenSendKey, preferredInputDeviceUID: self.preferredInputDeviceUID, microphonePriority: self.microphonePriority, suppressedMicrophoneUIDs: self.suppressedMicrophoneUIDs.sorted(), @@ -3318,6 +3322,18 @@ final class SettingsStore: ObservableObject { self.enableAIStreaming = payload.enableAIStreaming self.copyTranscriptionToClipboard = payload.copyTranscriptionToClipboard self.textInsertionMode = payload.textInsertionMode + if let spokenSendEnabled = payload.spokenSendEnabled { + self.spokenSendEnabled = spokenSendEnabled + } + if let spokenSendImmediatelyEnabled = payload.spokenSendImmediatelyEnabled { + self.spokenSendImmediatelyEnabled = spokenSendImmediatelyEnabled + } + if let spokenSendPhrase = payload.spokenSendPhrase { + self.spokenSendPhrase = spokenSendPhrase + } + if let spokenSendKey = payload.spokenSendKey { + self.spokenSendKey = spokenSendKey + } self.preferredInputDeviceUID = payload.preferredInputDeviceUID self.suppressedMicrophoneUIDs = Set(payload.suppressedMicrophoneUIDs ?? []) if let microphonePriority = payload.microphonePriority { @@ -5094,6 +5110,10 @@ private extension SettingsStore { static let enableAIStreaming = "EnableAIStreaming" static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard" static let textInsertionMode = "TextInsertionMode" + static let spokenSendEnabled = "SpokenSendEnabled" + static let spokenSendImmediatelyEnabled = "SpokenSendImmediatelyEnabled" + static let spokenSendPhrase = "SpokenSendPhrase" + static let spokenSendKey = "SpokenSendKey" static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled" static let betaReleasesEnabled = "BetaReleasesEnabled" static let lastUpdateCheckDate = "LastUpdateCheckDate" @@ -5222,6 +5242,77 @@ private extension SettingsStore { } extension SettingsStore { + enum SpokenSendKey: String, CaseIterable, Identifiable, Codable { + case enter + case shiftEnter + case commandEnter + + var id: String { + self.rawValue + } + + var displayName: String { + switch self { + case .enter: + return "Enter" + case .shiftEnter: + return "Shift + Enter" + case .commandEnter: + return "Command + Enter" + } + } + + var eventFlags: CGEventFlags { + switch self { + case .enter: + return [] + case .shiftEnter: + return .maskShift + case .commandEnter: + return .maskCommand + } + } + } + + var spokenSendEnabled: Bool { + get { self.defaults.object(forKey: Keys.spokenSendEnabled) as? Bool ?? false } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.spokenSendEnabled) + } + } + + var spokenSendImmediatelyEnabled: Bool { + get { self.defaults.object(forKey: Keys.spokenSendImmediatelyEnabled) as? Bool ?? true } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.spokenSendImmediatelyEnabled) + } + } + + var spokenSendPhrase: String { + get { self.defaults.string(forKey: Keys.spokenSendPhrase) ?? "send it" } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.spokenSendPhrase) + } + } + + var spokenSendKey: SpokenSendKey { + get { + guard let raw = self.defaults.string(forKey: Keys.spokenSendKey), + let key = SpokenSendKey(rawValue: raw) + else { + return .enter + } + return key + } + set { + objectWillChange.send() + self.defaults.set(newValue.rawValue, forKey: Keys.spokenSendKey) + } + } + enum TextInsertionMode: String, CaseIterable, Identifiable, Codable { case standard case reliablePaste diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 200e5297..4e4157a5 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -4554,6 +4554,46 @@ final class ASRService: ObservableObject { ) } + func typeOutputPlanToActiveFieldAndWait( + _ plan: DictationLiteralOutputPlan, + preferredTargetPID: pid_t?, + textReadyAt: TimeInterval? = nil, + tracksDictionaryCorrections: Bool = false, + postInsertionKey: SettingsStore.SpokenSendKey? = nil, + requiredFocusTarget: TypingService.CapturedFocusTarget? = nil + ) async -> TypingService.DeliveryOutcome { + let requestedAt = ProcessInfo.processInfo.systemUptime + let textReadyAge = textReadyAt.map { Int(((requestedAt - $0) * 1000).rounded()) } + let text = plan.plainText + DebugLogger.shared.benchmark( + "TYPING_BENCH", + message: "asr_type_request chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyAgeMs=\(textReadyAge.map { String($0) } ?? "nil")", + source: "TypingBenchmark" + ) + let outcome = await withCheckedContinuation { continuation in + self.typingService.typeOutputPlanInstantly( + plan, + preferredTargetPID: preferredTargetPID, + textReadyAt: textReadyAt, + tracksDictionaryCorrections: tracksDictionaryCorrections, + postInsertionKey: postInsertionKey, + requiredFocusTarget: requiredFocusTarget + ) { outcome in + continuation.resume(returning: outcome) + } + } + let dispatchedAt = ProcessInfo.processInfo.systemUptime + let textReadyToDispatchMs = textReadyAt.map { + String(Int(((dispatchedAt - $0) * 1000).rounded())) + } ?? "nil" + DebugLogger.shared.benchmark( + "TYPING_BENCH", + message: "asr_type_dispatched chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyToDispatchMs=\(textReadyToDispatchMs)", + source: "TypingBenchmark" + ) + return outcome + } + /// Removes filler sounds from transcribed text static func removeFillerWords(_ text: String) -> String { guard SettingsStore.shared.removeFillerWordsEnabled else { return text } diff --git a/Sources/Fluid/Services/GlobalHotkeyManager.swift b/Sources/Fluid/Services/GlobalHotkeyManager.swift index ff952d88..79dac766 100644 --- a/Sources/Fluid/Services/GlobalHotkeyManager.swift +++ b/Sources/Fluid/Services/GlobalHotkeyManager.swift @@ -782,6 +782,10 @@ final class GlobalHotkeyManager: NSObject { return tapRecoveryResult } + if Self.isSynthesizedTypingEvent(event) { + return Unmanaged.passUnretained(event) + } + if self.isShortcutCaptureActiveProvider?() ?? false { self.resetModifierOnlyShortcutTracking() return Unmanaged.passUnretained(event) @@ -1282,6 +1286,10 @@ final class GlobalHotkeyManager: NSObject { } } + nonisolated static func isSynthesizedTypingEvent(_ event: CGEvent) -> Bool { + event.getIntegerValueField(.eventSourceUserData) == TypingService.synthesizedEventUserData + } + private func handlePrimaryDictationTriggerDown() { switch self.hotkeyMode { case .hold: diff --git a/Sources/Fluid/Services/MenuBarManager.swift b/Sources/Fluid/Services/MenuBarManager.swift index 4d331d2c..b1df4f7a 100644 --- a/Sources/Fluid/Services/MenuBarManager.swift +++ b/Sources/Fluid/Services/MenuBarManager.swift @@ -119,6 +119,18 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { .receive(on: DispatchQueue.main) .sink { [weak self] newText in guard self != nil else { return } + if asrService.isRunning, + NotchContentState.shared.mode == .dictation, + SettingsStore.shared.spokenSendEnabled, + !SettingsStore.shared.spokenSendImmediatelyEnabled + { + let detected = SpokenSendParser.parse( + newText, + phrase: SettingsStore.shared.spokenSendPhrase, + enabled: true + ).shouldSend + NotchContentState.shared.setSpokenSendIndicatorState(detected ? .detected : .hidden) + } if NotchOverlayManager.shared.shouldShowOrTrackLivePreviewText { NotchOverlayManager.shared.updateTranscriptionText(newText) } diff --git a/Sources/Fluid/Services/NotchOverlayManager.swift b/Sources/Fluid/Services/NotchOverlayManager.swift index 799570a9..6a754fae 100644 --- a/Sources/Fluid/Services/NotchOverlayManager.swift +++ b/Sources/Fluid/Services/NotchOverlayManager.swift @@ -368,6 +368,7 @@ final class NotchOverlayManager { // Safety: reset processing state when hiding NotchContentState.shared.setProcessing(false) + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) // Handle visible or showing states (can hide while still expanding) guard self.state == .visible || self.state == .showing, self.notch != nil else { diff --git a/Sources/Fluid/Services/SpokenSendParser.swift b/Sources/Fluid/Services/SpokenSendParser.swift new file mode 100644 index 00000000..6b3e9604 --- /dev/null +++ b/Sources/Fluid/Services/SpokenSendParser.swift @@ -0,0 +1,95 @@ +import Foundation + +struct SpokenSendParseResult: Equatable { + let text: String + let shouldSend: Bool +} + +enum SpokenSendParser { + static let immediateStopSettleDuration: TimeInterval = 1.5 + static let immediateStopSettleNanoseconds: UInt64 = 1_500_000_000 + static let immediateStopRequiredSilenceDuration: TimeInterval = 0.35 + static let immediateStopVoiceActivityGraceDuration: TimeInterval = 0.15 + + static func shouldStopImmediately( + _ text: String, + phrase: String, + spokenSendEnabled: Bool, + sendImmediatelyEnabled: Bool + ) -> Bool { + sendImmediatelyEnabled && + self.parse(text, phrase: phrase, enabled: spokenSendEnabled).shouldSend + } + + static func canCompleteImmediateStop( + _ text: String, + phrase: String, + spokenSendEnabled: Bool, + sendImmediatelyEnabled: Bool, + receivedFreshTranscript: Bool, + quietDuration: TimeInterval + ) -> Bool { + receivedFreshTranscript && + quietDuration >= self.immediateStopRequiredSilenceDuration && + self.shouldStopImmediately( + text, + phrase: phrase, + spokenSendEnabled: spokenSendEnabled, + sendImmediatelyEnabled: sendImmediatelyEnabled + ) + } + + static func shouldCancelCountdownForVoiceActivity( + countdownStartedAt: TimeInterval, + voiceActivityAt: TimeInterval + ) -> Bool { + voiceActivityAt - countdownStartedAt >= self.immediateStopVoiceActivityGraceDuration + } + + static func parse(_ text: String, phrase: String, enabled: Bool) -> SpokenSendParseResult { + guard enabled else { + return SpokenSendParseResult(text: text, shouldSend: false) + } + + let phraseWords = phrase + .trimmingCharacters(in: .whitespacesAndNewlines) + .split(whereSeparator: \.isWhitespace) + .map(String.init) + guard !phraseWords.isEmpty else { + return SpokenSendParseResult(text: text, shouldSend: false) + } + + let phrasePattern = phraseWords + .map(NSRegularExpression.escapedPattern(for:)) + .joined(separator: #"\s+"#) + let trailingPunctuation = #"[\s\p{P}]*$"# + + if let literalRegex = try? NSRegularExpression( + pattern: #"(?i)(? Bool { + guard let preferredTargetPID, preferredTargetPID > 0, + requiredTargetPID == preferredTargetPID + else { + return false + } + return !isSecureTextField && modifiersReleased && exactFocusIsActive + } + // Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 // or UserDefaults bool for key "enableTypingLogs". private static var isLoggingEnabled: Bool { @@ -131,7 +186,7 @@ final class TypingService { /// Best-effort: returns the PID owning the currently focused accessibility element. /// This is more reliable than NSWorkspace.frontmostApplication for floating overlays/launchers. - static func captureSystemFocusedPID() -> pid_t? { + static func captureSystemFocusTarget() -> CapturedFocusTarget? { // Accessibility is required to query system-focused AX element. guard AXIsProcessTrusted() else { self.storeFocusSnapshot(nil) @@ -167,7 +222,57 @@ final class TypingService { ?? Self.copyAXElementAttribute(from: appElement, attribute: kAXMainWindowAttribute as CFString) Self.storeFocusSnapshot(FocusSnapshot(pid: pid, window: window, element: element)) Self.logFocusState("[TypingService] Captured focus snapshot") - return pid + return CapturedFocusTarget(pid: pid, window: window, element: element) + } + + static func captureSystemFocusedPID() -> pid_t? { + self.captureSystemFocusTarget()?.pid + } + + static func isExactFocusTargetActive(_ target: CapturedFocusTarget) -> Bool { + guard AXIsProcessTrusted() else { return false } + + let systemWideElement = AXUIElementCreateSystemWide() + var focusedElementRef: CFTypeRef? + let result = AXUIElementCopyAttributeValue( + systemWideElement, + kAXFocusedUIElementAttribute as CFString, + &focusedElementRef + ) + guard result == .success, let focusedElementRef, + CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() + else { + return false + } + + let currentElement = unsafeBitCast(focusedElementRef, to: AXUIElement.self) + return CFEqual(currentElement, target.element) + } + + @discardableResult + static func restoreFocusTarget(_ target: CapturedFocusTarget) -> Bool { + guard AXIsProcessTrusted() else { return false } + let appElement = AXUIElementCreateApplication(target.pid) + + if let window = target.window { + _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString) + _ = AXUIElementSetAttributeValue(appElement, kAXMainWindowAttribute as CFString, window) + _ = AXUIElementSetAttributeValue(appElement, kAXFocusedWindowAttribute as CFString, window) + usleep(40_000) + } + + for _ in 0..<3 { + let result = AXUIElementSetAttributeValue( + target.element, + kAXFocusedAttribute as CFString, + kCFBooleanTrue + ) + if result == .success, self.isExactFocusTargetActive(target) { + return true + } + usleep(50_000) + } + return self.isExactFocusTargetActive(target) } /// Best-effort: returns the text immediately before the caret in the currently focused @@ -301,7 +406,10 @@ final class TypingService { _ plan: DictationLiteralOutputPlan, preferredTargetPID: pid_t?, textReadyAt: TimeInterval?, - tracksDictionaryCorrections: Bool = false + tracksDictionaryCorrections: Bool = false, + postInsertionKey: SettingsStore.SpokenSendKey? = nil, + requiredFocusTarget: CapturedFocusTarget? = nil, + completion: ((DeliveryOutcome) -> Void)? = nil ) { let requestedAt = ProcessInfo.processInfo.systemUptime let text = plan.plainText @@ -319,9 +427,10 @@ final class TypingService { self.log("[TypingService] ENTRY: typeTextInstantly called with text length: \(text.count)") self.log("[TypingService] Text preview: \"\(String(text.prefix(100)))\"") - guard text.isEmpty == false else { + guard text.isEmpty == false || postInsertionKey != nil else { self.bench("request_return reason=empty_text") self.log("[TypingService] ERROR: Empty text provided, aborting") + completion?(.rejected) return } @@ -329,6 +438,7 @@ final class TypingService { guard !self.isCurrentlyTyping else { self.bench("request_return reason=already_typing") self.log("[TypingService] WARNING: Skipping text injection - already in progress") + completion?(.rejected) return } @@ -337,6 +447,7 @@ final class TypingService { self.bench("request_return reason=accessibility_not_trusted") self.log("[TypingService] ERROR: Accessibility permissions required for text injection") self.log("[TypingService] Current accessibility status: \(AXIsProcessTrusted())") + completion?(.rejected) return } @@ -344,6 +455,7 @@ final class TypingService { self.isCurrentlyTyping = true DispatchQueue.global(qos: .userInitiated).async { + var outcome: DeliveryOutcome = .insertionFailed let workerStartedAt = ProcessInfo.processInfo.systemUptime self.bench("worker_start queueDelayMs=\(Self.elapsedMs(from: requestedAt, to: workerStartedAt))") @@ -354,6 +466,7 @@ final class TypingService { "complete totalMs=\(Self.elapsedMs(from: requestedAt, to: completedAt)) textReadyToCompleteMs=\(textReadyAt.map { String(Self.elapsedMs(from: $0, to: completedAt)) } ?? "nil")" ) self.log("[TypingService] Typing operation completed, isCurrentlyTyping set to false") + completion?(outcome) } self.log("[TypingService] Starting async text insertion process") @@ -361,21 +474,57 @@ final class TypingService { usleep(useconds_t(settleDelayMs * 1000)) } self.bench("settle_delay_done delayMs=\(settleDelayMs) elapsedMs=\(Self.elapsedMs(since: requestedAt))") - self.log("[TypingService] Delay completed, calling insertTextInstantly") - let insertStartedAt = ProcessInfo.processInfo.systemUptime - self.bench("insert_call") - self.insertTextInstantly(text, preferredTargetPID: preferredTargetPID) - self.bench( - "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" - ) - if tracksDictionaryCorrections { - Task { @MainActor in - AutomaticDictionaryCorrectionTracker.shared.beginObservingInsertion( - text, - targetPID: preferredTargetPID - ) + let hasTextToInsert = !text.isEmpty + if hasTextToInsert { + self.log("[TypingService] Delay completed, calling insertTextInstantly") + let insertStartedAt = ProcessInfo.processInfo.systemUptime + self.bench("insert_call") + let inserted = self.insertTextInstantly(text, preferredTargetPID: preferredTargetPID) + self.bench( + "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" + ) + guard inserted else { + outcome = .insertionFailed + return + } + + outcome = .inserted + if tracksDictionaryCorrections, postInsertionKey == nil { + Task { @MainActor in + AutomaticDictionaryCorrectionTracker.shared.beginObservingInsertion( + text, + targetPID: preferredTargetPID + ) + } } } + + guard let postInsertionKey else { return } + guard let preferredTargetPID, let requiredFocusTarget else { + outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + return + } + let modifiersReleased = self.waitForPhysicalModifiersToRelease(timeout: 2) + let exactFocusIsActive = Self.isExactFocusTargetActive(requiredFocusTarget) + guard Self.canDispatchPostInsertionAction( + preferredTargetPID: preferredTargetPID, + requiredTargetPID: requiredFocusTarget.pid, + isSecureTextField: requiredFocusTarget.isSecureTextField, + modifiersReleased: modifiersReleased, + exactFocusIsActive: exactFocusIsActive + ) else { + outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + return + } + + usleep(50_000) + guard Self.isExactFocusTargetActive(requiredFocusTarget), + self.postReturnKey(postInsertionKey, targetPID: preferredTargetPID) + else { + outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + return + } + outcome = hasTextToInsert ? .insertedAndActionDispatched : .actionDispatched } } @@ -393,7 +542,7 @@ final class TypingService { // MARK: - Internal insertion pipeline - private func insertTextInstantly(_ text: String, preferredTargetPID: pid_t?) { + private func insertTextInstantly(_ text: String, preferredTargetPID: pid_t?) -> Bool { self.log("[TypingService] insertTextInstantly called with \(text.count) characters") self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"") @@ -403,7 +552,7 @@ final class TypingService { self.log("[TypingService] Ghostty target detected in standard mode (PID \(ghosttyTargetPID)); forcing Reliable Paste path") if self.tryReliablePasteInsertion(text, preferredTargetPID: ghosttyTargetPID) { self.log("[TypingService] SUCCESS: Ghostty Reliable Paste path completed") - return + return true } self.log("[TypingService] Ghostty Reliable Paste path fell through to direct-typing fallbacks") } @@ -412,14 +561,14 @@ final class TypingService { self.log("[TypingService] Reliable Paste mode enabled") if self.tryReliablePasteInsertion(text, preferredTargetPID: preferredTargetPID) { self.log("[TypingService] SUCCESS: Reliable Paste mode completed") - return + return true } self.log("[TypingService] Reliable Paste mode fell through to direct-typing fallbacks") } else if let preferredTargetPID, preferredTargetPID > 0 { self.log("[TypingService] Experimental Direct Typing mode: trying preferred PID unicode insertion first") if self.insertTextBulkInstant(text, targetPID: preferredTargetPID) { self.log("[TypingService] SUCCESS: Preferred PID CGEvent insertion completed") - return + return true } self.log("[TypingService] Preferred PID CGEvent insertion failed, continuing fallback pipeline") } @@ -453,7 +602,7 @@ final class TypingService { self.log("[TypingService] Trying CGEvent insertion targeting focused PID \(focusedPID)") if self.insertTextBulkInstant(text, targetPID: focusedPID) { self.log("[TypingService] SUCCESS: CGEvent focused-PID insertion completed") - return + return true } } @@ -461,7 +610,7 @@ final class TypingService { self.log("[TypingService] Trying Accessibility focused-element insertion") if self.insertTextViaAccessibility(text) { self.log("[TypingService] SUCCESS: Accessibility insertion completed") - return + return true } // HID Fallback if PID targeting failed @@ -469,7 +618,7 @@ final class TypingService { self.log("[TypingService] No focused PID available, trying HID CGEvent insertion") if self.insertTextBulkHIDInstant(text) { self.log("[TypingService] SUCCESS: CGEvent HID insertion completed") - return + return true } } @@ -477,7 +626,7 @@ final class TypingService { self.log("[TypingService] CGEvent failed, trying clipboard fallback") if self.insertTextViaClipboard(text) { self.log("[TypingService] SUCCESS: Clipboard insertion completed") - return + return true } // Last resort: Character-by-character @@ -490,6 +639,37 @@ final class TypingService { usleep(1000) } self.log("[TypingService] Character-by-character typing completed") + return true + } + + private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) -> Bool { + let relevant: CGEventFlags = [.maskCommand, .maskControl, .maskAlternate, .maskShift, .maskSecondaryFn] + let startedAt = ProcessInfo.processInfo.systemUptime + while ProcessInfo.processInfo.systemUptime - startedAt < timeout { + if CGEventSource.flagsState(.combinedSessionState).isDisjoint(with: relevant) { + return true + } + usleep(15_000) + } + return false + } + + private func postReturnKey(_ key: SettingsStore.SpokenSendKey, targetPID: pid_t) -> Bool { + let returnKeyCode = CGKeyCode(kVK_Return) + guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: false) + else { + return false + } + + keyDown.flags = key.eventFlags + keyUp.flags = key.eventFlags + keyDown.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) + keyUp.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) + keyDown.postToPid(targetPID) + usleep(10_000) + keyUp.postToPid(targetPID) + return true } private func tryReliablePasteInsertion(_ text: String, preferredTargetPID: pid_t?) -> Bool { diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index fbd91b5d..3c8e493a 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -871,6 +871,9 @@ struct SettingsView: View { } Divider().opacity(0.2) + self.spokenSendSettings + Divider().opacity(0.2) + self.optionToggleRow( title: "Save Transcription History", description: "Save transcriptions for stats tracking. Disable for privacy.", @@ -2811,6 +2814,82 @@ struct FlowLayout: Layout { } } +private extension SettingsView { + var spokenSendSettings: some View { + Group { + self.optionToggleRow( + title: "Spoken Send", + description: "Say a phrase at the end of dictation to send with your chosen Enter command.", + isOn: Binding( + get: { self.settings.spokenSendEnabled }, + set: { self.settings.spokenSendEnabled = $0 } + ) + ) + + if self.settings.spokenSendEnabled { + VStack(spacing: 10) { + self.optionToggleRow( + title: "Send Immediately", + description: "Stop listening and send as soon as the phrase is recognized. May not work with all voice models; Parakeet is recommended.", + isOn: Binding( + get: { self.settings.spokenSendImmediatelyEnabled }, + set: { self.settings.spokenSendImmediatelyEnabled = $0 } + ) + ) + + HStack(alignment: .center) { + VStack(alignment: .leading, spacing: 2) { + Text("Send Phrase") + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.settingsTitleText) + Text("Matched only at the end. Say “literal \(self.settings.spokenSendPhrase)” to dictate it normally.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.settingsSecondaryText) + } + + Spacer() + + TextField( + "send it", + text: Binding( + get: { self.settings.spokenSendPhrase }, + set: { self.settings.spokenSendPhrase = $0 } + ) + ) + .textFieldStyle(.roundedBorder) + .frame(width: 170) + } + + HStack(alignment: .center) { + VStack(alignment: .leading, spacing: 2) { + Text("Send Command") + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.settingsTitleText) + Text("Choose the Enter behavior expected by the destination app.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.settingsSecondaryText) + } + + Spacer() + + Picker("", selection: Binding( + get: { self.settings.spokenSendKey }, + set: { self.settings.spokenSendKey = $0 } + )) { + ForEach(SettingsStore.SpokenSendKey.allCases) { key in + Text(key.displayName).tag(key) + } + } + .pickerStyle(.menu) + .frame(width: 170, alignment: .trailing) + } + } + .padding(.leading, 12) + } + } + } +} + // MARK: - Analytics modal confirmation struct AnalyticsConfirmationView: View { diff --git a/Sources/Fluid/Views/BottomOverlayView.swift b/Sources/Fluid/Views/BottomOverlayView.swift index 63a27835..b794b5d6 100644 --- a/Sources/Fluid/Views/BottomOverlayView.swift +++ b/Sources/Fluid/Views/BottomOverlayView.swift @@ -123,6 +123,7 @@ final class BottomOverlayWindowController { // offscreen. Revealing the neutral shell first causes a visible flash // that reads as the overlay appearing twice. NotchContentState.shared.setBottomOverlayPresented(true) + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) NotchContentState.shared.mode = mode switch mode { case .dictation: NotchContentState.shared.promptPickerMode = .dictate @@ -2137,6 +2138,16 @@ struct BottomOverlayView: View { } } + private var showsSpokenSendIndicator: Bool { + self.contentState.mode == .dictation && + self.settings.spokenSendEnabled && + self.contentState.spokenSendIndicatorState.isVisible + } + + private var spokenSendIndicatorSize: CGFloat { + max(self.layout.modeFontSize + 3, 13) + } + private static let transientOverlayStatusTexts: Set = [ "Transcribing", "Refining", @@ -2989,7 +3000,7 @@ struct BottomOverlayView: View { } // Waveform + Mode label row - HStack(spacing: self.layout.hPadding / 1.5) { + HStack(spacing: self.isPillSize ? 4 : self.layout.hPadding / 1.5) { // Target app icon (the app where text will be typed) let appIcon = self.displayedAppIcon let showModelLoading = self.layout.showsModeLabel && !self.appServices.asr.isAsrReady && @@ -3016,16 +3027,43 @@ struct BottomOverlayView: View { // Waveform visualization BottomWaveformView(color: self.modeColor, layout: self.layout) - .frame(width: self.layout.waveformWidth, height: self.layout.waveformHeight) + .frame( + width: self.isPillSize && self.showsSpokenSendIndicator + ? 34 + : self.layout.waveformWidth, + height: self.layout.waveformHeight + ) + + if self.isPillSize, self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.modeColor, + size: 14 + ) + .id(self.contentState.spokenSendCountdownID) + .transition(.scale(scale: 0.8).combined(with: .opacity)) + } // Mode label + model load hint if self.layout.showsModeLabel { VStack(alignment: .leading, spacing: 2) { - Text(self.modeLabel) - .font(.system(size: self.layout.modeFontSize, weight: .semibold)) - .foregroundStyle(self.modeColor) - .lineLimit(1) - .fixedSize(horizontal: true, vertical: false) + HStack(spacing: 5) { + Text(self.modeLabel) + .font(.system(size: self.layout.modeFontSize, weight: .semibold)) + .foregroundStyle(self.modeColor) + .lineLimit(1) + .fixedSize(horizontal: true, vertical: false) + + if self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.modeColor, + size: self.spokenSendIndicatorSize + ) + .id(self.contentState.spokenSendCountdownID) + .transition(.scale(scale: 0.8).combined(with: .opacity)) + } + } if !self.appServices.asr.isAsrReady && (self.appServices.asr.isLoadingModel || self.appServices.asr.isDownloadingModel) @@ -3037,6 +3075,10 @@ struct BottomOverlayView: View { .lineLimit(1) } } + .animation( + self.reduceMotion ? nil : .easeOut(duration: 0.14), + value: self.contentState.spokenSendIndicatorState + ) } } } diff --git a/Sources/Fluid/Views/NotchContentViews.swift b/Sources/Fluid/Views/NotchContentViews.swift index bf3e69c9..d7cbd2f0 100644 --- a/Sources/Fluid/Views/NotchContentViews.swift +++ b/Sources/Fluid/Views/NotchContentViews.swift @@ -10,6 +10,85 @@ import Combine import QuartzCore import SwiftUI +enum SpokenSendIndicatorState: Equatable { + case hidden + case detected + case countingDown + case sending + case sent + case failed + + var isVisible: Bool { + self != .hidden + } +} + +struct SpokenSendIndicatorView: View { + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @State private var progress: CGFloat = 0 + + let state: SpokenSendIndicatorState + let color: Color + let size: CGFloat + + var body: some View { + Group { + switch self.state { + case .hidden: + EmptyView() + case .detected: + self.symbol("paperplane.fill", color: self.color, accessibilityLabel: "Send detected") + case .countingDown: + ZStack { + Circle() + .stroke(self.color.opacity(0.22), lineWidth: self.lineWidth) + + Circle() + .trim(from: 0, to: self.progress) + .stroke( + self.color, + style: StrokeStyle(lineWidth: self.lineWidth, lineCap: .round) + ) + .rotationEffect(.degrees(-90)) + + self.symbol("paperplane.fill", color: self.color, accessibilityLabel: "Waiting to send") + } + .onAppear { + guard !self.reduceMotion else { + self.progress = 1 + return + } + withAnimation(.linear(duration: SpokenSendParser.immediateStopSettleDuration)) { + self.progress = 1 + } + } + case .sending: + ZStack { + Circle() + .stroke(self.color, lineWidth: self.lineWidth) + self.symbol("paperplane.fill", color: self.color, accessibilityLabel: "Sending") + } + case .sent: + self.symbol("checkmark.circle.fill", color: self.color, accessibilityLabel: "Sent") + case .failed: + self.symbol("exclamationmark.circle.fill", color: .orange, accessibilityLabel: "Send skipped") + } + } + .frame(width: self.size, height: self.size) + } + + private var lineWidth: CGFloat { + max(1, self.size * 0.085) + } + + private func symbol(_ name: String, color: Color, accessibilityLabel: String) -> some View { + Image(systemName: name) + .font(.system(size: max(6, self.size * 0.46), weight: .semibold)) + .foregroundStyle(color) + .accessibilityLabel(accessibilityLabel) + } +} + // MARK: - Observable state for notch content (Singleton) @MainActor @@ -25,6 +104,8 @@ class NotchContentState: ObservableObject { @Published var isAIProcessingFailureVisible: Bool = false @Published private(set) var aiProcessingFailureMessage: String = "AI Enhancement failed" @Published private(set) var canRetryAIProcessingFailure: Bool = true + @Published private(set) var spokenSendIndicatorState: SpokenSendIndicatorState = .hidden + @Published private(set) var spokenSendCountdownID: UInt64 = 0 @Published var activeDictationShortcutSlot: SettingsStore.DictationShortcutSlot? = nil @Published var promptModeOverrideProfileName: String? = nil // Name shown in overlay when prompt mode hotkey is active @Published var promptModeOverrideProfileID: String? = nil // ID of the active override profile (for checkmark in menu) @@ -111,6 +192,18 @@ class NotchContentState: ObservableObject { self.isAIProcessingFailureVisible = false } + func setSpokenSendIndicatorState(_ state: SpokenSendIndicatorState) { + guard self.spokenSendIndicatorState != state else { return } + self.spokenSendIndicatorState = state + } + + @discardableResult + func beginSpokenSendCountdown() -> UInt64 { + self.spokenSendCountdownID &+= 1 + self.spokenSendIndicatorState = .countingDown + return self.spokenSendCountdownID + } + /// Update transcription and recompute cached lines func updateTranscription(_ text: String) { let boundedText = Self.tailCharacters(in: text, maxCharacters: Self.maxStoredTranscriptionCharacters) @@ -418,6 +511,12 @@ struct NotchExpandedView: View { self.contentState.mode.notchColor } + private var showsSpokenSendIndicator: Bool { + self.contentState.mode == .dictation && + self.settings.spokenSendEnabled && + self.contentState.spokenSendIndicatorState.isVisible + } + private var presentationPolicy: NotchOverlayManager.NotchPresentationPolicy { NotchOverlayManager.shared.currentNotchPresentationPolicy } @@ -904,9 +1003,20 @@ struct NotchExpandedView: View { .frame(width: 48, height: 18) self.promptSelectorControl + + if self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.modeColor, + size: 14 + ) + .id(self.contentState.spokenSendCountdownID) + .transition(.scale(scale: 0.8).combined(with: .opacity)) + } } .frame(maxWidth: .infinity, alignment: .center) .offset(x: 4, y: 0) + .animation(.easeOut(duration: 0.14), value: self.contentState.spokenSendIndicatorState) self.promptHoverMenuRow @@ -1160,13 +1270,33 @@ struct NotchCompactLeadingView: View { struct NotchCompactTrailingView: View { let audioPublisher: AnyPublisher @ObservedObject private var contentState = NotchContentState.shared + @ObservedObject private var settings = SettingsStore.shared + + private var showsSpokenSendIndicator: Bool { + self.contentState.mode == .dictation && + self.settings.spokenSendEnabled && + self.contentState.spokenSendIndicatorState.isVisible + } var body: some View { - CompactNotchWaveformView( - audioPublisher: self.audioPublisher, - color: self.contentState.mode.notchColor - ) + HStack(spacing: 3) { + CompactNotchWaveformView( + audioPublisher: self.audioPublisher, + color: self.contentState.mode.notchColor + ) + .frame(width: self.showsSpokenSendIndicator ? 18 : 34, height: 16) + + if self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.contentState.mode.notchColor, + size: 13 + ) + .id(self.contentState.spokenSendCountdownID) + } + } .frame(width: 34, height: 16) + .animation(.easeOut(duration: 0.14), value: self.contentState.spokenSendIndicatorState) } } diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift new file mode 100644 index 00000000..5d9f4555 --- /dev/null +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -0,0 +1,316 @@ +import CoreGraphics +@testable import FluidVoice_Debug +import XCTest + +final class SpokenSendTests: XCTestCase { + func testDisabledFeatureLeavesTextUntouched() { + XCTAssertEqual( + SpokenSendParser.parse("Hello send it", phrase: "send it", enabled: false), + SpokenSendParseResult(text: "Hello send it", shouldSend: false) + ) + } + + func testTerminalPhraseIsRemovedAndArmsSend() { + XCTAssertEqual( + SpokenSendParser.parse("Hello there, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Hello there,", shouldSend: true) + ) + } + + func testCapitalizationAndFullStopDoNotAffectSend() { + XCTAssertEqual( + SpokenSendParser.parse("Ready to go, SEND IT.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready to go,", shouldSend: true) + ) + } + + func testNearbyTrailingPunctuationDoesNotAffectSend() { + XCTAssertEqual( + SpokenSendParser.parse(#"Ready to go — send it…")]"#, phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready to go —", shouldSend: true) + ) + } + + func testPhraseInMiddleDoesNotArmSend() { + XCTAssertEqual( + SpokenSendParser.parse("Send it when you are ready", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Send it when you are ready", shouldSend: false) + ) + } + + func testImmediateStopRequiresChildOption() { + XCTAssertTrue( + SpokenSendParser.shouldStopImmediately( + "Ready, send it.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + XCTAssertFalse( + SpokenSendParser.shouldStopImmediately( + "Ready, send it.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: false + ) + ) + } + + func testImmediateStopDoesNotTriggerForPhraseInMiddle() { + XCTAssertFalse( + SpokenSendParser.shouldStopImmediately( + "Send it when you are ready", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + } + + func testTerminalASRRefinementStaysArmed() { + XCTAssertTrue( + SpokenSendParser.shouldStopImmediately( + "Ready, send it", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + XCTAssertTrue( + SpokenSendParser.shouldStopImmediately( + "Ready, SEND IT.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + XCTAssertFalse( + SpokenSendParser.shouldStopImmediately( + "Ready, send it after I finish this sentence.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + } + + func testImmediateStopCompletionRequiresFreshTranscriptAndSilence() { + let arguments = ( + text: "Ready, send it.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + + XCTAssertFalse( + SpokenSendParser.canCompleteImmediateStop( + arguments.text, + phrase: arguments.phrase, + spokenSendEnabled: arguments.spokenSendEnabled, + sendImmediatelyEnabled: arguments.sendImmediatelyEnabled, + receivedFreshTranscript: false, + quietDuration: 2 + ) + ) + XCTAssertFalse( + SpokenSendParser.canCompleteImmediateStop( + arguments.text, + phrase: arguments.phrase, + spokenSendEnabled: arguments.spokenSendEnabled, + sendImmediatelyEnabled: arguments.sendImmediatelyEnabled, + receivedFreshTranscript: true, + quietDuration: 0 + ) + ) + XCTAssertTrue( + SpokenSendParser.canCompleteImmediateStop( + arguments.text, + phrase: arguments.phrase, + spokenSendEnabled: arguments.spokenSendEnabled, + sendImmediatelyEnabled: arguments.sendImmediatelyEnabled, + receivedFreshTranscript: true, + quietDuration: SpokenSendParser.immediateStopRequiredSilenceDuration + ) + ) + } + + func testImmediateStopCompletionCancelsForContinuedSpeech() { + XCTAssertFalse( + SpokenSendParser.canCompleteImmediateStop( + "Ready, send it after I finish this sentence.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true, + receivedFreshTranscript: true, + quietDuration: 2 + ) + ) + } + + func testVoiceActivityGraceIgnoresOnlyTheRecognitionTail() { + let startedAt: TimeInterval = 100 + XCTAssertFalse( + SpokenSendParser.shouldCancelCountdownForVoiceActivity( + countdownStartedAt: startedAt, + voiceActivityAt: startedAt + 0.05 + ) + ) + XCTAssertTrue( + SpokenSendParser.shouldCancelCountdownForVoiceActivity( + countdownStartedAt: startedAt, + voiceActivityAt: startedAt + SpokenSendParser.immediateStopVoiceActivityGraceDuration + ) + ) + } + + func testLiteralEscapeKeepsPhraseWithoutSending() { + XCTAssertEqual( + SpokenSendParser.parse("Please type literal send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Please type send it", shouldSend: false) + ) + } + + func testPhraseOnlySubmitsExistingDraftWithoutInsertingCommand() { + XCTAssertEqual( + SpokenSendParser.parse("Send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "", shouldSend: true) + ) + } + + func testCustomPhraseAllowsFlexibleWhitespaceAndCase() { + XCTAssertEqual( + SpokenSendParser.parse("Looks good PLEASE SUBMIT", phrase: "please submit", enabled: true), + SpokenSendParseResult(text: "Looks good", shouldSend: true) + ) + } + + func testAvailableSendCommandsMapToExpectedFlags() { + XCTAssertEqual(SettingsStore.SpokenSendKey.enter.eventFlags, []) + XCTAssertEqual(SettingsStore.SpokenSendKey.shiftEnter.eventFlags, .maskShift) + XCTAssertEqual(SettingsStore.SpokenSendKey.commandEnter.eventFlags, .maskCommand) + } + + func testGeneratedSendCommandsAreExcludedFromFluidVoiceHotkeys() throws { + for key in SettingsStore.SpokenSendKey.allCases { + let event = try XCTUnwrap( + CGEvent( + keyboardEventSource: nil, + virtualKey: 36, + keyDown: true + ) + ) + event.flags = key.eventFlags + event.setIntegerValueField( + .eventSourceUserData, + value: TypingService.synthesizedEventUserData + ) + + XCTAssertTrue( + GlobalHotkeyManager.isSynthesizedTypingEvent(event), + "\(key.displayName) must bypass FluidVoice hotkey matching" + ) + } + + let physicalEvent = try XCTUnwrap( + CGEvent( + keyboardEventSource: nil, + virtualKey: 36, + keyDown: true + ) + ) + XCTAssertFalse(GlobalHotkeyManager.isSynthesizedTypingEvent(physicalEvent)) + } + + func testPostInsertionActionRequiresExactNonSecureFocus() { + XCTAssertTrue( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 43, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: true, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: false, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: false + ) + ) + } + + func testDeliveryOutcomesReportInsertionAndSendSeparately() { + XCTAssertTrue(TypingService.DeliveryOutcome.insertedAndActionDispatched.didInsert) + XCTAssertTrue(TypingService.DeliveryOutcome.insertedAndActionDispatched.didDispatchAction) + XCTAssertTrue(TypingService.DeliveryOutcome.actionDispatched.didDispatchAction) + XCTAssertFalse(TypingService.DeliveryOutcome.actionDispatched.didInsert) + XCTAssertTrue(TypingService.DeliveryOutcome.insertedActionSuppressed.didInsert) + XCTAssertFalse(TypingService.DeliveryOutcome.insertedActionSuppressed.didDispatchAction) + } + + func testOverlayIndicatorVisibilityCoversEveryActiveState() { + XCTAssertFalse(SpokenSendIndicatorState.hidden.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.detected.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.countingDown.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.sending.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.sent.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.failed.isVisible) + } + + @MainActor + func testSettingsBackupIncludesSpokenSendConfiguration() async { + let settings = SettingsStore.shared + let originalEnabled = settings.spokenSendEnabled + let originalImmediate = settings.spokenSendImmediatelyEnabled + let originalPhrase = settings.spokenSendPhrase + let originalKey = settings.spokenSendKey + defer { + settings.spokenSendEnabled = originalEnabled + settings.spokenSendImmediatelyEnabled = originalImmediate + settings.spokenSendPhrase = originalPhrase + settings.spokenSendKey = originalKey + } + + settings.spokenSendEnabled = true + settings.spokenSendImmediatelyEnabled = false + settings.spokenSendPhrase = "ship it" + settings.spokenSendKey = .commandEnter + + let document = await BackupService.shared.makeBackupDocument() + XCTAssertEqual(document.settings.spokenSendEnabled, true) + XCTAssertEqual(document.settings.spokenSendImmediatelyEnabled, false) + XCTAssertEqual(document.settings.spokenSendPhrase, "ship it") + XCTAssertEqual(document.settings.spokenSendKey, .commandEnter) + } +} From 2f6cd1f9c27bb62786b1b1495977d404a2723686 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Wed, 29 Jul 2026 16:21:00 -0700 Subject: [PATCH 02/15] polish spoken send behavior --- Sources/Fluid/ContentView.swift | 2 +- Sources/Fluid/Services/SpokenSendParser.swift | 31 +++++++++- Sources/Fluid/UI/SettingsView.swift | 2 +- Sources/Fluid/Views/BottomOverlayView.swift | 21 ++++--- Sources/Fluid/Views/NotchContentViews.swift | 21 ++++--- .../SpokenSendTests.swift | 56 +++++++++++++++++-- 6 files changed, 107 insertions(+), 26 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 91721d8e..41c57cb1 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -2696,7 +2696,7 @@ struct ContentView: View { } private func handleSpokenSendAudioLevel(_ level: CGFloat) { - guard level > 0 else { return } + guard SpokenSendParser.isMeaningfulVoiceActivity(level) else { return } let activityAt = ProcessInfo.processInfo.systemUptime self.spokenSendLastVoiceActivityAt = activityAt diff --git a/Sources/Fluid/Services/SpokenSendParser.swift b/Sources/Fluid/Services/SpokenSendParser.swift index 6b3e9604..00b4b9b9 100644 --- a/Sources/Fluid/Services/SpokenSendParser.swift +++ b/Sources/Fluid/Services/SpokenSendParser.swift @@ -9,7 +9,8 @@ enum SpokenSendParser { static let immediateStopSettleDuration: TimeInterval = 1.5 static let immediateStopSettleNanoseconds: UInt64 = 1_500_000_000 static let immediateStopRequiredSilenceDuration: TimeInterval = 0.35 - static let immediateStopVoiceActivityGraceDuration: TimeInterval = 0.15 + static let immediateStopVoiceActivityGraceDuration: TimeInterval = 0.35 + static let immediateStopVoiceActivityLevelThreshold: CGFloat = 0.12 static func shouldStopImmediately( _ text: String, @@ -46,6 +47,10 @@ enum SpokenSendParser { voiceActivityAt - countdownStartedAt >= self.immediateStopVoiceActivityGraceDuration } + static func isMeaningfulVoiceActivity(_ level: CGFloat) -> Bool { + level >= self.immediateStopVoiceActivityLevelThreshold + } + static func parse(_ text: String, phrase: String, enabled: Bool) -> SpokenSendParseResult { guard enabled else { return SpokenSendParseResult(text: text, shouldSend: false) @@ -88,8 +93,28 @@ enum SpokenSendParser { return SpokenSendParseResult(text: text, shouldSend: false) } - let cleaned = String(text[.. Bool { + guard let last = text.last else { return false } + return [".", ",", ";", ":", "?", "!", "…", "-", "–", "—"].contains(last) + } + + private static func polishCommandPrefix(_ text: String) -> String { + var polished = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard let last = polished.last, [",", ";", ":", "-", "–", "—"].contains(last) else { + return polished + } + + polished.removeLast() + return polished.trimmingCharacters(in: .whitespacesAndNewlines) + "." + } } diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 3c8e493a..3d8bbee4 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -2842,7 +2842,7 @@ private extension SettingsView { Text("Send Phrase") .font(self.theme.typography.bodyStrong) .foregroundStyle(self.settingsTitleText) - Text("Matched only at the end. Say “literal \(self.settings.spokenSendPhrase)” to dictate it normally.") + Text("Say it alone, or after punctuation, at the end. Say “literal \(self.settings.spokenSendPhrase)” to dictate it normally.") .font(self.theme.typography.bodySmall) .foregroundStyle(self.settingsSecondaryText) } diff --git a/Sources/Fluid/Views/BottomOverlayView.swift b/Sources/Fluid/Views/BottomOverlayView.swift index b794b5d6..607b9f0e 100644 --- a/Sources/Fluid/Views/BottomOverlayView.swift +++ b/Sources/Fluid/Views/BottomOverlayView.swift @@ -3026,13 +3026,17 @@ struct BottomOverlayView: View { .opacity((appIcon != nil || showModelLoading || !self.layout.showsModeLabel) ? 1 : 0) // Waveform visualization - BottomWaveformView(color: self.modeColor, layout: self.layout) - .frame( - width: self.isPillSize && self.showsSpokenSendIndicator - ? 34 - : self.layout.waveformWidth, - height: self.layout.waveformHeight - ) + BottomWaveformView( + color: self.modeColor, + layout: self.layout, + visibleBarCount: self.isPillSize && self.showsSpokenSendIndicator ? 6 : nil + ) + .frame( + width: self.isPillSize && self.showsSpokenSendIndicator + ? 32 + : self.layout.waveformWidth, + height: self.layout.waveformHeight + ) if self.isPillSize, self.showsSpokenSendIndicator { SpokenSendIndicatorView( @@ -3291,6 +3295,7 @@ struct BottomOverlayView: View { struct BottomWaveformView: View { let color: Color let layout: BottomOverlayView.LayoutConstants + let visibleBarCount: Int? @ObservedObject private var contentState = NotchContentState.shared // Initialize with max possible bar count (11 for large) to prevent index-out-of-range before onAppear @@ -3298,7 +3303,7 @@ struct BottomWaveformView: View { @State private var noiseThreshold: CGFloat = .init(SettingsStore.shared.visualizerNoiseThreshold) private var barCount: Int { - self.layout.barCount + self.visibleBarCount ?? self.layout.barCount } private var barWidth: CGFloat { diff --git a/Sources/Fluid/Views/NotchContentViews.swift b/Sources/Fluid/Views/NotchContentViews.swift index d7cbd2f0..10184687 100644 --- a/Sources/Fluid/Views/NotchContentViews.swift +++ b/Sources/Fluid/Views/NotchContentViews.swift @@ -1279,12 +1279,13 @@ struct NotchCompactTrailingView: View { } var body: some View { - HStack(spacing: 3) { + HStack(spacing: self.showsSpokenSendIndicator ? 4 : 0) { CompactNotchWaveformView( audioPublisher: self.audioPublisher, - color: self.contentState.mode.notchColor + color: self.contentState.mode.notchColor, + barCount: self.showsSpokenSendIndicator ? 4 : 8 ) - .frame(width: self.showsSpokenSendIndicator ? 18 : 34, height: 16) + .frame(width: self.showsSpokenSendIndicator ? 16 : 34, height: 16) if self.showsSpokenSendIndicator { SpokenSendIndicatorView( @@ -1851,14 +1852,19 @@ struct ExpandedModeWaveformView: View { } struct CompactNotchWaveformView: View { + private static let storedBarCount = 8 + let audioPublisher: AnyPublisher let color: Color + var barCount: Int = 8 @StateObject private var data: AudioVisualizationData @ObservedObject private var contentState = NotchContentState.shared - @State private var barHeights: [CGFloat] = Array(repeating: 3, count: 8) + @State private var barHeights: [CGFloat] = Array( + repeating: 3, + count: CompactNotchWaveformView.storedBarCount + ) - private let barCount = 8 private let barWidth: CGFloat = 2.5 private let barSpacing: CGFloat = 2 private let minHeight: CGFloat = 3 @@ -1866,9 +1872,10 @@ struct CompactNotchWaveformView: View { private let noiseThreshold: CGFloat = 0.05 private let processingFlatHeight: CGFloat = 3 - init(audioPublisher: AnyPublisher, color: Color) { + init(audioPublisher: AnyPublisher, color: Color, barCount: Int = 8) { self.audioPublisher = audioPublisher self.color = color + self.barCount = min(max(barCount, 1), Self.storedBarCount) _data = StateObject(wrappedValue: AudioVisualizationData(audioLevelPublisher: audioPublisher)) } @@ -1950,7 +1957,7 @@ struct CompactNotchWaveformView: View { private func resetBarsToBaseline(animated: Bool) { let apply = { - self.barHeights = Array(repeating: self.minHeight, count: self.barCount) + self.barHeights = Array(repeating: self.minHeight, count: Self.storedBarCount) } if animated { diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index 5d9f4555..11c67527 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -13,21 +13,21 @@ final class SpokenSendTests: XCTestCase { func testTerminalPhraseIsRemovedAndArmsSend() { XCTAssertEqual( SpokenSendParser.parse("Hello there, send it.", phrase: "send it", enabled: true), - SpokenSendParseResult(text: "Hello there,", shouldSend: true) + SpokenSendParseResult(text: "Hello there.", shouldSend: true) ) } func testCapitalizationAndFullStopDoNotAffectSend() { XCTAssertEqual( SpokenSendParser.parse("Ready to go, SEND IT.", phrase: "send it", enabled: true), - SpokenSendParseResult(text: "Ready to go,", shouldSend: true) + SpokenSendParseResult(text: "Ready to go.", shouldSend: true) ) } func testNearbyTrailingPunctuationDoesNotAffectSend() { XCTAssertEqual( SpokenSendParser.parse(#"Ready to go — send it…")]"#, phrase: "send it", enabled: true), - SpokenSendParseResult(text: "Ready to go —", shouldSend: true) + SpokenSendParseResult(text: "Ready to go.", shouldSend: true) ) } @@ -38,6 +38,40 @@ final class SpokenSendTests: XCTestCase { ) } + func testNaturalSentenceEndingWithPhraseDoesNotArmSend() { + XCTAssertEqual( + SpokenSendParser.parse("I wanna send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "I wanna send it.", shouldSend: false) + ) + } + + func testPunctuationBreakDisambiguatesTerminalCommand() { + XCTAssertEqual( + SpokenSendParser.parse("I wanna send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "I wanna send it.", shouldSend: true) + ) + } + + func testOpeningPunctuationDoesNotDisambiguateTerminalCommand() { + for text in ["I wanna (send it.", #"I wanna "send it.""#, "I_wanna_send it."] { + XCTAssertEqual( + SpokenSendParser.parse(text, phrase: "send it", enabled: true), + SpokenSendParseResult(text: text, shouldSend: false) + ) + } + } + + func testFinalQuestionOrExclamationMarkIsPreserved() { + XCTAssertEqual( + SpokenSendParser.parse("Are we ready? send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Are we ready?", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ship it! send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ship it!", shouldSend: true) + ) + } + func testImmediateStopRequiresChildOption() { XCTAssertTrue( SpokenSendParser.shouldStopImmediately( @@ -159,7 +193,17 @@ final class SpokenSendTests: XCTestCase { XCTAssertTrue( SpokenSendParser.shouldCancelCountdownForVoiceActivity( countdownStartedAt: startedAt, - voiceActivityAt: startedAt + SpokenSendParser.immediateStopVoiceActivityGraceDuration + voiceActivityAt: startedAt + SpokenSendParser.immediateStopVoiceActivityGraceDuration + 0.001 + ) + ) + XCTAssertFalse( + SpokenSendParser.isMeaningfulVoiceActivity( + SpokenSendParser.immediateStopVoiceActivityLevelThreshold.nextDown + ) + ) + XCTAssertTrue( + SpokenSendParser.isMeaningfulVoiceActivity( + SpokenSendParser.immediateStopVoiceActivityLevelThreshold ) ) } @@ -180,8 +224,8 @@ final class SpokenSendTests: XCTestCase { func testCustomPhraseAllowsFlexibleWhitespaceAndCase() { XCTAssertEqual( - SpokenSendParser.parse("Looks good PLEASE SUBMIT", phrase: "please submit", enabled: true), - SpokenSendParseResult(text: "Looks good", shouldSend: true) + SpokenSendParser.parse("Looks good. PLEASE SUBMIT", phrase: "please submit", enabled: true), + SpokenSendParseResult(text: "Looks good.", shouldSend: true) ) } From 87c554f7a76e371b18821087a011fb1b2686efba Mon Sep 17 00:00:00 2001 From: altic-dev Date: Wed, 29 Jul 2026 16:49:55 -0700 Subject: [PATCH 03/15] polish spoken send cleanup: strip junk punctuation, allow repeated phrase, any prefix --- Sources/Fluid/Services/SpokenSendParser.swift | 43 +++++++++++------- Sources/Fluid/UI/SettingsView.swift | 2 +- .../SpokenSendTests.swift | 45 ++++++++++++++----- 3 files changed, 62 insertions(+), 28 deletions(-) diff --git a/Sources/Fluid/Services/SpokenSendParser.swift b/Sources/Fluid/Services/SpokenSendParser.swift index 00b4b9b9..982e411c 100644 --- a/Sources/Fluid/Services/SpokenSendParser.swift +++ b/Sources/Fluid/Services/SpokenSendParser.swift @@ -83,38 +83,51 @@ enum SpokenSendParser { } guard let commandRegex = try? NSRegularExpression( - pattern: #"(?i)(? Bool { - guard let last = text.last else { return false } - return [".", ",", ";", ":", "?", "!", "…", "-", "–", "—"].contains(last) - } - private static func polishCommandPrefix(_ text: String) -> String { var polished = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard let last = polished.last, [",", ";", ":", "-", "–", "—"].contains(last) else { - return polished + while let last = polished.last, [",", ";", ":", "-", "–", "—"].contains(last) { + polished.removeLast() + while polished.last?.isWhitespace == true { + polished.removeLast() + } } - polished.removeLast() - return polished.trimmingCharacters(in: .whitespacesAndNewlines) + "." + guard let last = polished.last else { + return polished + } + if [".", "?", "!", "…"].contains(last) { + return polished + } + return polished + "." } } diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index 3d8bbee4..e8cc99e2 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -2842,7 +2842,7 @@ private extension SettingsView { Text("Send Phrase") .font(self.theme.typography.bodyStrong) .foregroundStyle(self.settingsTitleText) - Text("Say it alone, or after punctuation, at the end. Say “literal \(self.settings.spokenSendPhrase)” to dictate it normally.") + Text("Say it at the end. Say “literal \(self.settings.spokenSendPhrase)” to dictate it normally.") .font(self.theme.typography.bodySmall) .foregroundStyle(self.settingsSecondaryText) } diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index 11c67527..0428e7d5 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -38,27 +38,41 @@ final class SpokenSendTests: XCTestCase { ) } - func testNaturalSentenceEndingWithPhraseDoesNotArmSend() { + func testTerminalPhraseDoesNotRequireLeadingOrTrailingPunctuation() { XCTAssertEqual( - SpokenSendParser.parse("I wanna send it.", phrase: "send it", enabled: true), - SpokenSendParseResult(text: "I wanna send it.", shouldSend: false) + SpokenSendParser.parse("Ready to go send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready to go.", shouldSend: true) ) } - func testPunctuationBreakDisambiguatesTerminalCommand() { + func testRepeatedTerminalPhrasesAreAllRemoved() { XCTAssertEqual( SpokenSendParser.parse("I wanna send it, send it.", phrase: "send it", enabled: true), - SpokenSendParseResult(text: "I wanna send it.", shouldSend: true) + SpokenSendParseResult(text: "I wanna.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready SEND IT send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "", shouldSend: true) ) } - func testOpeningPunctuationDoesNotDisambiguateTerminalCommand() { - for text in ["I wanna (send it.", #"I wanna "send it.""#, "I_wanna_send it."] { - XCTAssertEqual( - SpokenSendParser.parse(text, phrase: "send it", enabled: true), - SpokenSendParseResult(text: text, shouldSend: false) - ) - } + func testRepeatedTrailingSeparatorsCollapseToOneSentenceEnding() { + XCTAssertEqual( + SpokenSendParser.parse("Ready,,,,; — send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready.,,,;— send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready?,,, send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready?", shouldSend: true) + ) } func testFinalQuestionOrExclamationMarkIsPreserved() { @@ -215,6 +229,13 @@ final class SpokenSendTests: XCTestCase { ) } + func testLiteralEscapeBeforeRepeatedCommandKeepsOnePhraseAndSends() { + XCTAssertEqual( + SpokenSendParser.parse("Please type literal send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Please type send it.", shouldSend: true) + ) + } + func testPhraseOnlySubmitsExistingDraftWithoutInsertingCommand() { XCTAssertEqual( SpokenSendParser.parse("Send it", phrase: "send it", enabled: true), From c7a9e9709618467ea736008fd62f1c47ac791673 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 16:44:11 -0700 Subject: [PATCH 04/15] fix spoken send safety --- Sources/Fluid/ContentView.swift | 26 ++++++++++++++-- Sources/Fluid/Services/TypingService.swift | 36 ++++++++++++---------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 41c57cb1..9f2fb4d0 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -286,6 +286,7 @@ struct ContentView: View { @State private var spokenSendPartialRevision: UInt64 = 0 @State private var spokenSendCountdownStartedAt: TimeInterval? @State private var spokenSendLastVoiceActivityAt: TimeInterval = 0 + @State private var spokenSendVoiceActivityCancellable: AnyCancellable? private var isRecordingAnyShortcutCapture: Bool { self.activeShortcutRecordingTarget != nil @@ -362,9 +363,6 @@ struct ContentView: View { .onReceive(self.asr.$partialTranscription) { text in self.handleSpokenSendPartialTranscription(text) } - .onReceive(self.asr.audioLevelPublisher) { level in - self.handleSpokenSendAudioLevel(level) - } .toolbar { if !self.settings.shouldShowOnboarding { ToolbarItemGroup(placement: .primaryAction) { @@ -2611,6 +2609,7 @@ struct ContentView: View { private func advanceOverlayLifecycle() { self.spokenSendAutoStopTask?.cancel() self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() self.spokenSendAutoStopTriggered = false self.spokenSendPartialRevision = 0 self.spokenSendCountdownStartedAt = nil @@ -2636,6 +2635,7 @@ struct ContentView: View { guard shouldStop else { self.spokenSendAutoStopTask?.cancel() self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() self.spokenSendCountdownStartedAt = nil if !self.spokenSendAutoStopTriggered, NotchContentState.shared.spokenSendIndicatorState == .countingDown @@ -2653,6 +2653,8 @@ struct ContentView: View { let expectedPartialRevision = self.spokenSendPartialRevision let countdownStartedAt = ProcessInfo.processInfo.systemUptime self.spokenSendCountdownStartedAt = countdownStartedAt + self.spokenSendLastVoiceActivityAt = countdownStartedAt + self.startSpokenSendVoiceActivityMonitoring() let expectedCountdownID = NotchContentState.shared.beginSpokenSendCountdown() self.spokenSendAutoStopTask = Task { @MainActor in // Keep one countdown across harmless streaming refinements such as punctuation or casing. @@ -2678,6 +2680,7 @@ struct ContentView: View { NotchContentState.shared.spokenSendCountdownID == expectedCountdownID { self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() self.spokenSendCountdownStartedAt = nil if NotchContentState.shared.spokenSendIndicatorState == .countingDown { NotchContentState.shared.setSpokenSendIndicatorState(.hidden) @@ -2687,6 +2690,7 @@ struct ContentView: View { } self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() self.spokenSendCountdownStartedAt = nil self.spokenSendAutoStopTriggered = true NotchContentState.shared.setSpokenSendIndicatorState(.sending) @@ -2712,12 +2716,28 @@ struct ContentView: View { self.spokenSendAutoStopTask?.cancel() self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() self.spokenSendCountdownStartedAt = nil if NotchContentState.shared.spokenSendIndicatorState == .countingDown { NotchContentState.shared.setSpokenSendIndicatorState(.hidden) } } + private func startSpokenSendVoiceActivityMonitoring() { + guard self.spokenSendVoiceActivityCancellable == nil else { return } + self.spokenSendVoiceActivityCancellable = self.asr.audioLevelPublisher + .filter(SpokenSendParser.isMeaningfulVoiceActivity) + .receive(on: RunLoop.main) + .sink { level in + self.handleSpokenSendAudioLevel(level) + } + } + + private func stopSpokenSendVoiceActivityMonitoring() { + self.spokenSendVoiceActivityCancellable?.cancel() + self.spokenSendVoiceActivityCancellable = nil + } + private func hideOverlayAsync(reason: String) { let expectedOverlayLifecycleID = self.overlayLifecycleID self.appBench("overlay_hide_request reason=\(reason) lifecycle=\(expectedOverlayLifecycleID)") diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 1e3bc66f..83b70b5e 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -475,6 +475,25 @@ final class TypingService { } self.bench("settle_delay_done delayMs=\(settleDelayMs) elapsedMs=\(Self.elapsedMs(since: requestedAt))") let hasTextToInsert = !text.isEmpty + if postInsertionKey != nil { + guard let preferredTargetPID, let requiredFocusTarget else { + outcome = .actionSuppressed + return + } + let modifiersReleased = self.waitForPhysicalModifiersToRelease(timeout: 2) + let exactFocusIsActive = Self.isExactFocusTargetActive(requiredFocusTarget) + guard Self.canDispatchPostInsertionAction( + preferredTargetPID: preferredTargetPID, + requiredTargetPID: requiredFocusTarget.pid, + isSecureTextField: requiredFocusTarget.isSecureTextField, + modifiersReleased: modifiersReleased, + exactFocusIsActive: exactFocusIsActive + ) else { + outcome = .actionSuppressed + return + } + } + if hasTextToInsert { self.log("[TypingService] Delay completed, calling insertTextInstantly") let insertStartedAt = ProcessInfo.processInfo.systemUptime @@ -500,22 +519,7 @@ final class TypingService { } guard let postInsertionKey else { return } - guard let preferredTargetPID, let requiredFocusTarget else { - outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed - return - } - let modifiersReleased = self.waitForPhysicalModifiersToRelease(timeout: 2) - let exactFocusIsActive = Self.isExactFocusTargetActive(requiredFocusTarget) - guard Self.canDispatchPostInsertionAction( - preferredTargetPID: preferredTargetPID, - requiredTargetPID: requiredFocusTarget.pid, - isSecureTextField: requiredFocusTarget.isSecureTextField, - modifiersReleased: modifiersReleased, - exactFocusIsActive: exactFocusIsActive - ) else { - outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed - return - } + guard let preferredTargetPID, let requiredFocusTarget else { return } usleep(50_000) guard Self.isExactFocusTargetActive(requiredFocusTarget), From 6c42196f2686cec5772801fcd377dacd06fa824e Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:05:55 -0700 Subject: [PATCH 05/15] harden spoken send delivery --- Sources/Fluid/ContentView.swift | 9 +- ...RService+SpokenPunctuationFormatting.swift | 28 +++- Sources/Fluid/Services/TypingService.swift | 158 ++++++++++++++++-- .../SpokenSendTests.swift | 44 +++++ 4 files changed, 214 insertions(+), 25 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 9f2fb4d0..b4aff0f2 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -1623,11 +1623,10 @@ struct ContentView: View { private func isSpokenSendBlockedApp( _ appInfo: (name: String, bundleId: String, windowTitle: String) ) -> Bool { - let identity = "\(appInfo.name) \(appInfo.bundleId)".lowercased() - return identity.contains("terminal") - || identity.contains("iterm") - || identity.contains("warp") - || identity.contains("ghostty") + TerminalAppClassifier.isTerminal( + appName: appInfo.name, + bundleID: appInfo.bundleId + ) } private func deliverSpokenSend( diff --git a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift index 37640bea..b931b314 100644 --- a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift +++ b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift @@ -1,5 +1,24 @@ import Foundation +enum TerminalAppClassifier { + private static let identityMarkers = [ + "terminal", + "iterm", + "warp", + "ghostty", + "kitty", + "alacritty", + "wezterm", + ] + + static func isTerminal(appName: String?, bundleID: String?) -> Bool { + let identity = [appName, bundleID] + .compactMap { $0?.lowercased() } + .joined(separator: " ") + return self.identityMarkers.contains { identity.contains($0) } + } +} + extension ASRService { static func applySpokenPunctuationFormatting( _ text: String, @@ -30,7 +49,8 @@ private enum SpokenPunctuationFormatter { let haystack = [self.appName, self.bundleID, self.windowTitle] .compactMap { $0?.lowercased() } .joined(separator: " ") - return haystack.contains("codex") || + return TerminalAppClassifier.isTerminal(appName: self.appName, bundleID: self.bundleID) || + haystack.contains("codex") || haystack.contains("chatgpt") || haystack.contains("claude") || haystack.contains("cursor") || @@ -38,12 +58,6 @@ private enum SpokenPunctuationFormatter { haystack.contains("xcode") || haystack.contains("visual studio code") || haystack.contains("vscode") || - haystack.contains("terminal") || - haystack.contains("iterm") || - haystack.contains("warp") || - haystack.contains("ghostty") || - haystack.contains("kitty") || - haystack.contains("alacritty") || haystack.contains("slack") || haystack.contains("discord") || haystack.contains("teams") diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 83b70b5e..5c4794d8 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -49,14 +49,15 @@ final class TypingService { requiredTargetPID: pid_t?, isSecureTextField: Bool, modifiersReleased: Bool, - exactFocusIsActive: Bool + exactFocusIsActive: Bool, + insertionConfirmed: Bool = true ) -> Bool { guard let preferredTargetPID, preferredTargetPID > 0, requiredTargetPID == preferredTargetPID else { return false } - return !isSecureTextField && modifiersReleased && exactFocusIsActive + return !isSecureTextField && modifiersReleased && exactFocusIsActive && insertionConfirmed } // Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 @@ -89,6 +90,7 @@ final class TypingService { private struct FocusedTextSnapshot { let pid: pid_t + let element: AXUIElement let bundleIdentifier: String? let value: String? let selectedRange: CFRange? @@ -103,6 +105,18 @@ final class TypingService { case caretMovedExpectedDistance = "caret_moved_expected_distance" case timeout case unavailable + + var isConfirmed: Bool { + switch self { + case .appScriptContainsText, + .appScriptCaretMovedExpectedDistance, + .fieldContainsText, + .caretMovedExpectedDistance: + return true + case .timeout, .unavailable: + return false + } + } } private static let focusSnapshotQueue = DispatchQueue(label: "TypingService.FocusSnapshot") @@ -498,7 +512,11 @@ final class TypingService { self.log("[TypingService] Delay completed, calling insertTextInstantly") let insertStartedAt = ProcessInfo.processInfo.systemUptime self.bench("insert_call") - let inserted = self.insertTextInstantly(text, preferredTargetPID: preferredTargetPID) + let inserted = self.insertTextInstantly( + text, + preferredTargetPID: preferredTargetPID, + requiredFocusTarget: postInsertionKey == nil ? nil : requiredFocusTarget + ) self.bench( "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" ) @@ -522,8 +540,15 @@ final class TypingService { guard let preferredTargetPID, let requiredFocusTarget else { return } usleep(50_000) - guard Self.isExactFocusTargetActive(requiredFocusTarget), - self.postReturnKey(postInsertionKey, targetPID: preferredTargetPID) + guard Self.canDispatchPostInsertionAction( + preferredTargetPID: preferredTargetPID, + requiredTargetPID: requiredFocusTarget.pid, + isSecureTextField: requiredFocusTarget.isSecureTextField, + modifiersReleased: self.waitForPhysicalModifiersToRelease(timeout: 0.2), + exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget), + insertionConfirmed: !hasTextToInsert || outcome.didInsert + ), + self.postReturnKey(postInsertionKey, targetPID: preferredTargetPID) else { outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed return @@ -546,10 +571,22 @@ final class TypingService { // MARK: - Internal insertion pipeline - private func insertTextInstantly(_ text: String, preferredTargetPID: pid_t?) -> Bool { + private func insertTextInstantly( + _ text: String, + preferredTargetPID: pid_t?, + requiredFocusTarget: CapturedFocusTarget? + ) -> Bool { self.log("[TypingService] insertTextInstantly called with \(text.count) characters") self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"") + if let requiredFocusTarget { + return self.insertTextIntoExactFocusTarget( + text, + preferredTargetPID: preferredTargetPID, + requiredFocusTarget: requiredFocusTarget + ) + } + if self.textInsertionMode == .standard, let ghosttyTargetPID = self.ghosttyTargetPID(preferredTargetPID: preferredTargetPID) { @@ -634,16 +671,108 @@ final class TypingService { } // Last resort: Character-by-character - self.log("[TypingService] WARNING: All methods failed, trying character-by-character") + self.log("[TypingService] WARNING: All methods failed, trying verified character-by-character") + let focusedTextSnapshot = self.captureFocusedTextSnapshot() + var postedEveryCharacter = true for (index, char) in text.enumerated() { if index % 10 == 0 { self.log("[TypingService] Typing character \(index + 1)/\(text.count)") } - self.typeCharacter(char) + postedEveryCharacter = self.typeCharacter(char) && postedEveryCharacter usleep(1000) } - self.log("[TypingService] Character-by-character typing completed") - return true + guard postedEveryCharacter else { + self.log("[TypingService] Character-by-character event creation failed") + return false + } + let verification = self.waitForFocusedTextVerification( + from: focusedTextSnapshot, + expectedText: text, + timeoutMicros: 500_000 + ) + self.log("[TypingService] Character-by-character verification: \(verification.rawValue)") + return verification.isConfirmed + } + + private func insertTextIntoExactFocusTarget( + _ text: String, + preferredTargetPID: pid_t?, + requiredFocusTarget: CapturedFocusTarget + ) -> Bool { + guard preferredTargetPID == requiredFocusTarget.pid, + !requiredFocusTarget.isSecureTextField, + Self.isExactFocusTargetActive(requiredFocusTarget) + else { + self.log("[TypingService] Exact-target insertion rejected before dispatch") + return false + } + + let eventAttempt: ExactTargetEventAttempt + switch self.textInsertionMode { + case .standard: + eventAttempt = self.performVerifiedExactTargetEventInsertion( + text, + target: requiredFocusTarget + ) { + self.insertTextBulkInstant(text, targetPID: requiredFocusTarget.pid) + } + case .reliablePaste: + eventAttempt = self.performVerifiedExactTargetEventInsertion( + text, + target: requiredFocusTarget + ) { + self.insertTextViaClipboardToPid( + text, + targetPID: requiredFocusTarget.pid, + activateTargetFirst: false + ) + } + } + + switch eventAttempt { + case .confirmed: + return true + case .dispatchedUnverified: + self.log("[TypingService] Exact-target event insertion could not be confirmed; suppressing action") + return false + case .notDispatched: + break + } + + guard Self.isExactFocusTargetActive(requiredFocusTarget) else { + self.log("[TypingService] Exact target changed before Accessibility fallback") + return false + } + return self.tryAllTextInsertionMethods(requiredFocusTarget.element, text) + } + + private enum ExactTargetEventAttempt { + case notDispatched + case dispatchedUnverified + case confirmed + } + + private func performVerifiedExactTargetEventInsertion( + _ text: String, + target: CapturedFocusTarget, + action: () -> Bool + ) -> ExactTargetEventAttempt { + guard Self.isExactFocusTargetActive(target) else { return .notDispatched } + guard let snapshot = self.captureFocusedTextSnapshot(), + snapshot.pid == target.pid, + action() + else { + return .notDispatched + } + guard Self.isExactFocusTargetActive(target) else { return .dispatchedUnverified } + + let verification = self.waitForFocusedTextVerification( + from: snapshot, + expectedText: text, + timeoutMicros: 500_000 + ) + self.log("[TypingService] Exact-target event verification: \(verification.rawValue)") + return verification.isConfirmed ? .confirmed : .dispatchedUnverified } private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) -> Bool { @@ -1253,6 +1382,7 @@ final class TypingService { let appScriptSnapshot = self.captureAppScriptTextSnapshot(forBundleIdentifier: bundleIdentifier) return FocusedTextSnapshot( pid: focusInfo.pid, + element: focusInfo.element, bundleIdentifier: bundleIdentifier, value: self.getElementStringValue(focusInfo.element), selectedRange: self.getSelectedTextRange(focusInfo.element), @@ -1311,7 +1441,8 @@ final class TypingService { waited += pollMicros guard let current = self.captureFocusedTextSnapshot(), - current.pid == snapshot.pid + current.pid == snapshot.pid, + CFEqual(current.element, snapshot.element) else { continue } @@ -1512,7 +1643,7 @@ final class TypingService { } } - private func typeCharacter(_ char: Character) { + private func typeCharacter(_ char: Character) -> Bool { let charString = String(char) let utf16Array = Array(charString.utf16) @@ -1521,7 +1652,7 @@ final class TypingService { let keyUpEvent = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { self.log("[TypingService] ERROR: Failed to create CGEvents for character: \(char)") - return + return false } // Set the unicode string for both events @@ -1532,5 +1663,6 @@ final class TypingService { keyDownEvent.post(tap: .cghidEventTap) usleep(2000) // Short delay between key down and up (2ms) keyUpEvent.post(tap: .cghidEventTap) + return true } } diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index 0428e7d5..3ffcc545 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -256,6 +256,40 @@ final class SpokenSendTests: XCTestCase { XCTAssertEqual(SettingsStore.SpokenSendKey.commandEnter.eventFlags, .maskCommand) } + func testTerminalClassifierBlocksEveryRecognizedTerminal() { + let terminals = [ + ("Terminal", "com.apple.Terminal"), + ("iTerm2", "com.googlecode.iterm2"), + ("Warp", "dev.warp.Warp-Stable"), + ("Ghostty", "com.mitchellh.ghostty"), + ("kitty", "net.kovidgoyal.kitty"), + ("Alacritty", "org.alacritty"), + ("WezTerm", "com.github.wez.wezterm"), + ] + + for (name, bundleID) in terminals { + XCTAssertTrue( + TerminalAppClassifier.isTerminal(appName: name, bundleID: bundleID), + "Spoken Send must stay disabled in \(name)" + ) + } + } + + func testTerminalClassifierDoesNotBlockOrdinaryEditors() { + XCTAssertFalse( + TerminalAppClassifier.isTerminal( + appName: "TextEdit", + bundleID: "com.apple.TextEdit" + ) + ) + XCTAssertFalse( + TerminalAppClassifier.isTerminal( + appName: "Codex", + bundleID: "com.openai.codex" + ) + ) + } + func testGeneratedSendCommandsAreExcludedFromFluidVoiceHotkeys() throws { for key in SettingsStore.SpokenSendKey.allCases { let event = try XCTUnwrap( @@ -333,6 +367,16 @@ final class SpokenSendTests: XCTestCase { exactFocusIsActive: false ) ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: true, + insertionConfirmed: false + ) + ) } func testDeliveryOutcomesReportInsertionAndSendSeparately() { From 1b3201a757502bdee800ad8e901e694e077da50a Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:12:18 -0700 Subject: [PATCH 06/15] verify spoken send accessibility insertion --- Sources/Fluid/Services/TypingService.swift | 28 +++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 5c4794d8..555f20ac 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -743,7 +743,10 @@ final class TypingService { self.log("[TypingService] Exact target changed before Accessibility fallback") return false } - return self.tryAllTextInsertionMethods(requiredFocusTarget.element, text) + return self.performVerifiedExactTargetAccessibilityInsertion( + text, + target: requiredFocusTarget + ) } private enum ExactTargetEventAttempt { @@ -775,6 +778,29 @@ final class TypingService { return verification.isConfirmed ? .confirmed : .dispatchedUnverified } + private func performVerifiedExactTargetAccessibilityInsertion( + _ text: String, + target: CapturedFocusTarget + ) -> Bool { + guard Self.isExactFocusTargetActive(target), + let snapshot = self.captureFocusedTextSnapshot(), + snapshot.pid == target.pid, + CFEqual(snapshot.element, target.element), + self.tryAllTextInsertionMethods(target.element, text), + Self.isExactFocusTargetActive(target) + else { + return false + } + + let verification = self.waitForFocusedTextVerification( + from: snapshot, + expectedText: text, + timeoutMicros: 500_000 + ) + self.log("[TypingService] Exact-target Accessibility verification: \(verification.rawValue)") + return verification.isConfirmed + } + private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) -> Bool { let relevant: CGEventFlags = [.maskCommand, .maskControl, .maskAlternate, .maskShift, .maskSecondaryFn] let startedAt = ProcessInfo.processInfo.systemUptime From 68be26d0c0eee02a8395e976aac12db26a55759d Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:19:19 -0700 Subject: [PATCH 07/15] bind spoken send to captured field --- Sources/Fluid/Services/TypingService.swift | 84 ++++------------------ 1 file changed, 14 insertions(+), 70 deletions(-) diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 555f20ac..bae74801 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -548,7 +548,7 @@ final class TypingService { exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget), insertionConfirmed: !hasTextToInsert || outcome.didInsert ), - self.postReturnKey(postInsertionKey, targetPID: preferredTargetPID) + self.postReturnKey(postInsertionKey, target: requiredFocusTarget) else { outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed return @@ -707,77 +707,12 @@ final class TypingService { return false } - let eventAttempt: ExactTargetEventAttempt - switch self.textInsertionMode { - case .standard: - eventAttempt = self.performVerifiedExactTargetEventInsertion( - text, - target: requiredFocusTarget - ) { - self.insertTextBulkInstant(text, targetPID: requiredFocusTarget.pid) - } - case .reliablePaste: - eventAttempt = self.performVerifiedExactTargetEventInsertion( - text, - target: requiredFocusTarget - ) { - self.insertTextViaClipboardToPid( - text, - targetPID: requiredFocusTarget.pid, - activateTargetFirst: false - ) - } - } - - switch eventAttempt { - case .confirmed: - return true - case .dispatchedUnverified: - self.log("[TypingService] Exact-target event insertion could not be confirmed; suppressing action") - return false - case .notDispatched: - break - } - - guard Self.isExactFocusTargetActive(requiredFocusTarget) else { - self.log("[TypingService] Exact target changed before Accessibility fallback") - return false - } return self.performVerifiedExactTargetAccessibilityInsertion( text, target: requiredFocusTarget ) } - private enum ExactTargetEventAttempt { - case notDispatched - case dispatchedUnverified - case confirmed - } - - private func performVerifiedExactTargetEventInsertion( - _ text: String, - target: CapturedFocusTarget, - action: () -> Bool - ) -> ExactTargetEventAttempt { - guard Self.isExactFocusTargetActive(target) else { return .notDispatched } - guard let snapshot = self.captureFocusedTextSnapshot(), - snapshot.pid == target.pid, - action() - else { - return .notDispatched - } - guard Self.isExactFocusTargetActive(target) else { return .dispatchedUnverified } - - let verification = self.waitForFocusedTextVerification( - from: snapshot, - expectedText: text, - timeoutMicros: 500_000 - ) - self.log("[TypingService] Exact-target event verification: \(verification.rawValue)") - return verification.isConfirmed ? .confirmed : .dispatchedUnverified - } - private func performVerifiedExactTargetAccessibilityInsertion( _ text: String, target: CapturedFocusTarget @@ -813,7 +748,11 @@ final class TypingService { return false } - private func postReturnKey(_ key: SettingsStore.SpokenSendKey, targetPID: pid_t) -> Bool { + private func postReturnKey( + _ key: SettingsStore.SpokenSendKey, + target: CapturedFocusTarget + ) -> Bool { + guard Self.isExactFocusTargetActive(target) else { return false } let returnKeyCode = CGKeyCode(kVK_Return) guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: true), let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: false) @@ -825,10 +764,15 @@ final class TypingService { keyUp.flags = key.eventFlags keyDown.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) keyUp.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) - keyDown.postToPid(targetPID) + guard Self.isExactFocusTargetActive(target) else { return false } + keyDown.postToPid(target.pid) usleep(10_000) - keyUp.postToPid(targetPID) - return true + guard Self.isExactFocusTargetActive(target) else { + keyUp.postToPid(target.pid) + return false + } + keyUp.postToPid(target.pid) + return Self.isExactFocusTargetActive(target) } private func tryReliablePasteInsertion(_ text: String, preferredTargetPID: pid_t?) -> Bool { From 6cb9fe9979e7cff640e358b812781af25e53ce57 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:22:04 -0700 Subject: [PATCH 08/15] close spoken send review gaps --- Sources/Fluid/ContentView.swift | 17 ++++++---- Sources/Fluid/Services/SpokenSendParser.swift | 32 +++++++++++++++++++ .../SpokenSendTests.swift | 22 +++++++++++++ 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index b4aff0f2..2375cb75 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -1620,12 +1620,15 @@ struct ContentView: View { return (name: "Unknown", bundleId: "unknown", windowTitle: "") } - private func isSpokenSendBlockedApp( - _ appInfo: (name: String, bundleId: String, windowTitle: String) - ) -> Bool { - TerminalAppClassifier.isTerminal( - appName: appInfo.name, - bundleID: appInfo.bundleId + private func isSpokenSendBlockedTarget(_ target: TypingService.CapturedFocusTarget?) -> Bool { + guard let target, + let app = NSRunningApplication(processIdentifier: target.pid) + else { + return true + } + return TerminalAppClassifier.isTerminal( + appName: app.localizedName ?? "", + bundleID: app.bundleIdentifier ?? "" ) } @@ -2469,7 +2472,7 @@ struct ContentView: View { && aiFallbackReason == nil && (sendsExistingDraft || !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) && targetMatchesRecordingFocus - && !self.isSpokenSendBlockedApp(appInfo) + && !self.isSpokenSendBlockedTarget(self.recordingFocusTarget) // Dispatch insertion as soon as the destination app is ready; the // overlay hides asynchronously after output so it cannot delay paste. if typingTarget.shouldRestoreOriginalFocus { diff --git a/Sources/Fluid/Services/SpokenSendParser.swift b/Sources/Fluid/Services/SpokenSendParser.swift index 982e411c..1d6063ed 100644 --- a/Sources/Fluid/Services/SpokenSendParser.swift +++ b/Sources/Fluid/Services/SpokenSendParser.swift @@ -100,6 +100,11 @@ enum SpokenSendParser { } var commandPrefix = String(text[.. String { var polished = text.trimmingCharacters(in: .whitespacesAndNewlines) while let last = polished.last, [",", ";", ":", "-", "–", "—"].contains(last) { diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index 3ffcc545..2ec9f6f9 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -31,6 +31,28 @@ final class SpokenSendTests: XCTestCase { ) } + func testPairedDelimitersAroundTerminalPhraseAreRemoved() { + XCTAssertEqual( + SpokenSendParser.parse("Ready (send it)", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready “send it”", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse(#"Ready ["send it"]"#, phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + } + + func testClosingQuoteBeforeTerminalPhraseIsPreserved() { + XCTAssertEqual( + SpokenSendParser.parse(#"He said "Ready" send it"#, phrase: "send it", enabled: true), + SpokenSendParseResult(text: #"He said "Ready"."#, shouldSend: true) + ) + } + func testPhraseInMiddleDoesNotArmSend() { XCTAssertEqual( SpokenSendParser.parse("Send it when you are ready", phrase: "send it", enabled: true), From cbd502ee092efeb953b6810139039cb631ad80b8 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:33:35 -0700 Subject: [PATCH 09/15] fix spoken send lifecycle gaps --- Sources/Fluid/ContentView.swift | 63 +++++++++++++++---- ...RService+SpokenPunctuationFormatting.swift | 2 + Sources/Fluid/Services/SpokenSendParser.swift | 51 ++++++++++++--- Sources/Fluid/Services/TypingService.swift | 28 +++------ .../SpokenSendTests.swift | 27 ++++++++ 5 files changed, 132 insertions(+), 39 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 2375cb75..9c22919a 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -287,6 +287,9 @@ struct ContentView: View { @State private var spokenSendCountdownStartedAt: TimeInterval? @State private var spokenSendLastVoiceActivityAt: TimeInterval = 0 @State private var spokenSendVoiceActivityCancellable: AnyCancellable? + @State private var spokenSendVoiceActivityGeneration: UInt64 = 0 + @State private var activeSpokenSendVoiceActivityID: UInt64? + @State private var isStoppingAndProcessingTranscription = false private var isRecordingAnyShortcutCapture: Bool { self.activeShortcutRecordingTarget != nil @@ -2088,6 +2091,16 @@ struct ContentView: View { // MARK: - Stop and Process Transcription private func stopAndProcessTranscription(route: DictationOutputRoute = .normal) async { + guard !self.isStoppingAndProcessingTranscription else { + DebugLogger.shared.debug("Ignoring duplicate stop-and-process request", source: "ContentView") + return + } + self.isStoppingAndProcessingTranscription = true + defer { self.isStoppingAndProcessingTranscription = false } + await self.performStopAndProcessTranscription(route: route) + } + + private func performStopAndProcessTranscription(route: DictationOutputRoute) async { DebugLogger.shared.debug("stopAndProcessTranscription called", source: "ContentView") DebugLogger.shared.info("Output route selected: \(route.rawValue)", source: "ContentView") self.appBench("stop_path_enter route=\(route.rawValue)") @@ -2107,7 +2120,7 @@ struct ContentView: View { !wasCommandMode && !promptTest.isActive && !shouldUseAIOnStop && - !self.settings.spokenSendEnabled + !self.spokenSendIsPendingForCurrentUtterance var didRequestOverlayHideOnStop = false DebugLogger.shared.info( "Routing decision snapshot | activeMode=\(modeAtStop.rawValue) | rewrite=\(wasRewriteMode) | command=\(wasCommandMode) | overlay=\(NotchContentState.shared.mode.rawValue)", @@ -2491,7 +2504,7 @@ struct ContentView: View { targetPID: typingTarget.pid, textReadyAt: finalTextReadyAt ) - didTypeExternally = deliveryOutcome.didInsert + didTypeExternally = deliveryOutcome.didInsert || deliveryOutcome.didDispatchAction } else { self.asr.typeOutputPlanToActiveField( finalOutputPlan, @@ -2512,8 +2525,7 @@ struct ContentView: View { try? await Task.sleep(nanoseconds: 650_000_000) } } - NotchOverlayManager.shared.updateTranscriptionText("") - NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + self.clearSpokenSendStatusIfNeeded(requested: spokenSendRequested) if !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop { self.hideOverlayAfterOutput() } @@ -2608,6 +2620,22 @@ struct ContentView: View { NotchContentState.shared.setSpokenSendIndicatorState(shouldSend ? .detected : .hidden) } + private var spokenSendIsPendingForCurrentUtterance: Bool { + guard self.settings.spokenSendEnabled else { return false } + if self.spokenSendAutoStopTriggered { return true } + return SpokenSendParser.parse( + self.asr.partialTranscription, + phrase: self.settings.spokenSendPhrase, + enabled: true + ).shouldSend + } + + private func clearSpokenSendStatusIfNeeded(requested: Bool) { + guard requested else { return } + NotchOverlayManager.shared.updateTranscriptionText("") + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + private func advanceOverlayLifecycle() { self.spokenSendAutoStopTask?.cancel() self.spokenSendAutoStopTask = nil @@ -2656,7 +2684,7 @@ struct ContentView: View { let countdownStartedAt = ProcessInfo.processInfo.systemUptime self.spokenSendCountdownStartedAt = countdownStartedAt self.spokenSendLastVoiceActivityAt = countdownStartedAt - self.startSpokenSendVoiceActivityMonitoring() + let expectedVoiceActivityMonitoringID = self.startSpokenSendVoiceActivityMonitoring() let expectedCountdownID = NotchContentState.shared.beginSpokenSendCountdown() self.spokenSendAutoStopTask = Task { @MainActor in // Keep one countdown across harmless streaming refinements such as punctuation or casing. @@ -2678,11 +2706,11 @@ struct ContentView: View { quietDuration: quietDuration ) else { + self.stopSpokenSendVoiceActivityMonitoring(matching: expectedVoiceActivityMonitoringID) if self.overlayLifecycleID == expectedOverlayLifecycleID, NotchContentState.shared.spokenSendCountdownID == expectedCountdownID { self.spokenSendAutoStopTask = nil - self.stopSpokenSendVoiceActivityMonitoring() self.spokenSendCountdownStartedAt = nil if NotchContentState.shared.spokenSendIndicatorState == .countingDown { NotchContentState.shared.setSpokenSendIndicatorState(.hidden) @@ -2692,7 +2720,7 @@ struct ContentView: View { } self.spokenSendAutoStopTask = nil - self.stopSpokenSendVoiceActivityMonitoring() + self.stopSpokenSendVoiceActivityMonitoring(matching: expectedVoiceActivityMonitoringID) self.spokenSendCountdownStartedAt = nil self.spokenSendAutoStopTriggered = true NotchContentState.shared.setSpokenSendIndicatorState(.sending) @@ -2725,19 +2753,30 @@ struct ContentView: View { } } - private func startSpokenSendVoiceActivityMonitoring() { - guard self.spokenSendVoiceActivityCancellable == nil else { return } + private func startSpokenSendVoiceActivityMonitoring() -> UInt64 { + if let activeSpokenSendVoiceActivityID { + return activeSpokenSendVoiceActivityID + } + self.spokenSendVoiceActivityGeneration &+= 1 + let monitoringID = self.spokenSendVoiceActivityGeneration + self.activeSpokenSendVoiceActivityID = monitoringID self.spokenSendVoiceActivityCancellable = self.asr.audioLevelPublisher .filter(SpokenSendParser.isMeaningfulVoiceActivity) - .receive(on: RunLoop.main) - .sink { level in + .receive(on: DispatchQueue.main) + .sink { [monitoringID] level in + guard self.activeSpokenSendVoiceActivityID == monitoringID else { return } self.handleSpokenSendAudioLevel(level) } + return monitoringID } - private func stopSpokenSendVoiceActivityMonitoring() { + private func stopSpokenSendVoiceActivityMonitoring(matching monitoringID: UInt64? = nil) { + if let monitoringID, self.activeSpokenSendVoiceActivityID != monitoringID { + return + } self.spokenSendVoiceActivityCancellable?.cancel() self.spokenSendVoiceActivityCancellable = nil + self.activeSpokenSendVoiceActivityID = nil } private func hideOverlayAsync(reason: String) { diff --git a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift index b931b314..aa50b639 100644 --- a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift +++ b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift @@ -9,6 +9,8 @@ enum TerminalAppClassifier { "kitty", "alacritty", "wezterm", + "hyper", + "termius", ] static func isTerminal(appName: String?, bundleID: String?) -> Bool { diff --git a/Sources/Fluid/Services/SpokenSendParser.swift b/Sources/Fluid/Services/SpokenSendParser.swift index 1d6063ed..5c04ae56 100644 --- a/Sources/Fluid/Services/SpokenSendParser.swift +++ b/Sources/Fluid/Services/SpokenSendParser.swift @@ -11,6 +11,11 @@ enum SpokenSendParser { static let immediateStopRequiredSilenceDuration: TimeInterval = 0.35 static let immediateStopVoiceActivityGraceDuration: TimeInterval = 0.35 static let immediateStopVoiceActivityLevelThreshold: CGFloat = 0.12 + private static let regexCache: NSCache = { + let cache = NSCache() + cache.countLimit = 12 + return cache + }() static func shouldStopImmediately( _ text: String, @@ -69,8 +74,8 @@ enum SpokenSendParser { .joined(separator: #"\s+"#) let trailingPunctuation = #"[\s\p{P}]*$"# - if let literalRegex = try? NSRegularExpression( - pattern: #"(?i)(? String { + guard let regex = Self.regex( + #"(?i)(? NSRegularExpression? { + let key = pattern as NSString + if let cached = self.regexCache.object(forKey: key) { + return cached + } + guard let compiled = try? NSRegularExpression(pattern: pattern) else { + return nil + } + self.regexCache.setObject(compiled, forKey: key) + return compiled + } + private static func removePairedOpeningDelimiters( from text: inout String, closedBy commandSuffix: String diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index bae74801..125f235c 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -5,6 +5,7 @@ import Foundation final class TypingService { nonisolated static let synthesizedEventUserData: Int64 = 0x46565353 + nonisolated static let postInsertionModifierReleaseTimeout: TimeInterval = 2 struct CapturedFocusTarget { let pid: pid_t @@ -489,24 +490,6 @@ final class TypingService { } self.bench("settle_delay_done delayMs=\(settleDelayMs) elapsedMs=\(Self.elapsedMs(since: requestedAt))") let hasTextToInsert = !text.isEmpty - if postInsertionKey != nil { - guard let preferredTargetPID, let requiredFocusTarget else { - outcome = .actionSuppressed - return - } - let modifiersReleased = self.waitForPhysicalModifiersToRelease(timeout: 2) - let exactFocusIsActive = Self.isExactFocusTargetActive(requiredFocusTarget) - guard Self.canDispatchPostInsertionAction( - preferredTargetPID: preferredTargetPID, - requiredTargetPID: requiredFocusTarget.pid, - isSecureTextField: requiredFocusTarget.isSecureTextField, - modifiersReleased: modifiersReleased, - exactFocusIsActive: exactFocusIsActive - ) else { - outcome = .actionSuppressed - return - } - } if hasTextToInsert { self.log("[TypingService] Delay completed, calling insertTextInstantly") @@ -537,14 +520,19 @@ final class TypingService { } guard let postInsertionKey else { return } - guard let preferredTargetPID, let requiredFocusTarget else { return } + guard let preferredTargetPID, let requiredFocusTarget else { + outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + return + } usleep(50_000) guard Self.canDispatchPostInsertionAction( preferredTargetPID: preferredTargetPID, requiredTargetPID: requiredFocusTarget.pid, isSecureTextField: requiredFocusTarget.isSecureTextField, - modifiersReleased: self.waitForPhysicalModifiersToRelease(timeout: 0.2), + modifiersReleased: self.waitForPhysicalModifiersToRelease( + timeout: Self.postInsertionModifierReleaseTimeout + ), exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget), insertionConfirmed: !hasTextToInsert || outcome.didInsert ), diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index 2ec9f6f9..73ab43db 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -258,6 +258,31 @@ final class SpokenSendTests: XCTestCase { ) } + func testLiteralEscapeIsStrippedWhenPhraseIsNotTerminal() { + XCTAssertEqual( + SpokenSendParser.parse("I said literal send it to him", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "I said send it to him", shouldSend: false) + ) + } + + func testEarlierLiteralEscapeIsStrippedWhenFinalCommandSends() { + XCTAssertEqual( + SpokenSendParser.parse( + "I said literal send it to him and then send it", + phrase: "send it", + enabled: true + ), + SpokenSendParseResult(text: "I said send it to him and then.", shouldSend: true) + ) + } + + func testMidSentencePhraseWithoutLiteralMarkerIsUnchanged() { + XCTAssertEqual( + SpokenSendParser.parse("Please send it to Dana", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Please send it to Dana", shouldSend: false) + ) + } + func testPhraseOnlySubmitsExistingDraftWithoutInsertingCommand() { XCTAssertEqual( SpokenSendParser.parse("Send it", phrase: "send it", enabled: true), @@ -287,6 +312,8 @@ final class SpokenSendTests: XCTestCase { ("kitty", "net.kovidgoyal.kitty"), ("Alacritty", "org.alacritty"), ("WezTerm", "com.github.wez.wezterm"), + ("Hyper", "co.zeit.hyper"), + ("Termius", "com.termius-dmg.mac"), ] for (name, bundleID) in terminals { From 626c953bb2cf54d7fecb4a9cacec23994bfb2d32 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:40:37 -0700 Subject: [PATCH 10/15] tighten spoken send action safety --- ...RService+SpokenPunctuationFormatting.swift | 13 ++++ Sources/Fluid/Services/TypingService.swift | 77 +++++++++++++++++-- .../SpokenSendTests.swift | 26 +++++++ 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift index aa50b639..954cca83 100644 --- a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift +++ b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift @@ -12,6 +12,14 @@ enum TerminalAppClassifier { "hyper", "termius", ] + private static let contextMarkers = [ + "terminal", + "shell", + "ssh", + "console", + "xterm", + "hterm", + ] static func isTerminal(appName: String?, bundleID: String?) -> Bool { let identity = [appName, bundleID] @@ -19,6 +27,11 @@ enum TerminalAppClassifier { .joined(separator: " ") return self.identityMarkers.contains { identity.contains($0) } } + + static func isTerminalContext(labels: [String]) -> Bool { + let context = labels.joined(separator: " ").lowercased() + return self.contextMarkers.contains { context.contains($0) } + } } extension ASRService { diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 125f235c..bbfe24b7 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -49,6 +49,7 @@ final class TypingService { preferredTargetPID: pid_t?, requiredTargetPID: pid_t?, isSecureTextField: Bool, + isTerminalLikeContext: Bool = false, modifiersReleased: Bool, exactFocusIsActive: Bool, insertionConfirmed: Bool = true @@ -58,7 +59,11 @@ final class TypingService { else { return false } - return !isSecureTextField && modifiersReleased && exactFocusIsActive && insertionConfirmed + return !isSecureTextField && + !isTerminalLikeContext && + modifiersReleased && + exactFocusIsActive && + insertionConfirmed } // Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 @@ -530,11 +535,14 @@ final class TypingService { preferredTargetPID: preferredTargetPID, requiredTargetPID: requiredFocusTarget.pid, isSecureTextField: requiredFocusTarget.isSecureTextField, + isTerminalLikeContext: Self.hasTerminalLikeContext(requiredFocusTarget), modifiersReleased: self.waitForPhysicalModifiersToRelease( timeout: Self.postInsertionModifierReleaseTimeout ), exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget), - insertionConfirmed: !hasTextToInsert || outcome.didInsert + insertionConfirmed: hasTextToInsert + ? outcome.didInsert + : Self.hasNonemptyEditableDraft(requiredFocusTarget) ), self.postReturnKey(postInsertionKey, target: requiredFocusTarget) else { @@ -755,12 +763,8 @@ final class TypingService { guard Self.isExactFocusTargetActive(target) else { return false } keyDown.postToPid(target.pid) usleep(10_000) - guard Self.isExactFocusTargetActive(target) else { - keyUp.postToPid(target.pid) - return false - } keyUp.postToPid(target.pid) - return Self.isExactFocusTargetActive(target) + return true } private func tryReliablePasteInsertion(_ text: String, preferredTargetPID: pid_t?) -> Bool { @@ -814,6 +818,65 @@ final class TypingService { return value as? String } + private static func accessibilityContextLabels( + for element: AXUIElement, + window: AXUIElement? + ) -> [String] { + let attributes = [ + kAXRoleDescriptionAttribute as CFString, + kAXDescriptionAttribute as CFString, + kAXTitleAttribute as CFString, + kAXIdentifierAttribute as CFString, + kAXHelpAttribute as CFString, + ] + var labels: [String] = [] + var current: AXUIElement? = element + for _ in 0..<6 { + guard let currentElement = current else { break } + labels.append(contentsOf: attributes.compactMap { + Self.stringAXAttribute(from: currentElement, attribute: $0) + }) + if let window, CFEqual(currentElement, window) { + break + } + current = Self.copyAXElementAttribute( + from: currentElement, + attribute: kAXParentAttribute as CFString + ) + } + if let window { + labels.append(contentsOf: attributes.compactMap { + Self.stringAXAttribute(from: window, attribute: $0) + }) + } + return labels + } + + private static func hasNonemptyEditableDraft(_ target: CapturedFocusTarget) -> Bool { + let editableRoles = ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox"] + guard self.isExactFocusTargetActive(target), + !target.isSecureTextField, + let role = self.stringAXAttribute( + from: target.element, + attribute: kAXRoleAttribute as CFString + ), + editableRoles.contains(role), + let value = self.stringAXAttribute( + from: target.element, + attribute: kAXValueAttribute as CFString + ) + else { + return false + } + return !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + + private static func hasTerminalLikeContext(_ target: CapturedFocusTarget) -> Bool { + TerminalAppClassifier.isTerminalContext( + labels: self.accessibilityContextLabels(for: target.element, window: target.window) + ) + } + private static func currentFocusDebugDescription() -> String { let systemWideElement = AXUIElementCreateSystemWide() var focusedElementRef: CFTypeRef? diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index 73ab43db..ea820833 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -339,6 +339,22 @@ final class SpokenSendTests: XCTestCase { ) } + func testTerminalClassifierRecognizesHostedTerminalContext() { + XCTAssertTrue( + TerminalAppClassifier.isTerminalContext( + labels: ["Visual Studio Code", "Integrated Terminal", "AXTextArea"] + ) + ) + XCTAssertTrue( + TerminalAppClassifier.isTerminalContext(labels: ["Web SSH Console"]) + ) + XCTAssertFalse( + TerminalAppClassifier.isTerminalContext( + labels: ["Visual Studio Code", "Source Editor", "AXTextArea"] + ) + ) + } + func testGeneratedSendCommandsAreExcludedFromFluidVoiceHotkeys() throws { for key in SettingsStore.SpokenSendKey.allCases { let event = try XCTUnwrap( @@ -398,6 +414,16 @@ final class SpokenSendTests: XCTestCase { exactFocusIsActive: true ) ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + isTerminalLikeContext: true, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) XCTAssertFalse( TypingService.canDispatchPostInsertionAction( preferredTargetPID: 42, From 5fae6f120e7b68e62cdd6ee80fa45a9547d24b82 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 17:52:54 -0700 Subject: [PATCH 11/15] preserve drafts during spoken send --- Sources/Fluid/Services/TypingService.swift | 58 +++++++++++++++++-- .../SpokenSendTests.swift | 26 +++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index bbfe24b7..b3a336b9 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -66,6 +66,27 @@ final class TypingService { insertionConfirmed } + nonisolated static func valueByInserting( + _ insertion: String, + into original: String, + selectedRange: CFRange + ) -> String? { + let originalValue = original as NSString + guard selectedRange.location >= 0, + selectedRange.length >= 0, + selectedRange.location <= originalValue.length, + selectedRange.length <= originalValue.length - selectedRange.location + else { + return nil + } + let value = NSMutableString(string: original) + value.replaceCharacters( + in: NSRange(location: selectedRange.location, length: selectedRange.length), + with: insertion + ) + return value as String + } + // Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 // or UserDefaults bool for key "enableTypingLogs". private static var isLoggingEnabled: Bool { @@ -717,19 +738,44 @@ final class TypingService { let snapshot = self.captureFocusedTextSnapshot(), snapshot.pid == target.pid, CFEqual(snapshot.element, target.element), - self.tryAllTextInsertionMethods(target.element, text), + let originalValue = snapshot.value, + let selectedRange = snapshot.selectedRange, + let expectedValue = Self.valueByInserting( + text, + into: originalValue, + selectedRange: selectedRange + ), + self.insertTextAtCursorUsingSelectedRange(target.element, text), Self.isExactFocusTargetActive(target) else { return false } - let verification = self.waitForFocusedTextVerification( - from: snapshot, - expectedText: text, + let verified = self.waitForExactTargetValue( + expectedValue, + target: target, timeoutMicros: 500_000 ) - self.log("[TypingService] Exact-target Accessibility verification: \(verification.rawValue)") - return verification.isConfirmed + self.log("[TypingService] Exact-target preserving insertion verified: \(verified)") + return verified + } + + private func waitForExactTargetValue( + _ expectedValue: String, + target: CapturedFocusTarget, + timeoutMicros: useconds_t + ) -> Bool { + let pollMicros: useconds_t = 50_000 + var waited: useconds_t = 0 + while waited < timeoutMicros { + usleep(pollMicros) + waited += pollMicros + guard Self.isExactFocusTargetActive(target) else { continue } + if self.getElementStringValue(target.element) == expectedValue { + return true + } + } + return false } private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) -> Bool { diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index ea820833..ba42d95a 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -463,6 +463,32 @@ final class SpokenSendTests: XCTestCase { XCTAssertFalse(TypingService.DeliveryOutcome.insertedActionSuppressed.didDispatchAction) } + func testExactTargetInsertionPreservesUnselectedDraftText() { + XCTAssertEqual( + TypingService.valueByInserting( + "new ", + into: "existing draft", + selectedRange: CFRange(location: 9, length: 0) + ), + "existing new draft" + ) + XCTAssertEqual( + TypingService.valueByInserting( + "replacement", + into: "keep old keep", + selectedRange: CFRange(location: 5, length: 3) + ), + "keep replacement keep" + ) + XCTAssertNil( + TypingService.valueByInserting( + "unsafe", + into: "draft", + selectedRange: CFRange(location: 99, length: 0) + ) + ) + } + func testOverlayIndicatorVisibilityCoversEveryActiveState() { XCTAssertFalse(SpokenSendIndicatorState.hidden.isVisible) XCTAssertTrue(SpokenSendIndicatorState.detected.isVisible) From 3db21a0f5dcbf784fa7130412b0d54534fe8d46a Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 18:06:51 -0700 Subject: [PATCH 12/15] revalidate spoken send before dispatch --- ...RService+SpokenPunctuationFormatting.swift | 1 + Sources/Fluid/Services/TypingService.swift | 72 +++++++++++++------ 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift index 954cca83..3fbde1d6 100644 --- a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift +++ b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift @@ -65,6 +65,7 @@ private enum SpokenPunctuationFormatter { .compactMap { $0?.lowercased() } .joined(separator: " ") return TerminalAppClassifier.isTerminal(appName: self.appName, bundleID: self.bundleID) || + TerminalAppClassifier.isTerminalContext(labels: [self.windowTitle ?? ""]) || haystack.contains("codex") || haystack.contains("chatgpt") || haystack.contains("claude") || diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index b3a336b9..6a6203d5 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -516,6 +516,7 @@ final class TypingService { } self.bench("settle_delay_done delayMs=\(settleDelayMs) elapsedMs=\(Self.elapsedMs(since: requestedAt))") let hasTextToInsert = !text.isEmpty + var expectedInsertedValue: String? if hasTextToInsert { self.log("[TypingService] Delay completed, calling insertTextInstantly") @@ -524,7 +525,8 @@ final class TypingService { let inserted = self.insertTextInstantly( text, preferredTargetPID: preferredTargetPID, - requiredFocusTarget: postInsertionKey == nil ? nil : requiredFocusTarget + requiredFocusTarget: postInsertionKey == nil ? nil : requiredFocusTarget, + expectedExactTargetValue: &expectedInsertedValue ) self.bench( "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" @@ -552,20 +554,30 @@ final class TypingService { } usleep(50_000) + let modifiersReleased = self.waitForPhysicalModifiersToRelease( + timeout: Self.postInsertionModifierReleaseTimeout + ) + let expectedActionValue: String? = if hasTextToInsert { + expectedInsertedValue + } else { + Self.nonemptyEditableDraftValue(requiredFocusTarget) + } + let insertionConfirmed = expectedActionValue != nil && + self.getElementStringValue(requiredFocusTarget.element) == expectedActionValue guard Self.canDispatchPostInsertionAction( preferredTargetPID: preferredTargetPID, requiredTargetPID: requiredFocusTarget.pid, isSecureTextField: requiredFocusTarget.isSecureTextField, isTerminalLikeContext: Self.hasTerminalLikeContext(requiredFocusTarget), - modifiersReleased: self.waitForPhysicalModifiersToRelease( - timeout: Self.postInsertionModifierReleaseTimeout - ), + modifiersReleased: modifiersReleased, exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget), - insertionConfirmed: hasTextToInsert - ? outcome.didInsert - : Self.hasNonemptyEditableDraft(requiredFocusTarget) + insertionConfirmed: insertionConfirmed ), - self.postReturnKey(postInsertionKey, target: requiredFocusTarget) + self.postReturnKey( + postInsertionKey, + target: requiredFocusTarget, + expectedValue: expectedActionValue + ) else { outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed return @@ -591,7 +603,8 @@ final class TypingService { private func insertTextInstantly( _ text: String, preferredTargetPID: pid_t?, - requiredFocusTarget: CapturedFocusTarget? + requiredFocusTarget: CapturedFocusTarget?, + expectedExactTargetValue: inout String? ) -> Bool { self.log("[TypingService] insertTextInstantly called with \(text.count) characters") self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"") @@ -600,7 +613,8 @@ final class TypingService { return self.insertTextIntoExactFocusTarget( text, preferredTargetPID: preferredTargetPID, - requiredFocusTarget: requiredFocusTarget + requiredFocusTarget: requiredFocusTarget, + expectedValue: &expectedExactTargetValue ) } @@ -714,7 +728,8 @@ final class TypingService { private func insertTextIntoExactFocusTarget( _ text: String, preferredTargetPID: pid_t?, - requiredFocusTarget: CapturedFocusTarget + requiredFocusTarget: CapturedFocusTarget, + expectedValue: inout String? ) -> Bool { guard preferredTargetPID == requiredFocusTarget.pid, !requiredFocusTarget.isSecureTextField, @@ -726,13 +741,15 @@ final class TypingService { return self.performVerifiedExactTargetAccessibilityInsertion( text, - target: requiredFocusTarget + target: requiredFocusTarget, + expectedValue: &expectedValue ) } private func performVerifiedExactTargetAccessibilityInsertion( _ text: String, - target: CapturedFocusTarget + target: CapturedFocusTarget, + expectedValue: inout String? ) -> Bool { guard Self.isExactFocusTargetActive(target), let snapshot = self.captureFocusedTextSnapshot(), @@ -740,7 +757,7 @@ final class TypingService { CFEqual(snapshot.element, target.element), let originalValue = snapshot.value, let selectedRange = snapshot.selectedRange, - let expectedValue = Self.valueByInserting( + let expectedValueAfterInsertion = Self.valueByInserting( text, into: originalValue, selectedRange: selectedRange @@ -752,11 +769,14 @@ final class TypingService { } let verified = self.waitForExactTargetValue( - expectedValue, + expectedValueAfterInsertion, target: target, timeoutMicros: 500_000 ) self.log("[TypingService] Exact-target preserving insertion verified: \(verified)") + if verified { + expectedValue = expectedValueAfterInsertion + } return verified } @@ -792,9 +812,15 @@ final class TypingService { private func postReturnKey( _ key: SettingsStore.SpokenSendKey, - target: CapturedFocusTarget + target: CapturedFocusTarget, + expectedValue: String? ) -> Bool { - guard Self.isExactFocusTargetActive(target) else { return false } + guard let expectedValue, + Self.isExactFocusTargetActive(target), + self.getElementStringValue(target.element) == expectedValue + else { + return false + } let returnKeyCode = CGKeyCode(kVK_Return) guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: true), let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: false) @@ -806,7 +832,11 @@ final class TypingService { keyUp.flags = key.eventFlags keyDown.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) keyUp.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) - guard Self.isExactFocusTargetActive(target) else { return false } + guard Self.isExactFocusTargetActive(target), + self.getElementStringValue(target.element) == expectedValue + else { + return false + } keyDown.postToPid(target.pid) usleep(10_000) keyUp.postToPid(target.pid) @@ -898,7 +928,7 @@ final class TypingService { return labels } - private static func hasNonemptyEditableDraft(_ target: CapturedFocusTarget) -> Bool { + private static func nonemptyEditableDraftValue(_ target: CapturedFocusTarget) -> String? { let editableRoles = ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox"] guard self.isExactFocusTargetActive(target), !target.isSecureTextField, @@ -912,9 +942,9 @@ final class TypingService { attribute: kAXValueAttribute as CFString ) else { - return false + return nil } - return !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + return value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : value } private static func hasTerminalLikeContext(_ target: CapturedFocusTarget) -> Bool { From ffb7df0d08f456f9c7d24958feeba95b628c8761 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 18:40:37 -0700 Subject: [PATCH 13/15] fix: remove dictation latency regressions --- Sources/Fluid/ContentView.swift | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 9c22919a..0265ea0c 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -2157,6 +2157,9 @@ struct ContentView: View { TranscriptionSoundPlayer.shared.playStopSound() }) self.appBench("asr_stop_return elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - asrStopStartedAt) * 1000).rounded()))") + // The duplicate-stop race ends when ASR reports that it is no longer running. + // Do not hold this guard across delivery or a new recording's stop can be swallowed. + self.isStoppingAndProcessingTranscription = false let audioSnapshot = self.asr.consumeLastCompletedAudioSnapshot() DebugLogger.shared.info( "Stop transcription result | chars=\(transcribedText.count) | empty=\(transcribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)", @@ -2489,7 +2492,7 @@ struct ContentView: View { // Dispatch insertion as soon as the destination app is ready; the // overlay hides asynchronously after output so it cannot delay paste. if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + await self.restoreFocusToRecordingTarget(requireExactTarget: spokenSendAllowed) } if spokenSendAllowed { NotchContentState.shared.setSpokenSendIndicatorState(.sending) @@ -3455,12 +3458,17 @@ struct ContentView: View { /// Best-effort: re-activate the app that was focused when recording started. /// Skips the AX restore work when the captured text element is already focused. - private func restoreFocusToRecordingTarget() async { + private func restoreFocusToRecordingTarget(requireExactTarget: Bool = false) async { guard let pid = NotchContentState.shared.recordingTargetPID else { return } let startedAt = ProcessInfo.processInfo.systemUptime self.appBench("focus_restore_start targetPID=\(pid)") if let focusTarget = self.recordingFocusTarget, focusTarget.pid == pid { - if TypingService.isExactFocusTargetActive(focusTarget) { + // Ordinary dictation keeps the legacy PID/editable-focus check because some apps + // vend a fresh AX element for the same field. Spoken Send remains exact-target only. + let targetIsStillActive = requireExactTarget + ? TypingService.isExactFocusTargetActive(focusTarget) + : TypingService.isCapturedFocusStillActive(for: pid) + if targetIsStillActive { self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") return } @@ -4051,13 +4059,6 @@ extension ContentView { return } self.advanceOverlayLifecycle() - if self.asr.micStatus == .authorized { - self.appBench("overlay_mode_request mode=Dictation") - self.menuBarManager.setOverlayMode(.dictation) - self.menuBarManager.showRecordingOverlayImmediately() - self.appBench("overlay_mode_requested mode=Dictation") - self.appBench("overlay_phase phase=connecting") - } Task { let asrStartStartedAt = ProcessInfo.processInfo.systemUptime DebugLogger.shared.benchmark("APP_BENCH", message: "asr_start_call", source: "AppBenchmark") @@ -4067,6 +4068,12 @@ extension ContentView { } self.captureRecordingContext() self.prewarmPrivateAIDictationIfNeeded(for: slot) + // Capture owns the critical path. Showing the overlay before asr.start() + // previously blocked the main actor for 58-87 ms before first PCM. + self.appBench("overlay_mode_request mode=Dictation") + self.menuBarManager.setOverlayMode(.dictation) + self.menuBarManager.showRecordingOverlayImmediately() + self.appBench("overlay_mode_requested mode=Dictation") self.appBench("overlay_phase phase=recording trigger=first_pcm") }) if startOutcome == .failed { From b15d5e19a4674b90c485e64281bc4a01b70016c6 Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 19:20:03 -0700 Subject: [PATCH 14/15] snapshot spoken send draft before wait --- Sources/Fluid/Services/TypingService.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index 6a6203d5..d0e4ec7b 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -553,6 +553,9 @@ final class TypingService { return } + let capturedActionOnlyDraftValue = hasTextToInsert + ? nil + : Self.nonemptyEditableDraftValue(requiredFocusTarget) usleep(50_000) let modifiersReleased = self.waitForPhysicalModifiersToRelease( timeout: Self.postInsertionModifierReleaseTimeout @@ -560,7 +563,7 @@ final class TypingService { let expectedActionValue: String? = if hasTextToInsert { expectedInsertedValue } else { - Self.nonemptyEditableDraftValue(requiredFocusTarget) + capturedActionOnlyDraftValue } let insertionConfirmed = expectedActionValue != nil && self.getElementStringValue(requiredFocusTarget.element) == expectedActionValue From 457cbcad55eec68d9dfa7b8e9189ac0babbc527f Mon Sep 17 00:00:00 2001 From: altic-dev Date: Mon, 10 Aug 2026 20:22:40 -0700 Subject: [PATCH 15/15] isolate spoken send delivery sessions --- Sources/Fluid/ContentView.swift | 113 +++++++++++++++--- .../SpokenSendTests.swift | 35 ++++++ 2 files changed, 132 insertions(+), 16 deletions(-) diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index 0265ea0c..077d1238 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -244,6 +244,7 @@ struct ContentView: View { @State private var recordingAppInfo: (name: String, bundleId: String, windowTitle: String)? = nil @State private var recordingPrecedingText: String = "" @State private var recordingFocusTarget: TypingService.CapturedFocusTarget? = nil + @State private var recordingSessionID = UUID() // Command Mode State // @State private var showCommandMode: Bool = false @@ -290,6 +291,7 @@ struct ContentView: View { @State private var spokenSendVoiceActivityGeneration: UInt64 = 0 @State private var activeSpokenSendVoiceActivityID: UInt64? @State private var isStoppingAndProcessingTranscription = false + @State private var stopProcessingOperationID: UUID? private var isRecordingAnyShortcutCapture: Bool { self.activeShortcutRecordingTarget != nil @@ -1623,6 +1625,20 @@ struct ContentView: View { return (name: "Unknown", bundleId: "unknown", windowTitle: "") } + nonisolated static func canDeliverCompletedRecording( + stoppedSessionID: UUID, + currentSessionID: UUID + ) -> Bool { + stoppedSessionID == currentSessionID + } + + nonisolated static func canFinishStopProcessingOperation( + completingOperationID: UUID, + currentOperationID: UUID? + ) -> Bool { + completingOperationID == currentOperationID + } + private func isSpokenSendBlockedTarget(_ target: TypingService.CapturedFocusTarget?) -> Bool { guard let target, let app = NSRunningApplication(processIdentifier: target.pid) @@ -1638,7 +1654,8 @@ struct ContentView: View { private func deliverSpokenSend( _ outputPlan: DictationLiteralOutputPlan, targetPID: pid_t?, - textReadyAt: TimeInterval + textReadyAt: TimeInterval, + requiredFocusTarget: TypingService.CapturedFocusTarget? ) async -> TypingService.DeliveryOutcome { let sendsExistingDraft = outputPlan.plainText.isEmpty let outcome = await self.asr.typeOutputPlanToActiveFieldAndWait( @@ -1646,7 +1663,7 @@ struct ContentView: View { preferredTargetPID: targetPID, textReadyAt: textReadyAt, postInsertionKey: self.settings.spokenSendKey, - requiredFocusTarget: self.recordingFocusTarget + requiredFocusTarget: requiredFocusTarget ) if outcome.didDispatchAction { NotchContentState.shared.setSpokenSendIndicatorState(.sent) @@ -1685,6 +1702,7 @@ struct ContentView: View { private func captureRecordingTargetContext() { // Capture the focused target PID BEFORE any overlay/UI changes. // Used to restore focus when the user interacts with overlay dropdowns. + self.recordingSessionID = UUID() let focusTarget = TypingService.captureSystemFocusTarget() self.recordingFocusTarget = focusTarget let focusedPID = focusTarget?.pid @@ -1719,7 +1737,12 @@ struct ContentView: View { } private func resolveTypingTargetPID() -> (pid: pid_t?, shouldRestoreOriginalFocus: Bool) { - let originalPID = NotchContentState.shared.recordingTargetPID + self.resolveTypingTargetPID(originalPID: NotchContentState.shared.recordingTargetPID) + } + + private func resolveTypingTargetPID( + originalPID: pid_t? + ) -> (pid: pid_t?, shouldRestoreOriginalFocus: Bool) { let currentFocusedPID = TypingService.captureSystemFocusedPID() ?? NSWorkspace.shared.frontmostApplication?.processIdentifier @@ -2095,16 +2118,28 @@ struct ContentView: View { DebugLogger.shared.debug("Ignoring duplicate stop-and-process request", source: "ContentView") return } + let operationID = UUID() + self.stopProcessingOperationID = operationID self.isStoppingAndProcessingTranscription = true - defer { self.isStoppingAndProcessingTranscription = false } - await self.performStopAndProcessTranscription(route: route) + defer { self.finishStopProcessingOperationIfCurrent(operationID) } + await self.performStopAndProcessTranscription(route: route, operationID: operationID) } - private func performStopAndProcessTranscription(route: DictationOutputRoute) async { + // swiftlint:disable:next function_body_length + private func performStopAndProcessTranscription( + route: DictationOutputRoute, + operationID: UUID + ) async { DebugLogger.shared.debug("stopAndProcessTranscription called", source: "ContentView") DebugLogger.shared.info("Output route selected: \(route.rawValue)", source: "ContentView") self.appBench("stop_path_enter route=\(route.rawValue)") + let stoppedSessionID = self.recordingSessionID + let stoppedFocusTarget = self.recordingFocusTarget + let stoppedTargetPID = NotchContentState.shared.recordingTargetPID + let stoppedAppInfo = self.recordingAppInfo + let stoppedPrecedingText = self.recordingPrecedingText + // Check if we're in rewrite or command mode let modeAtStop = self.activeRecordingMode let wasRewriteMode = modeAtStop == .edit || self.isRecordingForRewrite @@ -2159,7 +2194,7 @@ struct ContentView: View { self.appBench("asr_stop_return elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - asrStopStartedAt) * 1000).rounded()))") // The duplicate-stop race ends when ASR reports that it is no longer running. // Do not hold this guard across delivery or a new recording's stop can be swallowed. - self.isStoppingAndProcessingTranscription = false + self.finishStopProcessingOperationIfCurrent(operationID) let audioSnapshot = self.asr.consumeLastCompletedAudioSnapshot() DebugLogger.shared.info( "Stop transcription result | chars=\(transcribedText.count) | empty=\(transcribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)", @@ -2252,7 +2287,7 @@ struct ContentView: View { var finalText: String var aiFallbackReason: String? var postProcessingModel: String? - let appInfo = self.recordingAppInfo ?? self.getCurrentAppInfo() + let appInfo = stoppedAppInfo ?? self.getCurrentAppInfo() let punctuationFormattedText = ASRService.applySpokenPunctuationFormatting( transcribedText, appName: appInfo.name, @@ -2369,13 +2404,23 @@ struct ContentView: View { finalText = ASRService.applyGAAVFormatting(finalText) // Apply Continuous Dictation Mode after GAAV so smart caps use the field // context captured at recording start, and the trailing space enables chaining. - finalText = ASRService.applyContinuousDictationFormatting(finalText, precedingText: self.recordingPrecedingText) + finalText = ASRService.applyContinuousDictationFormatting(finalText, precedingText: stoppedPrecedingText) finalText = ASRService.applyTerminalLiteralAutocompleteSpacing( finalText, appName: appInfo.name, bundleID: appInfo.bundleId, windowTitle: appInfo.windowTitle ) + guard Self.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: self.recordingSessionID + ) else { + DebugLogger.shared.warning( + "Completed dictation output suppressed because a newer recording session started", + source: "ContentView" + ) + return + } self.recordingPrecedingText = "" self.asr.finalText = finalText if route == .onboardingSandbox, @@ -2480,19 +2525,33 @@ struct ContentView: View { ) if shouldTypeExternally { - let typingTarget = self.resolveTypingTargetPID() let spokenSendRequested = spokenSendParse.shouldSend + let typingTarget = self.resolveTypingTargetPID(originalPID: stoppedTargetPID) let targetMatchesRecordingFocus = typingTarget.pid != nil - && typingTarget.pid == self.recordingFocusTarget?.pid + && typingTarget.pid == stoppedFocusTarget?.pid let spokenSendAllowed = spokenSendRequested && aiFallbackReason == nil && (sendsExistingDraft || !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) && targetMatchesRecordingFocus - && !self.isSpokenSendBlockedTarget(self.recordingFocusTarget) + && !self.isSpokenSendBlockedTarget(stoppedFocusTarget) // Dispatch insertion as soon as the destination app is ready; the // overlay hides asynchronously after output so it cannot delay paste. if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget(requireExactTarget: spokenSendAllowed) + await self.restoreFocusToRecordingTarget( + requireExactTarget: spokenSendAllowed, + targetPID: stoppedTargetPID, + focusTarget: stoppedFocusTarget + ) + } + guard Self.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: self.recordingSessionID + ) else { + DebugLogger.shared.warning( + "Dictation delivery suppressed because a newer recording session started", + source: "ContentView" + ) + return } if spokenSendAllowed { NotchContentState.shared.setSpokenSendIndicatorState(.sending) @@ -2505,7 +2564,8 @@ struct ContentView: View { let deliveryOutcome = await self.deliverSpokenSend( finalOutputPlan, targetPID: typingTarget.pid, - textReadyAt: finalTextReadyAt + textReadyAt: finalTextReadyAt, + requiredFocusTarget: stoppedFocusTarget ) didTypeExternally = deliveryOutcome.didInsert || deliveryOutcome.didDispatchAction } else { @@ -2575,6 +2635,15 @@ struct ContentView: View { } } + private func finishStopProcessingOperationIfCurrent(_ operationID: UUID) { + guard Self.canFinishStopProcessingOperation( + completingOperationID: operationID, + currentOperationID: self.stopProcessingOperationID + ) else { return } + self.stopProcessingOperationID = nil + self.isStoppingAndProcessingTranscription = false + } + private func hideOverlayAfterOutput() { self.hideOverlayAsync(reason: "after_output") } @@ -3459,10 +3528,22 @@ struct ContentView: View { /// Best-effort: re-activate the app that was focused when recording started. /// Skips the AX restore work when the captured text element is already focused. private func restoreFocusToRecordingTarget(requireExactTarget: Bool = false) async { - guard let pid = NotchContentState.shared.recordingTargetPID else { return } + await self.restoreFocusToRecordingTarget( + requireExactTarget: requireExactTarget, + targetPID: NotchContentState.shared.recordingTargetPID, + focusTarget: self.recordingFocusTarget + ) + } + + private func restoreFocusToRecordingTarget( + requireExactTarget: Bool, + targetPID: pid_t?, + focusTarget: TypingService.CapturedFocusTarget? + ) async { + guard let pid = targetPID else { return } let startedAt = ProcessInfo.processInfo.systemUptime self.appBench("focus_restore_start targetPID=\(pid)") - if let focusTarget = self.recordingFocusTarget, focusTarget.pid == pid { + if let focusTarget, focusTarget.pid == pid { // Ordinary dictation keeps the legacy PID/editable-focus check because some apps // vend a fresh AX element for the same field. Spoken Send remains exact-target only. let targetIsStillActive = requireExactTarget diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift index ba42d95a..95165fea 100644 --- a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -463,6 +463,41 @@ final class SpokenSendTests: XCTestCase { XCTAssertFalse(TypingService.DeliveryOutcome.insertedActionSuppressed.didDispatchAction) } + func testCompletedRecordingIsSuppressedAfterANewerSessionStarts() { + let stoppedSessionID = UUID() + + XCTAssertTrue( + ContentView.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: stoppedSessionID + ) + ) + XCTAssertFalse( + ContentView.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: UUID() + ) + ) + } + + func testOlderStopOperationCannotClearNewerStopGuard() { + let olderOperationID = UUID() + let newerOperationID = UUID() + + XCTAssertTrue( + ContentView.canFinishStopProcessingOperation( + completingOperationID: newerOperationID, + currentOperationID: newerOperationID + ) + ) + XCTAssertFalse( + ContentView.canFinishStopProcessingOperation( + completingOperationID: olderOperationID, + currentOperationID: newerOperationID + ) + ) + } + func testExactTargetInsertionPreservesUnselectedDraftText() { XCTAssertEqual( TypingService.valueByInserting(