diff --git a/Fluid.xcodeproj/project.pbxproj b/Fluid.xcodeproj/project.pbxproj index f281dd4c..407c9464 100644 --- a/Fluid.xcodeproj/project.pbxproj +++ b/Fluid.xcodeproj/project.pbxproj @@ -22,6 +22,7 @@ C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */; }; DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */; }; 7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */; }; + B51800000000000000000002 /* SpokenSendTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = B51800000000000000000001 /* SpokenSendTests.swift */; }; 7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; }; 7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; }; 7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; }; @@ -59,6 +60,7 @@ D1A600000000000000000201 /* SpeakerTurnMergingTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpeakerTurnMergingTests.swift; sourceTree = ""; }; C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioEngineRetirementDrainTests.swift; sourceTree = ""; }; DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectAudioReliabilityTests.swift; sourceTree = ""; }; + B51800000000000000000001 /* SpokenSendTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SpokenSendTests.swift; sourceTree = ""; }; 7C078D8F2E3B339200FB7CAC /* FluidVoice Debug.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "FluidVoice Debug.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 7C91B0022F42AA0100C0DEF0 /* HotkeyShortcutTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HotkeyShortcutTests.swift; sourceTree = ""; }; 7CDB0A202F3C4D5600FB7CAD /* FluidDictationIntegrationTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FluidDictationIntegrationTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -141,6 +143,7 @@ C0DE63600000000000000001 /* AudioEngineRetirementDrainTests.swift */, DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */, 7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */, + B51800000000000000000001 /* SpokenSendTests.swift */, ); path = FluidDictationIntegrationTests; sourceTree = ""; @@ -303,6 +306,7 @@ C0DE63600000000000000002 /* AudioEngineRetirementDrainTests.swift in Sources */, DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */, 7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */, + B51800000000000000000002 /* SpokenSendTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/Sources/Fluid/ContentView.swift b/Sources/Fluid/ContentView.swift index a3cacb21..077d1238 100644 --- a/Sources/Fluid/ContentView.swift +++ b/Sources/Fluid/ContentView.swift @@ -243,6 +243,8 @@ struct ContentView: View { @State private var playgroundUsed: Bool = SettingsStore.shared.playgroundUsed @State private var recordingAppInfo: (name: String, bundleId: String, windowTitle: String)? = nil @State private var recordingPrecedingText: String = "" + @State private var recordingFocusTarget: TypingService.CapturedFocusTarget? = nil + @State private var recordingSessionID = UUID() // Command Mode State // @State private var showCommandMode: Bool = false @@ -280,6 +282,16 @@ struct ContentView: View { @State private var accessibilityGuideRequestID: UUID? @State private var prewarmDictationTask: Task? @State private var overlayLifecycleID: UInt64 = 0 + @State private var spokenSendAutoStopTask: Task? + @State private var spokenSendAutoStopTriggered = false + @State private var spokenSendPartialRevision: UInt64 = 0 + @State private var spokenSendCountdownStartedAt: TimeInterval? + @State private var spokenSendLastVoiceActivityAt: TimeInterval = 0 + @State private var spokenSendVoiceActivityCancellable: AnyCancellable? + @State private var spokenSendVoiceActivityGeneration: UInt64 = 0 + @State private var activeSpokenSendVoiceActivityID: UInt64? + @State private var isStoppingAndProcessingTranscription = false + @State private var stopProcessingOperationID: UUID? private var isRecordingAnyShortcutCapture: Bool { self.activeShortcutRecordingTarget != nil @@ -353,6 +365,9 @@ struct ContentView: View { .onReceive(NotificationCenter.default.publisher(for: .settingsBackupDidRestore)) { _ in self.reloadSettingsStateAfterBackupRestore() } + .onReceive(self.asr.$partialTranscription) { text in + self.handleSpokenSendPartialTranscription(text) + } .toolbar { if !self.settings.shouldShowOnboarding { ToolbarItemGroup(placement: .primaryAction) { @@ -1610,6 +1625,65 @@ struct ContentView: View { return (name: "Unknown", bundleId: "unknown", windowTitle: "") } + nonisolated static func canDeliverCompletedRecording( + stoppedSessionID: UUID, + currentSessionID: UUID + ) -> Bool { + stoppedSessionID == currentSessionID + } + + nonisolated static func canFinishStopProcessingOperation( + completingOperationID: UUID, + currentOperationID: UUID? + ) -> Bool { + completingOperationID == currentOperationID + } + + private func isSpokenSendBlockedTarget(_ target: TypingService.CapturedFocusTarget?) -> Bool { + guard let target, + let app = NSRunningApplication(processIdentifier: target.pid) + else { + return true + } + return TerminalAppClassifier.isTerminal( + appName: app.localizedName ?? "", + bundleID: app.bundleIdentifier ?? "" + ) + } + + private func deliverSpokenSend( + _ outputPlan: DictationLiteralOutputPlan, + targetPID: pid_t?, + textReadyAt: TimeInterval, + requiredFocusTarget: TypingService.CapturedFocusTarget? + ) async -> TypingService.DeliveryOutcome { + let sendsExistingDraft = outputPlan.plainText.isEmpty + let outcome = await self.asr.typeOutputPlanToActiveFieldAndWait( + outputPlan, + preferredTargetPID: targetPID, + textReadyAt: textReadyAt, + postInsertionKey: self.settings.spokenSendKey, + requiredFocusTarget: requiredFocusTarget + ) + if outcome.didDispatchAction { + NotchContentState.shared.setSpokenSendIndicatorState(.sent) + try? await Task.sleep(nanoseconds: 350_000_000) + return outcome + } + + NotchContentState.shared.setSpokenSendIndicatorState(.failed) + DebugLogger.shared.warning( + "Spoken Send skipped because delivery safety checks did not pass", + source: "ContentView" + ) + let message = outcome.didInsert + ? "Text inserted — send skipped" + : sendsExistingDraft ? "Couldn't send" : "Couldn't insert or send" + NotchOverlayManager.shared.updateTranscriptionText(message) + try? await Task.sleep(nanoseconds: 650_000_000) + return outcome + } + /// Best-effort frontmost window title lookup for the current app private func getFrontmostWindowTitle(ownerPid: pid_t) -> String? { let options: CGWindowListOption = [.optionOnScreenOnly, .excludeDesktopElements] @@ -1628,7 +1702,10 @@ struct ContentView: View { private func captureRecordingTargetContext() { // Capture the focused target PID BEFORE any overlay/UI changes. // Used to restore focus when the user interacts with overlay dropdowns. - let focusedPID = TypingService.captureSystemFocusedPID() + self.recordingSessionID = UUID() + let focusTarget = TypingService.captureSystemFocusTarget() + self.recordingFocusTarget = focusTarget + let focusedPID = focusTarget?.pid ?? NSWorkspace.shared.frontmostApplication?.processIdentifier NotchContentState.shared.recordingTargetPID = focusedPID @@ -1660,7 +1737,12 @@ struct ContentView: View { } private func resolveTypingTargetPID() -> (pid: pid_t?, shouldRestoreOriginalFocus: Bool) { - let originalPID = NotchContentState.shared.recordingTargetPID + self.resolveTypingTargetPID(originalPID: NotchContentState.shared.recordingTargetPID) + } + + private func resolveTypingTargetPID( + originalPID: pid_t? + ) -> (pid: pid_t?, shouldRestoreOriginalFocus: Bool) { let currentFocusedPID = TypingService.captureSystemFocusedPID() ?? NSWorkspace.shared.frontmostApplication?.processIdentifier @@ -2032,10 +2114,32 @@ struct ContentView: View { // MARK: - Stop and Process Transcription private func stopAndProcessTranscription(route: DictationOutputRoute = .normal) async { + guard !self.isStoppingAndProcessingTranscription else { + DebugLogger.shared.debug("Ignoring duplicate stop-and-process request", source: "ContentView") + return + } + let operationID = UUID() + self.stopProcessingOperationID = operationID + self.isStoppingAndProcessingTranscription = true + defer { self.finishStopProcessingOperationIfCurrent(operationID) } + await self.performStopAndProcessTranscription(route: route, operationID: operationID) + } + + // swiftlint:disable:next function_body_length + private func performStopAndProcessTranscription( + route: DictationOutputRoute, + operationID: UUID + ) async { DebugLogger.shared.debug("stopAndProcessTranscription called", source: "ContentView") DebugLogger.shared.info("Output route selected: \(route.rawValue)", source: "ContentView") self.appBench("stop_path_enter route=\(route.rawValue)") + let stoppedSessionID = self.recordingSessionID + let stoppedFocusTarget = self.recordingFocusTarget + let stoppedTargetPID = NotchContentState.shared.recordingTargetPID + let stoppedAppInfo = self.recordingAppInfo + let stoppedPrecedingText = self.recordingPrecedingText + // Check if we're in rewrite or command mode let modeAtStop = self.activeRecordingMode let wasRewriteMode = modeAtStop == .edit || self.isRecordingForRewrite @@ -2050,7 +2154,8 @@ struct ContentView: View { !wasRewriteMode && !wasCommandMode && !promptTest.isActive && - !shouldUseAIOnStop + !shouldUseAIOnStop && + !self.spokenSendIsPendingForCurrentUtterance var didRequestOverlayHideOnStop = false DebugLogger.shared.info( "Routing decision snapshot | activeMode=\(modeAtStop.rawValue) | rewrite=\(wasRewriteMode) | command=\(wasCommandMode) | overlay=\(NotchContentState.shared.mode.rawValue)", @@ -2087,6 +2192,9 @@ struct ContentView: View { TranscriptionSoundPlayer.shared.playStopSound() }) self.appBench("asr_stop_return elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - asrStopStartedAt) * 1000).rounded()))") + // The duplicate-stop race ends when ASR reports that it is no longer running. + // Do not hold this guard across delivery or a new recording's stop can be swallowed. + self.finishStopProcessingOperationIfCurrent(operationID) let audioSnapshot = self.asr.consumeLastCompletedAudioSnapshot() DebugLogger.shared.info( "Stop transcription result | chars=\(transcribedText.count) | empty=\(transcribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty)", @@ -2179,17 +2287,26 @@ struct ContentView: View { var finalText: String var aiFallbackReason: String? var postProcessingModel: String? - let appInfo = self.recordingAppInfo ?? self.getCurrentAppInfo() - let normalizedTranscribedText = ASRService.applySpokenPunctuationFormatting( + let appInfo = stoppedAppInfo ?? self.getCurrentAppInfo() + let punctuationFormattedText = ASRService.applySpokenPunctuationFormatting( transcribedText, appName: appInfo.name, bundleID: appInfo.bundleId, windowTitle: appInfo.windowTitle ) + let spokenSendParse = SpokenSendParser.parse( + punctuationFormattedText, + phrase: self.settings.spokenSendPhrase, + enabled: route == .normal && self.settings.spokenSendEnabled + ) + self.updateSpokenSendIndicatorForFinalParse(shouldSend: spokenSendParse.shouldSend) + let normalizedTranscribedText = spokenSendParse.text + let sendsExistingDraft = spokenSendParse.shouldSend && + normalizedTranscribedText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty - let shouldUseAI = activeDictationSlot.map { + let shouldUseAI = !sendsExistingDraft && (activeDictationSlot.map { DictationAIPostProcessingGate.isConfigured(for: $0, appBundleID: appInfo.bundleId) - } ?? DictationAIPostProcessingGate.isConfigured(for: .primary, appBundleID: appInfo.bundleId) + } ?? DictationAIPostProcessingGate.isConfigured(for: .primary, appBundleID: appInfo.bundleId)) let transcriptionModelInfo = self.currentTranscriptionModelInfo() if shouldUseAI { @@ -2287,13 +2404,23 @@ struct ContentView: View { finalText = ASRService.applyGAAVFormatting(finalText) // Apply Continuous Dictation Mode after GAAV so smart caps use the field // context captured at recording start, and the trailing space enables chaining. - finalText = ASRService.applyContinuousDictationFormatting(finalText, precedingText: self.recordingPrecedingText) + finalText = ASRService.applyContinuousDictationFormatting(finalText, precedingText: stoppedPrecedingText) finalText = ASRService.applyTerminalLiteralAutocompleteSpacing( finalText, appName: appInfo.name, bundleID: appInfo.bundleId, windowTitle: appInfo.windowTitle ) + guard Self.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: self.recordingSessionID + ) else { + DebugLogger.shared.warning( + "Completed dictation output suppressed because a newer recording session started", + source: "ContentView" + ) + return + } self.recordingPrecedingText = "" self.asr.finalText = finalText if route == .onboardingSandbox, @@ -2322,7 +2449,7 @@ struct ContentView: View { "mode": AnalyticsMode.dictation.rawValue, "words_bucket": AnalyticsBuckets.bucketWords(AnalyticsBuckets.wordCount(in: finalText)), "ai_used": shouldUseAI, - "ai_changed_text": transcribedText != finalText, + "ai_changed_text": normalizedTranscribedText != finalText, "transcription_provider": transcriptionModelInfo.provider, "transcription_model": transcriptionModelInfo.model, ] @@ -2350,13 +2477,13 @@ struct ContentView: View { let isFluidFrontmost = frontmostApp?.bundleIdentifier == Bundle.main.bundleIdentifier // Save to transcription history (transcription mode only, if enabled) - if shouldPersistOutputs, SettingsStore.shared.saveTranscriptionHistory { + if shouldPersistOutputs, !sendsExistingDraft, SettingsStore.shared.saveTranscriptionHistory { let historyEntryID = UUID() let historyTimestamp = Date() TranscriptionHistoryStore.shared.addEntry( id: historyEntryID, timestamp: historyTimestamp, - rawText: transcribedText, + rawText: spokenSendParse.shouldSend ? normalizedTranscribedText : transcribedText, processedText: finalText, appName: appInfo.name, windowTitle: appInfo.windowTitle, @@ -2374,6 +2501,7 @@ struct ContentView: View { // When FluidVoice itself is frontmost, the bound editor already receives `finalText`. // Avoid re-inserting or overwriting the clipboard in that self-target case. let shouldCopyToClipboard = shouldPersistOutputs && + !sendsExistingDraft && SettingsStore.shared.copyTranscriptionToClipboard && !isFluidFrontmost @@ -2397,22 +2525,70 @@ struct ContentView: View { ) if shouldTypeExternally { - let typingTarget = self.resolveTypingTargetPID() + let spokenSendRequested = spokenSendParse.shouldSend + let typingTarget = self.resolveTypingTargetPID(originalPID: stoppedTargetPID) + let targetMatchesRecordingFocus = typingTarget.pid != nil + && typingTarget.pid == stoppedFocusTarget?.pid + let spokenSendAllowed = spokenSendRequested + && aiFallbackReason == nil + && (sendsExistingDraft || !finalText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + && targetMatchesRecordingFocus + && !self.isSpokenSendBlockedTarget(stoppedFocusTarget) // Dispatch insertion as soon as the destination app is ready; the // overlay hides asynchronously after output so it cannot delay paste. if typingTarget.shouldRestoreOriginalFocus { - await self.restoreFocusToRecordingTarget() + await self.restoreFocusToRecordingTarget( + requireExactTarget: spokenSendAllowed, + targetPID: stoppedTargetPID, + focusTarget: stoppedFocusTarget + ) + } + guard Self.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: self.recordingSessionID + ) else { + DebugLogger.shared.warning( + "Dictation delivery suppressed because a newer recording session started", + source: "ContentView" + ) + return + } + if spokenSendAllowed { + NotchContentState.shared.setSpokenSendIndicatorState(.sending) + NotchOverlayManager.shared.updateTranscriptionText("Sending") } self.appBench( "text_ready_to_type_request elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - finalTextReadyAt) * 1000).rounded()))" ) - self.asr.typeOutputPlanToActiveField( - finalOutputPlan, - preferredTargetPID: typingTarget.pid, - textReadyAt: finalTextReadyAt, - tracksDictionaryCorrections: true - ) - didTypeExternally = true + if spokenSendAllowed { + let deliveryOutcome = await self.deliverSpokenSend( + finalOutputPlan, + targetPID: typingTarget.pid, + textReadyAt: finalTextReadyAt, + requiredFocusTarget: stoppedFocusTarget + ) + didTypeExternally = deliveryOutcome.didInsert || deliveryOutcome.didDispatchAction + } else { + self.asr.typeOutputPlanToActiveField( + finalOutputPlan, + preferredTargetPID: typingTarget.pid, + textReadyAt: finalTextReadyAt, + tracksDictionaryCorrections: true + ) + didTypeExternally = true + } + if spokenSendRequested, !spokenSendAllowed { + NotchContentState.shared.setSpokenSendIndicatorState(.failed) + DebugLogger.shared.warning( + "Spoken Send skipped because delivery safety checks did not pass", + source: "ContentView" + ) + if aiFallbackReason == nil { + NotchOverlayManager.shared.updateTranscriptionText("Text inserted — send skipped") + try? await Task.sleep(nanoseconds: 650_000_000) + } + } + self.clearSpokenSendStatusIfNeeded(requested: spokenSendRequested) if !shouldShowAIProcessingFailure, !didRequestOverlayHideOnStop { self.hideOverlayAfterOutput() } @@ -2459,6 +2635,15 @@ struct ContentView: View { } } + private func finishStopProcessingOperationIfCurrent(_ operationID: UUID) { + guard Self.canFinishStopProcessingOperation( + completingOperationID: operationID, + currentOperationID: self.stopProcessingOperationID + ) else { return } + self.stopProcessingOperationID = nil + self.isStoppingAndProcessingTranscription = false + } + private func hideOverlayAfterOutput() { self.hideOverlayAsync(reason: "after_output") } @@ -2500,11 +2685,172 @@ struct ContentView: View { return true } + private func updateSpokenSendIndicatorForFinalParse(shouldSend: Bool) { + if shouldSend, NotchContentState.shared.spokenSendIndicatorState == .sending { + return + } + NotchContentState.shared.setSpokenSendIndicatorState(shouldSend ? .detected : .hidden) + } + + private var spokenSendIsPendingForCurrentUtterance: Bool { + guard self.settings.spokenSendEnabled else { return false } + if self.spokenSendAutoStopTriggered { return true } + return SpokenSendParser.parse( + self.asr.partialTranscription, + phrase: self.settings.spokenSendPhrase, + enabled: true + ).shouldSend + } + + private func clearSpokenSendStatusIfNeeded(requested: Bool) { + guard requested else { return } + NotchOverlayManager.shared.updateTranscriptionText("") + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + private func advanceOverlayLifecycle() { + self.spokenSendAutoStopTask?.cancel() + self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() + self.spokenSendAutoStopTriggered = false + self.spokenSendPartialRevision = 0 + self.spokenSendCountdownStartedAt = nil + self.spokenSendLastVoiceActivityAt = ProcessInfo.processInfo.systemUptime self.overlayLifecycleID &+= 1 NotchContentState.shared.clearAIProcessingFailure() } + private func handleSpokenSendPartialTranscription(_ text: String) { + self.spokenSendPartialRevision &+= 1 + let isDictationMode = self.activeRecordingMode == .dictate || self.activeRecordingMode == .promptMode + let shouldStop = isDictationMode && + self.currentDictationOutputRouteForHotkeyStop() == .normal && + self.asr.isRunning && + !self.spokenSendAutoStopTriggered && + SpokenSendParser.shouldStopImmediately( + text, + phrase: self.settings.spokenSendPhrase, + spokenSendEnabled: self.settings.spokenSendEnabled, + sendImmediatelyEnabled: self.settings.spokenSendImmediatelyEnabled + ) + + guard shouldStop else { + self.spokenSendAutoStopTask?.cancel() + self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() + self.spokenSendCountdownStartedAt = nil + if !self.spokenSendAutoStopTriggered, + NotchContentState.shared.spokenSendIndicatorState == .countingDown + { + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + return + } + + guard self.spokenSendAutoStopTask == nil else { + return + } + + let expectedOverlayLifecycleID = self.overlayLifecycleID + let expectedPartialRevision = self.spokenSendPartialRevision + let countdownStartedAt = ProcessInfo.processInfo.systemUptime + self.spokenSendCountdownStartedAt = countdownStartedAt + self.spokenSendLastVoiceActivityAt = countdownStartedAt + let expectedVoiceActivityMonitoringID = self.startSpokenSendVoiceActivityMonitoring() + let expectedCountdownID = NotchContentState.shared.beginSpokenSendCountdown() + self.spokenSendAutoStopTask = Task { @MainActor in + // Keep one countdown across harmless streaming refinements such as punctuation or casing. + try? await Task.sleep(nanoseconds: SpokenSendParser.immediateStopSettleNanoseconds) + let quietDuration = ProcessInfo.processInfo.systemUptime - self.spokenSendLastVoiceActivityAt + guard !Task.isCancelled, + self.overlayLifecycleID == expectedOverlayLifecycleID, + NotchContentState.shared.spokenSendCountdownID == expectedCountdownID, + self.asr.isRunning, + self.activeRecordingMode == .dictate || self.activeRecordingMode == .promptMode, + self.currentDictationOutputRouteForHotkeyStop() == .normal, + !self.spokenSendAutoStopTriggered, + SpokenSendParser.canCompleteImmediateStop( + self.asr.partialTranscription, + phrase: self.settings.spokenSendPhrase, + spokenSendEnabled: self.settings.spokenSendEnabled, + sendImmediatelyEnabled: self.settings.spokenSendImmediatelyEnabled, + receivedFreshTranscript: self.spokenSendPartialRevision > expectedPartialRevision, + quietDuration: quietDuration + ) + else { + self.stopSpokenSendVoiceActivityMonitoring(matching: expectedVoiceActivityMonitoringID) + if self.overlayLifecycleID == expectedOverlayLifecycleID, + NotchContentState.shared.spokenSendCountdownID == expectedCountdownID + { + self.spokenSendAutoStopTask = nil + self.spokenSendCountdownStartedAt = nil + if NotchContentState.shared.spokenSendIndicatorState == .countingDown { + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + } + return + } + + self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring(matching: expectedVoiceActivityMonitoringID) + self.spokenSendCountdownStartedAt = nil + self.spokenSendAutoStopTriggered = true + NotchContentState.shared.setSpokenSendIndicatorState(.sending) + DebugLogger.shared.info("Spoken Send countdown completed; stopping dictation", source: "ContentView") + await self.stopAndProcessTranscription(route: .normal) + } + } + + private func handleSpokenSendAudioLevel(_ level: CGFloat) { + guard SpokenSendParser.isMeaningfulVoiceActivity(level) else { return } + + let activityAt = ProcessInfo.processInfo.systemUptime + self.spokenSendLastVoiceActivityAt = activityAt + guard let countdownStartedAt = self.spokenSendCountdownStartedAt, + SpokenSendParser.shouldCancelCountdownForVoiceActivity( + countdownStartedAt: countdownStartedAt, + voiceActivityAt: activityAt + ), + self.spokenSendAutoStopTask != nil + else { + return + } + + self.spokenSendAutoStopTask?.cancel() + self.spokenSendAutoStopTask = nil + self.stopSpokenSendVoiceActivityMonitoring() + self.spokenSendCountdownStartedAt = nil + if NotchContentState.shared.spokenSendIndicatorState == .countingDown { + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) + } + } + + private func startSpokenSendVoiceActivityMonitoring() -> UInt64 { + if let activeSpokenSendVoiceActivityID { + return activeSpokenSendVoiceActivityID + } + self.spokenSendVoiceActivityGeneration &+= 1 + let monitoringID = self.spokenSendVoiceActivityGeneration + self.activeSpokenSendVoiceActivityID = monitoringID + self.spokenSendVoiceActivityCancellable = self.asr.audioLevelPublisher + .filter(SpokenSendParser.isMeaningfulVoiceActivity) + .receive(on: DispatchQueue.main) + .sink { [monitoringID] level in + guard self.activeSpokenSendVoiceActivityID == monitoringID else { return } + self.handleSpokenSendAudioLevel(level) + } + return monitoringID + } + + private func stopSpokenSendVoiceActivityMonitoring(matching monitoringID: UInt64? = nil) { + if let monitoringID, self.activeSpokenSendVoiceActivityID != monitoringID { + return + } + self.spokenSendVoiceActivityCancellable?.cancel() + self.spokenSendVoiceActivityCancellable = nil + self.activeSpokenSendVoiceActivityID = nil + } + private func hideOverlayAsync(reason: String) { let expectedOverlayLifecycleID = self.overlayLifecycleID self.appBench("overlay_hide_request reason=\(reason) lifecycle=\(expectedOverlayLifecycleID)") @@ -3181,10 +3527,39 @@ struct ContentView: View { /// Best-effort: re-activate the app that was focused when recording started. /// Skips the AX restore work when the captured text element is already focused. - private func restoreFocusToRecordingTarget() async { - guard let pid = NotchContentState.shared.recordingTargetPID else { return } + private func restoreFocusToRecordingTarget(requireExactTarget: Bool = false) async { + await self.restoreFocusToRecordingTarget( + requireExactTarget: requireExactTarget, + targetPID: NotchContentState.shared.recordingTargetPID, + focusTarget: self.recordingFocusTarget + ) + } + + private func restoreFocusToRecordingTarget( + requireExactTarget: Bool, + targetPID: pid_t?, + focusTarget: TypingService.CapturedFocusTarget? + ) async { + guard let pid = targetPID else { return } let startedAt = ProcessInfo.processInfo.systemUptime self.appBench("focus_restore_start targetPID=\(pid)") + if let focusTarget, focusTarget.pid == pid { + // Ordinary dictation keeps the legacy PID/editable-focus check because some apps + // vend a fresh AX element for the same field. Spoken Send remains exact-target only. + let targetIsStillActive = requireExactTarget + ? TypingService.isExactFocusTargetActive(focusTarget) + : TypingService.isCapturedFocusStillActive(for: pid) + if targetIsStillActive { + self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") + return + } + let activated = TypingService.activateApp(pid: pid) + let focusedElementRestored = TypingService.restoreFocusTarget(focusTarget) + self.appBench( + "focus_restore_result activated=\(activated) element=\(focusedElementRestored) elapsedMs=\(Int(((ProcessInfo.processInfo.systemUptime - startedAt) * 1000).rounded()))" + ) + return + } if TypingService.isCapturedFocusStillActive(for: pid) { self.appBench("focus_restore_result activated=false element=true elapsedMs=0 reason=already_focused") DebugLogger.shared.debug( @@ -3765,13 +4140,6 @@ extension ContentView { return } self.advanceOverlayLifecycle() - if self.asr.micStatus == .authorized { - self.appBench("overlay_mode_request mode=Dictation") - self.menuBarManager.setOverlayMode(.dictation) - self.menuBarManager.showRecordingOverlayImmediately() - self.appBench("overlay_mode_requested mode=Dictation") - self.appBench("overlay_phase phase=connecting") - } Task { let asrStartStartedAt = ProcessInfo.processInfo.systemUptime DebugLogger.shared.benchmark("APP_BENCH", message: "asr_start_call", source: "AppBenchmark") @@ -3781,6 +4149,12 @@ extension ContentView { } self.captureRecordingContext() self.prewarmPrivateAIDictationIfNeeded(for: slot) + // Capture owns the critical path. Showing the overlay before asr.start() + // previously blocked the main actor for 58-87 ms before first PCM. + self.appBench("overlay_mode_request mode=Dictation") + self.menuBarManager.setOverlayMode(.dictation) + self.menuBarManager.showRecordingOverlayImmediately() + self.appBench("overlay_mode_requested mode=Dictation") self.appBench("overlay_phase phase=recording trigger=first_pcm") }) if startOutcome == .failed { diff --git a/Sources/Fluid/Persistence/BackupService.swift b/Sources/Fluid/Persistence/BackupService.swift index 4953a598..0d104176 100644 --- a/Sources/Fluid/Persistence/BackupService.swift +++ b/Sources/Fluid/Persistence/BackupService.swift @@ -62,6 +62,10 @@ struct SettingsBackupPayload: Codable, Equatable { let enableAIStreaming: Bool let copyTranscriptionToClipboard: Bool let textInsertionMode: SettingsStore.TextInsertionMode + let spokenSendEnabled: Bool? + let spokenSendImmediatelyEnabled: Bool? + let spokenSendPhrase: String? + let spokenSendKey: SettingsStore.SpokenSendKey? let preferredInputDeviceUID: String? // Optional so backups created before microphone priority ordering still decode. // swiftlint:disable:next discouraged_optional_collection diff --git a/Sources/Fluid/Persistence/SettingsStore.swift b/Sources/Fluid/Persistence/SettingsStore.swift index 50b378d4..8a04e4b2 100644 --- a/Sources/Fluid/Persistence/SettingsStore.swift +++ b/Sources/Fluid/Persistence/SettingsStore.swift @@ -3195,6 +3195,10 @@ final class SettingsStore: ObservableObject { enableAIStreaming: self.enableAIStreaming, copyTranscriptionToClipboard: self.copyTranscriptionToClipboard, textInsertionMode: self.textInsertionMode, + spokenSendEnabled: self.spokenSendEnabled, + spokenSendImmediatelyEnabled: self.spokenSendImmediatelyEnabled, + spokenSendPhrase: self.spokenSendPhrase, + spokenSendKey: self.spokenSendKey, preferredInputDeviceUID: self.preferredInputDeviceUID, microphonePriority: self.microphonePriority, suppressedMicrophoneUIDs: self.suppressedMicrophoneUIDs.sorted(), @@ -3318,6 +3322,18 @@ final class SettingsStore: ObservableObject { self.enableAIStreaming = payload.enableAIStreaming self.copyTranscriptionToClipboard = payload.copyTranscriptionToClipboard self.textInsertionMode = payload.textInsertionMode + if let spokenSendEnabled = payload.spokenSendEnabled { + self.spokenSendEnabled = spokenSendEnabled + } + if let spokenSendImmediatelyEnabled = payload.spokenSendImmediatelyEnabled { + self.spokenSendImmediatelyEnabled = spokenSendImmediatelyEnabled + } + if let spokenSendPhrase = payload.spokenSendPhrase { + self.spokenSendPhrase = spokenSendPhrase + } + if let spokenSendKey = payload.spokenSendKey { + self.spokenSendKey = spokenSendKey + } self.preferredInputDeviceUID = payload.preferredInputDeviceUID self.suppressedMicrophoneUIDs = Set(payload.suppressedMicrophoneUIDs ?? []) if let microphonePriority = payload.microphonePriority { @@ -5094,6 +5110,10 @@ private extension SettingsStore { static let enableAIStreaming = "EnableAIStreaming" static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard" static let textInsertionMode = "TextInsertionMode" + static let spokenSendEnabled = "SpokenSendEnabled" + static let spokenSendImmediatelyEnabled = "SpokenSendImmediatelyEnabled" + static let spokenSendPhrase = "SpokenSendPhrase" + static let spokenSendKey = "SpokenSendKey" static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled" static let betaReleasesEnabled = "BetaReleasesEnabled" static let lastUpdateCheckDate = "LastUpdateCheckDate" @@ -5222,6 +5242,77 @@ private extension SettingsStore { } extension SettingsStore { + enum SpokenSendKey: String, CaseIterable, Identifiable, Codable { + case enter + case shiftEnter + case commandEnter + + var id: String { + self.rawValue + } + + var displayName: String { + switch self { + case .enter: + return "Enter" + case .shiftEnter: + return "Shift + Enter" + case .commandEnter: + return "Command + Enter" + } + } + + var eventFlags: CGEventFlags { + switch self { + case .enter: + return [] + case .shiftEnter: + return .maskShift + case .commandEnter: + return .maskCommand + } + } + } + + var spokenSendEnabled: Bool { + get { self.defaults.object(forKey: Keys.spokenSendEnabled) as? Bool ?? false } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.spokenSendEnabled) + } + } + + var spokenSendImmediatelyEnabled: Bool { + get { self.defaults.object(forKey: Keys.spokenSendImmediatelyEnabled) as? Bool ?? true } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.spokenSendImmediatelyEnabled) + } + } + + var spokenSendPhrase: String { + get { self.defaults.string(forKey: Keys.spokenSendPhrase) ?? "send it" } + set { + objectWillChange.send() + self.defaults.set(newValue, forKey: Keys.spokenSendPhrase) + } + } + + var spokenSendKey: SpokenSendKey { + get { + guard let raw = self.defaults.string(forKey: Keys.spokenSendKey), + let key = SpokenSendKey(rawValue: raw) + else { + return .enter + } + return key + } + set { + objectWillChange.send() + self.defaults.set(newValue.rawValue, forKey: Keys.spokenSendKey) + } + } + enum TextInsertionMode: String, CaseIterable, Identifiable, Codable { case standard case reliablePaste diff --git a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift index 37640bea..3fbde1d6 100644 --- a/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift +++ b/Sources/Fluid/Services/ASRService+SpokenPunctuationFormatting.swift @@ -1,5 +1,39 @@ import Foundation +enum TerminalAppClassifier { + private static let identityMarkers = [ + "terminal", + "iterm", + "warp", + "ghostty", + "kitty", + "alacritty", + "wezterm", + "hyper", + "termius", + ] + private static let contextMarkers = [ + "terminal", + "shell", + "ssh", + "console", + "xterm", + "hterm", + ] + + static func isTerminal(appName: String?, bundleID: String?) -> Bool { + let identity = [appName, bundleID] + .compactMap { $0?.lowercased() } + .joined(separator: " ") + return self.identityMarkers.contains { identity.contains($0) } + } + + static func isTerminalContext(labels: [String]) -> Bool { + let context = labels.joined(separator: " ").lowercased() + return self.contextMarkers.contains { context.contains($0) } + } +} + extension ASRService { static func applySpokenPunctuationFormatting( _ text: String, @@ -30,7 +64,9 @@ private enum SpokenPunctuationFormatter { let haystack = [self.appName, self.bundleID, self.windowTitle] .compactMap { $0?.lowercased() } .joined(separator: " ") - return haystack.contains("codex") || + return TerminalAppClassifier.isTerminal(appName: self.appName, bundleID: self.bundleID) || + TerminalAppClassifier.isTerminalContext(labels: [self.windowTitle ?? ""]) || + haystack.contains("codex") || haystack.contains("chatgpt") || haystack.contains("claude") || haystack.contains("cursor") || @@ -38,12 +74,6 @@ private enum SpokenPunctuationFormatter { haystack.contains("xcode") || haystack.contains("visual studio code") || haystack.contains("vscode") || - haystack.contains("terminal") || - haystack.contains("iterm") || - haystack.contains("warp") || - haystack.contains("ghostty") || - haystack.contains("kitty") || - haystack.contains("alacritty") || haystack.contains("slack") || haystack.contains("discord") || haystack.contains("teams") diff --git a/Sources/Fluid/Services/ASRService.swift b/Sources/Fluid/Services/ASRService.swift index 200e5297..4e4157a5 100644 --- a/Sources/Fluid/Services/ASRService.swift +++ b/Sources/Fluid/Services/ASRService.swift @@ -4554,6 +4554,46 @@ final class ASRService: ObservableObject { ) } + func typeOutputPlanToActiveFieldAndWait( + _ plan: DictationLiteralOutputPlan, + preferredTargetPID: pid_t?, + textReadyAt: TimeInterval? = nil, + tracksDictionaryCorrections: Bool = false, + postInsertionKey: SettingsStore.SpokenSendKey? = nil, + requiredFocusTarget: TypingService.CapturedFocusTarget? = nil + ) async -> TypingService.DeliveryOutcome { + let requestedAt = ProcessInfo.processInfo.systemUptime + let textReadyAge = textReadyAt.map { Int(((requestedAt - $0) * 1000).rounded()) } + let text = plan.plainText + DebugLogger.shared.benchmark( + "TYPING_BENCH", + message: "asr_type_request chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyAgeMs=\(textReadyAge.map { String($0) } ?? "nil")", + source: "TypingBenchmark" + ) + let outcome = await withCheckedContinuation { continuation in + self.typingService.typeOutputPlanInstantly( + plan, + preferredTargetPID: preferredTargetPID, + textReadyAt: textReadyAt, + tracksDictionaryCorrections: tracksDictionaryCorrections, + postInsertionKey: postInsertionKey, + requiredFocusTarget: requiredFocusTarget + ) { outcome in + continuation.resume(returning: outcome) + } + } + let dispatchedAt = ProcessInfo.processInfo.systemUptime + let textReadyToDispatchMs = textReadyAt.map { + String(Int(((dispatchedAt - $0) * 1000).rounded())) + } ?? "nil" + DebugLogger.shared.benchmark( + "TYPING_BENCH", + message: "asr_type_dispatched chars=\(text.count) preferredPID=\(preferredTargetPID.map { String($0) } ?? "nil") textReadyToDispatchMs=\(textReadyToDispatchMs)", + source: "TypingBenchmark" + ) + return outcome + } + /// Removes filler sounds from transcribed text static func removeFillerWords(_ text: String) -> String { guard SettingsStore.shared.removeFillerWordsEnabled else { return text } diff --git a/Sources/Fluid/Services/GlobalHotkeyManager.swift b/Sources/Fluid/Services/GlobalHotkeyManager.swift index ff952d88..79dac766 100644 --- a/Sources/Fluid/Services/GlobalHotkeyManager.swift +++ b/Sources/Fluid/Services/GlobalHotkeyManager.swift @@ -782,6 +782,10 @@ final class GlobalHotkeyManager: NSObject { return tapRecoveryResult } + if Self.isSynthesizedTypingEvent(event) { + return Unmanaged.passUnretained(event) + } + if self.isShortcutCaptureActiveProvider?() ?? false { self.resetModifierOnlyShortcutTracking() return Unmanaged.passUnretained(event) @@ -1282,6 +1286,10 @@ final class GlobalHotkeyManager: NSObject { } } + nonisolated static func isSynthesizedTypingEvent(_ event: CGEvent) -> Bool { + event.getIntegerValueField(.eventSourceUserData) == TypingService.synthesizedEventUserData + } + private func handlePrimaryDictationTriggerDown() { switch self.hotkeyMode { case .hold: diff --git a/Sources/Fluid/Services/MenuBarManager.swift b/Sources/Fluid/Services/MenuBarManager.swift index 4d331d2c..b1df4f7a 100644 --- a/Sources/Fluid/Services/MenuBarManager.swift +++ b/Sources/Fluid/Services/MenuBarManager.swift @@ -119,6 +119,18 @@ final class MenuBarManager: NSObject, ObservableObject, NSMenuDelegate { .receive(on: DispatchQueue.main) .sink { [weak self] newText in guard self != nil else { return } + if asrService.isRunning, + NotchContentState.shared.mode == .dictation, + SettingsStore.shared.spokenSendEnabled, + !SettingsStore.shared.spokenSendImmediatelyEnabled + { + let detected = SpokenSendParser.parse( + newText, + phrase: SettingsStore.shared.spokenSendPhrase, + enabled: true + ).shouldSend + NotchContentState.shared.setSpokenSendIndicatorState(detected ? .detected : .hidden) + } if NotchOverlayManager.shared.shouldShowOrTrackLivePreviewText { NotchOverlayManager.shared.updateTranscriptionText(newText) } diff --git a/Sources/Fluid/Services/NotchOverlayManager.swift b/Sources/Fluid/Services/NotchOverlayManager.swift index 799570a9..6a754fae 100644 --- a/Sources/Fluid/Services/NotchOverlayManager.swift +++ b/Sources/Fluid/Services/NotchOverlayManager.swift @@ -368,6 +368,7 @@ final class NotchOverlayManager { // Safety: reset processing state when hiding NotchContentState.shared.setProcessing(false) + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) // Handle visible or showing states (can hide while still expanding) guard self.state == .visible || self.state == .showing, self.notch != nil else { diff --git a/Sources/Fluid/Services/SpokenSendParser.swift b/Sources/Fluid/Services/SpokenSendParser.swift new file mode 100644 index 00000000..5c04ae56 --- /dev/null +++ b/Sources/Fluid/Services/SpokenSendParser.swift @@ -0,0 +1,202 @@ +import Foundation + +struct SpokenSendParseResult: Equatable { + let text: String + let shouldSend: Bool +} + +enum SpokenSendParser { + static let immediateStopSettleDuration: TimeInterval = 1.5 + static let immediateStopSettleNanoseconds: UInt64 = 1_500_000_000 + static let immediateStopRequiredSilenceDuration: TimeInterval = 0.35 + static let immediateStopVoiceActivityGraceDuration: TimeInterval = 0.35 + static let immediateStopVoiceActivityLevelThreshold: CGFloat = 0.12 + private static let regexCache: NSCache = { + let cache = NSCache() + cache.countLimit = 12 + return cache + }() + + static func shouldStopImmediately( + _ text: String, + phrase: String, + spokenSendEnabled: Bool, + sendImmediatelyEnabled: Bool + ) -> Bool { + sendImmediatelyEnabled && + self.parse(text, phrase: phrase, enabled: spokenSendEnabled).shouldSend + } + + static func canCompleteImmediateStop( + _ text: String, + phrase: String, + spokenSendEnabled: Bool, + sendImmediatelyEnabled: Bool, + receivedFreshTranscript: Bool, + quietDuration: TimeInterval + ) -> Bool { + receivedFreshTranscript && + quietDuration >= self.immediateStopRequiredSilenceDuration && + self.shouldStopImmediately( + text, + phrase: phrase, + spokenSendEnabled: spokenSendEnabled, + sendImmediatelyEnabled: sendImmediatelyEnabled + ) + } + + static func shouldCancelCountdownForVoiceActivity( + countdownStartedAt: TimeInterval, + voiceActivityAt: TimeInterval + ) -> Bool { + voiceActivityAt - countdownStartedAt >= self.immediateStopVoiceActivityGraceDuration + } + + static func isMeaningfulVoiceActivity(_ level: CGFloat) -> Bool { + level >= self.immediateStopVoiceActivityLevelThreshold + } + + static func parse(_ text: String, phrase: String, enabled: Bool) -> SpokenSendParseResult { + guard enabled else { + return SpokenSendParseResult(text: text, shouldSend: false) + } + + let phraseWords = phrase + .trimmingCharacters(in: .whitespacesAndNewlines) + .split(whereSeparator: \.isWhitespace) + .map(String.init) + guard !phraseWords.isEmpty else { + return SpokenSendParseResult(text: text, shouldSend: false) + } + + let phrasePattern = phraseWords + .map(NSRegularExpression.escapedPattern(for:)) + .joined(separator: #"\s+"#) + let trailingPunctuation = #"[\s\p{P}]*$"# + + if let literalRegex = Self.regex( + #"(?i)(? String { + guard let regex = Self.regex( + #"(?i)(? NSRegularExpression? { + let key = pattern as NSString + if let cached = self.regexCache.object(forKey: key) { + return cached + } + guard let compiled = try? NSRegularExpression(pattern: pattern) else { + return nil + } + self.regexCache.setObject(compiled, forKey: key) + return compiled + } + + private static func removePairedOpeningDelimiters( + from text: inout String, + closedBy commandSuffix: String + ) { + let delimiterPairs: [Character: Character] = [ + "(": ")", "[": "]", "{": "}", + "\"": "\"", "'": "'", "“": "”", "‘": "’", "«": "»", + ] + while true { + var candidate = text + while candidate.last?.isWhitespace == true { + candidate.removeLast() + } + guard let opener = candidate.last, + let closer = delimiterPairs[opener], + commandSuffix.contains(closer) + else { + return + } + candidate.removeLast() + text = candidate + while text.last?.isWhitespace == true { + text.removeLast() + } + } + } + + private static func polishCommandPrefix(_ text: String) -> String { + var polished = text.trimmingCharacters(in: .whitespacesAndNewlines) + while let last = polished.last, [",", ";", ":", "-", "–", "—"].contains(last) { + polished.removeLast() + while polished.last?.isWhitespace == true { + polished.removeLast() + } + } + + guard let last = polished.last else { + return polished + } + if [".", "?", "!", "…"].contains(last) { + return polished + } + return polished + "." + } +} diff --git a/Sources/Fluid/Services/TypingService.swift b/Sources/Fluid/Services/TypingService.swift index b3315b7a..d0e4ec7b 100644 --- a/Sources/Fluid/Services/TypingService.swift +++ b/Sources/Fluid/Services/TypingService.swift @@ -4,6 +4,89 @@ import Carbon.HIToolbox import Foundation final class TypingService { + nonisolated static let synthesizedEventUserData: Int64 = 0x46565353 + nonisolated static let postInsertionModifierReleaseTimeout: TimeInterval = 2 + + struct CapturedFocusTarget { + let pid: pid_t + let window: AXUIElement? + let element: AXUIElement + + var isSecureTextField: Bool { + let subrole = TypingService.stringAXAttribute( + from: self.element, + attribute: kAXSubroleAttribute as CFString + ) ?? "" + return subrole == (kAXSecureTextFieldSubrole as String) + || subrole.localizedCaseInsensitiveContains("secure") + } + } + + enum DeliveryOutcome: Equatable { + case rejected + case insertionFailed + case inserted + case actionSuppressed + case actionDispatched + case insertedActionSuppressed + case insertedAndActionDispatched + + var didInsert: Bool { + switch self { + case .inserted, .insertedActionSuppressed, .insertedAndActionDispatched: + return true + case .rejected, .insertionFailed, .actionSuppressed, .actionDispatched: + return false + } + } + + var didDispatchAction: Bool { + self == .actionDispatched || self == .insertedAndActionDispatched + } + } + + nonisolated static func canDispatchPostInsertionAction( + preferredTargetPID: pid_t?, + requiredTargetPID: pid_t?, + isSecureTextField: Bool, + isTerminalLikeContext: Bool = false, + modifiersReleased: Bool, + exactFocusIsActive: Bool, + insertionConfirmed: Bool = true + ) -> Bool { + guard let preferredTargetPID, preferredTargetPID > 0, + requiredTargetPID == preferredTargetPID + else { + return false + } + return !isSecureTextField && + !isTerminalLikeContext && + modifiersReleased && + exactFocusIsActive && + insertionConfirmed + } + + nonisolated static func valueByInserting( + _ insertion: String, + into original: String, + selectedRange: CFRange + ) -> String? { + let originalValue = original as NSString + guard selectedRange.location >= 0, + selectedRange.length >= 0, + selectedRange.location <= originalValue.length, + selectedRange.length <= originalValue.length - selectedRange.location + else { + return nil + } + let value = NSMutableString(string: original) + value.replaceCharacters( + in: NSRange(location: selectedRange.location, length: selectedRange.length), + with: insertion + ) + return value as String + } + // Logging toggle (off by default). Enable by setting env FLUID_TYPING_LOGS=1 // or UserDefaults bool for key "enableTypingLogs". private static var isLoggingEnabled: Bool { @@ -34,6 +117,7 @@ final class TypingService { private struct FocusedTextSnapshot { let pid: pid_t + let element: AXUIElement let bundleIdentifier: String? let value: String? let selectedRange: CFRange? @@ -48,6 +132,18 @@ final class TypingService { case caretMovedExpectedDistance = "caret_moved_expected_distance" case timeout case unavailable + + var isConfirmed: Bool { + switch self { + case .appScriptContainsText, + .appScriptCaretMovedExpectedDistance, + .fieldContainsText, + .caretMovedExpectedDistance: + return true + case .timeout, .unavailable: + return false + } + } } private static let focusSnapshotQueue = DispatchQueue(label: "TypingService.FocusSnapshot") @@ -131,7 +227,7 @@ final class TypingService { /// Best-effort: returns the PID owning the currently focused accessibility element. /// This is more reliable than NSWorkspace.frontmostApplication for floating overlays/launchers. - static func captureSystemFocusedPID() -> pid_t? { + static func captureSystemFocusTarget() -> CapturedFocusTarget? { // Accessibility is required to query system-focused AX element. guard AXIsProcessTrusted() else { self.storeFocusSnapshot(nil) @@ -167,7 +263,57 @@ final class TypingService { ?? Self.copyAXElementAttribute(from: appElement, attribute: kAXMainWindowAttribute as CFString) Self.storeFocusSnapshot(FocusSnapshot(pid: pid, window: window, element: element)) Self.logFocusState("[TypingService] Captured focus snapshot") - return pid + return CapturedFocusTarget(pid: pid, window: window, element: element) + } + + static func captureSystemFocusedPID() -> pid_t? { + self.captureSystemFocusTarget()?.pid + } + + static func isExactFocusTargetActive(_ target: CapturedFocusTarget) -> Bool { + guard AXIsProcessTrusted() else { return false } + + let systemWideElement = AXUIElementCreateSystemWide() + var focusedElementRef: CFTypeRef? + let result = AXUIElementCopyAttributeValue( + systemWideElement, + kAXFocusedUIElementAttribute as CFString, + &focusedElementRef + ) + guard result == .success, let focusedElementRef, + CFGetTypeID(focusedElementRef) == AXUIElementGetTypeID() + else { + return false + } + + let currentElement = unsafeBitCast(focusedElementRef, to: AXUIElement.self) + return CFEqual(currentElement, target.element) + } + + @discardableResult + static func restoreFocusTarget(_ target: CapturedFocusTarget) -> Bool { + guard AXIsProcessTrusted() else { return false } + let appElement = AXUIElementCreateApplication(target.pid) + + if let window = target.window { + _ = AXUIElementPerformAction(window, kAXRaiseAction as CFString) + _ = AXUIElementSetAttributeValue(appElement, kAXMainWindowAttribute as CFString, window) + _ = AXUIElementSetAttributeValue(appElement, kAXFocusedWindowAttribute as CFString, window) + usleep(40_000) + } + + for _ in 0..<3 { + let result = AXUIElementSetAttributeValue( + target.element, + kAXFocusedAttribute as CFString, + kCFBooleanTrue + ) + if result == .success, self.isExactFocusTargetActive(target) { + return true + } + usleep(50_000) + } + return self.isExactFocusTargetActive(target) } /// Best-effort: returns the text immediately before the caret in the currently focused @@ -301,7 +447,10 @@ final class TypingService { _ plan: DictationLiteralOutputPlan, preferredTargetPID: pid_t?, textReadyAt: TimeInterval?, - tracksDictionaryCorrections: Bool = false + tracksDictionaryCorrections: Bool = false, + postInsertionKey: SettingsStore.SpokenSendKey? = nil, + requiredFocusTarget: CapturedFocusTarget? = nil, + completion: ((DeliveryOutcome) -> Void)? = nil ) { let requestedAt = ProcessInfo.processInfo.systemUptime let text = plan.plainText @@ -319,9 +468,10 @@ final class TypingService { self.log("[TypingService] ENTRY: typeTextInstantly called with text length: \(text.count)") self.log("[TypingService] Text preview: \"\(String(text.prefix(100)))\"") - guard text.isEmpty == false else { + guard text.isEmpty == false || postInsertionKey != nil else { self.bench("request_return reason=empty_text") self.log("[TypingService] ERROR: Empty text provided, aborting") + completion?(.rejected) return } @@ -329,6 +479,7 @@ final class TypingService { guard !self.isCurrentlyTyping else { self.bench("request_return reason=already_typing") self.log("[TypingService] WARNING: Skipping text injection - already in progress") + completion?(.rejected) return } @@ -337,6 +488,7 @@ final class TypingService { self.bench("request_return reason=accessibility_not_trusted") self.log("[TypingService] ERROR: Accessibility permissions required for text injection") self.log("[TypingService] Current accessibility status: \(AXIsProcessTrusted())") + completion?(.rejected) return } @@ -344,6 +496,7 @@ final class TypingService { self.isCurrentlyTyping = true DispatchQueue.global(qos: .userInitiated).async { + var outcome: DeliveryOutcome = .insertionFailed let workerStartedAt = ProcessInfo.processInfo.systemUptime self.bench("worker_start queueDelayMs=\(Self.elapsedMs(from: requestedAt, to: workerStartedAt))") @@ -354,6 +507,7 @@ final class TypingService { "complete totalMs=\(Self.elapsedMs(from: requestedAt, to: completedAt)) textReadyToCompleteMs=\(textReadyAt.map { String(Self.elapsedMs(from: $0, to: completedAt)) } ?? "nil")" ) self.log("[TypingService] Typing operation completed, isCurrentlyTyping set to false") + completion?(outcome) } self.log("[TypingService] Starting async text insertion process") @@ -361,21 +515,77 @@ final class TypingService { usleep(useconds_t(settleDelayMs * 1000)) } self.bench("settle_delay_done delayMs=\(settleDelayMs) elapsedMs=\(Self.elapsedMs(since: requestedAt))") - self.log("[TypingService] Delay completed, calling insertTextInstantly") - let insertStartedAt = ProcessInfo.processInfo.systemUptime - self.bench("insert_call") - self.insertTextInstantly(text, preferredTargetPID: preferredTargetPID) - self.bench( - "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" - ) - if tracksDictionaryCorrections { - Task { @MainActor in - AutomaticDictionaryCorrectionTracker.shared.beginObservingInsertion( - text, - targetPID: preferredTargetPID - ) + let hasTextToInsert = !text.isEmpty + var expectedInsertedValue: String? + + if hasTextToInsert { + self.log("[TypingService] Delay completed, calling insertTextInstantly") + let insertStartedAt = ProcessInfo.processInfo.systemUptime + self.bench("insert_call") + let inserted = self.insertTextInstantly( + text, + preferredTargetPID: preferredTargetPID, + requiredFocusTarget: postInsertionKey == nil ? nil : requiredFocusTarget, + expectedExactTargetValue: &expectedInsertedValue + ) + self.bench( + "insert_return elapsedMs=\(Self.elapsedMs(since: insertStartedAt)) totalMs=\(Self.elapsedMs(since: requestedAt))" + ) + guard inserted else { + outcome = .insertionFailed + return } + + outcome = .inserted + if tracksDictionaryCorrections, postInsertionKey == nil { + Task { @MainActor in + AutomaticDictionaryCorrectionTracker.shared.beginObservingInsertion( + text, + targetPID: preferredTargetPID + ) + } + } + } + + guard let postInsertionKey else { return } + guard let preferredTargetPID, let requiredFocusTarget else { + outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + return + } + + let capturedActionOnlyDraftValue = hasTextToInsert + ? nil + : Self.nonemptyEditableDraftValue(requiredFocusTarget) + usleep(50_000) + let modifiersReleased = self.waitForPhysicalModifiersToRelease( + timeout: Self.postInsertionModifierReleaseTimeout + ) + let expectedActionValue: String? = if hasTextToInsert { + expectedInsertedValue + } else { + capturedActionOnlyDraftValue } + let insertionConfirmed = expectedActionValue != nil && + self.getElementStringValue(requiredFocusTarget.element) == expectedActionValue + guard Self.canDispatchPostInsertionAction( + preferredTargetPID: preferredTargetPID, + requiredTargetPID: requiredFocusTarget.pid, + isSecureTextField: requiredFocusTarget.isSecureTextField, + isTerminalLikeContext: Self.hasTerminalLikeContext(requiredFocusTarget), + modifiersReleased: modifiersReleased, + exactFocusIsActive: Self.isExactFocusTargetActive(requiredFocusTarget), + insertionConfirmed: insertionConfirmed + ), + self.postReturnKey( + postInsertionKey, + target: requiredFocusTarget, + expectedValue: expectedActionValue + ) + else { + outcome = hasTextToInsert ? .insertedActionSuppressed : .actionSuppressed + return + } + outcome = hasTextToInsert ? .insertedAndActionDispatched : .actionDispatched } } @@ -393,17 +603,31 @@ final class TypingService { // MARK: - Internal insertion pipeline - private func insertTextInstantly(_ text: String, preferredTargetPID: pid_t?) { + private func insertTextInstantly( + _ text: String, + preferredTargetPID: pid_t?, + requiredFocusTarget: CapturedFocusTarget?, + expectedExactTargetValue: inout String? + ) -> Bool { self.log("[TypingService] insertTextInstantly called with \(text.count) characters") self.log("[TypingService] Attempting to type text: \"\(text.prefix(50))\(text.count > 50 ? "..." : "")\"") + if let requiredFocusTarget { + return self.insertTextIntoExactFocusTarget( + text, + preferredTargetPID: preferredTargetPID, + requiredFocusTarget: requiredFocusTarget, + expectedValue: &expectedExactTargetValue + ) + } + if self.textInsertionMode == .standard, let ghosttyTargetPID = self.ghosttyTargetPID(preferredTargetPID: preferredTargetPID) { self.log("[TypingService] Ghostty target detected in standard mode (PID \(ghosttyTargetPID)); forcing Reliable Paste path") if self.tryReliablePasteInsertion(text, preferredTargetPID: ghosttyTargetPID) { self.log("[TypingService] SUCCESS: Ghostty Reliable Paste path completed") - return + return true } self.log("[TypingService] Ghostty Reliable Paste path fell through to direct-typing fallbacks") } @@ -412,14 +636,14 @@ final class TypingService { self.log("[TypingService] Reliable Paste mode enabled") if self.tryReliablePasteInsertion(text, preferredTargetPID: preferredTargetPID) { self.log("[TypingService] SUCCESS: Reliable Paste mode completed") - return + return true } self.log("[TypingService] Reliable Paste mode fell through to direct-typing fallbacks") } else if let preferredTargetPID, preferredTargetPID > 0 { self.log("[TypingService] Experimental Direct Typing mode: trying preferred PID unicode insertion first") if self.insertTextBulkInstant(text, targetPID: preferredTargetPID) { self.log("[TypingService] SUCCESS: Preferred PID CGEvent insertion completed") - return + return true } self.log("[TypingService] Preferred PID CGEvent insertion failed, continuing fallback pipeline") } @@ -453,7 +677,7 @@ final class TypingService { self.log("[TypingService] Trying CGEvent insertion targeting focused PID \(focusedPID)") if self.insertTextBulkInstant(text, targetPID: focusedPID) { self.log("[TypingService] SUCCESS: CGEvent focused-PID insertion completed") - return + return true } } @@ -461,7 +685,7 @@ final class TypingService { self.log("[TypingService] Trying Accessibility focused-element insertion") if self.insertTextViaAccessibility(text) { self.log("[TypingService] SUCCESS: Accessibility insertion completed") - return + return true } // HID Fallback if PID targeting failed @@ -469,7 +693,7 @@ final class TypingService { self.log("[TypingService] No focused PID available, trying HID CGEvent insertion") if self.insertTextBulkHIDInstant(text) { self.log("[TypingService] SUCCESS: CGEvent HID insertion completed") - return + return true } } @@ -477,19 +701,149 @@ final class TypingService { self.log("[TypingService] CGEvent failed, trying clipboard fallback") if self.insertTextViaClipboard(text) { self.log("[TypingService] SUCCESS: Clipboard insertion completed") - return + return true } // Last resort: Character-by-character - self.log("[TypingService] WARNING: All methods failed, trying character-by-character") + self.log("[TypingService] WARNING: All methods failed, trying verified character-by-character") + let focusedTextSnapshot = self.captureFocusedTextSnapshot() + var postedEveryCharacter = true for (index, char) in text.enumerated() { if index % 10 == 0 { self.log("[TypingService] Typing character \(index + 1)/\(text.count)") } - self.typeCharacter(char) + postedEveryCharacter = self.typeCharacter(char) && postedEveryCharacter usleep(1000) } - self.log("[TypingService] Character-by-character typing completed") + guard postedEveryCharacter else { + self.log("[TypingService] Character-by-character event creation failed") + return false + } + let verification = self.waitForFocusedTextVerification( + from: focusedTextSnapshot, + expectedText: text, + timeoutMicros: 500_000 + ) + self.log("[TypingService] Character-by-character verification: \(verification.rawValue)") + return verification.isConfirmed + } + + private func insertTextIntoExactFocusTarget( + _ text: String, + preferredTargetPID: pid_t?, + requiredFocusTarget: CapturedFocusTarget, + expectedValue: inout String? + ) -> Bool { + guard preferredTargetPID == requiredFocusTarget.pid, + !requiredFocusTarget.isSecureTextField, + Self.isExactFocusTargetActive(requiredFocusTarget) + else { + self.log("[TypingService] Exact-target insertion rejected before dispatch") + return false + } + + return self.performVerifiedExactTargetAccessibilityInsertion( + text, + target: requiredFocusTarget, + expectedValue: &expectedValue + ) + } + + private func performVerifiedExactTargetAccessibilityInsertion( + _ text: String, + target: CapturedFocusTarget, + expectedValue: inout String? + ) -> Bool { + guard Self.isExactFocusTargetActive(target), + let snapshot = self.captureFocusedTextSnapshot(), + snapshot.pid == target.pid, + CFEqual(snapshot.element, target.element), + let originalValue = snapshot.value, + let selectedRange = snapshot.selectedRange, + let expectedValueAfterInsertion = Self.valueByInserting( + text, + into: originalValue, + selectedRange: selectedRange + ), + self.insertTextAtCursorUsingSelectedRange(target.element, text), + Self.isExactFocusTargetActive(target) + else { + return false + } + + let verified = self.waitForExactTargetValue( + expectedValueAfterInsertion, + target: target, + timeoutMicros: 500_000 + ) + self.log("[TypingService] Exact-target preserving insertion verified: \(verified)") + if verified { + expectedValue = expectedValueAfterInsertion + } + return verified + } + + private func waitForExactTargetValue( + _ expectedValue: String, + target: CapturedFocusTarget, + timeoutMicros: useconds_t + ) -> Bool { + let pollMicros: useconds_t = 50_000 + var waited: useconds_t = 0 + while waited < timeoutMicros { + usleep(pollMicros) + waited += pollMicros + guard Self.isExactFocusTargetActive(target) else { continue } + if self.getElementStringValue(target.element) == expectedValue { + return true + } + } + return false + } + + private func waitForPhysicalModifiersToRelease(timeout: TimeInterval) -> Bool { + let relevant: CGEventFlags = [.maskCommand, .maskControl, .maskAlternate, .maskShift, .maskSecondaryFn] + let startedAt = ProcessInfo.processInfo.systemUptime + while ProcessInfo.processInfo.systemUptime - startedAt < timeout { + if CGEventSource.flagsState(.combinedSessionState).isDisjoint(with: relevant) { + return true + } + usleep(15_000) + } + return false + } + + private func postReturnKey( + _ key: SettingsStore.SpokenSendKey, + target: CapturedFocusTarget, + expectedValue: String? + ) -> Bool { + guard let expectedValue, + Self.isExactFocusTargetActive(target), + self.getElementStringValue(target.element) == expectedValue + else { + return false + } + let returnKeyCode = CGKeyCode(kVK_Return) + guard let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: returnKeyCode, keyDown: false) + else { + return false + } + + keyDown.flags = key.eventFlags + keyUp.flags = key.eventFlags + keyDown.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) + keyUp.setIntegerValueField(.eventSourceUserData, value: Self.synthesizedEventUserData) + guard Self.isExactFocusTargetActive(target), + self.getElementStringValue(target.element) == expectedValue + else { + return false + } + keyDown.postToPid(target.pid) + usleep(10_000) + keyUp.postToPid(target.pid) + return true } private func tryReliablePasteInsertion(_ text: String, preferredTargetPID: pid_t?) -> Bool { @@ -543,6 +897,65 @@ final class TypingService { return value as? String } + private static func accessibilityContextLabels( + for element: AXUIElement, + window: AXUIElement? + ) -> [String] { + let attributes = [ + kAXRoleDescriptionAttribute as CFString, + kAXDescriptionAttribute as CFString, + kAXTitleAttribute as CFString, + kAXIdentifierAttribute as CFString, + kAXHelpAttribute as CFString, + ] + var labels: [String] = [] + var current: AXUIElement? = element + for _ in 0..<6 { + guard let currentElement = current else { break } + labels.append(contentsOf: attributes.compactMap { + Self.stringAXAttribute(from: currentElement, attribute: $0) + }) + if let window, CFEqual(currentElement, window) { + break + } + current = Self.copyAXElementAttribute( + from: currentElement, + attribute: kAXParentAttribute as CFString + ) + } + if let window { + labels.append(contentsOf: attributes.compactMap { + Self.stringAXAttribute(from: window, attribute: $0) + }) + } + return labels + } + + private static func nonemptyEditableDraftValue(_ target: CapturedFocusTarget) -> String? { + let editableRoles = ["AXTextField", "AXTextArea", "AXSearchField", "AXComboBox"] + guard self.isExactFocusTargetActive(target), + !target.isSecureTextField, + let role = self.stringAXAttribute( + from: target.element, + attribute: kAXRoleAttribute as CFString + ), + editableRoles.contains(role), + let value = self.stringAXAttribute( + from: target.element, + attribute: kAXValueAttribute as CFString + ) + else { + return nil + } + return value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : value + } + + private static func hasTerminalLikeContext(_ target: CapturedFocusTarget) -> Bool { + TerminalAppClassifier.isTerminalContext( + labels: self.accessibilityContextLabels(for: target.element, window: target.window) + ) + } + private static func currentFocusDebugDescription() -> String { let systemWideElement = AXUIElementCreateSystemWide() var focusedElementRef: CFTypeRef? @@ -1069,6 +1482,7 @@ final class TypingService { let appScriptSnapshot = self.captureAppScriptTextSnapshot(forBundleIdentifier: bundleIdentifier) return FocusedTextSnapshot( pid: focusInfo.pid, + element: focusInfo.element, bundleIdentifier: bundleIdentifier, value: self.getElementStringValue(focusInfo.element), selectedRange: self.getSelectedTextRange(focusInfo.element), @@ -1127,7 +1541,8 @@ final class TypingService { waited += pollMicros guard let current = self.captureFocusedTextSnapshot(), - current.pid == snapshot.pid + current.pid == snapshot.pid, + CFEqual(current.element, snapshot.element) else { continue } @@ -1328,7 +1743,7 @@ final class TypingService { } } - private func typeCharacter(_ char: Character) { + private func typeCharacter(_ char: Character) -> Bool { let charString = String(char) let utf16Array = Array(charString.utf16) @@ -1337,7 +1752,7 @@ final class TypingService { let keyUpEvent = CGEvent(keyboardEventSource: nil, virtualKey: 0, keyDown: false) else { self.log("[TypingService] ERROR: Failed to create CGEvents for character: \(char)") - return + return false } // Set the unicode string for both events @@ -1348,5 +1763,6 @@ final class TypingService { keyDownEvent.post(tap: .cghidEventTap) usleep(2000) // Short delay between key down and up (2ms) keyUpEvent.post(tap: .cghidEventTap) + return true } } diff --git a/Sources/Fluid/UI/SettingsView.swift b/Sources/Fluid/UI/SettingsView.swift index fbd91b5d..e8cc99e2 100644 --- a/Sources/Fluid/UI/SettingsView.swift +++ b/Sources/Fluid/UI/SettingsView.swift @@ -871,6 +871,9 @@ struct SettingsView: View { } Divider().opacity(0.2) + self.spokenSendSettings + Divider().opacity(0.2) + self.optionToggleRow( title: "Save Transcription History", description: "Save transcriptions for stats tracking. Disable for privacy.", @@ -2811,6 +2814,82 @@ struct FlowLayout: Layout { } } +private extension SettingsView { + var spokenSendSettings: some View { + Group { + self.optionToggleRow( + title: "Spoken Send", + description: "Say a phrase at the end of dictation to send with your chosen Enter command.", + isOn: Binding( + get: { self.settings.spokenSendEnabled }, + set: { self.settings.spokenSendEnabled = $0 } + ) + ) + + if self.settings.spokenSendEnabled { + VStack(spacing: 10) { + self.optionToggleRow( + title: "Send Immediately", + description: "Stop listening and send as soon as the phrase is recognized. May not work with all voice models; Parakeet is recommended.", + isOn: Binding( + get: { self.settings.spokenSendImmediatelyEnabled }, + set: { self.settings.spokenSendImmediatelyEnabled = $0 } + ) + ) + + HStack(alignment: .center) { + VStack(alignment: .leading, spacing: 2) { + Text("Send Phrase") + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.settingsTitleText) + Text("Say it at the end. Say “literal \(self.settings.spokenSendPhrase)” to dictate it normally.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.settingsSecondaryText) + } + + Spacer() + + TextField( + "send it", + text: Binding( + get: { self.settings.spokenSendPhrase }, + set: { self.settings.spokenSendPhrase = $0 } + ) + ) + .textFieldStyle(.roundedBorder) + .frame(width: 170) + } + + HStack(alignment: .center) { + VStack(alignment: .leading, spacing: 2) { + Text("Send Command") + .font(self.theme.typography.bodyStrong) + .foregroundStyle(self.settingsTitleText) + Text("Choose the Enter behavior expected by the destination app.") + .font(self.theme.typography.bodySmall) + .foregroundStyle(self.settingsSecondaryText) + } + + Spacer() + + Picker("", selection: Binding( + get: { self.settings.spokenSendKey }, + set: { self.settings.spokenSendKey = $0 } + )) { + ForEach(SettingsStore.SpokenSendKey.allCases) { key in + Text(key.displayName).tag(key) + } + } + .pickerStyle(.menu) + .frame(width: 170, alignment: .trailing) + } + } + .padding(.leading, 12) + } + } + } +} + // MARK: - Analytics modal confirmation struct AnalyticsConfirmationView: View { diff --git a/Sources/Fluid/Views/BottomOverlayView.swift b/Sources/Fluid/Views/BottomOverlayView.swift index 63a27835..607b9f0e 100644 --- a/Sources/Fluid/Views/BottomOverlayView.swift +++ b/Sources/Fluid/Views/BottomOverlayView.swift @@ -123,6 +123,7 @@ final class BottomOverlayWindowController { // offscreen. Revealing the neutral shell first causes a visible flash // that reads as the overlay appearing twice. NotchContentState.shared.setBottomOverlayPresented(true) + NotchContentState.shared.setSpokenSendIndicatorState(.hidden) NotchContentState.shared.mode = mode switch mode { case .dictation: NotchContentState.shared.promptPickerMode = .dictate @@ -2137,6 +2138,16 @@ struct BottomOverlayView: View { } } + private var showsSpokenSendIndicator: Bool { + self.contentState.mode == .dictation && + self.settings.spokenSendEnabled && + self.contentState.spokenSendIndicatorState.isVisible + } + + private var spokenSendIndicatorSize: CGFloat { + max(self.layout.modeFontSize + 3, 13) + } + private static let transientOverlayStatusTexts: Set = [ "Transcribing", "Refining", @@ -2989,7 +3000,7 @@ struct BottomOverlayView: View { } // Waveform + Mode label row - HStack(spacing: self.layout.hPadding / 1.5) { + HStack(spacing: self.isPillSize ? 4 : self.layout.hPadding / 1.5) { // Target app icon (the app where text will be typed) let appIcon = self.displayedAppIcon let showModelLoading = self.layout.showsModeLabel && !self.appServices.asr.isAsrReady && @@ -3015,17 +3026,48 @@ struct BottomOverlayView: View { .opacity((appIcon != nil || showModelLoading || !self.layout.showsModeLabel) ? 1 : 0) // Waveform visualization - BottomWaveformView(color: self.modeColor, layout: self.layout) - .frame(width: self.layout.waveformWidth, height: self.layout.waveformHeight) + BottomWaveformView( + color: self.modeColor, + layout: self.layout, + visibleBarCount: self.isPillSize && self.showsSpokenSendIndicator ? 6 : nil + ) + .frame( + width: self.isPillSize && self.showsSpokenSendIndicator + ? 32 + : self.layout.waveformWidth, + height: self.layout.waveformHeight + ) + + if self.isPillSize, self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.modeColor, + size: 14 + ) + .id(self.contentState.spokenSendCountdownID) + .transition(.scale(scale: 0.8).combined(with: .opacity)) + } // Mode label + model load hint if self.layout.showsModeLabel { VStack(alignment: .leading, spacing: 2) { - Text(self.modeLabel) - .font(.system(size: self.layout.modeFontSize, weight: .semibold)) - .foregroundStyle(self.modeColor) - .lineLimit(1) - .fixedSize(horizontal: true, vertical: false) + HStack(spacing: 5) { + Text(self.modeLabel) + .font(.system(size: self.layout.modeFontSize, weight: .semibold)) + .foregroundStyle(self.modeColor) + .lineLimit(1) + .fixedSize(horizontal: true, vertical: false) + + if self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.modeColor, + size: self.spokenSendIndicatorSize + ) + .id(self.contentState.spokenSendCountdownID) + .transition(.scale(scale: 0.8).combined(with: .opacity)) + } + } if !self.appServices.asr.isAsrReady && (self.appServices.asr.isLoadingModel || self.appServices.asr.isDownloadingModel) @@ -3037,6 +3079,10 @@ struct BottomOverlayView: View { .lineLimit(1) } } + .animation( + self.reduceMotion ? nil : .easeOut(duration: 0.14), + value: self.contentState.spokenSendIndicatorState + ) } } } @@ -3249,6 +3295,7 @@ struct BottomOverlayView: View { struct BottomWaveformView: View { let color: Color let layout: BottomOverlayView.LayoutConstants + let visibleBarCount: Int? @ObservedObject private var contentState = NotchContentState.shared // Initialize with max possible bar count (11 for large) to prevent index-out-of-range before onAppear @@ -3256,7 +3303,7 @@ struct BottomWaveformView: View { @State private var noiseThreshold: CGFloat = .init(SettingsStore.shared.visualizerNoiseThreshold) private var barCount: Int { - self.layout.barCount + self.visibleBarCount ?? self.layout.barCount } private var barWidth: CGFloat { diff --git a/Sources/Fluid/Views/NotchContentViews.swift b/Sources/Fluid/Views/NotchContentViews.swift index bf3e69c9..10184687 100644 --- a/Sources/Fluid/Views/NotchContentViews.swift +++ b/Sources/Fluid/Views/NotchContentViews.swift @@ -10,6 +10,85 @@ import Combine import QuartzCore import SwiftUI +enum SpokenSendIndicatorState: Equatable { + case hidden + case detected + case countingDown + case sending + case sent + case failed + + var isVisible: Bool { + self != .hidden + } +} + +struct SpokenSendIndicatorView: View { + @Environment(\.accessibilityReduceMotion) private var reduceMotion + @State private var progress: CGFloat = 0 + + let state: SpokenSendIndicatorState + let color: Color + let size: CGFloat + + var body: some View { + Group { + switch self.state { + case .hidden: + EmptyView() + case .detected: + self.symbol("paperplane.fill", color: self.color, accessibilityLabel: "Send detected") + case .countingDown: + ZStack { + Circle() + .stroke(self.color.opacity(0.22), lineWidth: self.lineWidth) + + Circle() + .trim(from: 0, to: self.progress) + .stroke( + self.color, + style: StrokeStyle(lineWidth: self.lineWidth, lineCap: .round) + ) + .rotationEffect(.degrees(-90)) + + self.symbol("paperplane.fill", color: self.color, accessibilityLabel: "Waiting to send") + } + .onAppear { + guard !self.reduceMotion else { + self.progress = 1 + return + } + withAnimation(.linear(duration: SpokenSendParser.immediateStopSettleDuration)) { + self.progress = 1 + } + } + case .sending: + ZStack { + Circle() + .stroke(self.color, lineWidth: self.lineWidth) + self.symbol("paperplane.fill", color: self.color, accessibilityLabel: "Sending") + } + case .sent: + self.symbol("checkmark.circle.fill", color: self.color, accessibilityLabel: "Sent") + case .failed: + self.symbol("exclamationmark.circle.fill", color: .orange, accessibilityLabel: "Send skipped") + } + } + .frame(width: self.size, height: self.size) + } + + private var lineWidth: CGFloat { + max(1, self.size * 0.085) + } + + private func symbol(_ name: String, color: Color, accessibilityLabel: String) -> some View { + Image(systemName: name) + .font(.system(size: max(6, self.size * 0.46), weight: .semibold)) + .foregroundStyle(color) + .accessibilityLabel(accessibilityLabel) + } +} + // MARK: - Observable state for notch content (Singleton) @MainActor @@ -25,6 +104,8 @@ class NotchContentState: ObservableObject { @Published var isAIProcessingFailureVisible: Bool = false @Published private(set) var aiProcessingFailureMessage: String = "AI Enhancement failed" @Published private(set) var canRetryAIProcessingFailure: Bool = true + @Published private(set) var spokenSendIndicatorState: SpokenSendIndicatorState = .hidden + @Published private(set) var spokenSendCountdownID: UInt64 = 0 @Published var activeDictationShortcutSlot: SettingsStore.DictationShortcutSlot? = nil @Published var promptModeOverrideProfileName: String? = nil // Name shown in overlay when prompt mode hotkey is active @Published var promptModeOverrideProfileID: String? = nil // ID of the active override profile (for checkmark in menu) @@ -111,6 +192,18 @@ class NotchContentState: ObservableObject { self.isAIProcessingFailureVisible = false } + func setSpokenSendIndicatorState(_ state: SpokenSendIndicatorState) { + guard self.spokenSendIndicatorState != state else { return } + self.spokenSendIndicatorState = state + } + + @discardableResult + func beginSpokenSendCountdown() -> UInt64 { + self.spokenSendCountdownID &+= 1 + self.spokenSendIndicatorState = .countingDown + return self.spokenSendCountdownID + } + /// Update transcription and recompute cached lines func updateTranscription(_ text: String) { let boundedText = Self.tailCharacters(in: text, maxCharacters: Self.maxStoredTranscriptionCharacters) @@ -418,6 +511,12 @@ struct NotchExpandedView: View { self.contentState.mode.notchColor } + private var showsSpokenSendIndicator: Bool { + self.contentState.mode == .dictation && + self.settings.spokenSendEnabled && + self.contentState.spokenSendIndicatorState.isVisible + } + private var presentationPolicy: NotchOverlayManager.NotchPresentationPolicy { NotchOverlayManager.shared.currentNotchPresentationPolicy } @@ -904,9 +1003,20 @@ struct NotchExpandedView: View { .frame(width: 48, height: 18) self.promptSelectorControl + + if self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.modeColor, + size: 14 + ) + .id(self.contentState.spokenSendCountdownID) + .transition(.scale(scale: 0.8).combined(with: .opacity)) + } } .frame(maxWidth: .infinity, alignment: .center) .offset(x: 4, y: 0) + .animation(.easeOut(duration: 0.14), value: self.contentState.spokenSendIndicatorState) self.promptHoverMenuRow @@ -1160,13 +1270,34 @@ struct NotchCompactLeadingView: View { struct NotchCompactTrailingView: View { let audioPublisher: AnyPublisher @ObservedObject private var contentState = NotchContentState.shared + @ObservedObject private var settings = SettingsStore.shared + + private var showsSpokenSendIndicator: Bool { + self.contentState.mode == .dictation && + self.settings.spokenSendEnabled && + self.contentState.spokenSendIndicatorState.isVisible + } var body: some View { - CompactNotchWaveformView( - audioPublisher: self.audioPublisher, - color: self.contentState.mode.notchColor - ) + HStack(spacing: self.showsSpokenSendIndicator ? 4 : 0) { + CompactNotchWaveformView( + audioPublisher: self.audioPublisher, + color: self.contentState.mode.notchColor, + barCount: self.showsSpokenSendIndicator ? 4 : 8 + ) + .frame(width: self.showsSpokenSendIndicator ? 16 : 34, height: 16) + + if self.showsSpokenSendIndicator { + SpokenSendIndicatorView( + state: self.contentState.spokenSendIndicatorState, + color: self.contentState.mode.notchColor, + size: 13 + ) + .id(self.contentState.spokenSendCountdownID) + } + } .frame(width: 34, height: 16) + .animation(.easeOut(duration: 0.14), value: self.contentState.spokenSendIndicatorState) } } @@ -1721,14 +1852,19 @@ struct ExpandedModeWaveformView: View { } struct CompactNotchWaveformView: View { + private static let storedBarCount = 8 + let audioPublisher: AnyPublisher let color: Color + var barCount: Int = 8 @StateObject private var data: AudioVisualizationData @ObservedObject private var contentState = NotchContentState.shared - @State private var barHeights: [CGFloat] = Array(repeating: 3, count: 8) + @State private var barHeights: [CGFloat] = Array( + repeating: 3, + count: CompactNotchWaveformView.storedBarCount + ) - private let barCount = 8 private let barWidth: CGFloat = 2.5 private let barSpacing: CGFloat = 2 private let minHeight: CGFloat = 3 @@ -1736,9 +1872,10 @@ struct CompactNotchWaveformView: View { private let noiseThreshold: CGFloat = 0.05 private let processingFlatHeight: CGFloat = 3 - init(audioPublisher: AnyPublisher, color: Color) { + init(audioPublisher: AnyPublisher, color: Color, barCount: Int = 8) { self.audioPublisher = audioPublisher self.color = color + self.barCount = min(max(barCount, 1), Self.storedBarCount) _data = StateObject(wrappedValue: AudioVisualizationData(audioLevelPublisher: audioPublisher)) } @@ -1820,7 +1957,7 @@ struct CompactNotchWaveformView: View { private func resetBarsToBaseline(animated: Bool) { let apply = { - self.barHeights = Array(repeating: self.minHeight, count: self.barCount) + self.barHeights = Array(repeating: self.minHeight, count: Self.storedBarCount) } if animated { diff --git a/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift new file mode 100644 index 00000000..95165fea --- /dev/null +++ b/Tests/FluidDictationIntegrationTests/SpokenSendTests.swift @@ -0,0 +1,561 @@ +import CoreGraphics +@testable import FluidVoice_Debug +import XCTest + +final class SpokenSendTests: XCTestCase { + func testDisabledFeatureLeavesTextUntouched() { + XCTAssertEqual( + SpokenSendParser.parse("Hello send it", phrase: "send it", enabled: false), + SpokenSendParseResult(text: "Hello send it", shouldSend: false) + ) + } + + func testTerminalPhraseIsRemovedAndArmsSend() { + XCTAssertEqual( + SpokenSendParser.parse("Hello there, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Hello there.", shouldSend: true) + ) + } + + func testCapitalizationAndFullStopDoNotAffectSend() { + XCTAssertEqual( + SpokenSendParser.parse("Ready to go, SEND IT.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready to go.", shouldSend: true) + ) + } + + func testNearbyTrailingPunctuationDoesNotAffectSend() { + XCTAssertEqual( + SpokenSendParser.parse(#"Ready to go — send it…")]"#, phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready to go.", shouldSend: true) + ) + } + + func testPairedDelimitersAroundTerminalPhraseAreRemoved() { + XCTAssertEqual( + SpokenSendParser.parse("Ready (send it)", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready “send it”", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse(#"Ready ["send it"]"#, phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + } + + func testClosingQuoteBeforeTerminalPhraseIsPreserved() { + XCTAssertEqual( + SpokenSendParser.parse(#"He said "Ready" send it"#, phrase: "send it", enabled: true), + SpokenSendParseResult(text: #"He said "Ready"."#, shouldSend: true) + ) + } + + func testPhraseInMiddleDoesNotArmSend() { + XCTAssertEqual( + SpokenSendParser.parse("Send it when you are ready", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Send it when you are ready", shouldSend: false) + ) + } + + func testTerminalPhraseDoesNotRequireLeadingOrTrailingPunctuation() { + XCTAssertEqual( + SpokenSendParser.parse("Ready to go send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready to go.", shouldSend: true) + ) + } + + func testRepeatedTerminalPhrasesAreAllRemoved() { + XCTAssertEqual( + SpokenSendParser.parse("I wanna send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "I wanna.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready SEND IT send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "", shouldSend: true) + ) + } + + func testRepeatedTrailingSeparatorsCollapseToOneSentenceEnding() { + XCTAssertEqual( + SpokenSendParser.parse("Ready,,,,; — send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready.,,,;— send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready.", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ready?,,, send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ready?", shouldSend: true) + ) + } + + func testFinalQuestionOrExclamationMarkIsPreserved() { + XCTAssertEqual( + SpokenSendParser.parse("Are we ready? send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Are we ready?", shouldSend: true) + ) + XCTAssertEqual( + SpokenSendParser.parse("Ship it! send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Ship it!", shouldSend: true) + ) + } + + func testImmediateStopRequiresChildOption() { + XCTAssertTrue( + SpokenSendParser.shouldStopImmediately( + "Ready, send it.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + XCTAssertFalse( + SpokenSendParser.shouldStopImmediately( + "Ready, send it.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: false + ) + ) + } + + func testImmediateStopDoesNotTriggerForPhraseInMiddle() { + XCTAssertFalse( + SpokenSendParser.shouldStopImmediately( + "Send it when you are ready", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + } + + func testTerminalASRRefinementStaysArmed() { + XCTAssertTrue( + SpokenSendParser.shouldStopImmediately( + "Ready, send it", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + XCTAssertTrue( + SpokenSendParser.shouldStopImmediately( + "Ready, SEND IT.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + XCTAssertFalse( + SpokenSendParser.shouldStopImmediately( + "Ready, send it after I finish this sentence.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + ) + } + + func testImmediateStopCompletionRequiresFreshTranscriptAndSilence() { + let arguments = ( + text: "Ready, send it.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true + ) + + XCTAssertFalse( + SpokenSendParser.canCompleteImmediateStop( + arguments.text, + phrase: arguments.phrase, + spokenSendEnabled: arguments.spokenSendEnabled, + sendImmediatelyEnabled: arguments.sendImmediatelyEnabled, + receivedFreshTranscript: false, + quietDuration: 2 + ) + ) + XCTAssertFalse( + SpokenSendParser.canCompleteImmediateStop( + arguments.text, + phrase: arguments.phrase, + spokenSendEnabled: arguments.spokenSendEnabled, + sendImmediatelyEnabled: arguments.sendImmediatelyEnabled, + receivedFreshTranscript: true, + quietDuration: 0 + ) + ) + XCTAssertTrue( + SpokenSendParser.canCompleteImmediateStop( + arguments.text, + phrase: arguments.phrase, + spokenSendEnabled: arguments.spokenSendEnabled, + sendImmediatelyEnabled: arguments.sendImmediatelyEnabled, + receivedFreshTranscript: true, + quietDuration: SpokenSendParser.immediateStopRequiredSilenceDuration + ) + ) + } + + func testImmediateStopCompletionCancelsForContinuedSpeech() { + XCTAssertFalse( + SpokenSendParser.canCompleteImmediateStop( + "Ready, send it after I finish this sentence.", + phrase: "send it", + spokenSendEnabled: true, + sendImmediatelyEnabled: true, + receivedFreshTranscript: true, + quietDuration: 2 + ) + ) + } + + func testVoiceActivityGraceIgnoresOnlyTheRecognitionTail() { + let startedAt: TimeInterval = 100 + XCTAssertFalse( + SpokenSendParser.shouldCancelCountdownForVoiceActivity( + countdownStartedAt: startedAt, + voiceActivityAt: startedAt + 0.05 + ) + ) + XCTAssertTrue( + SpokenSendParser.shouldCancelCountdownForVoiceActivity( + countdownStartedAt: startedAt, + voiceActivityAt: startedAt + SpokenSendParser.immediateStopVoiceActivityGraceDuration + 0.001 + ) + ) + XCTAssertFalse( + SpokenSendParser.isMeaningfulVoiceActivity( + SpokenSendParser.immediateStopVoiceActivityLevelThreshold.nextDown + ) + ) + XCTAssertTrue( + SpokenSendParser.isMeaningfulVoiceActivity( + SpokenSendParser.immediateStopVoiceActivityLevelThreshold + ) + ) + } + + func testLiteralEscapeKeepsPhraseWithoutSending() { + XCTAssertEqual( + SpokenSendParser.parse("Please type literal send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Please type send it", shouldSend: false) + ) + } + + func testLiteralEscapeBeforeRepeatedCommandKeepsOnePhraseAndSends() { + XCTAssertEqual( + SpokenSendParser.parse("Please type literal send it, send it.", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Please type send it.", shouldSend: true) + ) + } + + func testLiteralEscapeIsStrippedWhenPhraseIsNotTerminal() { + XCTAssertEqual( + SpokenSendParser.parse("I said literal send it to him", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "I said send it to him", shouldSend: false) + ) + } + + func testEarlierLiteralEscapeIsStrippedWhenFinalCommandSends() { + XCTAssertEqual( + SpokenSendParser.parse( + "I said literal send it to him and then send it", + phrase: "send it", + enabled: true + ), + SpokenSendParseResult(text: "I said send it to him and then.", shouldSend: true) + ) + } + + func testMidSentencePhraseWithoutLiteralMarkerIsUnchanged() { + XCTAssertEqual( + SpokenSendParser.parse("Please send it to Dana", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "Please send it to Dana", shouldSend: false) + ) + } + + func testPhraseOnlySubmitsExistingDraftWithoutInsertingCommand() { + XCTAssertEqual( + SpokenSendParser.parse("Send it", phrase: "send it", enabled: true), + SpokenSendParseResult(text: "", shouldSend: true) + ) + } + + func testCustomPhraseAllowsFlexibleWhitespaceAndCase() { + XCTAssertEqual( + SpokenSendParser.parse("Looks good. PLEASE SUBMIT", phrase: "please submit", enabled: true), + SpokenSendParseResult(text: "Looks good.", shouldSend: true) + ) + } + + func testAvailableSendCommandsMapToExpectedFlags() { + XCTAssertEqual(SettingsStore.SpokenSendKey.enter.eventFlags, []) + XCTAssertEqual(SettingsStore.SpokenSendKey.shiftEnter.eventFlags, .maskShift) + XCTAssertEqual(SettingsStore.SpokenSendKey.commandEnter.eventFlags, .maskCommand) + } + + func testTerminalClassifierBlocksEveryRecognizedTerminal() { + let terminals = [ + ("Terminal", "com.apple.Terminal"), + ("iTerm2", "com.googlecode.iterm2"), + ("Warp", "dev.warp.Warp-Stable"), + ("Ghostty", "com.mitchellh.ghostty"), + ("kitty", "net.kovidgoyal.kitty"), + ("Alacritty", "org.alacritty"), + ("WezTerm", "com.github.wez.wezterm"), + ("Hyper", "co.zeit.hyper"), + ("Termius", "com.termius-dmg.mac"), + ] + + for (name, bundleID) in terminals { + XCTAssertTrue( + TerminalAppClassifier.isTerminal(appName: name, bundleID: bundleID), + "Spoken Send must stay disabled in \(name)" + ) + } + } + + func testTerminalClassifierDoesNotBlockOrdinaryEditors() { + XCTAssertFalse( + TerminalAppClassifier.isTerminal( + appName: "TextEdit", + bundleID: "com.apple.TextEdit" + ) + ) + XCTAssertFalse( + TerminalAppClassifier.isTerminal( + appName: "Codex", + bundleID: "com.openai.codex" + ) + ) + } + + func testTerminalClassifierRecognizesHostedTerminalContext() { + XCTAssertTrue( + TerminalAppClassifier.isTerminalContext( + labels: ["Visual Studio Code", "Integrated Terminal", "AXTextArea"] + ) + ) + XCTAssertTrue( + TerminalAppClassifier.isTerminalContext(labels: ["Web SSH Console"]) + ) + XCTAssertFalse( + TerminalAppClassifier.isTerminalContext( + labels: ["Visual Studio Code", "Source Editor", "AXTextArea"] + ) + ) + } + + func testGeneratedSendCommandsAreExcludedFromFluidVoiceHotkeys() throws { + for key in SettingsStore.SpokenSendKey.allCases { + let event = try XCTUnwrap( + CGEvent( + keyboardEventSource: nil, + virtualKey: 36, + keyDown: true + ) + ) + event.flags = key.eventFlags + event.setIntegerValueField( + .eventSourceUserData, + value: TypingService.synthesizedEventUserData + ) + + XCTAssertTrue( + GlobalHotkeyManager.isSynthesizedTypingEvent(event), + "\(key.displayName) must bypass FluidVoice hotkey matching" + ) + } + + let physicalEvent = try XCTUnwrap( + CGEvent( + keyboardEventSource: nil, + virtualKey: 36, + keyDown: true + ) + ) + XCTAssertFalse(GlobalHotkeyManager.isSynthesizedTypingEvent(physicalEvent)) + } + + func testPostInsertionActionRequiresExactNonSecureFocus() { + XCTAssertTrue( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 43, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: true, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + isTerminalLikeContext: true, + modifiersReleased: true, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: false, + exactFocusIsActive: true + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: false + ) + ) + XCTAssertFalse( + TypingService.canDispatchPostInsertionAction( + preferredTargetPID: 42, + requiredTargetPID: 42, + isSecureTextField: false, + modifiersReleased: true, + exactFocusIsActive: true, + insertionConfirmed: false + ) + ) + } + + func testDeliveryOutcomesReportInsertionAndSendSeparately() { + XCTAssertTrue(TypingService.DeliveryOutcome.insertedAndActionDispatched.didInsert) + XCTAssertTrue(TypingService.DeliveryOutcome.insertedAndActionDispatched.didDispatchAction) + XCTAssertTrue(TypingService.DeliveryOutcome.actionDispatched.didDispatchAction) + XCTAssertFalse(TypingService.DeliveryOutcome.actionDispatched.didInsert) + XCTAssertTrue(TypingService.DeliveryOutcome.insertedActionSuppressed.didInsert) + XCTAssertFalse(TypingService.DeliveryOutcome.insertedActionSuppressed.didDispatchAction) + } + + func testCompletedRecordingIsSuppressedAfterANewerSessionStarts() { + let stoppedSessionID = UUID() + + XCTAssertTrue( + ContentView.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: stoppedSessionID + ) + ) + XCTAssertFalse( + ContentView.canDeliverCompletedRecording( + stoppedSessionID: stoppedSessionID, + currentSessionID: UUID() + ) + ) + } + + func testOlderStopOperationCannotClearNewerStopGuard() { + let olderOperationID = UUID() + let newerOperationID = UUID() + + XCTAssertTrue( + ContentView.canFinishStopProcessingOperation( + completingOperationID: newerOperationID, + currentOperationID: newerOperationID + ) + ) + XCTAssertFalse( + ContentView.canFinishStopProcessingOperation( + completingOperationID: olderOperationID, + currentOperationID: newerOperationID + ) + ) + } + + func testExactTargetInsertionPreservesUnselectedDraftText() { + XCTAssertEqual( + TypingService.valueByInserting( + "new ", + into: "existing draft", + selectedRange: CFRange(location: 9, length: 0) + ), + "existing new draft" + ) + XCTAssertEqual( + TypingService.valueByInserting( + "replacement", + into: "keep old keep", + selectedRange: CFRange(location: 5, length: 3) + ), + "keep replacement keep" + ) + XCTAssertNil( + TypingService.valueByInserting( + "unsafe", + into: "draft", + selectedRange: CFRange(location: 99, length: 0) + ) + ) + } + + func testOverlayIndicatorVisibilityCoversEveryActiveState() { + XCTAssertFalse(SpokenSendIndicatorState.hidden.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.detected.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.countingDown.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.sending.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.sent.isVisible) + XCTAssertTrue(SpokenSendIndicatorState.failed.isVisible) + } + + @MainActor + func testSettingsBackupIncludesSpokenSendConfiguration() async { + let settings = SettingsStore.shared + let originalEnabled = settings.spokenSendEnabled + let originalImmediate = settings.spokenSendImmediatelyEnabled + let originalPhrase = settings.spokenSendPhrase + let originalKey = settings.spokenSendKey + defer { + settings.spokenSendEnabled = originalEnabled + settings.spokenSendImmediatelyEnabled = originalImmediate + settings.spokenSendPhrase = originalPhrase + settings.spokenSendKey = originalKey + } + + settings.spokenSendEnabled = true + settings.spokenSendImmediatelyEnabled = false + settings.spokenSendPhrase = "ship it" + settings.spokenSendKey = .commandEnter + + let document = await BackupService.shared.makeBackupDocument() + XCTAssertEqual(document.settings.spokenSendEnabled, true) + XCTAssertEqual(document.settings.spokenSendImmediatelyEnabled, false) + XCTAssertEqual(document.settings.spokenSendPhrase, "ship it") + XCTAssertEqual(document.settings.spokenSendKey, .commandEnter) + } +}