diff --git a/macos/Makefile b/macos/Makefile index a5328cc..2f79d61 100644 --- a/macos/Makefile +++ b/macos/Makefile @@ -11,7 +11,7 @@ BUNDLE_DIR = $(APP_BUNDLE)/Contents # Compiler settings SWIFT = swiftc -SWIFT_FLAGS = -O -framework Cocoa -framework AVFoundation -framework Carbon -framework CoreGraphics -framework ServiceManagement +SWIFT_FLAGS = -O -framework Cocoa -framework AVFoundation -framework Speech -framework Carbon -framework CoreGraphics -framework ServiceManagement .PHONY: all clean build bundle install run dmg diff --git a/macos/src/main.swift b/macos/src/main.swift index d11ac35..dd2937d 100644 --- a/macos/src/main.swift +++ b/macos/src/main.swift @@ -1,5 +1,6 @@ import Cocoa -import AVFoundation +@preconcurrency import AVFoundation +@preconcurrency import Speech import Carbon.HIToolbox import ServiceManagement import ApplicationServices @@ -11,6 +12,7 @@ struct Defaults { static let modelPath = "whisperModelPath" static let playSounds = "playSounds" static let replacements = "customReplacements" + static let livePreview = "livePreview" } // MARK: - Whisper Models @@ -411,6 +413,162 @@ final class WhisperServer { // A small floating, non-interactive panel shown while recording. Doubles as a // diagnostic: if the input stays silent it flags "no input" — exactly the // failure that silently produced empty recordings + whisper hallucinations. +// MARK: - Live Preview (on-device Apple dictation model) +// Streams a rough live transcript into the recording HUD using the system's +// on-device dictation model (SpeechAnalyzer + DictationTranscriber, macOS 26). +// It is only a preview: the pasted text still comes from Whisper. It captures +// the microphone with its own AVAudioEngine so the AVAudioRecorder path used +// for Whisper stays untouched; if anything here fails there is simply no preview. +final class LivePreview { + // Called on the main thread with the transcript so far + var onText: ((String) -> Void)? + + private var session = 0 + private var engine: AVAudioEngine? + private var analyzer: SpeechAnalyzer? + private var input: AsyncStream.Continuation? + private var resultsTask: Task? + + // App language code -> supported dictation locale ("hu" -> hu-HU); nil if unsupported + static func locale(for languageCode: String) async -> Locale? { + let requested = languageCode == SupportedLanguages.autoDetect ? Locale.current : Locale(identifier: languageCode) + return await DictationTranscriber.supportedLocale(equivalentTo: requested) + } + + private static func makeTranscriber(locale: Locale) -> DictationTranscriber { + DictationTranscriber(locale: locale, contentHints: [], transcriptionOptions: [.punctuation], + reportingOptions: [.volatileResults, .frequentFinalization], attributeOptions: []) + } + + // Download the on-device model for this language in the background if it's missing + static func installModelIfNeeded(languageCode: String) { + Task { + guard let locale = await locale(for: languageCode) else { return } + do { + if let request = try await AssetInventory.assetInstallationRequest(supporting: [makeTranscriber(locale: locale)]) { + NSLog("Live preview: installing dictation model for \(locale.identifier)") + try await request.downloadAndInstall() + NSLog("Live preview: dictation model installed for \(locale.identifier)") + } + } catch { + NSLog("Live preview: model install failed: \(error)") + } + } + } + + func start(languageCode: String, contextualStrings: [String]) { + stop() + let current = session + Task { @MainActor in + do { + guard let locale = await LivePreview.locale(for: languageCode) else { + NSLog("Live preview: no on-device dictation model for '\(languageCode)'") + return + } + let transcriber = LivePreview.makeTranscriber(locale: locale) + // Even when the model is already on disk the app may first have to register it; + // that takes a fraction of a second. A real download just means no preview this time. + if await AssetInventory.status(forModules: [transcriber]) != .installed, + let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { + NSLog("Live preview: preparing dictation model for \(locale.identifier)") + try await request.downloadAndInstall() + } + guard current == self.session, + let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else { return } + + // Start the microphone first; the stream buffers audio until the analyzer is running, + // so the first words aren't lost while the model spins up. + let (stream, continuation) = AsyncStream.makeStream() + let engine = AVAudioEngine() + let micFormat = engine.inputNode.outputFormat(forBus: 0) + guard micFormat.sampleRate > 0, current == self.session, + let converter = AVAudioConverter(from: micFormat, to: analyzerFormat) else { return } + engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: micFormat, + block: LivePreview.tapBlock(converter: converter, format: analyzerFormat, continuation: continuation)) + try engine.start() + self.engine = engine + self.input = continuation + + let analyzer = SpeechAnalyzer(modules: [transcriber]) + self.analyzer = analyzer + if !contextualStrings.isEmpty { + let context = AnalysisContext() + context.contextualStrings[.general] = contextualStrings + try await analyzer.setContext(context) + } + self.resultsTask = Task { @MainActor [weak self] in + // Finalized segments accumulate; the volatile tail is replaced on every update + var finalized = "" + do { + for try await result in transcriber.results { + let text = String(result.text.characters) + let shown = LivePreview.join(finalized, text) + if result.isFinal { finalized = shown } + guard let self = self, current == self.session else { return } + self.onText?(shown) + } + } catch { + NSLog("Live preview: results ended: \(error)") + } + } + guard current == self.session else { return } // stop() already tore everything down + try await analyzer.start(inputSequence: stream) + } catch { + NSLog("Live preview failed: \(error)") + if current == self.session { self.stop() } + } + } + } + + func stop() { + session += 1 + if let engine = engine { + engine.inputNode.removeTap(onBus: 0) + engine.stop() + } + engine = nil + input?.finish() + input = nil + resultsTask?.cancel() + resultsTask = nil + if let analyzer = analyzer { + Task { await analyzer.cancelAndFinishNow() } + } + analyzer = nil + } + + // Segments don't always carry a separating space + static func join(_ head: String, _ tail: String) -> String { + guard let last = head.last, let first = tail.first, !last.isWhitespace, !first.isWhitespace else { return head + tail } + return head + " " + tail + } + + // Runs on the audio thread: convert mic buffers to the analyzer's format. + // Built in a nonisolated static context so it carries no actor assumptions. + private static func tapBlock(converter: AVAudioConverter, format: AVAudioFormat, + continuation: AsyncStream.Continuation) -> AVAudioNodeTapBlock { + return { buffer, _ in + let ratio = format.sampleRate / buffer.format.sampleRate + let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 64 + guard let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { return } + var consumed = false + var error: NSError? + converter.convert(to: out, error: &error) { _, status in + if consumed { + status.pointee = .noDataNow + return nil + } + consumed = true + status.pointee = .haveData + return buffer + } + if error == nil, out.frameLength > 0 { + continuation.yield(AnalyzerInput(buffer: out)) + } + } + } +} + final class MeterView: NSView { enum Mode { case recording, transcribing } enum Notice { case loadingModel, modelLoaded } @@ -419,15 +577,55 @@ final class MeterView: NSView { private var silent = false var mode = Mode.recording { didSet { needsDisplay = true } } var notice: Notice? { didSet { needsDisplay = true } } + var previewText: String? { didSet { if previewText != oldValue { cachedPreview = nil; needsDisplay = true } } } + var hasPreview: Bool { previewBlock() != nil } - // Height for the current mode + notice + private static let maxPreviewLines = 3 + private static let previewFont = NSFont.systemFont(ofSize: 13) + private var cachedPreview: (width: CGFloat, text: NSAttributedString, height: CGFloat)? + + private var noticeHeight: CGFloat { + switch notice { + case .loadingModel?: return 35 + case .modelLoaded?: return 19 + case nil: return 0 + } + } + + // Height for the current mode, notice and live preview text var preferredHeight: CGFloat { let base: CGFloat = mode == .recording ? 72 : 44 - switch notice { - case .loadingModel?: return base + 34 - case .modelLoaded?: return base + 20 - case nil: return base + let preview = previewBlock().map { 6 + $0.height } ?? 0 + return base + noticeHeight + preview + } + + // The tail of the live transcript that fits in a few lines; older words scroll off the top + private func previewBlock() -> (text: NSAttributedString, height: CGFloat)? { + guard let raw = previewText, !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil } + let width = bounds.width - 32 + if let cached = cachedPreview, cached.width == width { return (cached.text, cached.height) } + + let attrs: [NSAttributedString.Key: Any] = [ + .font: MeterView.previewFont, + .foregroundColor: NSColor(calibratedWhite: 1, alpha: 0.92) + ] + func measure(_ string: NSAttributedString) -> CGFloat { + ceil(string.boundingRect(with: NSSize(width: width, height: 10_000), options: [.usesLineFragmentOrigin, .usesFontLeading]).height) } + let lineHeight = measure(NSAttributedString(string: "Ag", attributes: attrs)) + let maxHeight = lineHeight * CGFloat(MeterView.maxPreviewLines) + 1 + + let words = raw.split(whereSeparator: { $0.isWhitespace }) + var dropped = 0 + var text = NSAttributedString(string: words.joined(separator: " "), attributes: attrs) + var height = measure(text) + while height > maxHeight && dropped < words.count - 1 { + dropped += 1 + text = NSAttributedString(string: "…" + words[dropped...].joined(separator: " "), attributes: attrs) + height = measure(text) + } + cachedPreview = (width, text, height) + return (text, height) } func set(level: CGFloat, silent: Bool) { @@ -435,7 +633,7 @@ final class MeterView: NSView { self.silent = silent needsDisplay = true } - func reset() { level = 0; silent = false; mode = .recording; notice = nil; needsDisplay = true } + func reset() { level = 0; silent = false; mode = .recording; notice = nil; previewText = nil; needsDisplay = true } override func draw(_ dirtyRect: NSRect) { let bg = NSBezierPath(roundedRect: bounds, xRadius: 12, yRadius: 12) @@ -472,6 +670,12 @@ final class MeterView: NSView { break } + if let preview = previewBlock() { + let top = textY - 6 + preview.text.draw(with: NSRect(x: 16, y: top - preview.height, width: bounds.width - 32, height: preview.height), + options: [.usesLineFragmentOrigin, .usesFontLeading]) + } + guard mode == .recording else { return } let segs = 20 @@ -496,7 +700,7 @@ final class MeterView: NSView { final class RecordingHUD { private var window: NSWindow? - private let meter = MeterView(frame: NSRect(x: 0, y: 0, width: 320, height: 72)) + private let meter = MeterView(frame: NSRect(x: 0, y: 0, width: 420, height: 72)) func show() { meter.reset() @@ -512,6 +716,13 @@ final class RecordingHUD { layout() } + func setPreview(_ text: String?) { + meter.previewText = text + layout() + } + + var hasPreview: Bool { meter.hasPreview } + // Stay up after recording while a slow (model-loading) transcription runs func showTranscribing(notice: MeterView.Notice?) { makeWindowIfNeeded() @@ -533,7 +744,7 @@ final class RecordingHUD { private func makeWindowIfNeeded() { if window == nil { - let w = NSPanel(contentRect: NSRect(x: 0, y: 0, width: 320, height: 72), + let w = NSPanel(contentRect: NSRect(x: 0, y: 0, width: 420, height: 72), styleMask: [.borderless, .nonactivatingPanel], backing: .buffered, defer: false) w.isOpaque = false @@ -786,6 +997,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, var settingsWindowController: NSWindowController? var whisperServer: WhisperServer? let recordingHUD = RecordingHUD() + let livePreview = LivePreview() var meterTimer: Timer? var silenceStart: Date? weak var dictTable: NSTableView? @@ -814,6 +1026,11 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, set { UserDefaults.standard.set(newValue, forKey: Defaults.playSounds) } } + var livePreviewEnabled: Bool { + get { UserDefaults.standard.object(forKey: Defaults.livePreview) as? Bool ?? true } + set { UserDefaults.standard.set(newValue, forKey: Defaults.livePreview) } + } + // Custom dictionary: ordered list of ["from": recognised, "to": replacement]. var replacements: [[String: String]] { get { (UserDefaults.standard.array(forKey: Defaults.replacements) as? [[String: String]]) ?? [] } @@ -823,11 +1040,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, // Whisper initial prompt built from the desired ("to") terms, to bias // recognition toward the user's names/jargon. Returns nil if empty. func customPrompt() -> String? { - let terms = replacements.compactMap { $0["to"] } + let terms = dictionaryTerms() + return terms.isEmpty ? nil : terms.joined(separator: ", ") + } + + // The desired ("to") dictionary terms, also given to the live preview model + func dictionaryTerms() -> [String] { + replacements.compactMap { $0["to"] } .map { $0.replacingOccurrences(of: "\\n", with: " ") } .map { $0.trimmingCharacters(in: .whitespaces) } .filter { !$0.isEmpty } - return terms.isEmpty ? nil : terms.joined(separator: ", ") } // Apply the dictionary find/replace pairs to a transcript. "\n" in a @@ -858,6 +1080,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, } checkAccessibilityPermission() + if livePreviewEnabled { LivePreview.installModelIfNeeded(languageCode: language) } NSLog("WhisperDictate started. Press ⌃⌥D to toggle recording.") } @@ -1091,7 +1314,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, func createSettingsWindow() -> NSWindow { let window = NSWindow( - contentRect: NSRect(x: 0, y: 0, width: 450, height: 540), + contentRect: NSRect(x: 0, y: 0, width: 450, height: 580), styleMask: [.titled, .closable], backing: .buffered, defer: false @@ -1102,7 +1325,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, let contentView = NSView(frame: window.contentView!.bounds) - var y: CGFloat = 498 + var y: CGFloat = 538 let labelWidth: CGFloat = 120 let controlX: CGFloat = 140 let controlWidth: CGFloat = 280 @@ -1180,6 +1403,15 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, loginCheck.state = isLaunchAtLoginEnabled() ? .on : .off contentView.addSubview(loginCheck) + y -= 40 + + // Live preview + let previewCheck = NSButton(checkboxWithTitle: "Live preview while recording", target: self, action: #selector(livePreviewChanged(_:))) + previewCheck.frame = NSRect(x: controlX, y: y, width: controlWidth, height: 24) + previewCheck.state = livePreviewEnabled ? .on : .off + previewCheck.toolTip = "Shows a rough live transcript in the recording overlay using macOS's on-device dictation model. The pasted text still comes from Whisper." + contentView.addSubview(previewCheck) + // Custom dictionary (recognised -> replacement) let dictLabel = NSTextField(labelWithString: "Dictionary (heard → replacement):") dictLabel.frame = NSRect(x: 20, y: 304, width: 300, height: 20) @@ -1247,6 +1479,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, @objc func languageChanged(_ sender: NSPopUpButton) { if let code = sender.selectedItem?.representedObject as? String { language = code + if livePreviewEnabled { LivePreview.installModelIfNeeded(languageCode: code) } NSLog("Language changed to: \(language)") } } @@ -1330,6 +1563,11 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, } } + @objc func livePreviewChanged(_ sender: NSButton) { + livePreviewEnabled = sender.state == .on + if livePreviewEnabled { LivePreview.installModelIfNeeded(languageCode: language) } + } + @objc func playSoundsChanged(_ sender: NSButton) { playSounds = sender.state == .on } @@ -1562,6 +1800,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, NSLog("Recording started") startMeter() prepareModelForDictation() + startLivePreview() } catch { NSLog("Recording failed: \(error)") if playSounds { NSSound(named: "Basso")?.play() } @@ -1603,6 +1842,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, checkModelLoaded() } + // MARK: - Live preview + func startLivePreview() { + guard livePreviewEnabled else { return } + livePreview.onText = { [weak self] text in + guard let self = self, self.isRecording else { return } + self.recordingHUD.setPreview(self.applyReplacements(text)) + } + livePreview.start(languageCode: language, contextualStrings: dictionaryTerms()) + } + // MARK: - Model warm-up // Start getting the model into memory as soon as recording starts, so loading // overlaps with the user talking instead of following it. Only if that turns @@ -1659,10 +1908,13 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate, audioRecorder?.stop() isRecording = false isTranscribing = true - // Model still loading and already announced: keep explaining the wait - let keepHUD = modelLoading && loadNoticeShown + livePreview.stop() + // Keep the HUD up while Whisper works if there is something to show: + // the live preview text, or why this transcription is slower + let explainLoading = modelLoading && loadNoticeShown + let keepHUD = explainLoading || recordingHUD.hasPreview stopMeter(hideHUD: !keepHUD) - if keepHUD { recordingHUD.showTranscribing(notice: .loadingModel) } + if keepHUD { recordingHUD.showTranscribing(notice: explainLoading ? .loadingModel : nil) } statusItem.button?.title = "⏳" updateStatus("Transcribing...") if playSounds { NSSound(named: "Pop")?.play() }