Add live transcript preview in the recording HUD

While recording, the HUD now shows a rough live transcript using macOS's
on-device dictation model (SpeechAnalyzer + DictationTranscriber), which
also supports Hungarian. It is only a preview: Whisper still produces the
pasted text.

The preview captures the microphone with its own AVAudioEngine, leaving
the AVAudioRecorder path for Whisper untouched, so a preview failure just
means no preview. Audio is buffered from the moment recording starts so
the first words aren't lost while the model spins up; dictionary terms
are passed as contextual strings and replacements are applied to the
preview text. The dictation model for the selected language is
registered/downloaded at launch and on language change, and awaited
inline if needed.

The HUD is wider and shows the last three lines of the transcript; after
recording it stays up as "Transcribing…" until the final text is pasted.
A "Live preview while recording" setting (on by default) turns it off.
No extra permission is required beyond the microphone.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

macos/Makefile

@@ -11,7 +11,7 @@ BUNDLE_DIR = $(APP_BUNDLE)/Contents
 
 # Compiler settings
 SWIFT = swiftc
-SWIFT_FLAGS = -O -framework Cocoa -framework AVFoundation -framework Carbon -framework CoreGraphics -framework ServiceManagement
+SWIFT_FLAGS = -O -framework Cocoa -framework AVFoundation -framework Speech -framework Carbon -framework CoreGraphics -framework ServiceManagement
 
 .PHONY: all clean build bundle install run dmg
 

macos/src/main.swift

@@ -1,5 +1,6 @@
 import Cocoa
-import AVFoundation
+@preconcurrency import AVFoundation
+@preconcurrency import Speech
 import Carbon.HIToolbox
 import ServiceManagement
 import ApplicationServices
@@ -11,6 +12,7 @@ struct Defaults {
     static let modelPath = "whisperModelPath"
     static let playSounds = "playSounds"
     static let replacements = "customReplacements"
+    static let livePreview = "livePreview"
 }
 
 // MARK: - Whisper Models
@@ -411,6 +413,162 @@ final class WhisperServer {
 // A small floating, non-interactive panel shown while recording. Doubles as a
 // diagnostic: if the input stays silent it flags "no input" — exactly the
 // failure that silently produced empty recordings + whisper hallucinations.
+// MARK: - Live Preview (on-device Apple dictation model)
+// Streams a rough live transcript into the recording HUD using the system's
+// on-device dictation model (SpeechAnalyzer + DictationTranscriber, macOS 26).
+// It is only a preview: the pasted text still comes from Whisper. It captures
+// the microphone with its own AVAudioEngine so the AVAudioRecorder path used
+// for Whisper stays untouched; if anything here fails there is simply no preview.
+final class LivePreview {
+    // Called on the main thread with the transcript so far
+    var onText: ((String) -> Void)?
+
+    private var session = 0
+    private var engine: AVAudioEngine?
+    private var analyzer: SpeechAnalyzer?
+    private var input: AsyncStream<AnalyzerInput>.Continuation?
+    private var resultsTask: Task<Void, Never>?
+
+    // App language code -> supported dictation locale ("hu" -> hu-HU); nil if unsupported
+    static func locale(for languageCode: String) async -> Locale? {
+        let requested = languageCode == SupportedLanguages.autoDetect ? Locale.current : Locale(identifier: languageCode)
+        return await DictationTranscriber.supportedLocale(equivalentTo: requested)
+    }
+
+    private static func makeTranscriber(locale: Locale) -> DictationTranscriber {
+        DictationTranscriber(locale: locale, contentHints: [], transcriptionOptions: [.punctuation],
+                             reportingOptions: [.volatileResults, .frequentFinalization], attributeOptions: [])
+    }
+
+    // Download the on-device model for this language in the background if it's missing
+    static func installModelIfNeeded(languageCode: String) {
+        Task {
+            guard let locale = await locale(for: languageCode) else { return }
+            do {
+                if let request = try await AssetInventory.assetInstallationRequest(supporting: [makeTranscriber(locale: locale)]) {
+                    NSLog("Live preview: installing dictation model for \(locale.identifier)")
+                    try await request.downloadAndInstall()
+                    NSLog("Live preview: dictation model installed for \(locale.identifier)")
+                }
+            } catch {
+                NSLog("Live preview: model install failed: \(error)")
+            }
+        }
+    }
+
+    func start(languageCode: String, contextualStrings: [String]) {
+        stop()
+        let current = session
+        Task { @MainActor in
+            do {
+                guard let locale = await LivePreview.locale(for: languageCode) else {
+                    NSLog("Live preview: no on-device dictation model for '\(languageCode)'")
+                    return
+                }
+                let transcriber = LivePreview.makeTranscriber(locale: locale)
+                // Even when the model is already on disk the app may first have to register it;
+                // that takes a fraction of a second. A real download just means no preview this time.
+                if await AssetInventory.status(forModules: [transcriber]) != .installed,
+                   let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
+                    NSLog("Live preview: preparing dictation model for \(locale.identifier)")
+                    try await request.downloadAndInstall()
+                }
+                guard current == self.session,
+                      let analyzerFormat = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else { return }
+
+                // Start the microphone first; the stream buffers audio until the analyzer is running,
+                // so the first words aren't lost while the model spins up.
+                let (stream, continuation) = AsyncStream<AnalyzerInput>.makeStream()
+                let engine = AVAudioEngine()
+                let micFormat = engine.inputNode.outputFormat(forBus: 0)
+                guard micFormat.sampleRate > 0, current == self.session,
+                      let converter = AVAudioConverter(from: micFormat, to: analyzerFormat) else { return }
+                engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: micFormat,
+                                            block: LivePreview.tapBlock(converter: converter, format: analyzerFormat, continuation: continuation))
+                try engine.start()
+                self.engine = engine
+                self.input = continuation
+
+                let analyzer = SpeechAnalyzer(modules: [transcriber])
+                self.analyzer = analyzer
+                if !contextualStrings.isEmpty {
+                    let context = AnalysisContext()
+                    context.contextualStrings[.general] = contextualStrings
+                    try await analyzer.setContext(context)
+                }
+                self.resultsTask = Task { @MainActor [weak self] in
+                    // Finalized segments accumulate; the volatile tail is replaced on every update
+                    var finalized = ""
+                    do {
+                        for try await result in transcriber.results {
+                            let text = String(result.text.characters)
+                            let shown = LivePreview.join(finalized, text)
+                            if result.isFinal { finalized = shown }
+                            guard let self = self, current == self.session else { return }
+                            self.onText?(shown)
+                        }
+                    } catch {
+                        NSLog("Live preview: results ended: \(error)")
+                    }
+                }
+                guard current == self.session else { return }  // stop() already tore everything down
+                try await analyzer.start(inputSequence: stream)
+            } catch {
+                NSLog("Live preview failed: \(error)")
+                if current == self.session { self.stop() }
+            }
+        }
+    }
+
+    func stop() {
+        session += 1
+        if let engine = engine {
+            engine.inputNode.removeTap(onBus: 0)
+            engine.stop()
+        }
+        engine = nil
+        input?.finish()
+        input = nil
+        resultsTask?.cancel()
+        resultsTask = nil
+        if let analyzer = analyzer {
+            Task { await analyzer.cancelAndFinishNow() }
+        }
+        analyzer = nil
+    }
+
+    // Segments don't always carry a separating space
+    static func join(_ head: String, _ tail: String) -> String {
+        guard let last = head.last, let first = tail.first, !last.isWhitespace, !first.isWhitespace else { return head + tail }
+        return head + " " + tail
+    }
+
+    // Runs on the audio thread: convert mic buffers to the analyzer's format.
+    // Built in a nonisolated static context so it carries no actor assumptions.
+    private static func tapBlock(converter: AVAudioConverter, format: AVAudioFormat,
+                                 continuation: AsyncStream<AnalyzerInput>.Continuation) -> AVAudioNodeTapBlock {
+        return { buffer, _ in
+            let ratio = format.sampleRate / buffer.format.sampleRate
+            let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 64
+            guard let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { return }
+            var consumed = false
+            var error: NSError?
+            converter.convert(to: out, error: &error) { _, status in
+                if consumed {
+                    status.pointee = .noDataNow
+                    return nil
+                }
+                consumed = true
+                status.pointee = .haveData
+                return buffer
+            }
+            if error == nil, out.frameLength > 0 {
+                continuation.yield(AnalyzerInput(buffer: out))
+            }
+        }
+    }
+}
+
 final class MeterView: NSView {
     enum Mode { case recording, transcribing }
     enum Notice { case loadingModel, modelLoaded }
@@ -419,15 +577,55 @@ final class MeterView: NSView {
     private var silent = false
     var mode = Mode.recording { didSet { needsDisplay = true } }
     var notice: Notice? { didSet { needsDisplay = true } }
+    var previewText: String? { didSet { if previewText != oldValue { cachedPreview = nil; needsDisplay = true } } }
+    var hasPreview: Bool { previewBlock() != nil }
 
-    // Height for the current mode + notice
+    private static let maxPreviewLines = 3
+    private static let previewFont = NSFont.systemFont(ofSize: 13)
+    private var cachedPreview: (width: CGFloat, text: NSAttributedString, height: CGFloat)?
+
+    private var noticeHeight: CGFloat {
+        switch notice {
+        case .loadingModel?: return 35
+        case .modelLoaded?: return 19
+        case nil: return 0
+        }
+    }
+
+    // Height for the current mode, notice and live preview text
     var preferredHeight: CGFloat {
         let base: CGFloat = mode == .recording ? 72 : 44
-        switch notice {
-        case .loadingModel?: return base + 34
-        case .modelLoaded?: return base + 20
-        case nil: return base
+        let preview = previewBlock().map { 6 + $0.height } ?? 0
+        return base + noticeHeight + preview
+    }
+
+    // The tail of the live transcript that fits in a few lines; older words scroll off the top
+    private func previewBlock() -> (text: NSAttributedString, height: CGFloat)? {
+        guard let raw = previewText, !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { return nil }
+        let width = bounds.width - 32
+        if let cached = cachedPreview, cached.width == width { return (cached.text, cached.height) }
+
+        let attrs: [NSAttributedString.Key: Any] = [
+            .font: MeterView.previewFont,
+            .foregroundColor: NSColor(calibratedWhite: 1, alpha: 0.92)
+        ]
+        func measure(_ string: NSAttributedString) -> CGFloat {
+            ceil(string.boundingRect(with: NSSize(width: width, height: 10_000), options: [.usesLineFragmentOrigin, .usesFontLeading]).height)
         }
+        let lineHeight = measure(NSAttributedString(string: "Ag", attributes: attrs))
+        let maxHeight = lineHeight * CGFloat(MeterView.maxPreviewLines) + 1
+
+        let words = raw.split(whereSeparator: { $0.isWhitespace })
+        var dropped = 0
+        var text = NSAttributedString(string: words.joined(separator: " "), attributes: attrs)
+        var height = measure(text)
+        while height > maxHeight && dropped < words.count - 1 {
+            dropped += 1
+            text = NSAttributedString(string: "…" + words[dropped...].joined(separator: " "), attributes: attrs)
+            height = measure(text)
+        }
+        cachedPreview = (width, text, height)
+        return (text, height)
     }
 
     func set(level: CGFloat, silent: Bool) {
@@ -435,7 +633,7 @@ final class MeterView: NSView {
         self.silent = silent
         needsDisplay = true
     }
-    func reset() { level = 0; silent = false; mode = .recording; notice = nil; needsDisplay = true }
+    func reset() { level = 0; silent = false; mode = .recording; notice = nil; previewText = nil; needsDisplay = true }
 
     override func draw(_ dirtyRect: NSRect) {
         let bg = NSBezierPath(roundedRect: bounds, xRadius: 12, yRadius: 12)
@@ -472,6 +670,12 @@ final class MeterView: NSView {
             break
         }
 
+        if let preview = previewBlock() {
+            let top = textY - 6
+            preview.text.draw(with: NSRect(x: 16, y: top - preview.height, width: bounds.width - 32, height: preview.height),
+                              options: [.usesLineFragmentOrigin, .usesFontLeading])
+        }
+
         guard mode == .recording else { return }
 
         let segs = 20
@@ -496,7 +700,7 @@ final class MeterView: NSView {
 
 final class RecordingHUD {
     private var window: NSWindow?
-    private let meter = MeterView(frame: NSRect(x: 0, y: 0, width: 320, height: 72))
+    private let meter = MeterView(frame: NSRect(x: 0, y: 0, width: 420, height: 72))
 
     func show() {
         meter.reset()
@@ -512,6 +716,13 @@ final class RecordingHUD {
         layout()
     }
 
+    func setPreview(_ text: String?) {
+        meter.previewText = text
+        layout()
+    }
+
+    var hasPreview: Bool { meter.hasPreview }
+
     // Stay up after recording while a slow (model-loading) transcription runs
     func showTranscribing(notice: MeterView.Notice?) {
         makeWindowIfNeeded()
@@ -533,7 +744,7 @@ final class RecordingHUD {
 
     private func makeWindowIfNeeded() {
         if window == nil {
-            let w = NSPanel(contentRect: NSRect(x: 0, y: 0, width: 320, height: 72),
+            let w = NSPanel(contentRect: NSRect(x: 0, y: 0, width: 420, height: 72),
                             styleMask: [.borderless, .nonactivatingPanel],
                             backing: .buffered, defer: false)
             w.isOpaque = false
@@ -786,6 +997,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
     var settingsWindowController: NSWindowController?
     var whisperServer: WhisperServer?
     let recordingHUD = RecordingHUD()
+    let livePreview = LivePreview()
     var meterTimer: Timer?
     var silenceStart: Date?
     weak var dictTable: NSTableView?
@@ -814,6 +1026,11 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
         set { UserDefaults.standard.set(newValue, forKey: Defaults.playSounds) }
     }
 
+    var livePreviewEnabled: Bool {
+        get { UserDefaults.standard.object(forKey: Defaults.livePreview) as? Bool ?? true }
+        set { UserDefaults.standard.set(newValue, forKey: Defaults.livePreview) }
+    }
+
     // Custom dictionary: ordered list of ["from": recognised, "to": replacement].
     var replacements: [[String: String]] {
         get { (UserDefaults.standard.array(forKey: Defaults.replacements) as? [[String: String]]) ?? [] }
@@ -823,11 +1040,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
     // Whisper initial prompt built from the desired ("to") terms, to bias
     // recognition toward the user's names/jargon. Returns nil if empty.
     func customPrompt() -> String? {
-        let terms = replacements.compactMap { $0["to"] }
+        let terms = dictionaryTerms()
+        return terms.isEmpty ? nil : terms.joined(separator: ", ")
+    }
+
+    // The desired ("to") dictionary terms, also given to the live preview model
+    func dictionaryTerms() -> [String] {
+        replacements.compactMap { $0["to"] }
             .map { $0.replacingOccurrences(of: "\\n", with: " ") }
             .map { $0.trimmingCharacters(in: .whitespaces) }
             .filter { !$0.isEmpty }
-        return terms.isEmpty ? nil : terms.joined(separator: ", ")
     }
 
     // Apply the dictionary find/replace pairs to a transcript. "\n" in a
@@ -858,6 +1080,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
         }
 
         checkAccessibilityPermission()
+        if livePreviewEnabled { LivePreview.installModelIfNeeded(languageCode: language) }
 
         NSLog("WhisperDictate started. Press ⌃⌥D to toggle recording.")
     }
@@ -1091,7 +1314,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
 
     func createSettingsWindow() -> NSWindow {
         let window = NSWindow(
-            contentRect: NSRect(x: 0, y: 0, width: 450, height: 540),
+            contentRect: NSRect(x: 0, y: 0, width: 450, height: 580),
             styleMask: [.titled, .closable],
             backing: .buffered,
             defer: false
@@ -1102,7 +1325,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
 
         let contentView = NSView(frame: window.contentView!.bounds)
 
-        var y: CGFloat = 498
+        var y: CGFloat = 538
         let labelWidth: CGFloat = 120
         let controlX: CGFloat = 140
         let controlWidth: CGFloat = 280
@@ -1180,6 +1403,15 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
         loginCheck.state = isLaunchAtLoginEnabled() ? .on : .off
         contentView.addSubview(loginCheck)
 
+        y -= 40
+
+        // Live preview
+        let previewCheck = NSButton(checkboxWithTitle: "Live preview while recording", target: self, action: #selector(livePreviewChanged(_:)))
+        previewCheck.frame = NSRect(x: controlX, y: y, width: controlWidth, height: 24)
+        previewCheck.state = livePreviewEnabled ? .on : .off
+        previewCheck.toolTip = "Shows a rough live transcript in the recording overlay using macOS's on-device dictation model. The pasted text still comes from Whisper."
+        contentView.addSubview(previewCheck)
+
         // Custom dictionary (recognised -> replacement)
         let dictLabel = NSTextField(labelWithString: "Dictionary (heard → replacement):")
         dictLabel.frame = NSRect(x: 20, y: 304, width: 300, height: 20)
@@ -1247,6 +1479,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
     @objc func languageChanged(_ sender: NSPopUpButton) {
         if let code = sender.selectedItem?.representedObject as? String {
             language = code
+            if livePreviewEnabled { LivePreview.installModelIfNeeded(languageCode: code) }
             NSLog("Language changed to: \(language)")
         }
     }
@@ -1330,6 +1563,11 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
         }
     }
 
+    @objc func livePreviewChanged(_ sender: NSButton) {
+        livePreviewEnabled = sender.state == .on
+        if livePreviewEnabled { LivePreview.installModelIfNeeded(languageCode: language) }
+    }
+
     @objc func playSoundsChanged(_ sender: NSButton) {
         playSounds = sender.state == .on
     }
@@ -1562,6 +1800,7 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
             NSLog("Recording started")
             startMeter()
             prepareModelForDictation()
+            startLivePreview()
         } catch {
             NSLog("Recording failed: \(error)")
             if playSounds { NSSound(named: "Basso")?.play() }
@@ -1603,6 +1842,16 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
         checkModelLoaded()
     }
 
+    // MARK: - Live preview
+    func startLivePreview() {
+        guard livePreviewEnabled else { return }
+        livePreview.onText = { [weak self] text in
+            guard let self = self, self.isRecording else { return }
+            self.recordingHUD.setPreview(self.applyReplacements(text))
+        }
+        livePreview.start(languageCode: language, contextualStrings: dictionaryTerms())
+    }
+
     // MARK: - Model warm-up
     // Start getting the model into memory as soon as recording starts, so loading
     // overlaps with the user talking instead of following it. Only if that turns
@@ -1659,10 +1908,13 @@ class AppDelegate: NSObject, NSApplicationDelegate, URLSessionDownloadDelegate,
         audioRecorder?.stop()
         isRecording = false
         isTranscribing = true
-        // Model still loading and already announced: keep explaining the wait
-        let keepHUD = modelLoading && loadNoticeShown
+        livePreview.stop()
+        // Keep the HUD up while Whisper works if there is something to show:
+        // the live preview text, or why this transcription is slower
+        let explainLoading = modelLoading && loadNoticeShown
+        let keepHUD = explainLoading || recordingHUD.hasPreview
         stopMeter(hideHUD: !keepHUD)
-        if keepHUD { recordingHUD.showTranscribing(notice: .loadingModel) }
+        if keepHUD { recordingHUD.showTranscribing(notice: explainLoading ? .loadingModel : nil) }
         statusItem.button?.title = "⏳"
         updateStatus("Transcribing...")
         if playSounds { NSSound(named: "Pop")?.play() }