diff --git a/macos/src/main.swift b/macos/src/main.swift index dd2937d..da969c7 100644 --- a/macos/src/main.swift +++ b/macos/src/main.swift @@ -468,6 +468,7 @@ final class LivePreview { let transcriber = LivePreview.makeTranscriber(locale: locale) // Even when the model is already on disk the app may first have to register it; // that takes a fraction of a second. A real download just means no preview this time. + _ = try? await AssetInventory.reserve(locale: locale) if await AssetInventory.status(forModules: [transcriber]) != .installed, let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) { NSLog("Live preview: preparing dictation model for \(locale.identifier)") @@ -482,7 +483,9 @@ final class LivePreview { let engine = AVAudioEngine() let micFormat = engine.inputNode.outputFormat(forBus: 0) guard micFormat.sampleRate > 0, current == self.session, - let converter = AVAudioConverter(from: micFormat, to: analyzerFormat) else { return } + let monoFormat = AVAudioFormat(standardFormatWithSampleRate: micFormat.sampleRate, channels: 1), + let converter = AVAudioConverter(from: monoFormat, to: analyzerFormat) else { return } + NSLog("Live preview: mic %d ch @ %.0f Hz -> %@", micFormat.channelCount, micFormat.sampleRate, analyzerFormat.description) engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: micFormat, block: LivePreview.tapBlock(converter: converter, format: analyzerFormat, continuation: continuation)) try engine.start() @@ -543,11 +546,13 @@ final class LivePreview { return head + " " + tail } - // Runs on the audio thread: convert mic buffers to the analyzer's format. - // Built in a nonisolated static context so it carries no actor assumptions. + // Runs on the audio thread: take the voice channel as mono, convert it to the + // analyzer's format. Built in a static context so it carries no actor assumptions. private static func tapBlock(converter: AVAudioConverter, format: AVAudioFormat, continuation: AsyncStream.Continuation) -> AVAudioNodeTapBlock { - return { buffer, _ in + let downmixer = VoiceChannelPicker() + return { micBuffer, _ in + guard let buffer = downmixer.mono(from: micBuffer) else { return } let ratio = format.sampleRate / buffer.format.sampleRate let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 64 guard let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { return } @@ -569,6 +574,47 @@ final class LivePreview { } } +// Picks the input channel that carries the voice and returns it as a mono buffer. +// AVAudioConverter keeps only channel 1 when downmixing stereo, and outputs pure +// silence for layout-less multi-channel inputs (e.g. a 4-input USB interface), +// so the live preview model would hear nothing. The loudest channel is chosen on +// smoothed energy, and only replaced by one that is clearly (6 dB) louder. +final class VoiceChannelPicker { + private var energy: [Float] = [] + private var active = 0 + private var monoFormat: AVAudioFormat? + + func mono(from buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + let channels = Int(buffer.format.channelCount) + let frames = Int(buffer.frameLength) + guard frames > 0, let data = buffer.floatChannelData else { return nil } + if channels == 1 { return buffer } + + if energy.count != channels { + energy = Array(repeating: 0, count: channels) + active = 0 + } + for c in 0.. energy[active] * 4 { + active = loudest + } + + if monoFormat?.sampleRate != buffer.format.sampleRate { + monoFormat = AVAudioFormat(standardFormatWithSampleRate: buffer.format.sampleRate, channels: 1) + } + guard let format = monoFormat, + let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames)), + let dest = out.floatChannelData else { return nil } + dest[0].update(from: data[active], count: frames) + out.frameLength = AVAudioFrameCount(frames) + return out + } +} + final class MeterView: NSView { enum Mode { case recording, transcribing } enum Notice { case loadingModel, modelLoaded }