Fix silent live preview on stereo and multi-input audio interfaces
AVAudioConverter keeps only channel 1 when downmixing stereo and outputs pure silence for layout-less multi-channel inputs (e.g. a 4-input USB interface such as the UA Volt 476P), so the dictation model heard nothing and the HUD never showed text. Pick the channel that carries the voice (loudest on smoothed energy, switching only on a clear 6 dB margin) and convert it as mono. Also reserve the dictation locale, which the Speech framework warns will become required, and log the mic format. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -468,6 +468,7 @@ final class LivePreview {
|
|||||||
let transcriber = LivePreview.makeTranscriber(locale: locale)
|
let transcriber = LivePreview.makeTranscriber(locale: locale)
|
||||||
// Even when the model is already on disk the app may first have to register it;
|
// Even when the model is already on disk the app may first have to register it;
|
||||||
// that takes a fraction of a second. A real download just means no preview this time.
|
// that takes a fraction of a second. A real download just means no preview this time.
|
||||||
|
_ = try? await AssetInventory.reserve(locale: locale)
|
||||||
if await AssetInventory.status(forModules: [transcriber]) != .installed,
|
if await AssetInventory.status(forModules: [transcriber]) != .installed,
|
||||||
let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
||||||
NSLog("Live preview: preparing dictation model for \(locale.identifier)")
|
NSLog("Live preview: preparing dictation model for \(locale.identifier)")
|
||||||
@@ -482,7 +483,9 @@ final class LivePreview {
|
|||||||
let engine = AVAudioEngine()
|
let engine = AVAudioEngine()
|
||||||
let micFormat = engine.inputNode.outputFormat(forBus: 0)
|
let micFormat = engine.inputNode.outputFormat(forBus: 0)
|
||||||
guard micFormat.sampleRate > 0, current == self.session,
|
guard micFormat.sampleRate > 0, current == self.session,
|
||||||
let converter = AVAudioConverter(from: micFormat, to: analyzerFormat) else { return }
|
let monoFormat = AVAudioFormat(standardFormatWithSampleRate: micFormat.sampleRate, channels: 1),
|
||||||
|
let converter = AVAudioConverter(from: monoFormat, to: analyzerFormat) else { return }
|
||||||
|
NSLog("Live preview: mic %d ch @ %.0f Hz -> %@", micFormat.channelCount, micFormat.sampleRate, analyzerFormat.description)
|
||||||
engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: micFormat,
|
engine.inputNode.installTap(onBus: 0, bufferSize: 4096, format: micFormat,
|
||||||
block: LivePreview.tapBlock(converter: converter, format: analyzerFormat, continuation: continuation))
|
block: LivePreview.tapBlock(converter: converter, format: analyzerFormat, continuation: continuation))
|
||||||
try engine.start()
|
try engine.start()
|
||||||
@@ -543,11 +546,13 @@ final class LivePreview {
|
|||||||
return head + " " + tail
|
return head + " " + tail
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runs on the audio thread: convert mic buffers to the analyzer's format.
|
// Runs on the audio thread: take the voice channel as mono, convert it to the
|
||||||
// Built in a nonisolated static context so it carries no actor assumptions.
|
// analyzer's format. Built in a static context so it carries no actor assumptions.
|
||||||
private static func tapBlock(converter: AVAudioConverter, format: AVAudioFormat,
|
private static func tapBlock(converter: AVAudioConverter, format: AVAudioFormat,
|
||||||
continuation: AsyncStream<AnalyzerInput>.Continuation) -> AVAudioNodeTapBlock {
|
continuation: AsyncStream<AnalyzerInput>.Continuation) -> AVAudioNodeTapBlock {
|
||||||
return { buffer, _ in
|
let downmixer = VoiceChannelPicker()
|
||||||
|
return { micBuffer, _ in
|
||||||
|
guard let buffer = downmixer.mono(from: micBuffer) else { return }
|
||||||
let ratio = format.sampleRate / buffer.format.sampleRate
|
let ratio = format.sampleRate / buffer.format.sampleRate
|
||||||
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 64
|
let capacity = AVAudioFrameCount(Double(buffer.frameLength) * ratio) + 64
|
||||||
guard let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { return }
|
guard let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: capacity) else { return }
|
||||||
@@ -569,6 +574,47 @@ final class LivePreview {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Picks the input channel that carries the voice and returns it as a mono buffer.
|
||||||
|
// AVAudioConverter keeps only channel 1 when downmixing stereo, and outputs pure
|
||||||
|
// silence for layout-less multi-channel inputs (e.g. a 4-input USB interface),
|
||||||
|
// so the live preview model would hear nothing. The loudest channel is chosen on
|
||||||
|
// smoothed energy, and only replaced by one that is clearly (6 dB) louder.
|
||||||
|
final class VoiceChannelPicker {
|
||||||
|
private var energy: [Float] = []
|
||||||
|
private var active = 0
|
||||||
|
private var monoFormat: AVAudioFormat?
|
||||||
|
|
||||||
|
func mono(from buffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? {
|
||||||
|
let channels = Int(buffer.format.channelCount)
|
||||||
|
let frames = Int(buffer.frameLength)
|
||||||
|
guard frames > 0, let data = buffer.floatChannelData else { return nil }
|
||||||
|
if channels == 1 { return buffer }
|
||||||
|
|
||||||
|
if energy.count != channels {
|
||||||
|
energy = Array(repeating: 0, count: channels)
|
||||||
|
active = 0
|
||||||
|
}
|
||||||
|
for c in 0..<channels {
|
||||||
|
var sum: Float = 0
|
||||||
|
for i in 0..<frames { sum += data[c][i] * data[c][i] }
|
||||||
|
energy[c] = energy[c] * 0.9 + sum / Float(frames) * 0.1
|
||||||
|
}
|
||||||
|
if let loudest = energy.indices.max(by: { energy[$0] < energy[$1] }), energy[loudest] > energy[active] * 4 {
|
||||||
|
active = loudest
|
||||||
|
}
|
||||||
|
|
||||||
|
if monoFormat?.sampleRate != buffer.format.sampleRate {
|
||||||
|
monoFormat = AVAudioFormat(standardFormatWithSampleRate: buffer.format.sampleRate, channels: 1)
|
||||||
|
}
|
||||||
|
guard let format = monoFormat,
|
||||||
|
let out = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames)),
|
||||||
|
let dest = out.floatChannelData else { return nil }
|
||||||
|
dest[0].update(from: data[active], count: frames)
|
||||||
|
out.frameLength = AVAudioFrameCount(frames)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
final class MeterView: NSView {
|
final class MeterView: NSView {
|
||||||
enum Mode { case recording, transcribing }
|
enum Mode { case recording, transcribing }
|
||||||
enum Notice { case loadingModel, modelLoaded }
|
enum Notice { case loadingModel, modelLoaded }
|
||||||
|
|||||||
Reference in New Issue
Block a user