in src/backend/mod.rs [3537:4224]
fn setup(
&mut self,
shared_voice_processing_unit: &mut SharedVoiceProcessingUnitManager,
) -> Result<()> {
self.debug_assert_is_on_stream_queue();
if self
.input_stream_params
.prefs()
.contains(StreamPrefs::LOOPBACK)
|| self
.output_stream_params
.prefs()
.contains(StreamPrefs::LOOPBACK)
{
cubeb_log!("({:p}) Loopback not supported for audiounit.", self.stm_ptr);
return Err(Error::not_supported());
}
let same_clock_domain = self.same_clock_domain();
let (in_dev_info, out_dev_info) = self.create_audiounits(shared_voice_processing_unit)?;
let using_voice_processing_unit = self.using_voice_processing_unit();
assert!(!self.stm_ptr.is_null());
let stream = unsafe { &(*self.stm_ptr) };
#[cfg(feature = "audio-dump")]
unsafe {
ffi::cubeb_audio_dump_init(&mut self.audio_dump_session);
}
// Configure I/O stream
if self.has_input() {
assert!(!self.input_unit.is_null());
cubeb_log!(
"({:p}) Initializing input by device info: {:?}",
self.stm_ptr,
in_dev_info
);
let device_channel_count =
get_channel_count(self.input_device.id, DeviceType::INPUT).unwrap_or(0);
if device_channel_count < self.input_stream_params.channels() {
cubeb_log!(
"({:p}) Invalid input channel count; device={}, params={}",
self.stm_ptr,
device_channel_count,
self.input_stream_params.channels()
);
return Err(Error::invalid_parameter());
}
cubeb_log!(
"({:p}) Opening input side: rate {}, channels {}, format {:?}, layout {:?}, prefs {:?}, latency in frames {}, voice processing {}.",
self.stm_ptr,
self.input_stream_params.rate(),
self.input_stream_params.channels(),
self.input_stream_params.format(),
self.input_stream_params.layout(),
self.input_stream_params.prefs(),
stream.latency_frames,
using_voice_processing_unit
);
// Get input device hardware information.
let mut input_hw_desc = AudioStreamBasicDescription::default();
let mut size = mem::size_of::<AudioStreamBasicDescription>();
let r = audio_unit_get_property(
self.input_unit,
kAudioUnitProperty_StreamFormat,
if using_voice_processing_unit {
// With a VPIO unit the input scope includes AEC reference channels.
// We need to use the output scope of the input bus.
kAudioUnitScope_Output
} else {
// With a HAL unit the output scope for the input bus returns the number of
// output channels of the output device, i.e. it seems the bus is ignored.
kAudioUnitScope_Input
},
AU_IN_BUS,
&mut input_hw_desc,
&mut size,
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitGetProperty/input/kAudioUnitProperty_StreamFormat rv={}",
r
);
return Err(Error::error());
}
cubeb_log!(
"({:p}) Input hardware description: {:?}",
self.stm_ptr,
input_hw_desc
);
// Notice: when we are using aggregate device, input_hw_desc.mChannelsPerFrame is the
// total of all the input channel count of the devices added in the aggregate device.
// Because we set the input device first on the aggregate device, the input device's
// input channels will also be first among all the aggregate device's channels, when
// accessed in the input callback. By requesting only the input device's channels here,
// any other input channels, i.e. from the output device, will be truncated away.
let params = unsafe {
let mut p = *self.input_stream_params.as_ptr();
p.channels = device_channel_count;
// Input AudioUnit must be configured with device's sample rate.
// we will resample inside input callback.
p.rate = input_hw_desc.mSampleRate as _;
StreamParams::from(p)
};
self.input_dev_desc = create_stream_description(¶ms).inspect_err(|_| {
cubeb_log!(
"({:p}) Setting format description for input failed.",
self.stm_ptr
);
})?;
#[cfg(feature = "audio-dump")]
{
let name = format!("input-{:p}.wav", self.stm_ptr);
let cname = CString::new(name).expect("OK");
let rv = unsafe {
ffi::cubeb_audio_dump_stream_init(
self.audio_dump_session,
&mut self.audio_dump_input,
*params.as_ptr(),
cname.as_ptr(),
)
};
if rv == 0 {
assert_ne!(self.audio_dump_input, ptr::null_mut(),);
cubeb_log!("Successfully inited audio dump for input");
} else {
cubeb_log!("Failed to init audio dump for input");
}
}
assert_eq!(self.input_dev_desc.mSampleRate, input_hw_desc.mSampleRate);
// Use latency to set buffer size
assert_ne!(stream.latency_frames, 0);
if let Err(r) =
set_buffer_size_sync(self.input_unit, DeviceType::INPUT, stream.latency_frames)
{
cubeb_log!("({:p}) Error in change input buffer size.", self.stm_ptr);
return Err(r);
}
let r = audio_unit_set_property(
self.input_unit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Output,
AU_IN_BUS,
&self.input_dev_desc,
mem::size_of::<AudioStreamBasicDescription>(),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/input/kAudioUnitProperty_StreamFormat rv={}",
r
);
return Err(Error::error());
}
// Frames per buffer in the input callback.
let r = audio_unit_set_property(
self.input_unit,
kAudioUnitProperty_MaximumFramesPerSlice,
kAudioUnitScope_Global,
AU_IN_BUS,
&stream.latency_frames,
mem::size_of::<u32>(),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/input/kAudioUnitProperty_MaximumFramesPerSlice rv={}",
r
);
return Err(Error::error());
}
// When we use the aggregate device, the self.input_dev_desc.mChannelsPerFrame is the
// total input channel count of all the device added in the aggregate device. However,
// we only need the audio data captured by the requested input device, so we need to
// ignore some data captured by the audio input of the requested output device (e.g.,
// the requested output device is a USB headset with built-in mic), in the beginning of
// the raw data taken from input callback.
self.input_buffer_manager = Some(BufferManager::new(
self.input_stream_params.format(),
SAFE_MAX_LATENCY_FRAMES as usize,
self.input_dev_desc.mChannelsPerFrame as usize,
self.input_dev_desc
.mChannelsPerFrame
.saturating_sub(device_channel_count) as usize,
self.input_stream_params.channels() as usize,
));
let aurcbs_in = AURenderCallbackStruct {
inputProc: Some(audiounit_input_callback),
inputProcRefCon: self.stm_ptr as *mut c_void,
};
let r = audio_unit_set_property(
self.input_unit,
kAudioOutputUnitProperty_SetInputCallback,
kAudioUnitScope_Global,
AU_OUT_BUS,
&aurcbs_in,
mem::size_of_val(&aurcbs_in),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/input/kAudioOutputUnitProperty_SetInputCallback rv={}",
r
);
return Err(Error::error());
}
stream.frames_read.store(0, Ordering::SeqCst);
cubeb_log!(
"({:p}) Input audiounit init with device {} successfully.",
self.stm_ptr,
in_dev_info.id
);
}
if self.has_input() && !self.has_output() && using_voice_processing_unit {
// We must configure the output side of VPIO to match the input side, even if we don't use it.
let r = audio_unit_set_property(
self.input_unit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input,
AU_OUT_BUS,
&self.input_dev_desc,
mem::size_of::<AudioStreamBasicDescription>(),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/output/kAudioUnitProperty_StreamFormat rv={}",
r
);
return Err(Error::error());
}
}
if self.has_output() {
assert!(!self.output_unit.is_null());
cubeb_log!(
"({:p}) Initialize output by device info: {:?}",
self.stm_ptr,
out_dev_info
);
cubeb_log!(
"({:p}) Opening output side: rate {}, channels {}, format {:?}, layout {:?}, prefs {:?}, latency in frames {}, voice processing {}.",
self.stm_ptr,
self.output_stream_params.rate(),
self.output_stream_params.channels(),
self.output_stream_params.format(),
self.output_stream_params.layout(),
self.output_stream_params.prefs(),
stream.latency_frames,
using_voice_processing_unit
);
// Get output device hardware information.
let mut output_hw_desc = AudioStreamBasicDescription::default();
let mut size = mem::size_of::<AudioStreamBasicDescription>();
let r = audio_unit_get_property(
self.output_unit,
kAudioUnitProperty_StreamFormat,
if using_voice_processing_unit {
// With a VPIO unit the output scope includes all channels in the hw.
// The VPIO unit however is only MONO which the input scope reflects.
kAudioUnitScope_Input
} else {
// With a HAL unit the output scope for the output bus returns the number of
// output channels of the hw, as we want. The input scope seems limited to
// two channels.
kAudioUnitScope_Output
},
AU_OUT_BUS,
&mut output_hw_desc,
&mut size,
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitGetProperty/output/kAudioUnitProperty_StreamFormat rv={}",
r
);
return Err(Error::error());
}
cubeb_log!(
"({:p}) Output hardware description: {:?}",
self.stm_ptr,
output_hw_desc
);
// This has been observed in the wild.
if output_hw_desc.mChannelsPerFrame == 0 {
cubeb_log!(
"({:p}) Output hardware description channel count is zero",
self.stm_ptr
);
return Err(Error::error());
}
// Simple case of stereo output, map to the stereo pair (that might not be the first
// two channels). Fall back to regular mixing if this fails.
let mut maybe_need_mixer = true;
if self.output_stream_params.channels() == 2
&& self.output_stream_params.layout() == ChannelLayout::STEREO
{
let layout = AudioChannelLayout {
mChannelLayoutTag: kAudioChannelLayoutTag_Stereo,
..Default::default()
};
let r = audio_unit_set_property(
self.output_unit,
kAudioUnitProperty_AudioChannelLayout,
kAudioUnitScope_Input,
AU_OUT_BUS,
&layout,
mem::size_of::<AudioChannelLayout>(),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/output/kAudioUnitProperty_AudioChannelLayout rv={}",
r
);
}
maybe_need_mixer = r != NO_ERR;
}
// Notice: when we are using aggregate device, the output_hw_desc.mChannelsPerFrame is
// the total of all the output channel count of the devices added in the aggregate device.
// Due to our aggregate device settings, the data recorded by the input device's output
// channels will be appended at the end of the raw data given by the output callback.
let params = unsafe {
let mut p = *self.output_stream_params.as_ptr();
p.channels = if maybe_need_mixer {
output_hw_desc.mChannelsPerFrame
} else {
self.output_stream_params.channels()
};
if using_voice_processing_unit {
// VPIO will always use the sample rate of the input hw for both input and output,
// as reported to us. (We can override it but we cannot improve quality this way).
p.rate = self.input_dev_desc.mSampleRate as _;
}
StreamParams::from(p)
};
self.output_dev_desc = create_stream_description(¶ms).inspect_err(|_| {
cubeb_log!(
"({:p}) Could not initialize the audio stream description.",
self.stm_ptr
);
})?;
#[cfg(feature = "audio-dump")]
{
let name = format!("output-{:p}.wav", self.stm_ptr);
let cname = CString::new(name).expect("OK");
let rv = unsafe {
ffi::cubeb_audio_dump_stream_init(
self.audio_dump_session,
&mut self.audio_dump_output,
*params.as_ptr(),
cname.as_ptr(),
)
};
if rv == 0 {
assert_ne!(self.audio_dump_output, ptr::null_mut(),);
cubeb_log!("Successfully inited audio dump for output");
} else {
cubeb_log!("Failed to init audio dump for output");
}
}
let device_layout = self
.get_output_channel_layout()
.inspect_err(|_| {
cubeb_log!(
"({:p}) Could not get any channel layout. Defaulting to no channels.",
self.stm_ptr
);
})
.unwrap_or_default();
cubeb_log!(
"({:p} Using output device channel layout {:?}",
self.stm_ptr,
device_layout
);
if maybe_need_mixer {
// The mixer will be set up when
// 0. not playing simply stereo, or failing to set the channel layout to the stereo
// pair
// 1. using aggregate device whose input device has output channels
// 2. output device has more channels than we need, and stream isn't simply stereo
// 3. output device has different layout than the one we have
self.mixer = if self.output_dev_desc.mChannelsPerFrame
!= self.output_stream_params.channels()
|| device_layout != mixer::get_channel_order(self.output_stream_params.layout())
{
cubeb_log!("Incompatible channel layouts detected, setting up remixer");
// We will be remixing the data before it reaches the output device.
Some(Mixer::new(
self.output_stream_params.format(),
self.output_stream_params.channels() as usize,
self.output_stream_params.layout(),
self.output_dev_desc.mChannelsPerFrame as usize,
device_layout,
))
} else {
None
};
}
let r = audio_unit_set_property(
self.output_unit,
kAudioUnitProperty_StreamFormat,
kAudioUnitScope_Input,
AU_OUT_BUS,
&self.output_dev_desc,
mem::size_of::<AudioStreamBasicDescription>(),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/output/kAudioUnitProperty_StreamFormat rv={}",
r
);
return Err(Error::error());
}
// Use latency to set buffer size
assert_ne!(stream.latency_frames, 0);
if let Err(r) =
set_buffer_size_sync(self.output_unit, DeviceType::OUTPUT, stream.latency_frames)
{
cubeb_log!("({:p}) Error in change output buffer size.", self.stm_ptr);
return Err(r);
}
// Frames per buffer in the input callback.
let r = audio_unit_set_property(
self.output_unit,
kAudioUnitProperty_MaximumFramesPerSlice,
kAudioUnitScope_Global,
AU_OUT_BUS,
&stream.latency_frames,
mem::size_of::<u32>(),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/output/kAudioUnitProperty_MaximumFramesPerSlice rv={}",
r
);
return Err(Error::error());
}
let aurcbs_out = AURenderCallbackStruct {
inputProc: Some(audiounit_output_callback),
inputProcRefCon: self.stm_ptr as *mut c_void,
};
let r = audio_unit_set_property(
self.output_unit,
kAudioUnitProperty_SetRenderCallback,
kAudioUnitScope_Global,
AU_OUT_BUS,
&aurcbs_out,
mem::size_of_val(&aurcbs_out),
);
if r != NO_ERR {
cubeb_log!(
"AudioUnitSetProperty/output/kAudioUnitProperty_SetRenderCallback rv={}",
r
);
return Err(Error::error());
}
stream.frames_written.store(0, Ordering::SeqCst);
cubeb_log!(
"({:p}) Output audiounit init with device {} successfully.",
self.stm_ptr,
out_dev_info.id
);
}
// We use a resampler because input AudioUnit operates
// reliable only in the capture device sample rate.
// Resampler will convert it to the user sample rate
// and deliver it to the callback.
let target_sample_rate = if self.has_input() {
self.input_stream_params.rate()
} else {
assert!(self.has_output());
self.output_stream_params.rate()
};
let resampler_input_params = if self.has_input() {
let mut p = unsafe { *(self.input_stream_params.as_ptr()) };
p.rate = self.input_dev_desc.mSampleRate as u32;
Some(p)
} else {
None
};
let resampler_output_params = if self.has_output() {
let mut p = unsafe { *(self.output_stream_params.as_ptr()) };
p.rate = self.output_dev_desc.mSampleRate as u32;
Some(p)
} else {
None
};
// Only reclock if there is an input and we couldn't use an aggregate device, and the
// devices are not part of the same clock domain.
let reclock_policy = if self.aggregate_device.is_none()
&& !using_voice_processing_unit
&& !same_clock_domain
{
cubeb_log!(
"Reclocking duplex steam using_aggregate_device={} same_clock_domain={}",
self.aggregate_device.is_some(),
same_clock_domain
);
ffi::CUBEB_RESAMPLER_RECLOCK_INPUT
} else {
ffi::CUBEB_RESAMPLER_RECLOCK_NONE
};
self.resampler = Resampler::new(
self.stm_ptr as *mut ffi::cubeb_stream,
resampler_input_params,
resampler_output_params,
target_sample_rate,
stream.data_callback,
stream.user_ptr,
ffi::CUBEB_RESAMPLER_QUALITY_DESKTOP,
reclock_policy,
);
// In duplex, the input thread might be different from the output thread, and we're logging
// everything from the output thread: relay the audio input callback information using a
// ring buffer to diagnose issues.
if self.has_input() && self.has_output() {
self.input_logging = Some(InputCallbackLogger::new());
}
#[cfg(feature = "audio-dump")]
{
unsafe { ffi::cubeb_audio_dump_start(self.audio_dump_session) };
self.audio_dump_session_running = true;
}
if !self.input_unit.is_null() {
let r = audio_unit_initialize(self.input_unit);
if r != NO_ERR {
cubeb_log!("AudioUnitInitialize/input rv={}", r);
return Err(Error::error());
}
stream.input_device_latency_frames.store(
get_fixed_latency(self.input_device.id, DeviceType::INPUT),
Ordering::SeqCst,
);
}
if !self.output_unit.is_null() {
if !std::ptr::eq(self.input_unit, self.output_unit) {
let r = audio_unit_initialize(self.output_unit);
if r != NO_ERR {
cubeb_log!("AudioUnitInitialize/output rv={}", r);
return Err(Error::error());
}
}
stream.output_device_latency_frames.store(
get_fixed_latency(self.output_device.id, DeviceType::OUTPUT),
Ordering::SeqCst,
);
let mut unit_s: f64 = 0.0;
let mut size = mem::size_of_val(&unit_s);
if audio_unit_get_property(
self.output_unit,
kAudioUnitProperty_Latency,
kAudioUnitScope_Global,
0,
&mut unit_s,
&mut size,
) == NO_ERR
{
stream.output_device_latency_frames.fetch_add(
(unit_s * self.output_dev_desc.mSampleRate) as u32,
Ordering::SeqCst,
);
}
}
if using_voice_processing_unit {
// The VPIO AudioUnit automatically ducks other audio streams on the VPIO
// output device. Its ramp duration is 0.5s when ducking, so unduck similarly
// now.
// NOTE: On MacOS 14 the ducking happens on creation of the VPIO AudioUnit.
// On MacOS 10.15 it happens on both creation and initialization, which
// is why we defer the unducking until now.
#[allow(non_upper_case_globals)]
let mut device = match self.output_device.id {
kAudioObjectUnknown => None,
id => Some(id),
};
device = device.or_else(|| get_default_device(DeviceType::OUTPUT));
match device {
None => {
cubeb_log!(
"({:p}) No output device to undo vpio ducking on",
self.stm_ptr
);
}
Some(id) => {
let r = audio_device_duck(id, 1.0, ptr::null_mut(), 0.5);
if r != NO_ERR {
cubeb_log!(
"({:p}) Failed to undo ducking of voiceprocessing on output device {}. Proceeding... Error: {}",
self.stm_ptr,
id,
r
);
}
}
};
// Always try to remember the applied input mute state. If it cannot be applied
// to the new device pair, we notify the client of an error and it will have to
// open a new stream.
if let Err(r) = set_input_mute(self.input_unit, self.input_mute) {
cubeb_log!(
"({:p}) Failed to set mute state of voiceprocessing. Error: {}",
self.stm_ptr,
r
);
return Err(r);
}
}
if let Err(r) = self.install_system_changed_callback() {
cubeb_log!(
"({:p}) Could not install the device change callback.",
self.stm_ptr
);
return Err(r);
}
if let Err(r) = self.install_device_changed_callback() {
cubeb_log!(
"({:p}) Could not install all device change callback.",
self.stm_ptr
);
return Err(r);
}
// We have either default_input_listener or input_alive_listener.
// We cannot have both of them at the same time.
assert!(
!self.has_input()
|| ((self.default_input_listener.is_some() != self.input_alive_listener.is_some())
&& (self.default_input_listener.is_some()
|| self.input_alive_listener.is_some()))
);
// We have either default_output_listener or output_alive_listener.
// We cannot have both of them at the same time.
assert!(
!self.has_output()
|| ((self.default_output_listener.is_some()
!= self.output_alive_listener.is_some())
&& (self.default_output_listener.is_some()
|| self.output_alive_listener.is_some()))
);
Ok(())
}