Implement [RTCAudioDeviceModule deliverRecordedData:]

README.pixiv.md

@@ -11,6 +11,58 @@ This is a fork of WebRTC made by [pixiv Inc](https://www.pixiv.co.jp/).
 - Native APIs are availble for Android library.
 - `webrtc::VideoBuffer`, a simple `webrtc::VideoSinkInterface` which allows you
   to retrieve the last frame at an arbitrary time is introduced.
+- APIs necessary to deliver recorded audio data on iOS's broadcast extension is
+  added.
+
+# Delivering audio data on iOS's broadcast extension
+
+## The problem and solution
+
+A broadcast extension allows to broadcast your screen from iOS. However, an
+extension is an environment restricted compared to normal applications, and
+lacks the Audio Unit framework, which is required by the audio device module
+for iOS of WebRTC.
+
+A broadcast extension can still receive sounds of applications and one recorded
+with microphone via [`RPBroadcastSampleHandler`](https://developer.apple.com/documentation/replaykit/rpbroadcastsamplehandler)
+object. This fork fixes the audio device module on environments without the
+Audio Unit framework and adds interfaces to deliver data received via
+[`RPBroadcastSampleHandler`](https://developer.apple.com/documentation/replaykit/rpbroadcastsamplehandler).
+
+## Relevant interfaces
+
+Use interfaces described here to deliver audio with your broadcast extension.
+
+### `[RTCPeerConnectionFactory initWithEncoderFactory: decoderFactory: audioDeviceModule:]`
+
+This method is same to `[RTCPeerConnectionFactory initWithEncoderFactory: decoderFactory:]`,
+but has one more parameter, `audioDeviceModule`, whose type is `RTCAudioDeviceModule`.
+
+Use this method to provide `RTCAudioDeviceModule` delivering audio data.
+
+### `RTCAudioDeviceModule`
+
+This class is defined in `RTCAudioDeviceModule.h`. It wraps the native
+implementation delivering audio data. The implementation is extended to
+receive data not from the Audio Unit framework.
+
+### `[RTCAudioDeviceModule deliverRecordedData:]`
+
+This method actually delivers the data your provide. The type of the parameter
+is [`CMSampleBuffer`](https://developer.apple.com/documentation/coremedia/cmsamplebuffer-u71).
+
+Provide [`CMSampleBuffer`](https://developer.apple.com/documentation/coremedia/cmsamplebuffer-u71)
+acquired with [`[RPBroadcastSampleHandler processSampleBuffer: with:]`](https://developer.apple.com/documentation/replaykit/rpbroadcastsamplehandler/2123045-processsamplebuffer).
+
+The audio unit must be uninitialized and disabled to use this method.
+
+### `RTCAudioSession.useManualAudio`
+
+Set `YES` on broadcast extension. That prevents initializing the audio unit.
+
+### `RTCAudioSession.isAudioEnabled`
+
+Set `NO` on broadcast extension. That disables the audio unit.
 
 # .NET bindings
 

sdk/BUILD.gn

@@ -796,6 +796,9 @@ if (is_ios || is_mac) {
         "..:no_global_constructors",
       ]
       sources = [
+        "objc/api/peerconnection/RTCAudioDeviceModule+Private.h",
+        "objc/api/peerconnection/RTCAudioDeviceModule.h",
+        "objc/api/peerconnection/RTCAudioDeviceModule.mm",
         "objc/api/peerconnection/RTCAudioSource+Private.h",
         "objc/api/peerconnection/RTCAudioSource.h",
         "objc/api/peerconnection/RTCAudioSource.mm",
@@ -1271,6 +1274,7 @@ if (is_ios || is_mac) {
           "objc/helpers/RTCCameraPreviewView.h",
           "objc/helpers/RTCDispatcher.h",
           "objc/helpers/UIDevice+RTCDevice.h",
+          "objc/api/peerconnection/RTCAudioDeviceModule.h",
           "objc/api/peerconnection/RTCAudioSource.h",
           "objc/api/peerconnection/RTCAudioTrack.h",
           "objc/api/peerconnection/RTCConfiguration.h",
@@ -1389,6 +1393,7 @@ if (is_ios || is_mac) {
         output_name = "WebRTC"
 
         sources = [
+          "objc/api/peerconnection/RTCAudioDeviceModule.h",
           "objc/api/peerconnection/RTCAudioSource.h",
           "objc/api/peerconnection/RTCAudioTrack.h",
           "objc/api/peerconnection/RTCCertificate.h",

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h

@@ -11,6 +11,7 @@
 #import <Foundation/Foundation.h>
 
 #import "RTCMacros.h"
+#import "RTCAudioDeviceModule.h"
 
 NS_ASSUME_NONNULL_BEGIN
 
@@ -37,6 +38,10 @@ RTC_OBJC_EXPORT
 - (instancetype)initWithEncoderFactory:(nullable id<RTCVideoEncoderFactory>)encoderFactory
                         decoderFactory:(nullable id<RTCVideoDecoderFactory>)decoderFactory;
 
+- (instancetype)initWithEncoderFactory:(nullable id<RTCVideoEncoderFactory>)encoderFactory
+                        decoderFactory:(nullable id<RTCVideoDecoderFactory>)decoderFactory
+                     audioDeviceModule:(RTCAudioDeviceModule *)audioDeviceModule;
+
 /** Initialize an RTCAudioSource with constraints. */
 - (RTCAudioSource *)audioSourceWithConstraints:(nullable RTCMediaConstraints *)constraints;
 

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm

@@ -12,6 +12,7 @@
 #import "RTCPeerConnectionFactory+Private.h"
 #import "RTCPeerConnectionFactoryOptions+Private.h"
 
+#import "RTCAudioDeviceModule+Private.h"
 #import "RTCAudioSource+Private.h"
 #import "RTCAudioTrack+Private.h"
 #import "RTCMediaConstraints+Private.h"
@@ -119,6 +120,30 @@ - (instancetype)initWithEncoderFactory:(nullable id<RTCVideoEncoderFactory>)enco
                 mediaTransportFactory:nullptr];
 }
 
+- (instancetype)initWithEncoderFactory:(nullable id<RTCVideoEncoderFactory>)encoderFactory
+                        decoderFactory:(nullable id<RTCVideoDecoderFactory>)decoderFactory
+                     audioDeviceModule:(RTCAudioDeviceModule *)audioDeviceModule {
+#ifdef HAVE_NO_MEDIA
+  return [self initWithNoMedia];
+#else
+  std::unique_ptr<webrtc::VideoEncoderFactory> native_encoder_factory;
+  std::unique_ptr<webrtc::VideoDecoderFactory> native_decoder_factory;
+  if (encoderFactory) {
+    native_encoder_factory = webrtc::ObjCToNativeVideoEncoderFactory(encoderFactory);
+  }
+  if (decoderFactory) {
+    native_decoder_factory = webrtc::ObjCToNativeVideoDecoderFactory(decoderFactory);
+  }
+  return [self initWithNativeAudioEncoderFactory:webrtc::CreateBuiltinAudioEncoderFactory()
+                       nativeAudioDecoderFactory:webrtc::CreateBuiltinAudioDecoderFactory()
+                       nativeVideoEncoderFactory:std::move(native_encoder_factory)
+                       nativeVideoDecoderFactory:std::move(native_decoder_factory)
+                               audioDeviceModule:audioDeviceModule.nativeModule
+                           audioProcessingModule:nullptr
+                           mediaTransportFactory:nullptr];
+#endif
+}
+
 - (instancetype)initNative {
   if (self = [super init]) {
     _networkThread = rtc::Thread::CreateWithSocketServer();

sdk/objc/native/src/audio/audio_device_ios.h

@@ -11,6 +11,7 @@
 #ifndef SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_
 #define SDK_OBJC_NATIVE_SRC_AUDIO_AUDIO_DEVICE_IOS_H_
 
+#include <CoreMedia/CoreMedia.h>
 #include <memory>
 
 #include "audio_session_observer.h"
@@ -142,6 +143,8 @@ class AudioDeviceIOS : public AudioDeviceGeneric,
   void OnCanPlayOrRecordChange(bool can_play_or_record) override;
   void OnChangedOutputVolume() override;
 
+  void OnDeliverRecordedExternalData(CMSampleBufferRef sample_buffer);
+
   // VoiceProcessingAudioUnitObserver methods.
   OSStatus OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
                                  const AudioTimeStamp* time_stamp,

sdk/objc/native/src/audio/audio_device_ios.mm

@@ -9,13 +9,15 @@
  */
 
 #import <AVFoundation/AVFoundation.h>
+#import <CoreMedia/CoreMedia.h>
 #import <Foundation/Foundation.h>
 
 #include "audio_device_ios.h"
 
 #include <cmath>
 
 #include "api/array_view.h"
+#include "common_audio/signal_processing/include/signal_processing_library.h"
 #include "helpers.h"
 #include "modules/audio_device/fine_audio_buffer.h"
 #include "rtc_base/atomic_ops.h"
@@ -228,7 +230,10 @@ static void LogDeviceInfo() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(audio_is_initialized_);
   RTC_DCHECK(!playing_);
-  RTC_DCHECK(audio_unit_);
+  if (!audio_unit_) {
+    RTCLogError(@"StartPlayout failed because audio is disabled.");
+    return -1;
+  }
   if (fine_audio_buffer_) {
     fine_audio_buffer_->ResetPlayout();
   }
@@ -281,11 +286,10 @@ static void LogDeviceInfo() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   RTC_DCHECK(audio_is_initialized_);
   RTC_DCHECK(!recording_);
-  RTC_DCHECK(audio_unit_);
   if (fine_audio_buffer_) {
     fine_audio_buffer_->ResetRecord();
   }
-  if (!playing_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) {
+  if (!playing_ && audio_unit_ && audio_unit_->GetState() == VoiceProcessingAudioUnit::kInitialized) {
     if (!audio_unit_->Start()) {
       RTCLogError(@"StartRecording failed to start audio unit.");
       return -1;
@@ -303,7 +307,9 @@ static void LogDeviceInfo() {
     return 0;
   }
   if (!playing_) {
-    ShutdownPlayOrRecord();
+    if (audio_unit_) {
+      ShutdownPlayOrRecord();
+    }
     audio_is_initialized_ = false;
   }
   rtc::AtomicOps::ReleaseStore(&recording_, 0);
@@ -365,6 +371,64 @@ static void LogDeviceInfo() {
   thread_->Post(RTC_FROM_HERE, this, kMessageOutputVolumeChange);
 }
 
+void AudioDeviceIOS::OnDeliverRecordedExternalData(CMSampleBufferRef sample_buffer) {
+  RTC_DCHECK_RUN_ON(&io_thread_checker_);
+
+  if (audio_unit_) {
+    RTCLogError(@"External recorded data was provided while audio unit is disabled.");
+    return;
+  }
+
+  CMFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sample_buffer);
+  const AudioStreamBasicDescription *asbd = CMAudioFormatDescriptionGetStreamBasicDescription(description);
+  if (!asbd) {
+    RTCLogError("External recorded data was not in audio format.");
+    return;
+  }
+
+  if (asbd->mSampleRate != record_parameters_.sample_rate() ||
+      asbd->mChannelsPerFrame != record_parameters_.channels()) {
+    record_parameters_.reset(asbd->mSampleRate, asbd->mChannelsPerFrame);
+    UpdateAudioDeviceBuffer();
+
+    // Create a modified audio buffer class which allows us to ask for,
+    // or deliver, any number of samples (and not only multiple of 10ms) to match
+    // the native audio unit buffer size.
+    RTC_DCHECK(audio_device_buffer_);
+    fine_audio_buffer_.reset(new FineAudioBuffer(audio_device_buffer_));
+  }
+
+  CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(sample_buffer);
+  if (block_buffer == nil) {
+    return;
+  }
+
+  AudioBufferList buffer_list;
+  CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sample_buffer,
+                                                          nullptr,
+                                                          &buffer_list,
+                                                          sizeof(buffer_list),
+                                                          nullptr,
+                                                          nullptr,
+                                                          kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment,
+                                                          &block_buffer);
+
+  rtc::ArrayView<int16_t> view {
+    static_cast<int16_t*>(buffer_list.mBuffers[0].mData),
+    buffer_list.mBuffers[0].mDataByteSize / sizeof(int16_t)
+  };
+
+  if (asbd->mFormatFlags & kAudioFormatFlagIsBigEndian) {
+    for (auto& element : view) {
+      element = be16toh(element);
+    }
+  }
+
+  fine_audio_buffer_->DeliverRecordedData(view, kFixedRecordDelayEstimate);
+
+  CFRelease(block_buffer);
+}
+
 OSStatus AudioDeviceIOS::OnDeliverRecordedData(AudioUnitRenderActionFlags* flags,
                                                const AudioTimeStamp* time_stamp,
                                                UInt32 bus_number,
@@ -753,8 +817,10 @@ static void LogDeviceInfo() {
   // be initialized on initialization.
   if (!audio_is_initialized_) return;
 
-  // If we're initialized, we must have an audio unit.
-  RTC_DCHECK(audio_unit_);
+  if (!audio_unit_ && !CreateAudioUnit()) {
+    RTCLog(@"Failed to create audio unit.");
+    return;
+  }
 
   bool should_initialize_audio_unit = false;
   bool should_uninitialize_audio_unit = false;
@@ -860,11 +926,6 @@ static void LogDeviceInfo() {
   LOGI() << "InitPlayOrRecord";
   RTC_DCHECK_RUN_ON(&thread_checker_);
 
-  // There should be no audio unit at this point.
-  if (!CreateAudioUnit()) {
-    return false;
-  }
-
   RTCAudioSession* session = [RTCAudioSession sharedInstance];
   // Subscribe to audio session events.
   [session pushDelegate:audio_session_observer_];
@@ -883,6 +944,12 @@ static void LogDeviceInfo() {
   // If we are ready to play or record, and if the audio session can be
   // configured, then initialize the audio unit.
   if (session.canPlayOrRecord) {
+    // There should be no audio unit at this point.
+    if (!CreateAudioUnit()) {
+      [session unlockForConfiguration];
+      return false;
+    }
+
     if (!ConfigureAudioSession()) {
       // One possible reason for failure is if an attempt was made to use the
       // audio session during or after a Media Services failure.

sdk/objc/native/src/audio/audio_device_module_ios.h

@@ -21,6 +21,10 @@
 #include "rtc_base/checks.h"
 #include "rtc_base/critical_section.h"
 
+#if defined(WEBRTC_IOS)
+#include <CoreMedia/CoreMedia.h>
+#endif
+
 namespace webrtc {
 
 class AudioDeviceGeneric;
@@ -128,6 +132,7 @@ class AudioDeviceModuleIOS : public AudioDeviceModule {
   int32_t GetPlayoutUnderrunCount() const override;
 
 #if defined(WEBRTC_IOS)
+  void OnDeliverRecordedExternalData(CMSampleBufferRef sample_buffer);
   int GetPlayoutAudioParameters(AudioParameters* params) const override;
   int GetRecordAudioParameters(AudioParameters* params) const override;
 #endif  // WEBRTC_IOS

sdk/objc/native/src/audio/audio_device_module_ios.mm

@@ -651,6 +651,10 @@
   }
 
 #if defined(WEBRTC_IOS)
+  void AudioDeviceModuleIOS::OnDeliverRecordedExternalData(CMSampleBufferRef sample_buffer) {
+    audio_device_->OnDeliverRecordedExternalData(sample_buffer);
+  }
+
   int AudioDeviceModuleIOS::GetPlayoutAudioParameters(
       AudioParameters* params) const {
     RTC_LOG(INFO) << __FUNCTION__;