feat: add svc codecs (#263)

examples/wgpu_room/src/app.rs

@@ -162,8 +162,8 @@ impl LkApp {
             });
 
             ui.menu_button("Debug", |ui| {
-                if ui.button("Refresh stats").clicked() {
-                    // TODO
+                if ui.button("Log stats").clicked() {
+                    let _ = self.service.send(AsyncCmd::LogStats);
                 }
             });
         });

examples/wgpu_room/src/logo_track.rs

@@ -1,6 +1,7 @@
 use image::ImageFormat;
 use image::RgbaImage;
 use livekit::options::TrackPublishOptions;
+use livekit::options::VideoCodec;
 use livekit::prelude::*;
 use livekit::webrtc::video_source::RtcVideoSource;
 use livekit::webrtc::video_source::VideoResolution;

examples/wgpu_room/src/service.rs

@@ -33,6 +33,7 @@ pub enum AsyncCmd {
         publication: RemoteTrackPublication,
     },
     E2eeKeyRatchet,
+    LogStats,
 }
 
 #[derive(Debug)]
@@ -199,6 +200,32 @@ async fn service_task(inner: Arc<ServiceInner>, mut cmd_rx: mpsc::UnboundedRecei
                     }
                 }
             }
+            AsyncCmd::LogStats => {
+                if let Some(state) = running_state.as_ref() {
+                    for (_, publication) in state.room.local_participant().tracks() {
+                        if let Some(track) = publication.track() {
+                            log::info!(
+                                "track stats: LOCAL {:?} {:?}",
+                                track.sid(),
+                                track.get_stats().await,
+                            );
+                        }
+                    }
+
+                    for (_, participant) in state.room.participants() {
+                        for (_, publication) in participant.tracks() {
+                            if let Some(track) = publication.track() {
+                                log::info!(
+                                    "track stats: {:?} {:?} {:?}",
+                                    participant.identity(),
+                                    track.sid(),
+                                    track.get_stats().await,
+                                );
+                            }
+                        }
+                    }
+                }
+            }
         }
     }
 }

livekit-ffi/protocol/video_frame.proto

@@ -97,6 +97,7 @@ enum VideoCodec {
   VP8 = 0;
   H264 = 1;
   AV1 = 2;
+  VP9 = 3;
 }
 
 enum VideoRotation {

livekit-ffi/src/conversion/video_frame.rs

@@ -132,6 +132,7 @@ impl From<proto::VideoCodec> for VideoCodec {
             proto::VideoCodec::Vp8 => Self::VP8,
             proto::VideoCodec::H264 => Self::H264,
             proto::VideoCodec::Av1 => Self::AV1,
+            proto::VideoCodec::Vp9 => Self::VP9,
         }
     }
 }

livekit-ffi/src/livekit.proto.rs

@@ -1843,6 +1843,7 @@ pub enum VideoCodec {
     Vp8 = 0,
     H264 = 1,
     Av1 = 2,
+    Vp9 = 3,
 }
 impl VideoCodec {
     /// String value of the enum field names used in the ProtoBuf definition.
@@ -1854,6 +1855,7 @@ impl VideoCodec {
             VideoCodec::Vp8 => "VP8",
             VideoCodec::H264 => "H264",
             VideoCodec::Av1 => "AV1",
+            VideoCodec::Vp9 => "VP9",
         }
     }
     /// Creates an enum from field names used in the ProtoBuf definition.
@@ -1862,6 +1864,7 @@ impl VideoCodec {
             "VP8" => Some(Self::Vp8),
             "H264" => Some(Self::H264),
             "AV1" => Some(Self::Av1),
+            "VP9" => Some(Self::Vp9),
             _ => None,
         }
     }

livekit/src/room/options.rs

@@ -20,6 +20,7 @@ use livekit_protocol as proto;
 pub enum VideoCodec {
     VP8,
     H264,
+    VP9,
     AV1,
 }
 
@@ -28,6 +29,7 @@ impl VideoCodec {
         match self {
             VideoCodec::VP8 => "vp8",
             VideoCodec::H264 => "h264",
+            VideoCodec::VP9 => "vp9",
             VideoCodec::AV1 => "av1",
         }
     }

livekit/src/room/track/local_track.rs

@@ -42,6 +42,13 @@ impl LocalTrack {
             Self::Video(track) => track.rtc_track().into(),
         }
     }
+
+    pub async fn get_stats(&self) -> RoomResult<Vec<RtcStats>> {
+        match self {
+            Self::Audio(track) => track.get_stats().await,
+            Self::Video(track) => track.get_stats().await,
+        }
+    }
 }
 
 impl From<LocalTrack> for Track {

livekit/src/room/track/remote_track.rs

@@ -37,6 +37,13 @@ impl RemoteTrack {
             Self::Video(track) => track.rtc_track().into(),
         }
     }
+
+    pub async fn get_stats(&self) -> RoomResult<Vec<RtcStats>> {
+        match self {
+            Self::Audio(track) => track.get_stats().await,
+            Self::Video(track) => track.get_stats().await,
+        }
+    }
 }
 
 pub(super) async fn get_stats(inner: &Arc<TrackInner>) -> RoomResult<Vec<RtcStats>> {

webrtc-sys/libwebrtc/build_linux.sh

@@ -91,6 +91,7 @@ args="is_debug=$debug  \
   rtc_build_tools=false \
   rtc_build_examples=false \
   rtc_libvpx_build_vp9=true \
+  enable_libaom=true \
   is_component_build=false \
   enable_stripping=true \
   use_goma=false \

webrtc-sys/libwebrtc/build_macos.sh

@@ -92,11 +92,11 @@ gn gen "$OUTPUT_DIR" --root="src" \
   rtc_build_examples=false \
   rtc_build_tools=false \
   rtc_libvpx_build_vp9=true \
+  enable_libaom=true \
   is_component_build=false \
   enable_stripping=true \
   rtc_enable_symbol_export=true \
   rtc_enable_objc_symbol_export=false \
-  enable_libaom = true \
   rtc_include_dav1d_in_internal_decoder_factory = true \
   rtc_use_h264=true \
   use_custom_libcxx=false \

webrtc-sys/libwebrtc/build_windows.cmd

@@ -65,7 +65,7 @@ if "!profile!" == "debug" (
 
 rem generate ninja for release
 call gn.bat gen %OUTPUT_DIR% --root="src" ^
-  --args="is_debug=!debug! is_clang=true target_cpu=\"!arch!\" use_custom_libcxx=false rtc_disable_check_msg=true rtc_include_tests=false rtc_build_examples=false rtc_build_tools=false is_component_build=false rtc_enable_protobuf=false rtc_use_h264=true ffmpeg_branding=\"Chrome\" symbol_level=0 enable_iterator_debugging=false"
+  --args="is_debug=!debug! is_clang=true target_cpu=\"!arch!\" use_custom_libcxx=false rtc_libvpx_build_vp9=true enable_libaom=true rtc_disable_check_msg=true rtc_include_tests=false rtc_build_examples=false rtc_build_tools=false is_component_build=false rtc_enable_protobuf=false rtc_use_h264=true ffmpeg_branding=\"Chrome\" symbol_level=0 enable_iterator_debugging=false"
 
 rem build
 ninja.exe -C %OUTPUT_DIR% :default

webrtc-sys/libwebrtc/patches/add_deps.patch

@@ -1,8 +1,29 @@
 diff --git a/BUILD.gn b/BUILD.gn
-index d5289b8..12685d1 100644
+index d5289b85d7..76823f6cff 100644
 --- a/BUILD.gn
 +++ b/BUILD.gn
-@@ -517,6 +517,10 @@ if (!build_with_chromium) {
+@@ -24,6 +24,9 @@
+ import("//build/config/linux/pkg_config.gni")
+ import("//build/config/sanitizers/sanitizers.gni")
+ import("webrtc.gni")
++import("//third_party/libaom/options.gni")
++
++
+ if (rtc_enable_protobuf) {
+   import("//third_party/protobuf/proto_library.gni")
+ }
+@@ -292,6 +295,10 @@ config("common_config") {
+     defines += [ "WEBRTC_INCLUDE_INTERNAL_AUDIO_DEVICE" ]
+   }
+
++  if (enable_libaom) {
++    defines += [ "RTC_USE_LIBAOM_AV1_ENCODER" ]
++  }
++
+   if (rtc_libvpx_build_vp9) {
+     defines += [ "RTC_ENABLE_VP9" ]
+   }
+@@ -517,6 +524,10 @@ if (!build_with_chromium) {
        "pc:rtc_pc",
        "sdk",
        "video",

webrtc-sys/src/video_decoder_factory.cpp

@@ -16,13 +16,19 @@
 
 #include "livekit/video_decoder_factory.h"
 
+#include "api/video_codecs/av1_profile.h"
 #include "api/video_codecs/sdp_video_format.h"
 #include "livekit/objc_video_factory.h"
 #include "media/base/media_constants.h"
 #include "modules/video_coding/codecs/h264/include/h264.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "rtc_base/logging.h"
 
+#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY)
+#include "modules/video_coding/codecs/av1/dav1d_decoder.h"  // nogncheck
+#endif
+
 #ifdef WEBRTC_ANDROID
 #include "livekit/android.h"
 #endif
@@ -52,10 +58,21 @@ std::vector<webrtc::SdpVideoFormat> VideoDecoderFactory::GetSupportedFormats()
   }
 
   formats.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
+  for (const webrtc::SdpVideoFormat& format :
+       webrtc::SupportedVP9DecoderCodecs())
+    formats.push_back(format);
   for (const webrtc::SdpVideoFormat& h264_format :
        webrtc::SupportedH264DecoderCodecs())
     formats.push_back(h264_format);
 
+#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY)
+  formats.push_back(webrtc::SdpVideoFormat(cricket::kAv1CodecName));
+  formats.push_back(webrtc::SdpVideoFormat(
+      cricket::kAv1CodecName,
+      {{webrtc::kAV1FmtpProfile,
+        AV1ProfileToString(webrtc::AV1Profile::kProfile1).data()}}));
+#endif
+
   return formats;
 }
 
@@ -86,9 +103,19 @@ std::unique_ptr<webrtc::VideoDecoder> VideoDecoderFactory::CreateVideoDecoder(
 
   if (absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName))
     return webrtc::VP8Decoder::Create();
+  if (absl::EqualsIgnoreCase(format.name, cricket::kVp9CodecName))
+    return webrtc::VP9Decoder::Create();
   if (absl::EqualsIgnoreCase(format.name, cricket::kH264CodecName))
     return webrtc::H264Decoder::Create();
 
+
+#if defined(RTC_DAV1D_IN_INTERNAL_DECODER_FACTORY)
+  if (absl::EqualsIgnoreCase(format.name, cricket::kAv1CodecName)) {
+    return webrtc::CreateDav1dDecoder();
+  }
+#endif
+
+
   RTC_LOG(LS_ERROR) << "No VideoDecoder found for " << format.name;
   return nullptr;
 }

webrtc-sys/src/video_encoder_factory.cpp

@@ -19,29 +19,34 @@
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_encoder.h"
 #include "api/video_codecs/video_encoder_factory_template.h"
-#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h"
 #include "livekit/objc_video_factory.h"
 #include "media/base/media_constants.h"
 #include "media/engine/simulcast_encoder_adapter.h"
 #include "rtc_base/logging.h"
-
+#if defined(RTC_USE_LIBAOM_AV1_ENCODER)
+#include "api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h"
+#endif
 #if defined(WEBRTC_USE_H264)
 #include "api/video_codecs/video_encoder_factory_template_open_h264_adapter.h"
 #endif
+#include "api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h"
+#include "api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h"
 
 #ifdef WEBRTC_ANDROID
 #include "livekit/android.h"
 #endif
 
 namespace livekit {
 
-using Factory =
-    webrtc::VideoEncoderFactoryTemplate<webrtc::LibvpxVp8EncoderTemplateAdapter
+using Factory = webrtc::VideoEncoderFactoryTemplate<
+    webrtc::LibvpxVp8EncoderTemplateAdapter,
 #if defined(WEBRTC_USE_H264)
-                                        ,
-                                        webrtc::OpenH264EncoderTemplateAdapter
+    webrtc::OpenH264EncoderTemplateAdapter,
+#endif
+#if defined(RTC_USE_LIBAOM_AV1_ENCODER)
+    webrtc::LibaomAv1EncoderTemplateAdapter,
 #endif
-                                        >;
+    webrtc::LibvpxVp9EncoderTemplateAdapter>;
 
 VideoEncoderFactory::InternalFactory::InternalFactory() {
 #ifdef __APPLE__