Improve debug output of VP9 and AV1 quality filters. (#2093)

* Improve debug output of VP9 and AV1 quality filters. * More improvements to Vp9 rewrite debugging. * Copy relevant fixes to AV1 projection.
jitsi · Feb 20, 2024 · af07121 · af07121
1 parent 7bd4888
commit af07121
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 8 deletions.
diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDAdaptiveSourceProjectionContext.kt
@@ -125,7 +125,21 @@ class Av1DDAdaptiveSourceProjectionContext(
             }
         }
 
-        val accept = frame.isAccepted && frame.projection?.accept(packet) == true
+        val accept = frame.isAccepted &&
+            if (frame.projection?.accept(packet) == true) {
+                true
+            } else {
+                if (frame.projection != null && frame.projection?.closedSeq != -1) {
+                    logger.debug(
+                        "Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}"
+                    )
+                } else if (frame.projection == null) {
+                    logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it")
+                } else {
+                    logger.warn("Not accepting $packet, even though frame projection is not closed")
+                }
+                false
+            }
 
         if (timeSeriesLogger.isTraceEnabled) {
             val pt = diagnosticContext.makeTimeSeriesPoint("rtp_av1")
@@ -277,7 +291,7 @@ class Av1DDAdaptiveSourceProjectionContext(
     }
 
     /**
-     * Create an projection for the first frame after an encoding switch.
+     * Create a projection for the first frame after an encoding switch.
      */
     private fun createEncodingSwitchProjection(
         frame: Av1DDFrame,

diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDFrameProjection.kt
@@ -80,7 +80,8 @@ class Av1DDFrameProjection internal constructor(
      * -1 if this projection is still "open" for new, later packets.
      * Projections can be closed when we switch away from their encodings.
      */
-    private var closedSeq = -1
+    var closedSeq = -1
+        private set
 
     /**
      * Ctor.

diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/av1/Av1DDQualityFilter.kt
@@ -416,6 +416,7 @@ internal class Av1DDQualityFilter(
     internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) {
         pt.addField("qf.currentIndex", Av1DDRtpLayerDesc.indexString(currentIndex))
             .addField("qf.internalTargetEncoding", internalTargetEncoding)
+            .addField("qf.internalTargetDt", internalTargetDt)
             .addField("qf.needsKeyframe", needsKeyframe)
             .addField(
                 "qf.mostRecentKeyframeGroupArrivalTimeMs",
@@ -439,6 +440,7 @@ internal class Av1DDQualityFilter(
                 mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1
             debugState["needsKeyframe"] = needsKeyframe
             debugState["internalTargetEncoding"] = internalTargetEncoding
+            debugState["internalTargetDt"] = internalTargetDt
             debugState["currentIndex"] = Av1DDRtpLayerDesc.indexString(currentIndex)
             return debugState
         }

diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9AdaptiveSourceProjectionContext.kt
@@ -131,14 +131,29 @@ class Vp9AdaptiveSourceProjectionContext(
             }
         }
 
-        val accept = frame.isAccepted && frame.projection?.accept(packet) == true
+        val accept = frame.isAccepted &&
+            if (frame.projection?.accept(packet) == true) {
+                true
+            } else {
+                if (frame.projection != null && frame.projection?.closedSeq != -1) {
+                    logger.debug(
+                        "Not accepting $packet: frame projection is closed at ${frame.projection?.closedSeq}"
+                    )
+                } else if (frame.projection == null) {
+                    logger.warn("Not accepting $packet: frame has no projection, even though QF accepted it")
+                } else {
+                    logger.warn("Not accepting $packet, even though frame projection is not closed")
+                }
+                false
+            }
 
         if (timeSeriesLogger.isTraceEnabled) {
             val pt = diagnosticContext.makeTimeSeriesPoint("rtp_vp9")
                 .addField("ssrc", packet.ssrc)
                 .addField("timestamp", packet.timestamp)
                 .addField("seq", packet.sequenceNumber)
                 .addField("pictureId", packet.pictureId)
+                .addField("pictureIdIndex", frame.index)
                 .addField("encoding", incomingEncoding)
                 .addField("spatialLayer", packet.spatialLayerIndex)
                 .addField("temporalLayer", packet.temporalLayerIndex)
@@ -255,7 +270,7 @@ class Vp9AdaptiveSourceProjectionContext(
     }
 
     /**
-     * Create an projection for the first frame after an encoding switch.
+     * Create a projection for the first frame after an encoding switch.
      */
     private fun createEncodingSwitchProjection(
         frame: Vp9Frame,

diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9FrameProjection.kt
@@ -89,7 +89,8 @@ internal constructor(
      * -1 if this projection is still "open" for new, later packets.
      * Projections can be closed when we switch away from their encodings.
      */
-    private var closedSeq = -1
+    var closedSeq = -1
+        private set
 
     /**
      * Ctor.

diff --git a/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt b/jvb/src/main/kotlin/org/jitsi/videobridge/cc/vp9/Vp9QualityFilter.kt
@@ -80,7 +80,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) {
     /**
      * Which spatial layers are currently being forwarded.
      */
-    private val layers: Array<Boolean> = Array(MAX_VP9_LAYERS) { false }
+    private val layers = BooleanArray(MAX_VP9_LAYERS)
 
     /**
      * Determines whether to accept or drop a VP9 frame.
@@ -428,6 +428,7 @@ internal class Vp9QualityFilter(parentLogger: Logger) {
     internal fun addDiagnosticContext(pt: DiagnosticContext.TimeSeriesPoint) {
         pt.addField("qf.currentIndex", indexString(currentIndex))
             .addField("qf.internalTargetEncoding", internalTargetEncoding)
+            .addField("qf.internalTargetSpatialId", internalTargetSpatialId)
             .addField("qf.needsKeyframe", needsKeyframe)
             .addField(
                 "qf.mostRecentKeyframeGroupArrivalTimeMs",
@@ -452,8 +453,10 @@ internal class Vp9QualityFilter(parentLogger: Logger) {
             debugState["mostRecentKeyframeGroupArrivalTimeMs"] =
                 mostRecentKeyframeGroupArrivalTime?.toEpochMilli() ?: -1
             debugState["needsKeyframe"] = needsKeyframe
-            debugState["internalTargetIndex"] = internalTargetEncoding
+            debugState["internalTargetEncoding"] = internalTargetEncoding
+            debugState["internalTargetSpatialId"] = internalTargetSpatialId
             debugState["currentIndex"] = indexString(currentIndex)
+            debugState["layersForwarded"] = layers.map { toString().first() }.joinToString(separator = "")
             return debugState
         }