video: adaptive bitrate when the car or its Wi-Fi falls behind

A quarter comes off when the frame rate sags while the encoder waits on acks;
two easy windows put a tenth back. If a cut doesn't help, the car's decoder is
the limit, so the cut is undone and adapting stops for that drive.

gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt

@@ -1089,11 +1089,15 @@ class GearslipRunner(
         }
 
         val useProjection = projection != null
+        // 60fps only over a cable: over Wi-Fi it doubles the data for little the driver sees.
+        videoFrameRate = if (config.frameRate == 1 && !wireless) 60 else 30
+        smoothWindows = 0
+        lastCut = null
+        decoderBound = false
         videoSource = VideoSource(
             width = width,
             height = height,
-            // 60fps only over a cable: over Wi-Fi it doubles the data for little the driver sees.
+            frameRate = videoFrameRate,
-            frameRate = if (config.frameRate == 1 && !wireless) 60 else 30,
             mode = if (useProjection) VideoSource.Mode.SURFACE else VideoSource.Mode.TEST_CARD,
             wireless = wireless,
             onCodecConfig = { csd ->
@@ -1378,8 +1382,10 @@ class GearslipRunner(
                     SessionReport.stream("${"%.1f".format(fps)} fps, sent=$sent acked=$acksSeen")
                     log.i(
                         "video: ${"%.1f".format(fps)} fps over 5s | sent=$sent waited=${waited}ms " +
-                            "acked=$acksSeen inFlight=$inFlight split=$splitMessages",
+                            "acked=$acksSeen inFlight=$inFlight split=$splitMessages " +
+                            "kbps=${(videoSource?.bitRate ?: 0) / 1000}",
                     )
+                    adaptBitRate(fps, waited)
 
                     // Self-healing safety net: force a fresh keyframe periodically regardless
                     // of whether we think one is owed. Protects against any future case where
@@ -1394,7 +1400,63 @@ class GearslipRunner(
         }.apply { isDaemon = true }.start()
     }
 
+    /** The frame rate the stream was set up for, which adaptive bitrate measures against. */
+    @Volatile private var videoFrameRate = 30
+
+    /** Five-second windows in a row the stream kept up, towards climbing back up. */
+    private var smoothWindows = 0
+    /** The bitrate and frame rate before the last cut, to see whether cutting helped. */
+    private var lastCut: Pair<Int, Double>? = null
+    /** Set once a cut didn't help: this car's decoder, not the link, sets the pace. */
+    private var decoderBound = false
+
+    /**
+     * Adaptive bitrate. A window where the encoder spent most of its time held back by the
+     * car's acks while the frame rate sagged means the car, or the Wi-Fi to it, can't take
+     * frames this size: a quarter comes off. Two easy windows in a row put a tenth back, up to
+     * where the stream started. A still screen (few frames, no waiting) changes nothing.
+     */
+    private fun adaptBitRate(fps: Double, waitedMs: Long) {
+        val source = videoSource ?: return
+        val target = videoFrameRate.toDouble()
+        val behind = waitedMs > ADAPT_WINDOW_MS / 2 && fps < target * 0.8
+        val easy = waitedMs < ADAPT_WINDOW_MS / 10
+        when {
+            behind && !decoderBound -> {
+                smoothWindows = 0
+                val cut = lastCut
+                if (cut != null && fps < cut.second * 1.05) {
+                    // The last cut bought no frames: the car's decoder is the limit, not the
+                    // link, and smaller frames would only look worse. Undo it and stop here.
+                    decoderBound = true
+                    source.setBitRate(cut.first)
+                    log.i("video: a lower bitrate didn't help the car keep up - back to ${cut.first / 1000} kbps, adapting off")
+                    return
+                }
+                val before = source.bitRate
+                source.setBitRate((before * 0.75).toInt())
+                if (source.bitRate != before) {
+                    lastCut = before to fps
+                    log.i("video: car falling behind - bitrate ${before / 1000} -> ${source.bitRate / 1000} kbps")
+                }
+            }
+            easy && source.bitRate < source.fullBitRate -> {
+                lastCut = null
+                if (++smoothWindows >= 2) {
+                    smoothWindows = 0
+                    val before = source.bitRate
+                    source.setBitRate((before * 1.1).toInt() + 1)
+                    log.i("video: keeping up - bitrate ${before / 1000} -> ${source.bitRate / 1000} kbps")
+                }
+            }
+            else -> smoothWindows = 0
+        }
+    }
+
     private companion object {
+        /** The stats window adaptive bitrate reads, matching the watchdog's sleep. */
+        const val ADAPT_WINDOW_MS = 5_000L
+
         // protobuf/aap_protobuf/service/control/ControlMessageType.proto:7-17
         const val MSG_VERSION_REQUEST = 1
         const val MSG_VERSION_RESPONSE = 2

gearslip/src/main/java/app/seb3thehacker/gearslip/VideoSource.kt

@@ -42,6 +42,14 @@ class VideoSource(
         private set
 
     private var codec: MediaCodec? = null
+
+    /** The bitrate the encoder started at, the most [setBitRate] ever goes back up to. */
+    var fullBitRate = 0
+        private set
+
+    /** What the encoder is asked for right now. */
+    @Volatile var bitRate = 0
+        private set
     private var worker: Thread? = null
     @Volatile private var running = false
     private var frameIndex = 0L
@@ -51,6 +59,8 @@ class VideoSource(
         val bitRate = defaultBitRate(width, height, frameRate, level).let {
             if (wireless) (it * WIRELESS_SHARE).toInt().coerceAtMost(WIRELESS_MAX_BIT_RATE) else it
         }
+        fullBitRate = bitRate
+        this.bitRate = bitRate
         fun format(tuned: Boolean) = MediaFormat.createVideoFormat(MIME, width, height).apply {
             setInteger(
                 MediaFormat.KEY_COLOR_FORMAT,
@@ -128,6 +138,19 @@ class VideoSource(
         }.onFailure { log.w("requestSyncFrame failed: ${it.message}") }
     }
 
+    /**
+     * Changes the bitrate while streaming, without a new keyframe or a restart. Used to back off
+     * when the car (or the Wi-Fi to it) can't keep up, and to climb back once it can.
+     */
+    fun setBitRate(bps: Int) {
+        val target = bps.coerceIn(fullBitRate / 4, fullBitRate)
+        if (target == bitRate) return
+        runCatching {
+            codec?.setParameters(Bundle().apply { putInt(MediaCodec.PARAMETER_KEY_VIDEO_BITRATE, target) })
+            bitRate = target
+        }.onFailure { log.w("setBitRate failed: ${it.message}") }
+    }
+
     fun stop() {
         running = false
         worker?.join(1_000)