listen through the car's mic, fix exit on Uconnect, follow the car's night mode

gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt

@@ -2,6 +2,7 @@ package app.seb3thehacker.gearslip
 
 import android.os.Build
 import app.seb3thehacker.gearslip.audio.AudioLink
+import app.seb3thehacker.gearslip.audio.CarMic
 import app.seb3thehacker.gearslip.car.CarAssistant
 import app.seb3thehacker.gearslip.car.CarEnvironment
 import app.seb3thehacker.gearslip.car.CarKeys
@@ -139,6 +140,9 @@ class GearslipRunner(
         audioLink?.close()
         audioLink = null
         CarSensors.clear()
+        CarMic.current = null
+        lastSensorNight = null
+        CarEnvironment.setSensorNight(null) // back to the clock until a car says otherwise
         projection?.onProjectionStopped()
         SessionStatus.disconnected()
     }
@@ -172,6 +176,8 @@ class GearslipRunner(
                 onInputMessage(messageId, body)
             } else if (frame.channel == sensorChannelId) {
                 onSensorMessage(messageId, body)
+            } else if (frame.channel == CarMic.current?.channelId) {
+                CarMic.current?.onMessage(messageId, body)
             } else if (frame.channel == audioLink?.channelId) {
                 // The audio channel is the one still being reverse-engineered, and its first few
                 // replies are what tell us why. Raw bytes for those, then quiet.
@@ -346,7 +352,9 @@ class GearslipRunner(
 
     private fun onServiceDiscoveryResponse(body: ByteArray) {
         log.i("<- ServiceDiscoveryResponse (${body.size} bytes, decrypted successfully)")
-        log.i("head unit describes itself as:\n" + Protobuf.describe(body))
+        // Field 5 is the car's serial number. Logs get shared, so it stays out of them.
+        val described = Protobuf.describe(body).replace(Regex("(?m)^  #5 string = .*$"), "  #5 string = (car serial, left out)")
+        log.i("head unit describes itself as:\n" + described)
         log.verdict(
             "VIABLE",
             "The head unit accepted the presented phone certificate, completed TLS, and is " +
@@ -405,6 +413,7 @@ class GearslipRunner(
         openChannel(video.serviceId)
         startAudioChannel(serviceDiscoveryResponse)
         startSensorChannel(serviceDiscoveryResponse)
+        startMicChannel(serviceDiscoveryResponse)
 
         val input = ServiceDiscovery.findInputService(serviceDiscoveryResponse, video.displayId)
         if (input == null) {
@@ -454,6 +463,18 @@ class GearslipRunner(
         openChannel(media.serviceId)
     }
 
+    /** The car's microphone, for voice: opened now, recording only while something listens. */
+    private fun startMicChannel(serviceDiscoveryResponse: ByteArray) {
+        val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run {
+            log.i("no microphone advertised - voice uses the phone's")
+            return
+        }
+        val config = mic.configs.first()
+        log.i("car microphone: channel=${mic.serviceId} ${mic.codecName} ${config.sampleRate}Hz/${config.bits}bit/x${config.channels}")
+        CarMic.current = CarMic(mic) { id, body -> send(id, body, encrypted = true, channel = mic.serviceId) }
+        openChannel(mic.serviceId)
+    }
+
     /**
      * Every sensor type the head unit advertised, subscribed to at once - there is no screen
      * to pick and choose yet, this is meant to show everything reachable. [CarSensors] answers
@@ -500,6 +521,22 @@ class GearslipRunner(
         }
     }
 
+    /**
+     * The car's own day/night signal (its headlights or light sensor): NightData { bool is_night = 1; }.
+     * It replaces the clock guess, so maps go dark when the car does.
+     */
+    private fun onNightSensor(body: ByteArray) {
+        val night = Wire.bytes(Wire.fields(body), SENSOR_NIGHT) ?: return
+        val isNight = Wire.varint(Wire.fields(night), 1) == 1L
+        if (isNight != lastSensorNight) {
+            lastSensorNight = isNight
+            log.i("car says it's ${if (isNight) "night" else "day"}")
+            CarEnvironment.setSensorNight(isNight)
+        }
+    }
+
+    private var lastSensorNight: Boolean? = null
+
     private fun onSensorMessage(messageId: Int, body: ByteArray) {
         when (messageId) {
             MSG_CHANNEL_OPEN_RESPONSE -> {
@@ -527,6 +564,7 @@ class GearslipRunner(
             MSG_SENSOR_EVENT_INDICATION -> {
                 CarSensors.onEvent(body)
                 logSensorEvent(body)
+                onNightSensor(body)
             }
 
             // SensorError { required SensorType sensor_type = 1; required SensorErrorType sensor_error_type = 2; }
@@ -574,6 +612,12 @@ class GearslipRunner(
 
             MSG_INPUT_REPORT -> {
                 val report = Wire.fields(body)
+                // The 2018 Uconnect agrees to Exit but keeps showing our last frame and sending
+                // touches, which looked like a crash. A touch means our screen is still up there.
+                if (videoStarted && !videoProjected && Wire.bytes(report, 3) != null) {
+                    log.w("the car said it took its screen back but still sends touches - showing ours again")
+                    onVideoFocus(VIDEO_FOCUS_PROJECTED)
+                }
                 Wire.bytes(report, 4)?.let(::onKeyEvent)
                 Wire.bytes(report, 6)?.let(::onRelativeEvent)
                 val touch = Wire.bytes(report, 3) ?: Wire.bytes(report, 7)
@@ -898,21 +942,31 @@ class GearslipRunner(
                 inFlight = 0
                 ackLock.notifyAll()
             }
+            val start = Protobuf.varintField(1, SESSION_ID.toLong()) +
+                Protobuf.varintField(2, (selectedConfigIndex ?: 0).toLong())
+            send(MSG_MEDIA_START, start, encrypted = true, channel = videoChannelId)
+            log.i("-> Start(session_id=$SESSION_ID)")
             lastCodecConfig?.let { send(MSG_MEDIA_CODEC_CONFIG, it, encrypted = true, channel = videoChannelId) }
             videoSource?.requestSyncFrame()
         } else {
-            log.i("the car is showing its own screen - video paused, audio carries on")
+            // A phone ends the stream when the car takes its screen. LIVI switches on the focus
+            // answer alone, but the 2018 Uconnect kept showing our last frame until it got Stop.
+            send(MSG_MEDIA_STOP, Protobuf.varintField(1, SESSION_ID.toLong()), encrypted = true, channel = videoChannelId)
+            log.i("the car is showing its own screen - video stopped, audio carries on")
         }
     }
 
     /** Android Auto's Exit: asks the car for its own interface, leaving the session running. */
     private fun requestNativeFocus() {
         if (!videoStarted) return
-        // VideoFocusRequestNotification { mode = 2; reason = 3 }
+        // VideoFocusRequestNotification { disp_channel_id = 1; mode = 2; reason = 3 }. The
-        val request = Protobuf.varintField(2, VIDEO_FOCUS_NATIVE.toLong()) +
+        // display's channel is deprecated, but the 2018 Uconnect (protocol 1.3) predates that:
+        // without it, it agreed to show its own screen and never did. LIVI ignores the field.
+        val request = Protobuf.varintField(1, videoChannelId.toLong()) +
+            Protobuf.varintField(2, VIDEO_FOCUS_NATIVE.toLong()) +
             Protobuf.varintField(3, VIDEO_FOCUS_REASON_LAUNCH_NATIVE.toLong())
         send(MSG_VIDEO_FOCUS_REQUEST, request, encrypted = true, channel = videoChannelId)
-        log.i("-> VideoFocusRequest(mode=NATIVE, reason=LAUNCH_NATIVE)")
+        log.i("-> VideoFocusRequest(display=$videoChannelId, mode=NATIVE, reason=LAUNCH_NATIVE)")
     }
 
     /** Arrival time of the last tap still waiting to be seen in a frame; 0 when none is. */
@@ -1137,6 +1191,7 @@ class GearslipRunner(
         const val MSG_SENSOR_EVENT_INDICATION = 32771
         const val MSG_SENSOR_ERROR = 32772
         private const val SENSOR_LOCATION = 1
+        private const val SENSOR_NIGHT = 10
         private const val SENSOR_LOG_EVERY_MS = 10_000L
 
         const val STREAM_MEDIA = 3

gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt

@@ -110,6 +110,34 @@ object ServiceDiscovery {
         return media.firstOrNull { it.codecType == CODEC_PCM } ?: media.firstOrNull()
     }
 
+    /**
+     * The car's microphone, from MediaSourceService (Service field 5):
+     *   MediaSourceService { MediaCodecType available_type = 1; AudioConfiguration audio_config = 2;
+     *                        bool available_while_in_call = 3; }
+     * Null when the head unit has none.
+     */
+    fun findMicService(response: ByteArray): AudioService? {
+        for (channel in Wire.allBytes(Wire.fields(response), 1)) {
+            val service = Wire.fields(channel)
+            val id = Wire.varint(service, 1)?.toInt() ?: continue
+            val source = Wire.fields(Wire.bytes(service, 5) ?: continue)
+            val f = Wire.fields(Wire.bytes(source, 2) ?: continue)
+            return AudioService(
+                serviceId = id,
+                streamType = 0,
+                codecType = Wire.varint(source, 1)?.toInt() ?: CODEC_PCM,
+                configs = listOf(
+                    AudioConfig(
+                        sampleRate = Wire.varint(f, 1)?.toInt() ?: 16_000,
+                        bits = Wire.varint(f, 2)?.toInt() ?: 16,
+                        channels = Wire.varint(f, 3)?.toInt() ?: 1,
+                    ),
+                ),
+            )
+        }
+        return null
+    }
+
     const val STREAM_MEDIA = 3
     const val CODEC_PCM = 1
 

gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt

@@ -0,0 +1,129 @@
+package app.seb3thehacker.gearslip.audio
+
+import app.seb3thehacker.gearslip.GearslipLog
+import app.seb3thehacker.gearslip.Protobuf
+import app.seb3thehacker.gearslip.ServiceDiscovery
+import app.seb3thehacker.gearslip.Wire
+import java.util.concurrent.LinkedBlockingQueue
+import java.util.concurrent.TimeUnit
+
+/**
+ * The car's own microphone, which most head units offer over the same link as the screen:
+ *
+ *   -> CHANNEL_OPEN_REQUEST          <- CHANNEL_OPEN_RESPONSE
+ *   -> Setup { config_index }        <- Config
+ *   -> MicrophoneRequest { open }    <- MicrophoneResponse { status, session_id }
+ *   <- Data { 8-byte timestamp, PCM }...   -> Ack { session_id, ack }
+ *   -> MicrophoneRequest { open = false }
+ *
+ * The channel opens with the session, but the car only records between [start] and [stop].
+ * Speech wants 16 kHz mono; a car that records at a multiple of that is averaged down to it, and
+ * one that doesn't isn't used, so voice falls back to the phone's microphone.
+ */
+class CarMic(
+    private val service: ServiceDiscovery.AudioService,
+    private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit,
+) {
+    val channelId get() = service.serviceId
+
+    private val config = service.configs.first()
+    private val step = config.sampleRate / RATE
+    private val usable = service.codecType == ServiceDiscovery.CODEC_PCM && config.bits == 16 &&
+        config.sampleRate % RATE == 0 && step >= 1 && config.channels in 1..2
+
+    @Volatile private var ready = false
+    @Volatile private var listening = false
+    @Volatile private var session = 0
+    private var packets = 0
+    private val chunks = LinkedBlockingQueue<ShortArray>()
+
+    fun onMessage(messageId: Int, body: ByteArray) {
+        when (messageId) {
+            MSG_CHANNEL_OPEN_RESPONSE -> {
+                val status = Protobuf.readInt32Field(body, 1) ?: 0
+                GearslipLog.i("mic: <- ChannelOpenResponse: status=$status")
+                if (status != 0) return
+                sendOnChannel(MSG_SETUP, Protobuf.varintField(1, 0L))
+                ready = usable
+                if (!usable) {
+                    GearslipLog.w(
+                        "mic: the car records ${service.codecName} ${config.sampleRate}Hz/${config.bits}bit/x${config.channels}, " +
+                            "which speech can't use - voice stays on the phone's microphone",
+                    )
+                }
+            }
+
+            MSG_CONFIG -> GearslipLog.i("mic: <- Config status=${Wire.varint(Wire.fields(body), 1)}")
+
+            MSG_MIC_RESPONSE -> {
+                val fields = Wire.fields(body)
+                session = Wire.varint(fields, 2)?.toInt() ?: 0
+                GearslipLog.i("mic: <- MicrophoneResponse status=${Wire.varint(fields, 1)} session=$session")
+            }
+
+            MSG_DATA -> {
+                sendOnChannel(MSG_ACK, Protobuf.varintField(1, session.toLong()) + Protobuf.varintField(2, 1L))
+                if (!listening || body.size <= TIMESTAMP_BYTES) return
+                if (packets++ == 0) GearslipLog.i("mic: first audio from the car (${body.size - TIMESTAMP_BYTES} bytes)")
+                chunks.offer(toSpeechRate(body))
+            }
+
+            else -> GearslipLog.i("mic: <- unhandled message id=$messageId (${body.size} bytes)")
+        }
+    }
+
+    /** Little-endian PCM after the timestamp, mixed to mono and averaged down to [RATE]. */
+    private fun toSpeechRate(body: ByteArray): ShortArray {
+        val frameBytes = 2 * config.channels
+        val frames = (body.size - TIMESTAMP_BYTES) / frameBytes
+        val out = ShortArray(frames / step)
+        for (i in out.indices) {
+            var sum = 0
+            for (s in 0 until step) {
+                val at = TIMESTAMP_BYTES + (i * step + s) * frameBytes
+                for (c in 0 until config.channels) {
+                    val o = at + 2 * c
+                    sum += (body[o].toInt() and 0xFF) or (body[o + 1].toInt() shl 8)
+                }
+            }
+            out[i] = (sum / (step * config.channels)).toShort()
+        }
+        return out
+    }
+
+    /** Asks the car to start recording. False when it can't, so the phone's microphone is used. */
+    fun start(): Boolean {
+        if (!ready) return false
+        chunks.clear()
+        packets = 0
+        listening = true
+        sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 1L) + Protobuf.varintField(4, 1L))
+        GearslipLog.i("mic: -> MicrophoneRequest(open)")
+        return true
+    }
+
+    /** The next piece of audio, or null if the car sent nothing for [timeoutMs]. */
+    fun read(timeoutMs: Long): ShortArray? = chunks.poll(timeoutMs, TimeUnit.MILLISECONDS)
+
+    fun stop() {
+        if (!listening) return
+        listening = false
+        sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 0L))
+        GearslipLog.i("mic: -> MicrophoneRequest(close) after $packets packets")
+    }
+
+    companion object {
+        /** The connected car's microphone, or null with no car or one that has none. */
+        @Volatile var current: CarMic? = null
+
+        const val RATE = 16_000
+        private const val TIMESTAMP_BYTES = 8
+        private const val MSG_CHANNEL_OPEN_RESPONSE = 8
+        private const val MSG_DATA = 0
+        private const val MSG_SETUP = 32768
+        private const val MSG_CONFIG = 32771
+        private const val MSG_ACK = 32772
+        private const val MSG_MIC_REQUEST = 32773
+        private const val MSG_MIC_RESPONSE = 32774
+    }
+}

gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt

@@ -14,6 +14,7 @@ import android.os.Handler
 import android.os.Looper
 import android.os.SystemClock
 import app.seb3thehacker.gearslip.GearslipLog
+import app.seb3thehacker.gearslip.audio.CarMic
 import java.util.Locale
 import java.util.concurrent.Executors
 import kotlin.math.log10
@@ -25,7 +26,8 @@ import kotlin.math.sqrt
  * none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply
  * of a few seconds comes back in well under a second on a recent phone.
  *
- * Whisper needs the whole phrase before it starts, so this records from the phone's microphone,
+ * Whisper needs the whole phrase before it starts, so this records from the car's microphone (the
+ * phone's when the car has none or sends nothing),
  * decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes
  * once. Everything runs on one worker thread; callbacks arrive on the main thread.
  */
@@ -45,6 +47,9 @@ object SpeechEngine {
     private const val CALIBRATE_MS = 300L
     private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word
     private const val CUE_VOLUME = 80
+    /** How long the car gets to start sending once asked, before the phone's microphone takes over. */
+    private const val CAR_FIRST_AUDIO_MS = 1_500L
+    private const val CAR_GAP_MS = 1_000L
     private const val CUE_MS = 150
     /** The car plays what the phone sends a moment late; waiting this out keeps the beep off the recording. */
     private const val CUE_TAIL_MS = 250L
@@ -132,14 +137,73 @@ object SpeechEngine {
      * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
      * loud cabin needs a louder voice than a quiet one to count as talking.
      */
+    /** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */
+    private interface Mic {
+        fun read(frame: ShortArray): Int
+        fun close()
+    }
+
     @SuppressLint("MissingPermission") // checked in listen()
-    private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? {
+    private fun phoneMic(): Mic {
         val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
         val recorder = AudioRecord(
             MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE,
             AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8),
         )
         check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" }
+        recorder.startRecording()
+        return object : Mic {
+            override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size)
+            override fun close() {
+                runCatching { recorder.stop() }
+                recorder.release()
+            }
+        }
+    }
+
+    /**
+     * The car's microphone, which hears the driver better than a phone in a cup holder. The car
+     * sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car
+     * microphone, or it sends nothing at first, so the phone's is used instead.
+     */
+    private fun carMic(): Mic? {
+        val car = CarMic.current ?: return null
+        if (!car.start()) return null
+        val first = car.read(CAR_FIRST_AUDIO_MS)
+        if (first == null) {
+            car.stop()
+            GearslipLog.w("speech: the car's microphone sent nothing - using the phone's")
+            return null
+        }
+        GearslipLog.i("speech: listening through the car's microphone")
+        return object : Mic {
+            var pending = first
+            var at = 0
+            override fun read(frame: ShortArray): Int {
+                var filled = 0
+                while (filled < frame.size) {
+                    val chunk = pending ?: car.read(CAR_GAP_MS) ?: break
+                    pending = chunk
+                    val n = minOf(frame.size - filled, chunk.size - at)
+                    System.arraycopy(chunk, at, frame, filled, n)
+                    filled += n
+                    at += n
+                    if (at == chunk.size) { pending = null; at = 0 }
+                }
+                // The car stopped sending mid-phrase: end the recording rather than wait forever.
+                return if (filled == 0) -1 else filled
+            }
+            override fun close() = car.stop()
+        }
+    }
+
+    /**
+     * Records until the driver stops talking. Returns the phrase from just before the first word,
+     * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
+     * loud cabin needs a louder voice than a quiet one to count as talking.
+     */
+    private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? {
+        val mic = carMic() ?: phoneMic()
         val samples = FloatArray((RATE * MAX_MS / 1000).toInt())
         var count = 0
         val frame = ShortArray(FRAME)
@@ -147,11 +211,11 @@ object SpeechEngine {
         var noiseFrames = 0
         var speechStart = -1
         var lastSpeech = 0
-        recorder.startRecording()
         try {
             while (id == run && count + FRAME <= samples.size) {
-                val n = recorder.read(frame, 0, FRAME)
+                val n = mic.read(frame)
-                if (n <= 0) continue
+                if (n < 0) break
+                if (n == 0) continue
                 var sum = 0.0
                 for (i in 0 until n) {
                     val v = frame[i] / 32768f
@@ -178,8 +242,7 @@ object SpeechEngine {
                 if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break
             }
         } finally {
-            runCatching { recorder.stop() }
+            mic.close()
-            recorder.release()
         }
         if (speechStart < 0) return null
         return samples.copyOfRange(speechStart, count)