From a4ab5b005e57ee16c8b9552e77b16b31f9ad02cf Mon Sep 17 00:00:00 2001 From: Seb3thehacker Date: Mon, 5 Oct 2026 17:03:43 -0700 Subject: [PATCH] listen through the car's mic, fix exit on Uconnect, follow the car's night mode --- .../seb3thehacker/gearslip/GearslipRunner.kt | 65 ++++++++- .../gearslip/ServiceDiscovery.kt | 28 ++++ .../seb3thehacker/gearslip/audio/CarMic.kt | 129 ++++++++++++++++++ .../gearslip/speech/SpeechEngine.kt | 77 ++++++++++- 4 files changed, 287 insertions(+), 12 deletions(-) create mode 100644 gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index 442fb79..b030191 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -2,6 +2,7 @@ package app.seb3thehacker.gearslip import android.os.Build import app.seb3thehacker.gearslip.audio.AudioLink +import app.seb3thehacker.gearslip.audio.CarMic import app.seb3thehacker.gearslip.car.CarAssistant import app.seb3thehacker.gearslip.car.CarEnvironment import app.seb3thehacker.gearslip.car.CarKeys @@ -139,6 +140,9 @@ class GearslipRunner( audioLink?.close() audioLink = null CarSensors.clear() + CarMic.current = null + lastSensorNight = null + CarEnvironment.setSensorNight(null) // back to the clock until a car says otherwise projection?.onProjectionStopped() SessionStatus.disconnected() } @@ -172,6 +176,8 @@ class GearslipRunner( onInputMessage(messageId, body) } else if (frame.channel == sensorChannelId) { onSensorMessage(messageId, body) + } else if (frame.channel == CarMic.current?.channelId) { + CarMic.current?.onMessage(messageId, body) } else if (frame.channel == audioLink?.channelId) { // The audio channel is the one still being reverse-engineered, and its first few // replies are what tell us why. Raw bytes for those, then quiet. @@ -346,7 +352,9 @@ class GearslipRunner( private fun onServiceDiscoveryResponse(body: ByteArray) { log.i("<- ServiceDiscoveryResponse (${body.size} bytes, decrypted successfully)") - log.i("head unit describes itself as:\n" + Protobuf.describe(body)) + // Field 5 is the car's serial number. Logs get shared, so it stays out of them. + val described = Protobuf.describe(body).replace(Regex("(?m)^ #5 string = .*$"), " #5 string = (car serial, left out)") + log.i("head unit describes itself as:\n" + described) log.verdict( "VIABLE", "The head unit accepted the presented phone certificate, completed TLS, and is " + @@ -405,6 +413,7 @@ class GearslipRunner( openChannel(video.serviceId) startAudioChannel(serviceDiscoveryResponse) startSensorChannel(serviceDiscoveryResponse) + startMicChannel(serviceDiscoveryResponse) val input = ServiceDiscovery.findInputService(serviceDiscoveryResponse, video.displayId) if (input == null) { @@ -454,6 +463,18 @@ class GearslipRunner( openChannel(media.serviceId) } + /** The car's microphone, for voice: opened now, recording only while something listens. */ + private fun startMicChannel(serviceDiscoveryResponse: ByteArray) { + val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run { + log.i("no microphone advertised - voice uses the phone's") + return + } + val config = mic.configs.first() + log.i("car microphone: channel=${mic.serviceId} ${mic.codecName} ${config.sampleRate}Hz/${config.bits}bit/x${config.channels}") + CarMic.current = CarMic(mic) { id, body -> send(id, body, encrypted = true, channel = mic.serviceId) } + openChannel(mic.serviceId) + } + /** * Every sensor type the head unit advertised, subscribed to at once - there is no screen * to pick and choose yet, this is meant to show everything reachable. [CarSensors] answers @@ -500,6 +521,22 @@ class GearslipRunner( } } + /** + * The car's own day/night signal (its headlights or light sensor): NightData { bool is_night = 1; }. + * It replaces the clock guess, so maps go dark when the car does. + */ + private fun onNightSensor(body: ByteArray) { + val night = Wire.bytes(Wire.fields(body), SENSOR_NIGHT) ?: return + val isNight = Wire.varint(Wire.fields(night), 1) == 1L + if (isNight != lastSensorNight) { + lastSensorNight = isNight + log.i("car says it's ${if (isNight) "night" else "day"}") + CarEnvironment.setSensorNight(isNight) + } + } + + private var lastSensorNight: Boolean? = null + private fun onSensorMessage(messageId: Int, body: ByteArray) { when (messageId) { MSG_CHANNEL_OPEN_RESPONSE -> { @@ -527,6 +564,7 @@ class GearslipRunner( MSG_SENSOR_EVENT_INDICATION -> { CarSensors.onEvent(body) logSensorEvent(body) + onNightSensor(body) } // SensorError { required SensorType sensor_type = 1; required SensorErrorType sensor_error_type = 2; } @@ -574,6 +612,12 @@ class GearslipRunner( MSG_INPUT_REPORT -> { val report = Wire.fields(body) + // The 2018 Uconnect agrees to Exit but keeps showing our last frame and sending + // touches, which looked like a crash. A touch means our screen is still up there. + if (videoStarted && !videoProjected && Wire.bytes(report, 3) != null) { + log.w("the car said it took its screen back but still sends touches - showing ours again") + onVideoFocus(VIDEO_FOCUS_PROJECTED) + } Wire.bytes(report, 4)?.let(::onKeyEvent) Wire.bytes(report, 6)?.let(::onRelativeEvent) val touch = Wire.bytes(report, 3) ?: Wire.bytes(report, 7) @@ -898,21 +942,31 @@ class GearslipRunner( inFlight = 0 ackLock.notifyAll() } + val start = Protobuf.varintField(1, SESSION_ID.toLong()) + + Protobuf.varintField(2, (selectedConfigIndex ?: 0).toLong()) + send(MSG_MEDIA_START, start, encrypted = true, channel = videoChannelId) + log.i("-> Start(session_id=$SESSION_ID)") lastCodecConfig?.let { send(MSG_MEDIA_CODEC_CONFIG, it, encrypted = true, channel = videoChannelId) } videoSource?.requestSyncFrame() } else { - log.i("the car is showing its own screen - video paused, audio carries on") + // A phone ends the stream when the car takes its screen. LIVI switches on the focus + // answer alone, but the 2018 Uconnect kept showing our last frame until it got Stop. + send(MSG_MEDIA_STOP, Protobuf.varintField(1, SESSION_ID.toLong()), encrypted = true, channel = videoChannelId) + log.i("the car is showing its own screen - video stopped, audio carries on") } } /** Android Auto's Exit: asks the car for its own interface, leaving the session running. */ private fun requestNativeFocus() { if (!videoStarted) return - // VideoFocusRequestNotification { mode = 2; reason = 3 } - val request = Protobuf.varintField(2, VIDEO_FOCUS_NATIVE.toLong()) + + // VideoFocusRequestNotification { disp_channel_id = 1; mode = 2; reason = 3 }. The + // display's channel is deprecated, but the 2018 Uconnect (protocol 1.3) predates that: + // without it, it agreed to show its own screen and never did. LIVI ignores the field. + val request = Protobuf.varintField(1, videoChannelId.toLong()) + + Protobuf.varintField(2, VIDEO_FOCUS_NATIVE.toLong()) + Protobuf.varintField(3, VIDEO_FOCUS_REASON_LAUNCH_NATIVE.toLong()) send(MSG_VIDEO_FOCUS_REQUEST, request, encrypted = true, channel = videoChannelId) - log.i("-> VideoFocusRequest(mode=NATIVE, reason=LAUNCH_NATIVE)") + log.i("-> VideoFocusRequest(display=$videoChannelId, mode=NATIVE, reason=LAUNCH_NATIVE)") } /** Arrival time of the last tap still waiting to be seen in a frame; 0 when none is. */ @@ -1137,6 +1191,7 @@ class GearslipRunner( const val MSG_SENSOR_EVENT_INDICATION = 32771 const val MSG_SENSOR_ERROR = 32772 private const val SENSOR_LOCATION = 1 + private const val SENSOR_NIGHT = 10 private const val SENSOR_LOG_EVERY_MS = 10_000L const val STREAM_MEDIA = 3 diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt index 9cf5361..3cf57e6 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt @@ -110,6 +110,34 @@ object ServiceDiscovery { return media.firstOrNull { it.codecType == CODEC_PCM } ?: media.firstOrNull() } + /** + * The car's microphone, from MediaSourceService (Service field 5): + * MediaSourceService { MediaCodecType available_type = 1; AudioConfiguration audio_config = 2; + * bool available_while_in_call = 3; } + * Null when the head unit has none. + */ + fun findMicService(response: ByteArray): AudioService? { + for (channel in Wire.allBytes(Wire.fields(response), 1)) { + val service = Wire.fields(channel) + val id = Wire.varint(service, 1)?.toInt() ?: continue + val source = Wire.fields(Wire.bytes(service, 5) ?: continue) + val f = Wire.fields(Wire.bytes(source, 2) ?: continue) + return AudioService( + serviceId = id, + streamType = 0, + codecType = Wire.varint(source, 1)?.toInt() ?: CODEC_PCM, + configs = listOf( + AudioConfig( + sampleRate = Wire.varint(f, 1)?.toInt() ?: 16_000, + bits = Wire.varint(f, 2)?.toInt() ?: 16, + channels = Wire.varint(f, 3)?.toInt() ?: 1, + ), + ), + ) + } + return null + } + const val STREAM_MEDIA = 3 const val CODEC_PCM = 1 diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt new file mode 100644 index 0000000..1236601 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt @@ -0,0 +1,129 @@ +package app.seb3thehacker.gearslip.audio + +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.Protobuf +import app.seb3thehacker.gearslip.ServiceDiscovery +import app.seb3thehacker.gearslip.Wire +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.TimeUnit + +/** + * The car's own microphone, which most head units offer over the same link as the screen: + * + * -> CHANNEL_OPEN_REQUEST <- CHANNEL_OPEN_RESPONSE + * -> Setup { config_index } <- Config + * -> MicrophoneRequest { open } <- MicrophoneResponse { status, session_id } + * <- Data { 8-byte timestamp, PCM }... -> Ack { session_id, ack } + * -> MicrophoneRequest { open = false } + * + * The channel opens with the session, but the car only records between [start] and [stop]. + * Speech wants 16 kHz mono; a car that records at a multiple of that is averaged down to it, and + * one that doesn't isn't used, so voice falls back to the phone's microphone. + */ +class CarMic( + private val service: ServiceDiscovery.AudioService, + private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit, +) { + val channelId get() = service.serviceId + + private val config = service.configs.first() + private val step = config.sampleRate / RATE + private val usable = service.codecType == ServiceDiscovery.CODEC_PCM && config.bits == 16 && + config.sampleRate % RATE == 0 && step >= 1 && config.channels in 1..2 + + @Volatile private var ready = false + @Volatile private var listening = false + @Volatile private var session = 0 + private var packets = 0 + private val chunks = LinkedBlockingQueue() + + fun onMessage(messageId: Int, body: ByteArray) { + when (messageId) { + MSG_CHANNEL_OPEN_RESPONSE -> { + val status = Protobuf.readInt32Field(body, 1) ?: 0 + GearslipLog.i("mic: <- ChannelOpenResponse: status=$status") + if (status != 0) return + sendOnChannel(MSG_SETUP, Protobuf.varintField(1, 0L)) + ready = usable + if (!usable) { + GearslipLog.w( + "mic: the car records ${service.codecName} ${config.sampleRate}Hz/${config.bits}bit/x${config.channels}, " + + "which speech can't use - voice stays on the phone's microphone", + ) + } + } + + MSG_CONFIG -> GearslipLog.i("mic: <- Config status=${Wire.varint(Wire.fields(body), 1)}") + + MSG_MIC_RESPONSE -> { + val fields = Wire.fields(body) + session = Wire.varint(fields, 2)?.toInt() ?: 0 + GearslipLog.i("mic: <- MicrophoneResponse status=${Wire.varint(fields, 1)} session=$session") + } + + MSG_DATA -> { + sendOnChannel(MSG_ACK, Protobuf.varintField(1, session.toLong()) + Protobuf.varintField(2, 1L)) + if (!listening || body.size <= TIMESTAMP_BYTES) return + if (packets++ == 0) GearslipLog.i("mic: first audio from the car (${body.size - TIMESTAMP_BYTES} bytes)") + chunks.offer(toSpeechRate(body)) + } + + else -> GearslipLog.i("mic: <- unhandled message id=$messageId (${body.size} bytes)") + } + } + + /** Little-endian PCM after the timestamp, mixed to mono and averaged down to [RATE]. */ + private fun toSpeechRate(body: ByteArray): ShortArray { + val frameBytes = 2 * config.channels + val frames = (body.size - TIMESTAMP_BYTES) / frameBytes + val out = ShortArray(frames / step) + for (i in out.indices) { + var sum = 0 + for (s in 0 until step) { + val at = TIMESTAMP_BYTES + (i * step + s) * frameBytes + for (c in 0 until config.channels) { + val o = at + 2 * c + sum += (body[o].toInt() and 0xFF) or (body[o + 1].toInt() shl 8) + } + } + out[i] = (sum / (step * config.channels)).toShort() + } + return out + } + + /** Asks the car to start recording. False when it can't, so the phone's microphone is used. */ + fun start(): Boolean { + if (!ready) return false + chunks.clear() + packets = 0 + listening = true + sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 1L) + Protobuf.varintField(4, 1L)) + GearslipLog.i("mic: -> MicrophoneRequest(open)") + return true + } + + /** The next piece of audio, or null if the car sent nothing for [timeoutMs]. */ + fun read(timeoutMs: Long): ShortArray? = chunks.poll(timeoutMs, TimeUnit.MILLISECONDS) + + fun stop() { + if (!listening) return + listening = false + sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 0L)) + GearslipLog.i("mic: -> MicrophoneRequest(close) after $packets packets") + } + + companion object { + /** The connected car's microphone, or null with no car or one that has none. */ + @Volatile var current: CarMic? = null + + const val RATE = 16_000 + private const val TIMESTAMP_BYTES = 8 + private const val MSG_CHANNEL_OPEN_RESPONSE = 8 + private const val MSG_DATA = 0 + private const val MSG_SETUP = 32768 + private const val MSG_CONFIG = 32771 + private const val MSG_ACK = 32772 + private const val MSG_MIC_REQUEST = 32773 + private const val MSG_MIC_RESPONSE = 32774 + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt index 8694e23..2fc90fa 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt @@ -14,6 +14,7 @@ import android.os.Handler import android.os.Looper import android.os.SystemClock import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.audio.CarMic import java.util.Locale import java.util.concurrent.Executors import kotlin.math.log10 @@ -25,7 +26,8 @@ import kotlin.math.sqrt * none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply * of a few seconds comes back in well under a second on a recent phone. * - * Whisper needs the whole phrase before it starts, so this records from the phone's microphone, + * Whisper needs the whole phrase before it starts, so this records from the car's microphone (the + * phone's when the car has none or sends nothing), * decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes * once. Everything runs on one worker thread; callbacks arrive on the main thread. */ @@ -45,6 +47,9 @@ object SpeechEngine { private const val CALIBRATE_MS = 300L private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word private const val CUE_VOLUME = 80 + /** How long the car gets to start sending once asked, before the phone's microphone takes over. */ + private const val CAR_FIRST_AUDIO_MS = 1_500L + private const val CAR_GAP_MS = 1_000L private const val CUE_MS = 150 /** The car plays what the phone sends a moment late; waiting this out keeps the beep off the recording. */ private const val CUE_TAIL_MS = 250L @@ -132,14 +137,73 @@ object SpeechEngine { * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a * loud cabin needs a louder voice than a quiet one to count as talking. */ + /** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */ + private interface Mic { + fun read(frame: ShortArray): Int + fun close() + } + @SuppressLint("MissingPermission") // checked in listen() - private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? { + private fun phoneMic(): Mic { val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) val recorder = AudioRecord( MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8), ) check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" } + recorder.startRecording() + return object : Mic { + override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size) + override fun close() { + runCatching { recorder.stop() } + recorder.release() + } + } + } + + /** + * The car's microphone, which hears the driver better than a phone in a cup holder. The car + * sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car + * microphone, or it sends nothing at first, so the phone's is used instead. + */ + private fun carMic(): Mic? { + val car = CarMic.current ?: return null + if (!car.start()) return null + val first = car.read(CAR_FIRST_AUDIO_MS) + if (first == null) { + car.stop() + GearslipLog.w("speech: the car's microphone sent nothing - using the phone's") + return null + } + GearslipLog.i("speech: listening through the car's microphone") + return object : Mic { + var pending = first + var at = 0 + override fun read(frame: ShortArray): Int { + var filled = 0 + while (filled < frame.size) { + val chunk = pending ?: car.read(CAR_GAP_MS) ?: break + pending = chunk + val n = minOf(frame.size - filled, chunk.size - at) + System.arraycopy(chunk, at, frame, filled, n) + filled += n + at += n + if (at == chunk.size) { pending = null; at = 0 } + } + // The car stopped sending mid-phrase: end the recording rather than wait forever. + return if (filled == 0) -1 else filled + } + override fun close() = car.stop() + } + } + + /** + * Records until the driver stops talking. Returns the phrase from just before the first word, + * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a + * loud cabin needs a louder voice than a quiet one to count as talking. + */ + private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? { + val mic = carMic() ?: phoneMic() val samples = FloatArray((RATE * MAX_MS / 1000).toInt()) var count = 0 val frame = ShortArray(FRAME) @@ -147,11 +211,11 @@ object SpeechEngine { var noiseFrames = 0 var speechStart = -1 var lastSpeech = 0 - recorder.startRecording() try { while (id == run && count + FRAME <= samples.size) { - val n = recorder.read(frame, 0, FRAME) - if (n <= 0) continue + val n = mic.read(frame) + if (n < 0) break + if (n == 0) continue var sum = 0.0 for (i in 0 until n) { val v = frame[i] / 32768f @@ -178,8 +242,7 @@ object SpeechEngine { if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break } } finally { - runCatching { recorder.stop() } - recorder.release() + mic.close() } if (speechStart < 0) return null return samples.copyOfRange(speechStart, count)