car audio: pause the phone's music when the car switches to the radio, send AAC to cars that take nothing else, and open GM's own channel in GM cars

gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt

@@ -84,6 +84,7 @@ class GearslipRunner(
     private var sensorTypes: List<Int> = emptyList()
     @Volatile private var audioLink: AudioLink? = null
     @Volatile private var guidanceLink: AudioLink? = null
+    @Volatile private var vendorChannelId = -1
     private var audioMessagesSeen = 0
     private var touchWidth = 0
     private var touchHeight = 0
@@ -213,6 +214,8 @@ class GearslipRunner(
                 guidanceLink?.onMessage(messageId, body)
             } else if (frame.channel == CarBluetooth.channelId) {
                 CarBluetooth.onMessage(messageId, body)
+            } else if (frame.channel == vendorChannelId) {
+                onVendorMessage(messageId, body)
             } else if (frame.channel == CarMic.current?.channelId) {
                 CarMic.current?.onMessage(messageId, body)
             } else if (frame.channel == audioLink?.channelId) {
@@ -470,6 +473,7 @@ class GearslipRunner(
         startMediaStatusChannel(serviceDiscoveryResponse)
         startClusterNavChannel(serviceDiscoveryResponse)
         startBluetooth(serviceDiscoveryResponse)
+        startVendorChannel(serviceDiscoveryResponse)
 
         val input = ServiceDiscovery.findInputService(serviceDiscoveryResponse, video.displayId)
         if (input == null) {
@@ -504,7 +508,7 @@ class GearslipRunner(
             return
         }
         if (media.codecType != ServiceDiscovery.CODEC_PCM) {
-            log.w("the car offers no PCM media sink, only ${media.codecName}; it may play our PCM as silence")
+            log.w("the car offers no PCM media sink, only ${media.codecName}; music goes as AAC")
         }
         val first = media.configs.firstOrNull()
         SessionReport.audio(
@@ -515,6 +519,9 @@ class GearslipRunner(
             media,
             sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = media.serviceId) },
             sendControl = { id, body -> send(id, body, encrypted = true) },
+            // The driver picked the radio or another source on the car: pause the phone's music
+            // as Android Auto does, instead of letting it play on to no one.
+            onTakenBack = { CarServices.pausePhoneMedia("the car switched to another source") },
         )
         openChannel(media.serviceId)
     }
@@ -571,6 +578,29 @@ class GearslipRunner(
     }
 
     /** Bluetooth pairing, for calls: the channel only opens if the driver asks to pair. */
+    /**
+     * GM's own channel ("com.gm.vip"), opened in GM cars only. Issue #18: a 2021 Bolt ends every
+     * session about 25 s in, and of the channels it offers, this is the one a Dacia that stays
+     * connected doesn't have. What GM sends on it is private, so Gearslip only opens it and logs
+     * what arrives, by message id and size.
+     */
+    private fun startVendorChannel(serviceDiscoveryResponse: ByteArray) {
+        val vendors = ServiceDiscovery.findVendorServices(serviceDiscoveryResponse)
+        vendors.forEach { log.i("vendor channel ${it.serviceId}: ${it.name}") }
+        val gm = vendors.firstOrNull { it.name.startsWith("com.gm.") } ?: return
+        vendorChannelId = gm.serviceId
+        log.i("-> ChannelOpenRequest on GM's channel ${gm.serviceId}")
+        openChannel(gm.serviceId)
+    }
+
+    private fun onVendorMessage(messageId: Int, body: ByteArray) {
+        if (messageId == MSG_CHANNEL_OPEN_RESPONSE) {
+            log.i("<- GM channel open: status=${Protobuf.readInt32Field(body, 1) ?: 0}")
+        } else {
+            log.i("<- GM channel message id=$messageId (${body.size} bytes)")
+        }
+    }
+
     private fun startBluetooth(serviceDiscoveryResponse: ByteArray) {
         val bluetooth = ServiceDiscovery.findBluetoothService(serviceDiscoveryResponse) ?: run {
             log.i("no Bluetooth service - the car can't be paired from here")

gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt

@@ -160,6 +160,18 @@ object ServiceDiscovery {
      */
     class BluetoothService(val serviceId: Int, val carAddress: String, val pairingMethods: List<Int>)
 
+    /** A carmaker's own channel (service field 12), for its apps on the phone: GM's is "com.gm.vip". */
+    class VendorService(val serviceId: Int, val name: String)
+
+    fun findVendorServices(response: ByteArray): List<VendorService> =
+        Wire.allBytes(Wire.fields(response), 1).mapNotNull { channel ->
+            val service = Wire.fields(channel)
+            val id = Wire.varint(service, 1)?.toInt() ?: return@mapNotNull null
+            val fields = Wire.fields(Wire.bytes(service, 12) ?: return@mapNotNull null)
+            val name = Wire.bytes(fields, 1)?.let { String(it, Charsets.UTF_8) } ?: return@mapNotNull null
+            VendorService(id, name)
+        }
+
     fun findBluetoothService(response: ByteArray): BluetoothService? {
         for (channel in Wire.allBytes(Wire.fields(response), 1)) {
             val service = Wire.fields(channel)
@@ -178,6 +190,8 @@ object ServiceDiscovery {
     const val STREAM_GUIDANCE = 1
     const val STREAM_MEDIA = 3
     const val CODEC_PCM = 1
+    const val CODEC_AAC_LC = 2
+    const val CODEC_AAC_LC_ADTS = 4
 
     private val STREAMS = mapOf(0 to "NONE", 1 to "GUIDANCE", 2 to "SYSTEM_AUDIO", 3 to "MEDIA", 4 to "TELEPHONY")
 

gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AacEncoder.kt

@@ -0,0 +1,94 @@
+package app.seb3thehacker.gearslip.audio
+
+import android.media.MediaCodec
+import android.media.MediaCodecInfo
+import android.media.MediaFormat
+
+/**
+ * Turns captured PCM into AAC-LC for a car whose music channel takes nothing else. Cars that
+ * offer PCM never get here: AAC costs a little delay and quality, and every car that works today
+ * is a PCM car.
+ *
+ * Android's encoder hands back raw access units, which is what AUDIO_AAC_LC expects. For
+ * AUDIO_AAC_LC_ADTS each unit gets the 7-byte ADTS header in front, so the car can find the
+ * frame edges itself. The encoder's first output is its configuration, not audio, and is skipped.
+ */
+internal class AacEncoder(private val sampleRate: Int, private val channels: Int, private val adts: Boolean) {
+
+    private val codec: MediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC).apply {
+        val format = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channels).apply {
+            setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC)
+            setInteger(MediaFormat.KEY_BIT_RATE, if (channels == 1) 96_000 else 192_000)
+            setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024)
+        }
+        configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
+        start()
+    }
+    private val info = MediaCodec.BufferInfo()
+    private var samplesIn = 0L
+
+    /** Encodes [size] bytes of 16-bit PCM and returns the AAC frames that came out, oldest first. */
+    fun encode(pcm: ByteArray, size: Int): List<ByteArray> {
+        var offset = 0
+        val out = ArrayList<ByteArray>()
+        while (offset < size) {
+            val index = codec.dequeueInputBuffer(INPUT_WAIT_US)
+            if (index < 0) {
+                drain(out)
+                continue
+            }
+            val buffer = codec.getInputBuffer(index) ?: continue
+            val n = minOf(buffer.remaining(), size - offset)
+            buffer.put(pcm, offset, n)
+            val presentationUs = samplesIn * 1_000_000L / sampleRate
+            codec.queueInputBuffer(index, 0, n, presentationUs, 0)
+            samplesIn += n / (2 * channels)
+            offset += n
+            drain(out)
+        }
+        return out
+    }
+
+    private fun drain(out: MutableList<ByteArray>) {
+        while (true) {
+            val index = codec.dequeueOutputBuffer(info, 0)
+            if (index < 0) return
+            if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && info.size > 0) {
+                val buffer = codec.getOutputBuffer(index)
+                if (buffer != null) {
+                    val frame = ByteArray(info.size)
+                    buffer.position(info.offset)
+                    buffer.get(frame)
+                    out.add(if (adts) adtsHeader(frame.size) + frame else frame)
+                }
+            }
+            codec.releaseOutputBuffer(index, false)
+        }
+    }
+
+    /** ADTS for one AAC-LC frame: sync word, no CRC, and the frame's length including this header. */
+    private fun adtsHeader(payload: Int): ByteArray {
+        val length = payload + 7
+        val freq = FREQUENCIES.indexOf(sampleRate).let { if (it < 0) 3 else it }
+        val profile = 1 // AAC-LC is object type 2, written as type minus one
+        return byteArrayOf(
+            0xFF.toByte(),
+            0xF1.toByte(),
+            ((profile shl 6) or (freq shl 2) or (channels shr 2)).toByte(),
+            (((channels and 3) shl 6) or (length shr 11)).toByte(),
+            ((length shr 3) and 0xFF).toByte(),
+            (((length and 7) shl 5) or 0x1F).toByte(),
+            0xFC.toByte(),
+        )
+    }
+
+    fun release() {
+        runCatching { codec.stop() }
+        runCatching { codec.release() }
+    }
+
+    private companion object {
+        const val INPUT_WAIT_US = 10_000L
+        val FREQUENCIES = listOf(96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350)
+    }
+}

gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt

@@ -29,6 +29,8 @@ class AudioLink(
     private val asksFocus: Boolean = true,
     private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) },
     private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) },
+    /** The car took the sound back after giving it, as when the driver switches to the radio. */
+    private val onTakenBack: () -> Unit = {},
 ) : CarAudio.Link {
 
     private var configIndex = 0
@@ -46,6 +48,9 @@ class AudioLink(
     private var dropped = 0
     @Volatile private var configTimer: Thread? = null
     private val ackLock = Object()
+    /** Only for a sink that takes AAC and nothing else; PCM sinks get the capture as it is. */
+    @Volatile private var aac: AacEncoder? = null
+    private val aacSink get() = service.codecType == ServiceDiscovery.CODEC_AAC_LC || service.codecType == ServiceDiscovery.CODEC_AAC_LC_ADTS
 
     private val config get() = service.configs.getOrNull(configIndex) ?: service.configs.firstOrNull()
     override val sampleRate get() = config?.sampleRate ?: 48_000
@@ -156,9 +161,12 @@ class AudioLink(
     fun onFocus(state: Int) {
         val name = FOCUS_STATES[state] ?: "unknown($state)"
         GearslipLog.i("$tag: <- AudioFocusResponse state=$name")
+        val had = focusGranted
         focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT ||
             state == FOCUS_STATE_GAIN_MEDIA_ONLY
         if (focusGranted) startStream() else stop()
+        // A ducking request leaves the music playing; a loss means another source has the speakers.
+        if (had && (state == FOCUS_STATE_LOSS || state == FOCUS_STATE_LOSS_TRANSIENT)) onTakenBack()
     }
 
     override fun start() {
@@ -181,6 +189,12 @@ class AudioLink(
         )
         inFlight = 0
         lastAckAt = System.currentTimeMillis()
+        if (aacSink && aac == null) {
+            aac = runCatching { AacEncoder(sampleRate, channels, adts = service.codecType == ServiceDiscovery.CODEC_AAC_LC_ADTS) }
+                .onSuccess { GearslipLog.i("$tag: the car takes ${service.codecName} only - encoding to AAC") }
+                .onFailure { GearslipLog.e("$tag: couldn't start the AAC encoder", it) }
+                .getOrNull()
+        }
         started = true
         GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels")
     }
@@ -195,6 +209,8 @@ class AudioLink(
         }
         focusRequested = false
         focusGranted = false
+        aac?.release()
+        aac = null
         GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes")
         chunks = 0
         bytes = 0
@@ -202,6 +218,25 @@ class AudioLink(
 
     override fun write(pcm: ByteArray, size: Int) {
         if (!started) return
+        val encoder = aac
+        if (encoder == null) {
+            send(pcm.copyOf(size))
+            return
+        }
+        val frames = try {
+            encoder.encode(pcm, size)
+        } catch (e: Exception) {
+            // A codec that dies mid-drive leaves the car silent, as it was before AAC; log it once.
+            GearslipLog.e("$tag: AAC encoding failed - no more audio this drive", e)
+            aac = null
+            started = false
+            return
+        }
+        frames.forEach(::send)
+    }
+
+    /** One Data message, once the car has room for it. */
+    private fun send(payload: ByteArray) {
         // Wait for the car to catch up rather than throw the chunk away: a dropped chunk is a gap
         // the driver hears, while a short wait is soaked up by the capture buffer behind us.
         synchronized(ackLock) {
@@ -228,7 +263,7 @@ class AudioLink(
         val micros = System.nanoTime() / 1_000
         for (i in 0 until 8) timestamp[i] = ((micros shr ((7 - i) * 8)) and 0xFF).toByte()
         try {
-            sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size))
+            sendOnChannel(MSG_DATA, timestamp + payload)
         } catch (e: java.io.IOException) {
             GearslipLog.e("$tag: the link died mid-stream", e)
             started = false
@@ -236,7 +271,7 @@ class AudioLink(
         }
         synchronized(ackLock) { inFlight++ }
         chunks++
-        bytes += size
+        bytes += payload.size
         // The first chunk proves the whole path; after that, an occasional line is enough to show
         // the car is still taking audio and to catch a window that has quietly closed.
         if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) {
@@ -272,6 +307,8 @@ class AudioLink(
         private const val FOCUS_STATE_GAIN = 1
         private const val FOCUS_STATE_GAIN_TRANSIENT = 2
         private const val FOCUS_STATE_GAIN_MEDIA_ONLY = 6
+        private const val FOCUS_STATE_LOSS = 3
+        private const val FOCUS_STATE_LOSS_TRANSIENT = 5
         private const val SESSION_ID = 2
         private const val STALL_MS = 2_000L
         private const val CHUNK_LOG_EVERY = 200 // about every four seconds

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt

@@ -221,7 +221,7 @@ object CarServices {
      * headphones are, so the apps won't pause themselves. Every playing session is paused, not
      * just the one Gearslip opened: anything the car was playing was going through the car.
      */
-    fun pausePhoneMedia() {
+    fun pausePhoneMedia(why: String = "the car left") {
         val ctx = context ?: return
         val manager = ctx.getSystemService(MediaSessionManager::class.java)
         val listener = ComponentName(ctx, GearslipNotificationListener::class.java)
@@ -237,7 +237,7 @@ object CarServices {
         }
         playing.forEach { runCatching { it.transportControls.pause() } }
         if (playing.isNotEmpty()) {
-            app.seb3thehacker.gearslip.GearslipLog.i("media: paused ${playing.joinToString { it.packageName }} as the car left")
+            app.seb3thehacker.gearslip.GearslipLog.i("media: paused ${playing.joinToString { it.packageName }}: $why")
         }
     }