diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index 0f63fa6..d06bb5c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -84,6 +84,7 @@ class GearslipRunner( private var sensorTypes: List = emptyList() @Volatile private var audioLink: AudioLink? = null @Volatile private var guidanceLink: AudioLink? = null + @Volatile private var vendorChannelId = -1 private var audioMessagesSeen = 0 private var touchWidth = 0 private var touchHeight = 0 @@ -213,6 +214,8 @@ class GearslipRunner( guidanceLink?.onMessage(messageId, body) } else if (frame.channel == CarBluetooth.channelId) { CarBluetooth.onMessage(messageId, body) + } else if (frame.channel == vendorChannelId) { + onVendorMessage(messageId, body) } else if (frame.channel == CarMic.current?.channelId) { CarMic.current?.onMessage(messageId, body) } else if (frame.channel == audioLink?.channelId) { @@ -470,6 +473,7 @@ class GearslipRunner( startMediaStatusChannel(serviceDiscoveryResponse) startClusterNavChannel(serviceDiscoveryResponse) startBluetooth(serviceDiscoveryResponse) + startVendorChannel(serviceDiscoveryResponse) val input = ServiceDiscovery.findInputService(serviceDiscoveryResponse, video.displayId) if (input == null) { @@ -504,7 +508,7 @@ class GearslipRunner( return } if (media.codecType != ServiceDiscovery.CODEC_PCM) { - log.w("the car offers no PCM media sink, only ${media.codecName}; it may play our PCM as silence") + log.w("the car offers no PCM media sink, only ${media.codecName}; music goes as AAC") } val first = media.configs.firstOrNull() SessionReport.audio( @@ -515,6 +519,9 @@ class GearslipRunner( media, sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = media.serviceId) }, sendControl = { id, body -> send(id, body, encrypted = true) }, + // The driver picked the radio or another source on the car: pause the phone's music + // as Android Auto does, instead of letting it play on to no one. + onTakenBack = { CarServices.pausePhoneMedia("the car switched to another source") }, ) openChannel(media.serviceId) } @@ -571,6 +578,29 @@ class GearslipRunner( } /** Bluetooth pairing, for calls: the channel only opens if the driver asks to pair. */ + /** + * GM's own channel ("com.gm.vip"), opened in GM cars only. Issue #18: a 2021 Bolt ends every + * session about 25 s in, and of the channels it offers, this is the one a Dacia that stays + * connected doesn't have. What GM sends on it is private, so Gearslip only opens it and logs + * what arrives, by message id and size. + */ + private fun startVendorChannel(serviceDiscoveryResponse: ByteArray) { + val vendors = ServiceDiscovery.findVendorServices(serviceDiscoveryResponse) + vendors.forEach { log.i("vendor channel ${it.serviceId}: ${it.name}") } + val gm = vendors.firstOrNull { it.name.startsWith("com.gm.") } ?: return + vendorChannelId = gm.serviceId + log.i("-> ChannelOpenRequest on GM's channel ${gm.serviceId}") + openChannel(gm.serviceId) + } + + private fun onVendorMessage(messageId: Int, body: ByteArray) { + if (messageId == MSG_CHANNEL_OPEN_RESPONSE) { + log.i("<- GM channel open: status=${Protobuf.readInt32Field(body, 1) ?: 0}") + } else { + log.i("<- GM channel message id=$messageId (${body.size} bytes)") + } + } + private fun startBluetooth(serviceDiscoveryResponse: ByteArray) { val bluetooth = ServiceDiscovery.findBluetoothService(serviceDiscoveryResponse) ?: run { log.i("no Bluetooth service - the car can't be paired from here") diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt index 0ecdc04..b9f5266 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt @@ -160,6 +160,18 @@ object ServiceDiscovery { */ class BluetoothService(val serviceId: Int, val carAddress: String, val pairingMethods: List) + /** A carmaker's own channel (service field 12), for its apps on the phone: GM's is "com.gm.vip". */ + class VendorService(val serviceId: Int, val name: String) + + fun findVendorServices(response: ByteArray): List = + Wire.allBytes(Wire.fields(response), 1).mapNotNull { channel -> + val service = Wire.fields(channel) + val id = Wire.varint(service, 1)?.toInt() ?: return@mapNotNull null + val fields = Wire.fields(Wire.bytes(service, 12) ?: return@mapNotNull null) + val name = Wire.bytes(fields, 1)?.let { String(it, Charsets.UTF_8) } ?: return@mapNotNull null + VendorService(id, name) + } + fun findBluetoothService(response: ByteArray): BluetoothService? { for (channel in Wire.allBytes(Wire.fields(response), 1)) { val service = Wire.fields(channel) @@ -178,6 +190,8 @@ object ServiceDiscovery { const val STREAM_GUIDANCE = 1 const val STREAM_MEDIA = 3 const val CODEC_PCM = 1 + const val CODEC_AAC_LC = 2 + const val CODEC_AAC_LC_ADTS = 4 private val STREAMS = mapOf(0 to "NONE", 1 to "GUIDANCE", 2 to "SYSTEM_AUDIO", 3 to "MEDIA", 4 to "TELEPHONY") diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AacEncoder.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AacEncoder.kt new file mode 100644 index 0000000..f80ae92 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AacEncoder.kt @@ -0,0 +1,94 @@ +package app.seb3thehacker.gearslip.audio + +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaFormat + +/** + * Turns captured PCM into AAC-LC for a car whose music channel takes nothing else. Cars that + * offer PCM never get here: AAC costs a little delay and quality, and every car that works today + * is a PCM car. + * + * Android's encoder hands back raw access units, which is what AUDIO_AAC_LC expects. For + * AUDIO_AAC_LC_ADTS each unit gets the 7-byte ADTS header in front, so the car can find the + * frame edges itself. The encoder's first output is its configuration, not audio, and is skipped. + */ +internal class AacEncoder(private val sampleRate: Int, private val channels: Int, private val adts: Boolean) { + + private val codec: MediaCodec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC).apply { + val format = MediaFormat.createAudioFormat(MediaFormat.MIMETYPE_AUDIO_AAC, sampleRate, channels).apply { + setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC) + setInteger(MediaFormat.KEY_BIT_RATE, if (channels == 1) 96_000 else 192_000) + setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 16 * 1024) + } + configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + start() + } + private val info = MediaCodec.BufferInfo() + private var samplesIn = 0L + + /** Encodes [size] bytes of 16-bit PCM and returns the AAC frames that came out, oldest first. */ + fun encode(pcm: ByteArray, size: Int): List { + var offset = 0 + val out = ArrayList() + while (offset < size) { + val index = codec.dequeueInputBuffer(INPUT_WAIT_US) + if (index < 0) { + drain(out) + continue + } + val buffer = codec.getInputBuffer(index) ?: continue + val n = minOf(buffer.remaining(), size - offset) + buffer.put(pcm, offset, n) + val presentationUs = samplesIn * 1_000_000L / sampleRate + codec.queueInputBuffer(index, 0, n, presentationUs, 0) + samplesIn += n / (2 * channels) + offset += n + drain(out) + } + return out + } + + private fun drain(out: MutableList) { + while (true) { + val index = codec.dequeueOutputBuffer(info, 0) + if (index < 0) return + if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG == 0 && info.size > 0) { + val buffer = codec.getOutputBuffer(index) + if (buffer != null) { + val frame = ByteArray(info.size) + buffer.position(info.offset) + buffer.get(frame) + out.add(if (adts) adtsHeader(frame.size) + frame else frame) + } + } + codec.releaseOutputBuffer(index, false) + } + } + + /** ADTS for one AAC-LC frame: sync word, no CRC, and the frame's length including this header. */ + private fun adtsHeader(payload: Int): ByteArray { + val length = payload + 7 + val freq = FREQUENCIES.indexOf(sampleRate).let { if (it < 0) 3 else it } + val profile = 1 // AAC-LC is object type 2, written as type minus one + return byteArrayOf( + 0xFF.toByte(), + 0xF1.toByte(), + ((profile shl 6) or (freq shl 2) or (channels shr 2)).toByte(), + (((channels and 3) shl 6) or (length shr 11)).toByte(), + ((length shr 3) and 0xFF).toByte(), + (((length and 7) shl 5) or 0x1F).toByte(), + 0xFC.toByte(), + ) + } + + fun release() { + runCatching { codec.stop() } + runCatching { codec.release() } + } + + private companion object { + const val INPUT_WAIT_US = 10_000L + val FREQUENCIES = listOf(96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350) + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index 9525bbc..e398b6d 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -29,6 +29,8 @@ class AudioLink( private val asksFocus: Boolean = true, private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) }, private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) }, + /** The car took the sound back after giving it, as when the driver switches to the radio. */ + private val onTakenBack: () -> Unit = {}, ) : CarAudio.Link { private var configIndex = 0 @@ -46,6 +48,9 @@ class AudioLink( private var dropped = 0 @Volatile private var configTimer: Thread? = null private val ackLock = Object() + /** Only for a sink that takes AAC and nothing else; PCM sinks get the capture as it is. */ + @Volatile private var aac: AacEncoder? = null + private val aacSink get() = service.codecType == ServiceDiscovery.CODEC_AAC_LC || service.codecType == ServiceDiscovery.CODEC_AAC_LC_ADTS private val config get() = service.configs.getOrNull(configIndex) ?: service.configs.firstOrNull() override val sampleRate get() = config?.sampleRate ?: 48_000 @@ -156,9 +161,12 @@ class AudioLink( fun onFocus(state: Int) { val name = FOCUS_STATES[state] ?: "unknown($state)" GearslipLog.i("$tag: <- AudioFocusResponse state=$name") + val had = focusGranted focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT || state == FOCUS_STATE_GAIN_MEDIA_ONLY if (focusGranted) startStream() else stop() + // A ducking request leaves the music playing; a loss means another source has the speakers. + if (had && (state == FOCUS_STATE_LOSS || state == FOCUS_STATE_LOSS_TRANSIENT)) onTakenBack() } override fun start() { @@ -181,6 +189,12 @@ class AudioLink( ) inFlight = 0 lastAckAt = System.currentTimeMillis() + if (aacSink && aac == null) { + aac = runCatching { AacEncoder(sampleRate, channels, adts = service.codecType == ServiceDiscovery.CODEC_AAC_LC_ADTS) } + .onSuccess { GearslipLog.i("$tag: the car takes ${service.codecName} only - encoding to AAC") } + .onFailure { GearslipLog.e("$tag: couldn't start the AAC encoder", it) } + .getOrNull() + } started = true GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels") } @@ -195,6 +209,8 @@ class AudioLink( } focusRequested = false focusGranted = false + aac?.release() + aac = null GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") chunks = 0 bytes = 0 @@ -202,6 +218,25 @@ class AudioLink( override fun write(pcm: ByteArray, size: Int) { if (!started) return + val encoder = aac + if (encoder == null) { + send(pcm.copyOf(size)) + return + } + val frames = try { + encoder.encode(pcm, size) + } catch (e: Exception) { + // A codec that dies mid-drive leaves the car silent, as it was before AAC; log it once. + GearslipLog.e("$tag: AAC encoding failed - no more audio this drive", e) + aac = null + started = false + return + } + frames.forEach(::send) + } + + /** One Data message, once the car has room for it. */ + private fun send(payload: ByteArray) { // Wait for the car to catch up rather than throw the chunk away: a dropped chunk is a gap // the driver hears, while a short wait is soaked up by the capture buffer behind us. synchronized(ackLock) { @@ -228,7 +263,7 @@ class AudioLink( val micros = System.nanoTime() / 1_000 for (i in 0 until 8) timestamp[i] = ((micros shr ((7 - i) * 8)) and 0xFF).toByte() try { - sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size)) + sendOnChannel(MSG_DATA, timestamp + payload) } catch (e: java.io.IOException) { GearslipLog.e("$tag: the link died mid-stream", e) started = false @@ -236,7 +271,7 @@ class AudioLink( } synchronized(ackLock) { inFlight++ } chunks++ - bytes += size + bytes += payload.size // The first chunk proves the whole path; after that, an occasional line is enough to show // the car is still taking audio and to catch a window that has quietly closed. if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) { @@ -272,6 +307,8 @@ class AudioLink( private const val FOCUS_STATE_GAIN = 1 private const val FOCUS_STATE_GAIN_TRANSIENT = 2 private const val FOCUS_STATE_GAIN_MEDIA_ONLY = 6 + private const val FOCUS_STATE_LOSS = 3 + private const val FOCUS_STATE_LOSS_TRANSIENT = 5 private const val SESSION_ID = 2 private const val STALL_MS = 2_000L private const val CHUNK_LOG_EVERY = 200 // about every four seconds diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt index 4d186be..4365baf 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt @@ -221,7 +221,7 @@ object CarServices { * headphones are, so the apps won't pause themselves. Every playing session is paused, not * just the one Gearslip opened: anything the car was playing was going through the car. */ - fun pausePhoneMedia() { + fun pausePhoneMedia(why: String = "the car left") { val ctx = context ?: return val manager = ctx.getSystemService(MediaSessionManager::class.java) val listener = ComponentName(ctx, GearslipNotificationListener::class.java) @@ -237,7 +237,7 @@ object CarServices { } playing.forEach { runCatching { it.transportControls.pause() } } if (playing.isNotEmpty()) { - app.seb3thehacker.gearslip.GearslipLog.i("media: paused ${playing.joinToString { it.packageName }} as the car left") + app.seb3thehacker.gearslip.GearslipLog.i("media: paused ${playing.joinToString { it.packageName }}: $why") } }