diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index 32103f7..e94d1b4 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -648,7 +648,9 @@ class GearslipRunner( tag = "guidance", sessionId = GUIDANCE_SESSION_ID, asksFocus = false, - onReady = { DirectionsVoice.attach(it) { audioLink?.streaming == true } }, + onReady = { + DirectionsVoice.attach(it, musicIsOurs = { audioLink?.streaming == true }, musicSink = { audioLink?.takeIf { link -> link.isReady } }) + }, onClose = { DirectionsVoice.detach(it) }, ) openChannel(guidance.serviceId) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index e86a0fb..8b9dc91 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -75,6 +75,9 @@ class AudioLink( /** Sending audio now, which for media means the car has handed us the speakers. */ val streaming get() = started + /** The car has set the sink up, so a stream can start. */ + val isReady get() = ready + @Volatile private var holdsSpeakers = false @Volatile private var focusAnswer: CountDownLatch? = null @Volatile private var answerDueBy = 0L @@ -113,6 +116,21 @@ class AudioLink( GearslipLog.i("$tag: -> AudioFocusRequest(RELEASE)") } + /** + * A spoken turn on the music channel while no music plays (see DirectionsVoice): the stream + * runs for the turn only, under ducking focus like the directions channel's, so the radio + * comes back after it rather than giving way to Android Auto for good. + */ + fun startForTurn() { + askForSpeakers() + startStream() + } + + fun stopForTurn() { + synchronized(startLock) { stopLocked(release = false) } + handBackSpeakers() + } + fun opened(status: Int) { GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status") if (status != 0) { @@ -218,7 +236,7 @@ class AudioLink( fun onFocus(state: Int) { val name = FOCUS_STATES[state] ?: "unknown($state)" GearslipLog.i("$tag: <- AudioFocusResponse state=$name") - if (!asksFocus) { + if (!asksFocus || expectsFocusAnswer()) { // Directions speak whatever the answer: a lost turn is worse than a loud one. answerDueBy = 0L focusAnswer?.countDown() diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt index ca5ec70..81d7ed8 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt @@ -137,6 +137,59 @@ object CarAudio { internal fun deliver(pcm: ByteArray, size: Int, peak: Float) { _level.value = peak - link?.write(pcm, size) + val voice = synchronized(voiceLock) { spoken } + if (voice == null) link?.write(pcm, size) else link?.write(mixVoice(pcm, size), size) } + + private val voiceLock = Any() + private var spoken: ByteArray? = null + private var spokenAt = 0 + private var spokenDone: java.util.concurrent.CountDownLatch? = null + + /** + * Mixes a spoken turn into the music from the next chunk on, with the music turned down + * under it, and waits until it has all gone out. [pcm] is 16-bit, in the sink's own format. + * False when no music is flowing to carry it, so the caller sends it some other way. + */ + fun speakOverMusic(pcm: ByteArray): Boolean { + if (_capture.value != Capture.ON || (link as? AudioLink)?.streaming != true) return false + val done = java.util.concurrent.CountDownLatch(1) + synchronized(voiceLock) { + spokenDone?.countDown() + spoken = pcm + spokenAt = 0 + spokenDone = done + } + val ms = pcm.size * 1000L / (2 * channels * sampleRate) + if (!done.await(ms + VOICE_SLACK_MS, java.util.concurrent.TimeUnit.MILLISECONDS)) { + // Capture stopped mid-turn; the rest of it is dropped rather than played late. + synchronized(voiceLock) { if (spokenDone === done) spoken = null } + } + return true + } + + private fun mixVoice(pcm: ByteArray, size: Int): ByteArray = synchronized(voiceLock) { + val voice = spoken ?: return pcm + val out = pcm.copyOf(size) + var i = 0 + while (i + 1 < size) { + val music = (out[i].toInt() and 0xFF) or (out[i + 1].toInt() shl 8) + val word = if (spokenAt + 1 < voice.size) (voice[spokenAt].toInt() and 0xFF) or (voice[spokenAt + 1].toInt() shl 8) else 0 + val mixed = (music * DUCK_PERCENT / 100 + word).coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()) + out[i] = mixed.toByte() + out[i + 1] = (mixed shr 8).toByte() + i += 2 + spokenAt += 2 + } + if (spokenAt >= voice.size) { + spoken = null + spokenDone?.countDown() + spokenDone = null + } + out + } + + /** How loud the music stays under a spoken turn. */ + private const val DUCK_PERCENT = 30 + private const val VOICE_SLACK_MS = 2_000L } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index 470d4af..627dcf5 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -57,6 +57,7 @@ object CarSettings { private const val KEY_NIGHT = "car_night_mode" private const val KEY_PIPE_AUDIO = "car_pipe_audio" private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions" +private const val KEY_DIRECTIONS_OVER_MUSIC = "car_directions_over_music" private const val KEY_BLUETOOTH_WARNING = "car_bluetooth_warning" private const val KEY_LOCATION_WARNING = "car_location_warning" private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked" @@ -101,6 +102,7 @@ private const val KEY_CLUSTER_VIDEO = "cluster_video_enabled" private val nightFlow = MutableStateFlow(NightMode.AUTO) private val pipeAudioFlow = MutableStateFlow(true) private val speakDirectionsFlow = MutableStateFlow(true) + private val directionsOverMusicFlow = MutableStateFlow(false) private val bluetoothWarningFlow = MutableStateFlow(true) private val locationWarningFlow = MutableStateFlow(true) private val lastNavFlow = MutableStateFlow(null) @@ -253,6 +255,9 @@ private const val KEY_CLUSTER_VIDEO = "cluster_video_enabled" /** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */ val speakDirections: StateFlow = speakDirectionsFlow + /** The turns go out on the music channel instead, for cars the directions channel upsets. */ + val directionsOverMusic: StateFlow = directionsOverMusicFlow + /** With Bluetooth audio, warn when the phone isn't connected to the car; see BluetoothAudioCheck. */ val bluetoothWarning: StateFlow = bluetoothWarningFlow @@ -344,6 +349,7 @@ private const val KEY_CLUSTER_VIDEO = "cluster_video_enabled" // Until the driver sets it, on for USB audio and off for Bluetooth, where the map app's // own voice already reaches the car. speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, pipeAudioFlow.value.toString()) == "true" + directionsOverMusicFlow.value = AppSettings.getString(app, KEY_DIRECTIONS_OVER_MUSIC, "false") == "true" bluetoothWarningFlow.value = AppSettings.getString(app, KEY_BLUETOOTH_WARNING, "true") == "true" locationWarningFlow.value = AppSettings.getString(app, KEY_LOCATION_WARNING, "true") == "true" nightFlow.value = runCatching { @@ -451,6 +457,11 @@ private const val KEY_CLUSTER_VIDEO = "cluster_video_enabled" AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString()) } + fun setDirectionsOverMusic(on: Boolean) { + directionsOverMusicFlow.value = on + AppSettings.putString(app, KEY_DIRECTIONS_OVER_MUSIC, on.toString()) + } + fun setBluetoothWarning(on: Boolean) { bluetoothWarningFlow.value = on AppSettings.putString(app, KEY_BLUETOOTH_WARNING, on.toString()) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index 4c8cb94..af19f12 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -67,6 +67,7 @@ fun CarSettingsScreen() { val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState() val speakDirections by CarSettings.speakDirections.collectAsState() + val directionsOverMusic by CarSettings.directionsOverMusic.collectAsState() val bluetoothWarning by CarSettings.bluetoothWarning.collectAsState() val locationWarning by CarSettings.locationWarning.collectAsState() val autoplay by CarSettings.autoplay.collectAsState() @@ -287,6 +288,22 @@ fun CarSettingsScreen() { ) } + if (section == chosen && speakDirections) item { + ChoiceRow( + stringResource(R.string.car_directions_over_music), + listOf(false to stringResource(R.string.off), true to stringResource(R.string.on)), + directionsOverMusic, + ) { CarSettings.setDirectionsOverMusic(it) } + } + + if (section == chosen && speakDirections) item { + Text( + stringResource(R.string.car_directions_over_music_note), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + if (section == chosen) item { ExperimentalFeatureRow( title = stringResource(R.string.car_phone_speech), diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt index 5b2c433..5283dda 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt @@ -14,6 +14,7 @@ import androidx.car.app.navigation.model.Maneuver import androidx.car.app.navigation.model.Step import app.seb3thehacker.gearslip.GearslipLog import app.seb3thehacker.gearslip.audio.AudioLink +import app.seb3thehacker.gearslip.audio.CarAudio import java.io.File import java.nio.ByteBuffer import java.nio.ByteOrder @@ -51,6 +52,8 @@ object DirectionsVoice { @Volatile private var link: AudioLink? = null @Volatile private var running = false @Volatile private var musicIsOurs: () -> Boolean = { false } + /** The car's music sink once it's set up, for [CarSettings.directionsOverMusic]. */ + @Volatile private var musicSink: () -> AudioLink? = { null } private val speaker = Executors.newSingleThreadScheduledExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } } /** Counts announcements, so a hand-back scheduled after one is dropped if another has begun. */ private var spoken = 0 @@ -95,9 +98,10 @@ object DirectionsVoice { } /** The car opened its directions sink: start following the map app's turns. */ - fun attach(link: AudioLink, musicIsOurs: () -> Boolean) { + fun attach(link: AudioLink, musicIsOurs: () -> Boolean, musicSink: () -> AudioLink? = { null }) { this.link = link this.musicIsOurs = musicIsOurs + this.musicSink = musicSink GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}") if (running) return running = true @@ -285,10 +289,16 @@ object DirectionsVoice { GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken") return } - val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run { + val wav = Wav.read(file) ?: run { GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read") return } + val music = musicSink() + if (CarSettings.directionsOverMusic.value && music != null) { + speakOverMusic(music, Wav.convert(wav, music.sampleRate, music.channels)) + return + } + val pcm = Wav.convert(wav, sink.sampleRate, sink.channels) // Logged before speaking too, so a car that dies mid-announcement shows what it was doing. GearslipLog.i("guidance: speaking a turn") // Held through back-to-back prompts rather than asked for and given back each time. @@ -302,18 +312,44 @@ object DirectionsVoice { GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)") } + /** + * A 2025 Nissan's Bosch unit drops the link about a second after any turn on its directions + * channel while its music channel is open, and plays music through nothing else (issue #35). + * So the directions channel is left alone: the turn rides in the music, or, with no music + * playing, goes out on the music channel by itself. + */ + private fun speakOverMusic(music: AudioLink, pcm: ByteArray) { + GearslipLog.i("guidance: speaking a turn over the music channel") + if (!CarAudio.speakOverMusic(pcm)) { + music.startForTurn() + try { + stream(music, pcm, alive = { musicSink() === music }, startAndStop = false) + } finally { + music.stopForTurn() + // Music that started during the turn found the stream running and didn't start its own. + if (CarAudio.capture.value == CarAudio.Capture.ON) music.start() + } + } + GearslipLog.i("guidance: spoke a turn over the music channel (${pcm.size / (2 * music.channels) * 1000 / music.sampleRate} ms)") + } + /** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */ - private fun stream(sink: AudioLink, pcm: ByteArray) { + private fun stream( + sink: AudioLink, + pcm: ByteArray, + alive: () -> Boolean = { link === sink }, + startAndStop: Boolean = true, + ) { val frame = 2 * sink.channels val chunk = sink.sampleRate / 50 * frame val chunkMs = 20L val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped - sink.start() + if (startAndStop) sink.start() try { val began = System.currentTimeMillis() var offset = 0 var sent = 0 - while (offset < padded.size && link === sink) { + while (offset < padded.size && alive()) { val size = minOf(chunk, padded.size - offset) sink.write(padded.copyOfRange(offset, offset + size), size) offset += size @@ -324,7 +360,7 @@ object DirectionsVoice { // Let the car play what it has before closing the stream on it. Thread.sleep(LEAD_MS) } finally { - sink.stop() + if (startAndStop) sink.stop() } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeCarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeCarSettings.kt index 8987df9..3e362d0 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeCarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeCarSettings.kt @@ -205,6 +205,7 @@ private fun GroupRows(group: CarGroup) { val units by CarSettings.units.collectAsState() val navBar by CarSettings.navBarMode.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState() + val directionsOverMusic by CarSettings.directionsOverMusic.collectAsState() val speakDirections by CarSettings.speakDirections.collectAsState() val systemSpeech by CarSettings.systemSpeech.collectAsState() val bluetoothWarning by CarSettings.bluetoothWarning.collectAsState() @@ -262,6 +263,15 @@ private fun GroupRows(group: CarGroup) { CarSettings::setSpeakDirections, note = stringResource(R.string.car_spoken_directions_note), ) + if (speakDirections) { + SettingsDivider() + Toggle( + stringResource(R.string.car_directions_over_music), + directionsOverMusic, + CarSettings::setDirectionsOverMusic, + note = stringResource(R.string.car_directions_over_music_note), + ) + } SettingsDivider() Toggle( stringResource(R.string.car_phone_speech), diff --git a/gearslip/src/main/res/values/strings.xml b/gearslip/src/main/res/values/strings.xml index 7a02d2d..8712577 100644 --- a/gearslip/src/main/res/values/strings.xml +++ b/gearslip/src/main/res/values/strings.xml @@ -468,6 +468,8 @@ Through Gearslip sends the phone\'s sound over the car connection, cable or wireless. Android asks to share your screen each drive, and a few apps block it. Bluetooth plays as the phone normally would, and Android never asks. + Directions over the music channel + For a car that drops the connection or restarts after a spoken turn. Gearslip mixes each turn into the music instead of using the car\'s directions channel. Gearslip reads each turn aloud through the car\'s speakers, and the music dips while it speaks. Turn it off if your map app\'s own voice already plays in the car. Listens through the phone\'s own speech service instead of Gearslip\'s. On most phones that\'s Google\'s, which may send your voice to Google. GrapheneOS has none, so Gearslip keeps using its own. Adds a Vehicle data tile: a live, raw dump of the head unit\'s sensor channel. Field labels (speed, RPM, fuel...) are carried over from other reverse-engineering projects and unverified against this car - treat them as a guess, not a fact, until you\'ve watched one change for yourself.