From 59f323ec06b942fc86eeb4cb01057c8c36d9cdf6 Mon Sep 17 00:00:00 2001 From: Seb3thehacker Date: Tue, 13 Oct 2026 12:26:31 -0700 Subject: [PATCH] directions: speak each turn on the car's guidance channel Map apps' own voice prompts can't reach the car over USB, so Gearslip reads the turn the cluster already gets and says it through the car's directions sink, which ducks the music. On by default, with a switch in car settings. --- .../seb3thehacker/gearslip/GearslipRunner.kt | 31 +- .../gearslip/ServiceDiscovery.kt | 1 + .../seb3thehacker/gearslip/audio/AudioLink.kt | 49 +-- .../seb3thehacker/gearslip/car/CarServices.kt | 1 + .../seb3thehacker/gearslip/car/CarSettings.kt | 11 + .../gearslip/car/CarSettingsScreen.kt | 13 + .../seb3thehacker/gearslip/car/ClusterNav.kt | 53 +-- .../gearslip/car/DirectionsVoice.kt | 326 ++++++++++++++++++ 8 files changed, 438 insertions(+), 47 deletions(-) create mode 100644 gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index 13c9cfc..c48df78 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -5,6 +5,7 @@ import app.seb3thehacker.gearslip.audio.AudioLink import app.seb3thehacker.gearslip.audio.CarMic import app.seb3thehacker.gearslip.media.CarMediaStatus import app.seb3thehacker.gearslip.car.ClusterNav +import app.seb3thehacker.gearslip.car.DirectionsVoice import app.seb3thehacker.gearslip.car.CarAssistant import app.seb3thehacker.gearslip.car.CarEnvironment import app.seb3thehacker.gearslip.car.CarKeys @@ -79,6 +80,7 @@ class GearslipRunner( @Volatile private var clusterNav: ClusterNav? = null private var sensorTypes: List = emptyList() @Volatile private var audioLink: AudioLink? = null + @Volatile private var guidanceLink: AudioLink? = null private var audioMessagesSeen = 0 private var touchWidth = 0 private var touchHeight = 0 @@ -154,6 +156,8 @@ class GearslipRunner( videoSource = null audioLink?.close() audioLink = null + guidanceLink?.close() + guidanceLink = null CarSensors.clear() CarMic.current = null mediaStatus?.stop() @@ -201,6 +205,8 @@ class GearslipRunner( clusterNav?.onMessage(messageId, body) } else if (frame.channel == mediaStatus?.channelId) { mediaStatus?.onMessage(messageId, body) + } else if (frame.channel == guidanceLink?.channelId) { + guidanceLink?.onMessage(messageId, body) } else if (frame.channel == CarMic.current?.channelId) { CarMic.current?.onMessage(messageId, body) } else if (frame.channel == audioLink?.channelId) { @@ -452,6 +458,7 @@ class GearslipRunner( openChannel(video.serviceId) startAudioChannel(serviceDiscoveryResponse) + startGuidanceChannel(serviceDiscoveryResponse) startSensorChannel(serviceDiscoveryResponse) startMicChannel(serviceDiscoveryResponse) startMediaStatusChannel(serviceDiscoveryResponse) @@ -473,7 +480,7 @@ class GearslipRunner( } } - /** Media audio rides its own sink; guidance, system and telephony sinks are left closed. */ + /** Media audio rides its own sink; spoken directions take the guidance sink, and system and telephony stay closed. */ private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) { val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) sinks.forEach { sink -> @@ -505,6 +512,27 @@ class GearslipRunner( openChannel(media.serviceId) } + /** The car's directions channel, where [DirectionsVoice] reads out each turn. */ + private fun startGuidanceChannel(serviceDiscoveryResponse: ByteArray) { + val guidance = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) + .filter { it.streamType == ServiceDiscovery.STREAM_GUIDANCE } + .firstOrNull { it.codecType == ServiceDiscovery.CODEC_PCM } ?: run { + log.i("no PCM directions sink - turns won't be spoken") + return + } + guidanceLink = AudioLink( + guidance, + sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = guidance.serviceId) }, + sendControl = { id, body -> send(id, body, encrypted = true) }, + tag = "guidance", + sessionId = GUIDANCE_SESSION_ID, + asksFocus = false, + onReady = { DirectionsVoice.attach(it) }, + onClose = { DirectionsVoice.detach(it) }, + ) + openChannel(guidance.serviceId) + } + /** The car's microphone, for voice: opened now, recording only while something listens. */ private fun startMicChannel(serviceDiscoveryResponse: ByteArray) { val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run { @@ -1292,6 +1320,7 @@ class GearslipRunner( const val ACK_STALL_MS = 2_000L const val AUDIO_TRACE_MESSAGES = 6 + const val GUIDANCE_SESSION_ID = 1 const val PERIODIC_KEYFRAME_MS = 5_000L } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt index 2aad847..529d99f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt @@ -153,6 +153,7 @@ object ServiceDiscovery { return null } + const val STREAM_GUIDANCE = 1 const val STREAM_MEDIA = 3 const val CODEC_PCM = 1 diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index 0ed7e80..9525bbc 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -22,6 +22,13 @@ class AudioLink( private val service: ServiceDiscovery.AudioService, private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit, private val sendControl: (messageId: Int, body: ByteArray) -> Unit, + /** Log name: "audio" for the media sink, "guidance" for the directions sink. */ + private val tag: String = "audio", + private val sessionId: Int = SESSION_ID, + /** Media asks the car for focus; directions just start, and the car ducks music under them. */ + private val asksFocus: Boolean = true, + private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) }, + private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) }, ) : CarAudio.Link { private var configIndex = 0 @@ -47,9 +54,9 @@ class AudioLink( val channelId get() = service.serviceId fun opened(status: Int) { - GearslipLog.i("audio: <- ChannelOpenResponse (media audio): status=$status") + GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status") if (status != 0) { - GearslipLog.w("audio: the head unit refused the media audio channel (status $status)") + GearslipLog.w("$tag: the head unit refused the $tag channel (status $status)") return } // Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink @@ -57,7 +64,7 @@ class AudioLink( // read from an older proto as a config index; 0 is no codec at all, and the 2024 VW // answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing. sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong())) - GearslipLog.i("audio: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}") + GearslipLog.i("$tag: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}") armConfigFallback() } @@ -89,12 +96,12 @@ class AudioLink( if (ready) return configTimer?.interrupt() GearslipLog.i( - "audio: $why - using the sink the head unit advertised " + + "$tag: $why - using the sink the head unit advertised " + "(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)", ) configIndex = 0 ready = true - CarAudio.attach(this) + onReady(this) } fun onMessage(messageId: Int, body: ByteArray) { @@ -106,17 +113,17 @@ class AudioLink( val status = Wire.varint(fields, 1)?.toInt() val window = Wire.varint(fields, 2)?.toInt() val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() } - GearslipLog.i("audio: <- Config status=$status max_unacked=$window indices=$indices") + GearslipLog.i("$tag: <- Config status=$status max_unacked=$window indices=$indices") if (window != null) maxUnacked = maxOf(1, window) configIndex = indices.firstOrNull() ?: 0 if (status == STATUS_READY) { configTimer?.interrupt() ready = true - GearslipLog.i("audio: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") - CarAudio.attach(this) + GearslipLog.i("$tag: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") + onReady(this) } else { GearslipLog.w( - "audio: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + + "$tag: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + "nothing will be sent until it is", ) } @@ -135,7 +142,7 @@ class AudioLink( else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) { useAdvertisedSink("<- empty Setup answer (id $messageId)") } else { - GearslipLog.i("audio: <- unhandled message id=$messageId (${body.size} bytes)") + GearslipLog.i("$tag: <- unhandled message id=$messageId (${body.size} bytes)") GearslipLog.hex(" audio msg", body, limit = 64) } } @@ -148,7 +155,7 @@ class AudioLink( */ fun onFocus(state: Int) { val name = FOCUS_STATES[state] ?: "unknown($state)" - GearslipLog.i("audio: <- AudioFocusResponse state=$name") + GearslipLog.i("$tag: <- AudioFocusResponse state=$name") focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT || state == FOCUS_STATE_GAIN_MEDIA_ONLY if (focusGranted) startStream() else stop() @@ -156,10 +163,10 @@ class AudioLink( override fun start() { if (!ready || started) return - if (!focusRequested) { + if (asksFocus && !focusRequested) { focusRequested = true sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong())) - GearslipLog.i("audio: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") + GearslipLog.i("$tag: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") } // Some head units grant the sink silently and never answer. Do not let that be the end of // it: start anyway, and let the acks say whether anything is being taken. @@ -170,12 +177,12 @@ class AudioLink( if (!ready || started) return sendOnChannel( MSG_START, - Protobuf.varintField(1, SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()), + Protobuf.varintField(1, sessionId.toLong()) + Protobuf.varintField(2, configIndex.toLong()), ) inFlight = 0 lastAckAt = System.currentTimeMillis() started = true - GearslipLog.i("audio: -> Start(session=$SESSION_ID, config=$configIndex) - streaming ${sampleRate}Hz x$channels") + GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels") } override fun stop() { @@ -183,12 +190,12 @@ class AudioLink( started = false synchronized(ackLock) { ackLock.notifyAll() } runCatching { - sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong())) + sendOnChannel(MSG_STOP, Protobuf.varintField(1, sessionId.toLong())) if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong())) } focusRequested = false focusGranted = false - GearslipLog.i("audio: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") + GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") chunks = 0 bytes = 0 } @@ -205,7 +212,7 @@ class AudioLink( // A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so // once, then now and then, so the rest of the log stays readable. if (stalls++ % 500 == 0) { - GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)") + GearslipLog.w("$tag: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)") } inFlight = 0 break @@ -223,7 +230,7 @@ class AudioLink( try { sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size)) } catch (e: java.io.IOException) { - GearslipLog.e("audio: the link died mid-stream", e) + GearslipLog.e("$tag: the link died mid-stream", e) started = false return } @@ -234,7 +241,7 @@ class AudioLink( // the car is still taking audio and to catch a window that has quietly closed. if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) { GearslipLog.i( - "audio: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + + "$tag: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + "dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}", ) } @@ -245,7 +252,7 @@ class AudioLink( configTimer = null stop() ready = false - CarAudio.detach(this) + onClose(this) } companion object { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt index a4992dc..587c4ea 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt @@ -65,6 +65,7 @@ object CarServices { app.seb3thehacker.gearslip.notify.CarMotion.start(appContext) app.seb3thehacker.gearslip.call.CarCalls.start(appContext) CarAssistant.init(appContext) + DirectionsVoice.init(appContext) CarKeys.init(appContext) VoiceReply.init(appContext) // Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest. diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index a0b3024..43c505c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -28,6 +28,7 @@ object CarSettings { private const val KEY_OPEN = "car_open_on_connect" private const val KEY_NIGHT = "car_night_mode" private const val KEY_PIPE_AUDIO = "car_pipe_audio" +private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions" private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked" private const val KEY_LAST_NAV = "car_last_nav" private const val KEY_LAST_MEDIA = "car_last_media" @@ -58,6 +59,7 @@ object CarSettings { private val openFlow = MutableStateFlow(HOME) private val nightFlow = MutableStateFlow(NightMode.AUTO) private val pipeAudioFlow = MutableStateFlow(true) + private val speakDirectionsFlow = MutableStateFlow(true) private val lastNavFlow = MutableStateFlow(null) private val lastMediaFlow = MutableStateFlow(null) private val autoplayFlow = MutableStateFlow(false) @@ -153,6 +155,9 @@ object CarSettings { */ val pipeAudio: StateFlow = pipeAudioFlow + /** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */ + val speakDirections: StateFlow = speakDirectionsFlow + /** * Whether the driver has already been offered Bluetooth after turning down the capture * dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged. @@ -205,6 +210,7 @@ object CarSettings { autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true" playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true" pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true" + speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, "true") == "true" nightFlow.value = runCatching { NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name)) }.getOrDefault(NightMode.AUTO) @@ -274,6 +280,11 @@ object CarSettings { if (on) CarAudio.request() else CarAudio.release() } + fun setSpeakDirections(on: Boolean) { + speakDirectionsFlow.value = on + AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString()) + } + fun setExperimentalFeaturesEnabled(on: Boolean) { experimentalFeaturesFlow.value = on AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString()) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index 2fec3e0..e14ec7a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -53,6 +53,7 @@ fun CarSettingsScreen() { val units by CarSettings.units.collectAsState() val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState() + val speakDirections by CarSettings.speakDirections.collectAsState() val autoplay by CarSettings.autoplay.collectAsState() val playerTitle by CarSettings.playerTitle.collectAsState() val theme by CarSettings.appTheme.collectAsState() @@ -137,6 +138,18 @@ fun CarSettingsScreen() { color = MaterialTheme.colorScheme.onSurfaceVariant, ) + ChoiceRow( + "Spoken directions", + listOf(false to "Off", true to "On"), + speakDirections, + ) { CarSettings.setSpeakDirections(it) } + Text( + "Gearslip reads each turn aloud through the car's speakers, and the music dips while it " + + "speaks. Turn it off if your map app's own voice already plays in the car.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + ExperimentalFeatureRow( title = "Use the phone's speech recognition", description = "Listens through the phone's own speech service instead of Gearslip's. On " + diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt index fda6c4c..8a4f8f6 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt @@ -60,23 +60,8 @@ class ClusterNav( running = false } - /** One turn as the car wants it, from whichever source the map app fills. */ - private class Turn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) - - private fun current(): Turn? { - val nav = CarServices.navOrNull() ?: return null - nav.trip.value?.let { trip -> - val step = trip.steps.firstOrNull() ?: return@let - val estimate = trip.stepTravelEstimates.firstOrNull() - return Turn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading) - } - val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null - val step = info.currentStep ?: return null - return Turn(step, info.currentDistance, null, info.isLoading) - } - private fun tick() { - val turn = current() + val turn = NavTurn.current() val status = when { turn == null -> STATUS_INACTIVE turn.rerouting -> STATUS_REROUTING @@ -114,7 +99,7 @@ class ClusterNav( * 7 yards), so the app's own display choice passes straight through. */ private fun sendDistance(distance: Distance, seconds: Long?) { - val meters = toMeters(distance) + val meters = NavTurn.toMeters(distance) val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}" if (key == sentDistance) return sentDistance = key @@ -127,14 +112,6 @@ class ClusterNav( ) } - private fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) { - Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0 - Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344 - Distance.UNIT_FEET -> 0.3048 - Distance.UNIT_YARDS -> 0.9144 - else -> 1.0 - }).toInt() - private fun CarText?.text() = this?.toString().orEmpty() companion object { @@ -194,3 +171,29 @@ class ClusterNav( } } } + +/** One turn as the car wants it, from whichever source the map app fills. */ +internal class NavTurn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) { + companion object { + /** The map app's next turn: from its trip (what clusters get), or its routing card. */ + fun current(): NavTurn? { + val nav = CarServices.navOrNull() ?: return null + nav.trip.value?.let { trip -> + val step = trip.steps.firstOrNull() ?: return@let + val estimate = trip.stepTravelEstimates.firstOrNull() + return NavTurn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading) + } + val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null + val step = info.currentStep ?: return null + return NavTurn(step, info.currentDistance, null, info.isLoading) + } + + fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) { + Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0 + Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344 + Distance.UNIT_FEET -> 0.3048 + Distance.UNIT_YARDS -> 0.9144 + else -> 1.0 + }).toInt() + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt new file mode 100644 index 0000000..6b9abf3 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt @@ -0,0 +1,326 @@ +package app.seb3thehacker.gearslip.car + +import android.content.Context +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import androidx.car.app.model.CarText +import androidx.car.app.model.Distance +import androidx.car.app.navigation.model.Maneuver +import androidx.car.app.navigation.model.Step +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.audio.AudioLink +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.math.roundToInt + +/** + * Spoken turn-by-turn on the car's directions channel (the GUIDANCE audio sink). + * + * A map app on the phone speaks its own prompts as navigation audio, which Android can't hand to + * Gearslip without root, and which the phone mutes while USB audio runs. So Gearslip reads the + * same turn the instrument cluster gets ([NavTurn]) and says it itself: once when the turn comes + * up, again a kilometre or so out on a long stretch, and once more just before it. The words are + * rendered with the phone's text-to-speech to a file, never played on the phone, and streamed to + * the car, which ducks the music under them the way it does for real Android Auto. + * + * Nothing spoken is logged: road names are where the driver is going. + */ +object DirectionsVoice { + + private enum class Stage { FAR, PREPARED, DONE } + + private var appContext: Context? = null + private var tts: TextToSpeech? = null + @Volatile private var ttsReady = false + private val pending = ConcurrentHashMap() + + @Volatile private var link: AudioLink? = null + @Volatile private var running = false + private val speaker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } } + @Volatile private var latest = 0 + + private var turnKey: String? = null + private var stage = Stage.DONE + + fun init(context: Context) { + if (tts != null) return + appContext = context.applicationContext + tts = TextToSpeech(appContext) { status -> ttsReady = status == TextToSpeech.SUCCESS }.apply { + setOnUtteranceProgressListener(object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) {} + override fun onDone(utteranceId: String?) { pending.remove(utteranceId)?.countDown() } + @Deprecated("Deprecated in Java, no replacement on this API level") + override fun onError(utteranceId: String?) { pending.remove(utteranceId)?.countDown() } + }) + } + } + + /** The car opened its directions sink: start following the map app's turns. */ + fun attach(link: AudioLink) { + this.link = link + GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}") + if (running) return + running = true + turnKey = null + Thread({ + while (running) { + runCatching { tick() }.onFailure { GearslipLog.w("guidance: ${it.message}") } + Thread.sleep(TICK_MS) + } + }, "gearslip-directions-watch").apply { isDaemon = true }.start() + } + + fun detach(link: AudioLink) { + if (this.link !== link) return + this.link = null + running = false + } + + private fun tick() { + if (!CarSettings.speakDirections.value) return + val turn = NavTurn.current() + if (turn == null) { + turnKey = null + return + } + if (turn.rerouting) return + val step = turn.step + val instruction = instruction(step) ?: return + val meters = turn.distance?.let(NavTurn::toMeters) + val close = (turn.seconds != null && turn.seconds <= NEAR_SECONDS) || (meters != null && meters <= NEAR_METERS) + + val key = "${step.road.text()}|${step.maneuver?.type}|${step.maneuver?.roundaboutExitNumber}|${step.cue.text()}" + if (key != turnKey) { + turnKey = key + if (step.maneuver?.type == Maneuver.TYPE_DEPART) { + stage = Stage.DONE + return + } + if (close || turn.distance == null) { + say(instruction) + stage = Stage.DONE + } else { + say("In ${spoken(turn.distance)}, $instruction") + stage = if (meters != null && meters > PREPARE_FROM_METERS) Stage.FAR else Stage.PREPARED + } + return + } + when { + stage != Stage.DONE && close -> { + say(instruction) + stage = Stage.DONE + } + stage == Stage.FAR && meters != null && meters <= PREPARE_AT_METERS -> { + say("In ${spoken(turn.distance)}, $instruction") + stage = Stage.PREPARED + } + } + } + + /** What to do, in words: built from the manoeuvre when the app names one, else the app's own text. */ + private fun instruction(step: Step): String? { + val road = step.road.text() + val onto = if (road.isBlank()) "" else " onto $road" + val toward = if (road.isBlank()) "" else " toward $road" + val maneuver = step.maneuver + val built = when (maneuver?.type) { + Maneuver.TYPE_TURN_SLIGHT_LEFT -> "Turn slightly left$onto" + Maneuver.TYPE_TURN_SLIGHT_RIGHT -> "Turn slightly right$onto" + Maneuver.TYPE_TURN_NORMAL_LEFT -> "Turn left$onto" + Maneuver.TYPE_TURN_NORMAL_RIGHT -> "Turn right$onto" + Maneuver.TYPE_TURN_SHARP_LEFT -> "Turn sharply left$onto" + Maneuver.TYPE_TURN_SHARP_RIGHT -> "Turn sharply right$onto" + Maneuver.TYPE_U_TURN_LEFT, Maneuver.TYPE_U_TURN_RIGHT -> "Make a U-turn" + Maneuver.TYPE_KEEP_LEFT -> "Keep left$toward" + Maneuver.TYPE_KEEP_RIGHT -> "Keep right$toward" + Maneuver.TYPE_FORK_LEFT -> "Keep left at the fork$toward" + Maneuver.TYPE_FORK_RIGHT -> "Keep right at the fork$toward" + Maneuver.TYPE_ON_RAMP_SLIGHT_LEFT, Maneuver.TYPE_ON_RAMP_NORMAL_LEFT, Maneuver.TYPE_ON_RAMP_SHARP_LEFT, + Maneuver.TYPE_ON_RAMP_U_TURN_LEFT -> "Take the ramp on the left$onto" + Maneuver.TYPE_ON_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_ON_RAMP_NORMAL_RIGHT, Maneuver.TYPE_ON_RAMP_SHARP_RIGHT, + Maneuver.TYPE_ON_RAMP_U_TURN_RIGHT -> "Take the ramp on the right$onto" + Maneuver.TYPE_OFF_RAMP_SLIGHT_LEFT, Maneuver.TYPE_OFF_RAMP_NORMAL_LEFT -> "Take the exit on the left$toward" + Maneuver.TYPE_OFF_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_OFF_RAMP_NORMAL_RIGHT -> "Take the exit on the right$toward" + Maneuver.TYPE_MERGE_LEFT -> "Merge left$onto" + Maneuver.TYPE_MERGE_RIGHT -> "Merge right$onto" + Maneuver.TYPE_MERGE_SIDE_UNSPECIFIED -> "Merge$onto" + Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW_WITH_ANGLE, + Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW_WITH_ANGLE -> + maneuver.roundaboutExitNumber.takeIf { it in 1..ORDINALS.size } + ?.let { "At the roundabout, take the ${ORDINALS[it - 1]} exit$onto" } + ?: "Go through the roundabout$onto" + Maneuver.TYPE_ROUNDABOUT_ENTER_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_CCW -> "Enter the roundabout" + Maneuver.TYPE_ROUNDABOUT_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_EXIT_CCW -> "Exit the roundabout$onto" + Maneuver.TYPE_STRAIGHT -> "Continue straight$onto" + Maneuver.TYPE_NAME_CHANGE -> "Continue$onto" + Maneuver.TYPE_FERRY_BOAT, Maneuver.TYPE_FERRY_BOAT_LEFT, Maneuver.TYPE_FERRY_BOAT_RIGHT -> "Take the ferry" + Maneuver.TYPE_FERRY_TRAIN, Maneuver.TYPE_FERRY_TRAIN_LEFT, Maneuver.TYPE_FERRY_TRAIN_RIGHT -> "Take the train" + Maneuver.TYPE_DESTINATION, Maneuver.TYPE_DESTINATION_STRAIGHT -> "Your destination is ahead" + Maneuver.TYPE_DESTINATION_LEFT -> "Your destination is on the left" + Maneuver.TYPE_DESTINATION_RIGHT -> "Your destination is on the right" + else -> null + } + return built ?: step.cue.text().ifBlank { null } + } + + /** The app's own rounded distance, read the way a person would say it. */ + private fun spoken(d: Distance): String { + val n = d.displayDistance + val number = if (n % 1.0 == 0.0) n.toInt().toString() else "%.1f".format(n) + val one = n == 1.0 + return when (d.displayUnit) { + Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> if (one) "1 kilometer" else "$number kilometers" + Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> when (n) { + 0.25 -> "a quarter mile" + 0.5 -> "half a mile" + 0.75 -> "three quarters of a mile" + 1.0 -> "1 mile" + else -> "$number miles" + } + Distance.UNIT_FEET -> "$number feet" + Distance.UNIT_YARDS -> "$number yards" + else -> "${n.roundToInt()} meters" + } + } + + /** Newest wins: a prompt still waiting when the next one arrives is already out of date. */ + private fun say(text: String) { + val id = ++latest + speaker.execute { + if (id != latest) return@execute + runCatching { play(text, id) }.onFailure { GearslipLog.w("guidance: could not speak a turn: ${it.message}") } + } + } + + private fun play(text: String, id: Int) { + val engine = tts ?: return + val context = appContext ?: return + val sink = link ?: return + if (!ttsReady) { + GearslipLog.w("guidance: text-to-speech isn't ready, a turn went unspoken") + return + } + val file = File(context.cacheDir, "directions.wav") + val utterance = "gearslip-directions-$id" + val done = CountDownLatch(1) + pending[utterance] = done + if (engine.synthesizeToFile(text, Bundle(), file, utterance) != TextToSpeech.SUCCESS) { + pending.remove(utterance) + return + } + if (!done.await(SYNTH_TIMEOUT_S, TimeUnit.SECONDS)) { + pending.remove(utterance) + GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken") + return + } + val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run { + GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read") + return + } + stream(sink, pcm) + GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)") + } + + /** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */ + private fun stream(sink: AudioLink, pcm: ByteArray) { + val frame = 2 * sink.channels + val chunk = sink.sampleRate / 50 * frame + val chunkMs = 20L + val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped + sink.start() + try { + val began = System.currentTimeMillis() + var offset = 0 + var sent = 0 + while (offset < padded.size && link === sink) { + val size = minOf(chunk, padded.size - offset) + sink.write(padded.copyOfRange(offset, offset + size), size) + offset += size + sent++ + val ahead = sent * chunkMs - (System.currentTimeMillis() - began) + if (ahead > LEAD_MS) Thread.sleep(ahead - LEAD_MS) + } + // Let the car play what it has before closing the stream on it. + Thread.sleep(LEAD_MS) + } finally { + sink.stop() + } + } + + private fun CarText?.text() = this?.toString().orEmpty() + + /** 16-bit PCM from the WAV text-to-speech writes, reshaped for the car's sink. */ + private object Wav { + class Pcm(val samples: ShortArray, val rate: Int, val channels: Int) + + fun read(file: File): Pcm? { + val bytes = file.readBytes() + if (bytes.size < 12 || String(bytes, 0, 4) != "RIFF" || String(bytes, 8, 4) != "WAVE") return null + val buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) + var pos = 12 + var rate = 0 + var channels = 0 + var bits = 0 + while (pos + 8 <= bytes.size) { + val name = String(bytes, pos, 4) + val size = buf.getInt(pos + 4) + val body = pos + 8 + // Some engines write a streaming WAV with a zero or oversized data length. + val end = if (size <= 0 || body + size > bytes.size) bytes.size else body + size + when (name) { + "fmt " -> { + channels = buf.getShort(body + 2).toInt() + rate = buf.getInt(body + 4) + bits = buf.getShort(body + 14).toInt() + } + "data" -> { + if (bits != 16 || rate <= 0 || channels <= 0) return null + val samples = ShortArray((end - body) / 2) + buf.position(body) + buf.asShortBuffer().get(samples, 0, samples.size) + return Pcm(samples, rate, channels) + } + } + pos = end + (end and 1) + } + return null + } + + /** Mixes to mono, resamples linearly, then copies to each of the car's channels. */ + fun convert(pcm: Pcm, rate: Int, channels: Int): ByteArray { + val frames = pcm.samples.size / pcm.channels + val mono = FloatArray(frames) { f -> + var sum = 0f + for (c in 0 until pcm.channels) sum += pcm.samples[f * pcm.channels + c] + sum / pcm.channels + } + val outFrames = (frames.toLong() * rate / pcm.rate).toInt() + val out = ByteBuffer.allocate(outFrames * channels * 2).order(ByteOrder.LITTLE_ENDIAN) + val step = pcm.rate.toDouble() / rate + for (i in 0 until outFrames) { + val at = i * step + val j = at.toInt().coerceAtMost(frames - 1) + val next = (j + 1).coerceAtMost(frames - 1) + val v = mono[j] + (mono[next] - mono[j]) * (at - j).toFloat() + val s = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() + repeat(channels) { out.putShort(s) } + } + return out.array() + } + } + + private const val TICK_MS = 500L + private const val NEAR_SECONDS = 12L + private const val NEAR_METERS = 120 + private const val PREPARE_FROM_METERS = 2_500 + private const val PREPARE_AT_METERS = 1_000 + private const val LEAD_MS = 100L + private const val SYNTH_TIMEOUT_S = 10L + private val ORDINALS = listOf("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth") +}