directions: speak each turn on the car's guidance channel

Map apps' own voice prompts can't reach the car over USB, so Gearslip reads
the turn the cluster already gets and says it through the car's directions
sink, which ducks the music. On by default, with a switch in car settings.
This commit is contained in:
Seb3thehacker committed 2026-10-13 12:26:31 -07:00
1 parent 0d0bfd3d66
commit 59f323ec06
8 files changed
+438 -47

No files matched your search

@@ -5,6 +5,7 @@ import app.seb3thehacker.gearslip.audio.AudioLink
import app.seb3thehacker.gearslip.audio.CarMic import app.seb3thehacker.gearslip.audio.CarMic
import app.seb3thehacker.gearslip.media.CarMediaStatus import app.seb3thehacker.gearslip.media.CarMediaStatus
import app.seb3thehacker.gearslip.car.ClusterNav import app.seb3thehacker.gearslip.car.ClusterNav
import app.seb3thehacker.gearslip.car.DirectionsVoice
import app.seb3thehacker.gearslip.car.CarAssistant import app.seb3thehacker.gearslip.car.CarAssistant
import app.seb3thehacker.gearslip.car.CarEnvironment import app.seb3thehacker.gearslip.car.CarEnvironment
import app.seb3thehacker.gearslip.car.CarKeys import app.seb3thehacker.gearslip.car.CarKeys
@@ -79,6 +80,7 @@ class GearslipRunner(
@Volatile private var clusterNav: ClusterNav? = null @Volatile private var clusterNav: ClusterNav? = null
private var sensorTypes: List<Int> = emptyList() private var sensorTypes: List<Int> = emptyList()
@Volatile private var audioLink: AudioLink? = null @Volatile private var audioLink: AudioLink? = null
@Volatile private var guidanceLink: AudioLink? = null
private var audioMessagesSeen = 0 private var audioMessagesSeen = 0
private var touchWidth = 0 private var touchWidth = 0
private var touchHeight = 0 private var touchHeight = 0
@@ -154,6 +156,8 @@ class GearslipRunner(
videoSource = null videoSource = null
audioLink?.close() audioLink?.close()
audioLink = null audioLink = null
guidanceLink?.close()
guidanceLink = null
CarSensors.clear() CarSensors.clear()
CarMic.current = null CarMic.current = null
mediaStatus?.stop() mediaStatus?.stop()
@@ -201,6 +205,8 @@ class GearslipRunner(
clusterNav?.onMessage(messageId, body) clusterNav?.onMessage(messageId, body)
} else if (frame.channel == mediaStatus?.channelId) { } else if (frame.channel == mediaStatus?.channelId) {
mediaStatus?.onMessage(messageId, body) mediaStatus?.onMessage(messageId, body)
} else if (frame.channel == guidanceLink?.channelId) {
guidanceLink?.onMessage(messageId, body)
} else if (frame.channel == CarMic.current?.channelId) { } else if (frame.channel == CarMic.current?.channelId) {
CarMic.current?.onMessage(messageId, body) CarMic.current?.onMessage(messageId, body)
} else if (frame.channel == audioLink?.channelId) { } else if (frame.channel == audioLink?.channelId) {
@@ -452,6 +458,7 @@ class GearslipRunner(
openChannel(video.serviceId) openChannel(video.serviceId)
startAudioChannel(serviceDiscoveryResponse) startAudioChannel(serviceDiscoveryResponse)
startGuidanceChannel(serviceDiscoveryResponse)
startSensorChannel(serviceDiscoveryResponse) startSensorChannel(serviceDiscoveryResponse)
startMicChannel(serviceDiscoveryResponse) startMicChannel(serviceDiscoveryResponse)
startMediaStatusChannel(serviceDiscoveryResponse) startMediaStatusChannel(serviceDiscoveryResponse)
@@ -473,7 +480,7 @@ class GearslipRunner(
} }
} }
/** Media audio rides its own sink; guidance, system and telephony sinks are left closed. */ /** Media audio rides its own sink; spoken directions take the guidance sink, and system and telephony stay closed. */
private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) { private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) {
val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse)
sinks.forEach { sink -> sinks.forEach { sink ->
@@ -505,6 +512,27 @@ class GearslipRunner(
openChannel(media.serviceId) openChannel(media.serviceId)
} }
/** The car's directions channel, where [DirectionsVoice] reads out each turn. */
private fun startGuidanceChannel(serviceDiscoveryResponse: ByteArray) {
val guidance = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse)
.filter { it.streamType == ServiceDiscovery.STREAM_GUIDANCE }
.firstOrNull { it.codecType == ServiceDiscovery.CODEC_PCM } ?: run {
log.i("no PCM directions sink - turns won't be spoken")
return
}
guidanceLink = AudioLink(
guidance,
sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = guidance.serviceId) },
sendControl = { id, body -> send(id, body, encrypted = true) },
tag = "guidance",
sessionId = GUIDANCE_SESSION_ID,
asksFocus = false,
onReady = { DirectionsVoice.attach(it) },
onClose = { DirectionsVoice.detach(it) },
)
openChannel(guidance.serviceId)
}
/** The car's microphone, for voice: opened now, recording only while something listens. */ /** The car's microphone, for voice: opened now, recording only while something listens. */
private fun startMicChannel(serviceDiscoveryResponse: ByteArray) { private fun startMicChannel(serviceDiscoveryResponse: ByteArray) {
val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run { val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run {
@@ -1292,6 +1320,7 @@ class GearslipRunner(
const val ACK_STALL_MS = 2_000L const val ACK_STALL_MS = 2_000L
const val AUDIO_TRACE_MESSAGES = 6 const val AUDIO_TRACE_MESSAGES = 6
const val GUIDANCE_SESSION_ID = 1
const val PERIODIC_KEYFRAME_MS = 5_000L const val PERIODIC_KEYFRAME_MS = 5_000L
} }
} }
@@ -153,6 +153,7 @@ object ServiceDiscovery {
return null return null
} }
const val STREAM_GUIDANCE = 1
const val STREAM_MEDIA = 3 const val STREAM_MEDIA = 3
const val CODEC_PCM = 1 const val CODEC_PCM = 1
@@ -22,6 +22,13 @@ class AudioLink(
private val service: ServiceDiscovery.AudioService, private val service: ServiceDiscovery.AudioService,
private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit, private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit,
private val sendControl: (messageId: Int, body: ByteArray) -> Unit, private val sendControl: (messageId: Int, body: ByteArray) -> Unit,
/** Log name: "audio" for the media sink, "guidance" for the directions sink. */
private val tag: String = "audio",
private val sessionId: Int = SESSION_ID,
/** Media asks the car for focus; directions just start, and the car ducks music under them. */
private val asksFocus: Boolean = true,
private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) },
private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) },
) : CarAudio.Link { ) : CarAudio.Link {
private var configIndex = 0 private var configIndex = 0
@@ -47,9 +54,9 @@ class AudioLink(
val channelId get() = service.serviceId val channelId get() = service.serviceId
fun opened(status: Int) { fun opened(status: Int) {
GearslipLog.i("audio: <- ChannelOpenResponse (media audio): status=$status") GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status")
if (status != 0) { if (status != 0) {
GearslipLog.w("audio: the head unit refused the media audio channel (status $status)") GearslipLog.w("$tag: the head unit refused the $tag channel (status $status)")
return return
} }
// Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink // Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink
@@ -57,7 +64,7 @@ class AudioLink(
// read from an older proto as a config index; 0 is no codec at all, and the 2024 VW // read from an older proto as a config index; 0 is no codec at all, and the 2024 VW
// answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing. // answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing.
sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong())) sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong()))
GearslipLog.i("audio: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}") GearslipLog.i("$tag: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}")
armConfigFallback() armConfigFallback()
} }
@@ -89,12 +96,12 @@ class AudioLink(
if (ready) return if (ready) return
configTimer?.interrupt() configTimer?.interrupt()
GearslipLog.i( GearslipLog.i(
"audio: $why - using the sink the head unit advertised " + "$tag: $why - using the sink the head unit advertised " +
"(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)", "(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)",
) )
configIndex = 0 configIndex = 0
ready = true ready = true
CarAudio.attach(this) onReady(this)
} }
fun onMessage(messageId: Int, body: ByteArray) { fun onMessage(messageId: Int, body: ByteArray) {
@@ -106,17 +113,17 @@ class AudioLink(
val status = Wire.varint(fields, 1)?.toInt() val status = Wire.varint(fields, 1)?.toInt()
val window = Wire.varint(fields, 2)?.toInt() val window = Wire.varint(fields, 2)?.toInt()
val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() } val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() }
GearslipLog.i("audio: <- Config status=$status max_unacked=$window indices=$indices") GearslipLog.i("$tag: <- Config status=$status max_unacked=$window indices=$indices")
if (window != null) maxUnacked = maxOf(1, window) if (window != null) maxUnacked = maxOf(1, window)
configIndex = indices.firstOrNull() ?: 0 configIndex = indices.firstOrNull() ?: 0
if (status == STATUS_READY) { if (status == STATUS_READY) {
configTimer?.interrupt() configTimer?.interrupt()
ready = true ready = true
GearslipLog.i("audio: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") GearslipLog.i("$tag: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit")
CarAudio.attach(this) onReady(this)
} else { } else {
GearslipLog.w( GearslipLog.w(
"audio: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + "$tag: the sink is not ready (status=$status, 1=WAIT 2=READY) - " +
"nothing will be sent until it is", "nothing will be sent until it is",
) )
} }
@@ -135,7 +142,7 @@ class AudioLink(
else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) { else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) {
useAdvertisedSink("<- empty Setup answer (id $messageId)") useAdvertisedSink("<- empty Setup answer (id $messageId)")
} else { } else {
GearslipLog.i("audio: <- unhandled message id=$messageId (${body.size} bytes)") GearslipLog.i("$tag: <- unhandled message id=$messageId (${body.size} bytes)")
GearslipLog.hex(" audio msg", body, limit = 64) GearslipLog.hex(" audio msg", body, limit = 64)
} }
} }
@@ -148,7 +155,7 @@ class AudioLink(
*/ */
fun onFocus(state: Int) { fun onFocus(state: Int) {
val name = FOCUS_STATES[state] ?: "unknown($state)" val name = FOCUS_STATES[state] ?: "unknown($state)"
GearslipLog.i("audio: <- AudioFocusResponse state=$name") GearslipLog.i("$tag: <- AudioFocusResponse state=$name")
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT || focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT ||
state == FOCUS_STATE_GAIN_MEDIA_ONLY state == FOCUS_STATE_GAIN_MEDIA_ONLY
if (focusGranted) startStream() else stop() if (focusGranted) startStream() else stop()
@@ -156,10 +163,10 @@ class AudioLink(
override fun start() { override fun start() {
if (!ready || started) return if (!ready || started) return
if (!focusRequested) { if (asksFocus && !focusRequested) {
focusRequested = true focusRequested = true
sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong())) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong()))
GearslipLog.i("audio: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") GearslipLog.i("$tag: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over")
} }
// Some head units grant the sink silently and never answer. Do not let that be the end of // Some head units grant the sink silently and never answer. Do not let that be the end of
// it: start anyway, and let the acks say whether anything is being taken. // it: start anyway, and let the acks say whether anything is being taken.
@@ -170,12 +177,12 @@ class AudioLink(
if (!ready || started) return if (!ready || started) return
sendOnChannel( sendOnChannel(
MSG_START, MSG_START,
Protobuf.varintField(1, SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()), Protobuf.varintField(1, sessionId.toLong()) + Protobuf.varintField(2, configIndex.toLong()),
) )
inFlight = 0 inFlight = 0
lastAckAt = System.currentTimeMillis() lastAckAt = System.currentTimeMillis()
started = true started = true
GearslipLog.i("audio: -> Start(session=$SESSION_ID, config=$configIndex) - streaming ${sampleRate}Hz x$channels") GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels")
} }
override fun stop() { override fun stop() {
@@ -183,12 +190,12 @@ class AudioLink(
started = false started = false
synchronized(ackLock) { ackLock.notifyAll() } synchronized(ackLock) { ackLock.notifyAll() }
runCatching { runCatching {
sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong())) sendOnChannel(MSG_STOP, Protobuf.varintField(1, sessionId.toLong()))
if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong())) if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong()))
} }
focusRequested = false focusRequested = false
focusGranted = false focusGranted = false
GearslipLog.i("audio: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes")
chunks = 0 chunks = 0
bytes = 0 bytes = 0
} }
@@ -205,7 +212,7 @@ class AudioLink(
// A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so // A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so
// once, then now and then, so the rest of the log stays readable. // once, then now and then, so the rest of the log stays readable.
if (stalls++ % 500 == 0) { if (stalls++ % 500 == 0) {
GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)") GearslipLog.w("$tag: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)")
} }
inFlight = 0 inFlight = 0
break break
@@ -223,7 +230,7 @@ class AudioLink(
try { try {
sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size)) sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size))
} catch (e: java.io.IOException) { } catch (e: java.io.IOException) {
GearslipLog.e("audio: the link died mid-stream", e) GearslipLog.e("$tag: the link died mid-stream", e)
started = false started = false
return return
} }
@@ -234,7 +241,7 @@ class AudioLink(
// the car is still taking audio and to catch a window that has quietly closed. // the car is still taking audio and to catch a window that has quietly closed.
if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) { if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) {
GearslipLog.i( GearslipLog.i(
"audio: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + "$tag: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " +
"dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}", "dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}",
) )
} }
@@ -245,7 +252,7 @@ class AudioLink(
configTimer = null configTimer = null
stop() stop()
ready = false ready = false
CarAudio.detach(this) onClose(this)
} }
companion object { companion object {
@@ -65,6 +65,7 @@ object CarServices {
app.seb3thehacker.gearslip.notify.CarMotion.start(appContext) app.seb3thehacker.gearslip.notify.CarMotion.start(appContext)
app.seb3thehacker.gearslip.call.CarCalls.start(appContext) app.seb3thehacker.gearslip.call.CarCalls.start(appContext)
CarAssistant.init(appContext) CarAssistant.init(appContext)
DirectionsVoice.init(appContext)
CarKeys.init(appContext) CarKeys.init(appContext)
VoiceReply.init(appContext) VoiceReply.init(appContext)
// Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest. // Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest.
@@ -28,6 +28,7 @@ object CarSettings {
private const val KEY_OPEN = "car_open_on_connect" private const val KEY_OPEN = "car_open_on_connect"
private const val KEY_NIGHT = "car_night_mode" private const val KEY_NIGHT = "car_night_mode"
private const val KEY_PIPE_AUDIO = "car_pipe_audio" private const val KEY_PIPE_AUDIO = "car_pipe_audio"
private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions"
private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked" private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked"
private const val KEY_LAST_NAV = "car_last_nav" private const val KEY_LAST_NAV = "car_last_nav"
private const val KEY_LAST_MEDIA = "car_last_media" private const val KEY_LAST_MEDIA = "car_last_media"
@@ -58,6 +59,7 @@ object CarSettings {
private val openFlow = MutableStateFlow(HOME) private val openFlow = MutableStateFlow(HOME)
private val nightFlow = MutableStateFlow(NightMode.AUTO) private val nightFlow = MutableStateFlow(NightMode.AUTO)
private val pipeAudioFlow = MutableStateFlow(true) private val pipeAudioFlow = MutableStateFlow(true)
private val speakDirectionsFlow = MutableStateFlow(true)
private val lastNavFlow = MutableStateFlow<String?>(null) private val lastNavFlow = MutableStateFlow<String?>(null)
private val lastMediaFlow = MutableStateFlow<String?>(null) private val lastMediaFlow = MutableStateFlow<String?>(null)
private val autoplayFlow = MutableStateFlow(false) private val autoplayFlow = MutableStateFlow(false)
@@ -153,6 +155,9 @@ object CarSettings {
*/ */
val pipeAudio: StateFlow<Boolean> = pipeAudioFlow val pipeAudio: StateFlow<Boolean> = pipeAudioFlow
/** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */
val speakDirections: StateFlow<Boolean> = speakDirectionsFlow
/** /**
* Whether the driver has already been offered Bluetooth after turning down the capture * Whether the driver has already been offered Bluetooth after turning down the capture
* dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged. * dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged.
@@ -205,6 +210,7 @@ object CarSettings {
autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true" autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true"
playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true" playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true"
pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true" pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true"
speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, "true") == "true"
nightFlow.value = runCatching { nightFlow.value = runCatching {
NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name)) NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name))
}.getOrDefault(NightMode.AUTO) }.getOrDefault(NightMode.AUTO)
@@ -274,6 +280,11 @@ object CarSettings {
if (on) CarAudio.request() else CarAudio.release() if (on) CarAudio.request() else CarAudio.release()
} }
fun setSpeakDirections(on: Boolean) {
speakDirectionsFlow.value = on
AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString())
}
fun setExperimentalFeaturesEnabled(on: Boolean) { fun setExperimentalFeaturesEnabled(on: Boolean) {
experimentalFeaturesFlow.value = on experimentalFeaturesFlow.value = on
AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString()) AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
@@ -53,6 +53,7 @@ fun CarSettingsScreen() {
val units by CarSettings.units.collectAsState() val units by CarSettings.units.collectAsState()
val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState() val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState()
val pipeAudio by CarSettings.pipeAudio.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState()
val speakDirections by CarSettings.speakDirections.collectAsState()
val autoplay by CarSettings.autoplay.collectAsState() val autoplay by CarSettings.autoplay.collectAsState()
val playerTitle by CarSettings.playerTitle.collectAsState() val playerTitle by CarSettings.playerTitle.collectAsState()
val theme by CarSettings.appTheme.collectAsState() val theme by CarSettings.appTheme.collectAsState()
@@ -137,6 +138,18 @@ fun CarSettingsScreen() {
color = MaterialTheme.colorScheme.onSurfaceVariant, color = MaterialTheme.colorScheme.onSurfaceVariant,
) )
ChoiceRow(
"Spoken directions",
listOf(false to "Off", true to "On"),
speakDirections,
) { CarSettings.setSpeakDirections(it) }
Text(
"Gearslip reads each turn aloud through the car's speakers, and the music dips while it " +
"speaks. Turn it off if your map app's own voice already plays in the car.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
ExperimentalFeatureRow( ExperimentalFeatureRow(
title = "Use the phone's speech recognition", title = "Use the phone's speech recognition",
description = "Listens through the phone's own speech service instead of Gearslip's. On " + description = "Listens through the phone's own speech service instead of Gearslip's. On " +
@@ -60,23 +60,8 @@ class ClusterNav(
running = false running = false
} }
/** One turn as the car wants it, from whichever source the map app fills. */
private class Turn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean)
private fun current(): Turn? {
val nav = CarServices.navOrNull() ?: return null
nav.trip.value?.let { trip ->
val step = trip.steps.firstOrNull() ?: return@let
val estimate = trip.stepTravelEstimates.firstOrNull()
return Turn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading)
}
val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null
val step = info.currentStep ?: return null
return Turn(step, info.currentDistance, null, info.isLoading)
}
private fun tick() { private fun tick() {
val turn = current() val turn = NavTurn.current()
val status = when { val status = when {
turn == null -> STATUS_INACTIVE turn == null -> STATUS_INACTIVE
turn.rerouting -> STATUS_REROUTING turn.rerouting -> STATUS_REROUTING
@@ -114,7 +99,7 @@ class ClusterNav(
* 7 yards), so the app's own display choice passes straight through. * 7 yards), so the app's own display choice passes straight through.
*/ */
private fun sendDistance(distance: Distance, seconds: Long?) { private fun sendDistance(distance: Distance, seconds: Long?) {
val meters = toMeters(distance) val meters = NavTurn.toMeters(distance)
val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}" val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}"
if (key == sentDistance) return if (key == sentDistance) return
sentDistance = key sentDistance = key
@@ -127,14 +112,6 @@ class ClusterNav(
) )
} }
private fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344
Distance.UNIT_FEET -> 0.3048
Distance.UNIT_YARDS -> 0.9144
else -> 1.0
}).toInt()
private fun CarText?.text() = this?.toString().orEmpty() private fun CarText?.text() = this?.toString().orEmpty()
companion object { companion object {
@@ -194,3 +171,29 @@ class ClusterNav(
} }
} }
} }
/** One turn as the car wants it, from whichever source the map app fills. */
internal class NavTurn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) {
companion object {
/** The map app's next turn: from its trip (what clusters get), or its routing card. */
fun current(): NavTurn? {
val nav = CarServices.navOrNull() ?: return null
nav.trip.value?.let { trip ->
val step = trip.steps.firstOrNull() ?: return@let
val estimate = trip.stepTravelEstimates.firstOrNull()
return NavTurn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading)
}
val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null
val step = info.currentStep ?: return null
return NavTurn(step, info.currentDistance, null, info.isLoading)
}
fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344
Distance.UNIT_FEET -> 0.3048
Distance.UNIT_YARDS -> 0.9144
else -> 1.0
}).toInt()
}
}
@@ -0,0 +1,326 @@
package app.seb3thehacker.gearslip.car
import android.content.Context
import android.os.Bundle
import android.speech.tts.TextToSpeech
import android.speech.tts.UtteranceProgressListener
import androidx.car.app.model.CarText
import androidx.car.app.model.Distance
import androidx.car.app.navigation.model.Maneuver
import androidx.car.app.navigation.model.Step
import app.seb3thehacker.gearslip.GearslipLog
import app.seb3thehacker.gearslip.audio.AudioLink
import java.io.File
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import kotlin.math.roundToInt
/**
* Spoken turn-by-turn on the car's directions channel (the GUIDANCE audio sink).
*
* A map app on the phone speaks its own prompts as navigation audio, which Android can't hand to
* Gearslip without root, and which the phone mutes while USB audio runs. So Gearslip reads the
* same turn the instrument cluster gets ([NavTurn]) and says it itself: once when the turn comes
* up, again a kilometre or so out on a long stretch, and once more just before it. The words are
* rendered with the phone's text-to-speech to a file, never played on the phone, and streamed to
* the car, which ducks the music under them the way it does for real Android Auto.
*
* Nothing spoken is logged: road names are where the driver is going.
*/
object DirectionsVoice {
private enum class Stage { FAR, PREPARED, DONE }
private var appContext: Context? = null
private var tts: TextToSpeech? = null
@Volatile private var ttsReady = false
private val pending = ConcurrentHashMap<String, CountDownLatch>()
@Volatile private var link: AudioLink? = null
@Volatile private var running = false
private val speaker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } }
@Volatile private var latest = 0
private var turnKey: String? = null
private var stage = Stage.DONE
fun init(context: Context) {
if (tts != null) return
appContext = context.applicationContext
tts = TextToSpeech(appContext) { status -> ttsReady = status == TextToSpeech.SUCCESS }.apply {
setOnUtteranceProgressListener(object : UtteranceProgressListener() {
override fun onStart(utteranceId: String?) {}
override fun onDone(utteranceId: String?) { pending.remove(utteranceId)?.countDown() }
@Deprecated("Deprecated in Java, no replacement on this API level")
override fun onError(utteranceId: String?) { pending.remove(utteranceId)?.countDown() }
})
}
}
/** The car opened its directions sink: start following the map app's turns. */
fun attach(link: AudioLink) {
this.link = link
GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}")
if (running) return
running = true
turnKey = null
Thread({
while (running) {
runCatching { tick() }.onFailure { GearslipLog.w("guidance: ${it.message}") }
Thread.sleep(TICK_MS)
}
}, "gearslip-directions-watch").apply { isDaemon = true }.start()
}
fun detach(link: AudioLink) {
if (this.link !== link) return
this.link = null
running = false
}
private fun tick() {
if (!CarSettings.speakDirections.value) return
val turn = NavTurn.current()
if (turn == null) {
turnKey = null
return
}
if (turn.rerouting) return
val step = turn.step
val instruction = instruction(step) ?: return
val meters = turn.distance?.let(NavTurn::toMeters)
val close = (turn.seconds != null && turn.seconds <= NEAR_SECONDS) || (meters != null && meters <= NEAR_METERS)
val key = "${step.road.text()}|${step.maneuver?.type}|${step.maneuver?.roundaboutExitNumber}|${step.cue.text()}"
if (key != turnKey) {
turnKey = key
if (step.maneuver?.type == Maneuver.TYPE_DEPART) {
stage = Stage.DONE
return
}
if (close || turn.distance == null) {
say(instruction)
stage = Stage.DONE
} else {
say("In ${spoken(turn.distance)}, $instruction")
stage = if (meters != null && meters > PREPARE_FROM_METERS) Stage.FAR else Stage.PREPARED
}
return
}
when {
stage != Stage.DONE && close -> {
say(instruction)
stage = Stage.DONE
}
stage == Stage.FAR && meters != null && meters <= PREPARE_AT_METERS -> {
say("In ${spoken(turn.distance)}, $instruction")
stage = Stage.PREPARED
}
}
}
/** What to do, in words: built from the manoeuvre when the app names one, else the app's own text. */
private fun instruction(step: Step): String? {
val road = step.road.text()
val onto = if (road.isBlank()) "" else " onto $road"
val toward = if (road.isBlank()) "" else " toward $road"
val maneuver = step.maneuver
val built = when (maneuver?.type) {
Maneuver.TYPE_TURN_SLIGHT_LEFT -> "Turn slightly left$onto"
Maneuver.TYPE_TURN_SLIGHT_RIGHT -> "Turn slightly right$onto"
Maneuver.TYPE_TURN_NORMAL_LEFT -> "Turn left$onto"
Maneuver.TYPE_TURN_NORMAL_RIGHT -> "Turn right$onto"
Maneuver.TYPE_TURN_SHARP_LEFT -> "Turn sharply left$onto"
Maneuver.TYPE_TURN_SHARP_RIGHT -> "Turn sharply right$onto"
Maneuver.TYPE_U_TURN_LEFT, Maneuver.TYPE_U_TURN_RIGHT -> "Make a U-turn"
Maneuver.TYPE_KEEP_LEFT -> "Keep left$toward"
Maneuver.TYPE_KEEP_RIGHT -> "Keep right$toward"
Maneuver.TYPE_FORK_LEFT -> "Keep left at the fork$toward"
Maneuver.TYPE_FORK_RIGHT -> "Keep right at the fork$toward"
Maneuver.TYPE_ON_RAMP_SLIGHT_LEFT, Maneuver.TYPE_ON_RAMP_NORMAL_LEFT, Maneuver.TYPE_ON_RAMP_SHARP_LEFT,
Maneuver.TYPE_ON_RAMP_U_TURN_LEFT -> "Take the ramp on the left$onto"
Maneuver.TYPE_ON_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_ON_RAMP_NORMAL_RIGHT, Maneuver.TYPE_ON_RAMP_SHARP_RIGHT,
Maneuver.TYPE_ON_RAMP_U_TURN_RIGHT -> "Take the ramp on the right$onto"
Maneuver.TYPE_OFF_RAMP_SLIGHT_LEFT, Maneuver.TYPE_OFF_RAMP_NORMAL_LEFT -> "Take the exit on the left$toward"
Maneuver.TYPE_OFF_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_OFF_RAMP_NORMAL_RIGHT -> "Take the exit on the right$toward"
Maneuver.TYPE_MERGE_LEFT -> "Merge left$onto"
Maneuver.TYPE_MERGE_RIGHT -> "Merge right$onto"
Maneuver.TYPE_MERGE_SIDE_UNSPECIFIED -> "Merge$onto"
Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW_WITH_ANGLE,
Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW_WITH_ANGLE ->
maneuver.roundaboutExitNumber.takeIf { it in 1..ORDINALS.size }
?.let { "At the roundabout, take the ${ORDINALS[it - 1]} exit$onto" }
?: "Go through the roundabout$onto"
Maneuver.TYPE_ROUNDABOUT_ENTER_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_CCW -> "Enter the roundabout"
Maneuver.TYPE_ROUNDABOUT_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_EXIT_CCW -> "Exit the roundabout$onto"
Maneuver.TYPE_STRAIGHT -> "Continue straight$onto"
Maneuver.TYPE_NAME_CHANGE -> "Continue$onto"
Maneuver.TYPE_FERRY_BOAT, Maneuver.TYPE_FERRY_BOAT_LEFT, Maneuver.TYPE_FERRY_BOAT_RIGHT -> "Take the ferry"
Maneuver.TYPE_FERRY_TRAIN, Maneuver.TYPE_FERRY_TRAIN_LEFT, Maneuver.TYPE_FERRY_TRAIN_RIGHT -> "Take the train"
Maneuver.TYPE_DESTINATION, Maneuver.TYPE_DESTINATION_STRAIGHT -> "Your destination is ahead"
Maneuver.TYPE_DESTINATION_LEFT -> "Your destination is on the left"
Maneuver.TYPE_DESTINATION_RIGHT -> "Your destination is on the right"
else -> null
}
return built ?: step.cue.text().ifBlank { null }
}
/** The app's own rounded distance, read the way a person would say it. */
private fun spoken(d: Distance): String {
val n = d.displayDistance
val number = if (n % 1.0 == 0.0) n.toInt().toString() else "%.1f".format(n)
val one = n == 1.0
return when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> if (one) "1 kilometer" else "$number kilometers"
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> when (n) {
0.25 -> "a quarter mile"
0.5 -> "half a mile"
0.75 -> "three quarters of a mile"
1.0 -> "1 mile"
else -> "$number miles"
}
Distance.UNIT_FEET -> "$number feet"
Distance.UNIT_YARDS -> "$number yards"
else -> "${n.roundToInt()} meters"
}
}
/** Newest wins: a prompt still waiting when the next one arrives is already out of date. */
private fun say(text: String) {
val id = ++latest
speaker.execute {
if (id != latest) return@execute
runCatching { play(text, id) }.onFailure { GearslipLog.w("guidance: could not speak a turn: ${it.message}") }
}
}
private fun play(text: String, id: Int) {
val engine = tts ?: return
val context = appContext ?: return
val sink = link ?: return
if (!ttsReady) {
GearslipLog.w("guidance: text-to-speech isn't ready, a turn went unspoken")
return
}
val file = File(context.cacheDir, "directions.wav")
val utterance = "gearslip-directions-$id"
val done = CountDownLatch(1)
pending[utterance] = done
if (engine.synthesizeToFile(text, Bundle(), file, utterance) != TextToSpeech.SUCCESS) {
pending.remove(utterance)
return
}
if (!done.await(SYNTH_TIMEOUT_S, TimeUnit.SECONDS)) {
pending.remove(utterance)
GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken")
return
}
val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run {
GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read")
return
}
stream(sink, pcm)
GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)")
}
/** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */
private fun stream(sink: AudioLink, pcm: ByteArray) {
val frame = 2 * sink.channels
val chunk = sink.sampleRate / 50 * frame
val chunkMs = 20L
val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped
sink.start()
try {
val began = System.currentTimeMillis()
var offset = 0
var sent = 0
while (offset < padded.size && link === sink) {
val size = minOf(chunk, padded.size - offset)
sink.write(padded.copyOfRange(offset, offset + size), size)
offset += size
sent++
val ahead = sent * chunkMs - (System.currentTimeMillis() - began)
if (ahead > LEAD_MS) Thread.sleep(ahead - LEAD_MS)
}
// Let the car play what it has before closing the stream on it.
Thread.sleep(LEAD_MS)
} finally {
sink.stop()
}
}
private fun CarText?.text() = this?.toString().orEmpty()
/** 16-bit PCM from the WAV text-to-speech writes, reshaped for the car's sink. */
private object Wav {
class Pcm(val samples: ShortArray, val rate: Int, val channels: Int)
fun read(file: File): Pcm? {
val bytes = file.readBytes()
if (bytes.size < 12 || String(bytes, 0, 4) != "RIFF" || String(bytes, 8, 4) != "WAVE") return null
val buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
var pos = 12
var rate = 0
var channels = 0
var bits = 0
while (pos + 8 <= bytes.size) {
val name = String(bytes, pos, 4)
val size = buf.getInt(pos + 4)
val body = pos + 8
// Some engines write a streaming WAV with a zero or oversized data length.
val end = if (size <= 0 || body + size > bytes.size) bytes.size else body + size
when (name) {
"fmt " -> {
channels = buf.getShort(body + 2).toInt()
rate = buf.getInt(body + 4)
bits = buf.getShort(body + 14).toInt()
}
"data" -> {
if (bits != 16 || rate <= 0 || channels <= 0) return null
val samples = ShortArray((end - body) / 2)
buf.position(body)
buf.asShortBuffer().get(samples, 0, samples.size)
return Pcm(samples, rate, channels)
}
}
pos = end + (end and 1)
}
return null
}
/** Mixes to mono, resamples linearly, then copies to each of the car's channels. */
fun convert(pcm: Pcm, rate: Int, channels: Int): ByteArray {
val frames = pcm.samples.size / pcm.channels
val mono = FloatArray(frames) { f ->
var sum = 0f
for (c in 0 until pcm.channels) sum += pcm.samples[f * pcm.channels + c]
sum / pcm.channels
}
val outFrames = (frames.toLong() * rate / pcm.rate).toInt()
val out = ByteBuffer.allocate(outFrames * channels * 2).order(ByteOrder.LITTLE_ENDIAN)
val step = pcm.rate.toDouble() / rate
for (i in 0 until outFrames) {
val at = i * step
val j = at.toInt().coerceAtMost(frames - 1)
val next = (j + 1).coerceAtMost(frames - 1)
val v = mono[j] + (mono[next] - mono[j]) * (at - j).toFloat()
val s = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort()
repeat(channels) { out.putShort(s) }
}
return out.array()
}
}
private const val TICK_MS = 500L
private const val NEAR_SECONDS = 12L
private const val NEAR_METERS = 120
private const val PREPARE_FROM_METERS = 2_500
private const val PREPARE_AT_METERS = 1_000
private const val LEAD_MS = 100L
private const val SYNTH_TIMEOUT_S = 10L
private val ORDINALS = listOf("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth")
}