directions: speak each turn on the car's guidance channel

Map apps' own voice prompts can't reach the car over USB, so Gearslip reads
the turn the cluster already gets and says it through the car's directions
sink, which ducks the music. On by default, with a switch in car settings.
This commit is contained in:
Seb3thehacker committed 2026-10-13 12:26:31 -07:00
1 parent 0d0bfd3d66
commit 59f323ec06
8 files changed
+438 -47

No files matched your search

@@ -5,6 +5,7 @@ import app.seb3thehacker.gearslip.audio.AudioLink
import app.seb3thehacker.gearslip.audio.CarMic
import app.seb3thehacker.gearslip.media.CarMediaStatus
import app.seb3thehacker.gearslip.car.ClusterNav
import app.seb3thehacker.gearslip.car.DirectionsVoice
import app.seb3thehacker.gearslip.car.CarAssistant
import app.seb3thehacker.gearslip.car.CarEnvironment
import app.seb3thehacker.gearslip.car.CarKeys
@@ -79,6 +80,7 @@ class GearslipRunner(
@Volatile private var clusterNav: ClusterNav? = null
private var sensorTypes: List<Int> = emptyList()
@Volatile private var audioLink: AudioLink? = null
@Volatile private var guidanceLink: AudioLink? = null
private var audioMessagesSeen = 0
private var touchWidth = 0
private var touchHeight = 0
@@ -154,6 +156,8 @@ class GearslipRunner(
videoSource = null
audioLink?.close()
audioLink = null
guidanceLink?.close()
guidanceLink = null
CarSensors.clear()
CarMic.current = null
mediaStatus?.stop()
@@ -201,6 +205,8 @@ class GearslipRunner(
clusterNav?.onMessage(messageId, body)
} else if (frame.channel == mediaStatus?.channelId) {
mediaStatus?.onMessage(messageId, body)
} else if (frame.channel == guidanceLink?.channelId) {
guidanceLink?.onMessage(messageId, body)
} else if (frame.channel == CarMic.current?.channelId) {
CarMic.current?.onMessage(messageId, body)
} else if (frame.channel == audioLink?.channelId) {
@@ -452,6 +458,7 @@ class GearslipRunner(
openChannel(video.serviceId)
startAudioChannel(serviceDiscoveryResponse)
startGuidanceChannel(serviceDiscoveryResponse)
startSensorChannel(serviceDiscoveryResponse)
startMicChannel(serviceDiscoveryResponse)
startMediaStatusChannel(serviceDiscoveryResponse)
@@ -473,7 +480,7 @@ class GearslipRunner(
}
}
/** Media audio rides its own sink; guidance, system and telephony sinks are left closed. */
/** Media audio rides its own sink; spoken directions take the guidance sink, and system and telephony stay closed. */
private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) {
val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse)
sinks.forEach { sink ->
@@ -505,6 +512,27 @@ class GearslipRunner(
openChannel(media.serviceId)
}
/** The car's directions channel, where [DirectionsVoice] reads out each turn. */
private fun startGuidanceChannel(serviceDiscoveryResponse: ByteArray) {
val guidance = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse)
.filter { it.streamType == ServiceDiscovery.STREAM_GUIDANCE }
.firstOrNull { it.codecType == ServiceDiscovery.CODEC_PCM } ?: run {
log.i("no PCM directions sink - turns won't be spoken")
return
}
guidanceLink = AudioLink(
guidance,
sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = guidance.serviceId) },
sendControl = { id, body -> send(id, body, encrypted = true) },
tag = "guidance",
sessionId = GUIDANCE_SESSION_ID,
asksFocus = false,
onReady = { DirectionsVoice.attach(it) },
onClose = { DirectionsVoice.detach(it) },
)
openChannel(guidance.serviceId)
}
/** The car's microphone, for voice: opened now, recording only while something listens. */
private fun startMicChannel(serviceDiscoveryResponse: ByteArray) {
val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run {
@@ -1292,6 +1320,7 @@ class GearslipRunner(
const val ACK_STALL_MS = 2_000L
const val AUDIO_TRACE_MESSAGES = 6
const val GUIDANCE_SESSION_ID = 1
const val PERIODIC_KEYFRAME_MS = 5_000L
}
}
@@ -153,6 +153,7 @@ object ServiceDiscovery {
return null
}
const val STREAM_GUIDANCE = 1
const val STREAM_MEDIA = 3
const val CODEC_PCM = 1
@@ -22,6 +22,13 @@ class AudioLink(
private val service: ServiceDiscovery.AudioService,
private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit,
private val sendControl: (messageId: Int, body: ByteArray) -> Unit,
/** Log name: "audio" for the media sink, "guidance" for the directions sink. */
private val tag: String = "audio",
private val sessionId: Int = SESSION_ID,
/** Media asks the car for focus; directions just start, and the car ducks music under them. */
private val asksFocus: Boolean = true,
private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) },
private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) },
) : CarAudio.Link {
private var configIndex = 0
@@ -47,9 +54,9 @@ class AudioLink(
val channelId get() = service.serviceId
fun opened(status: Int) {
GearslipLog.i("audio: <- ChannelOpenResponse (media audio): status=$status")
GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status")
if (status != 0) {
GearslipLog.w("audio: the head unit refused the media audio channel (status $status)")
GearslipLog.w("$tag: the head unit refused the $tag channel (status $status)")
return
}
// Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink
@@ -57,7 +64,7 @@ class AudioLink(
// read from an older proto as a config index; 0 is no codec at all, and the 2024 VW
// answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing.
sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong()))
GearslipLog.i("audio: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}")
GearslipLog.i("$tag: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}")
armConfigFallback()
}
@@ -89,12 +96,12 @@ class AudioLink(
if (ready) return
configTimer?.interrupt()
GearslipLog.i(
"audio: $why - using the sink the head unit advertised " +
"$tag: $why - using the sink the head unit advertised " +
"(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)",
)
configIndex = 0
ready = true
CarAudio.attach(this)
onReady(this)
}
fun onMessage(messageId: Int, body: ByteArray) {
@@ -106,17 +113,17 @@ class AudioLink(
val status = Wire.varint(fields, 1)?.toInt()
val window = Wire.varint(fields, 2)?.toInt()
val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() }
GearslipLog.i("audio: <- Config status=$status max_unacked=$window indices=$indices")
GearslipLog.i("$tag: <- Config status=$status max_unacked=$window indices=$indices")
if (window != null) maxUnacked = maxOf(1, window)
configIndex = indices.firstOrNull() ?: 0
if (status == STATUS_READY) {
configTimer?.interrupt()
ready = true
GearslipLog.i("audio: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit")
CarAudio.attach(this)
GearslipLog.i("$tag: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit")
onReady(this)
} else {
GearslipLog.w(
"audio: the sink is not ready (status=$status, 1=WAIT 2=READY) - " +
"$tag: the sink is not ready (status=$status, 1=WAIT 2=READY) - " +
"nothing will be sent until it is",
)
}
@@ -135,7 +142,7 @@ class AudioLink(
else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) {
useAdvertisedSink("<- empty Setup answer (id $messageId)")
} else {
GearslipLog.i("audio: <- unhandled message id=$messageId (${body.size} bytes)")
GearslipLog.i("$tag: <- unhandled message id=$messageId (${body.size} bytes)")
GearslipLog.hex(" audio msg", body, limit = 64)
}
}
@@ -148,7 +155,7 @@ class AudioLink(
*/
fun onFocus(state: Int) {
val name = FOCUS_STATES[state] ?: "unknown($state)"
GearslipLog.i("audio: <- AudioFocusResponse state=$name")
GearslipLog.i("$tag: <- AudioFocusResponse state=$name")
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT ||
state == FOCUS_STATE_GAIN_MEDIA_ONLY
if (focusGranted) startStream() else stop()
@@ -156,10 +163,10 @@ class AudioLink(
override fun start() {
if (!ready || started) return
if (!focusRequested) {
if (asksFocus && !focusRequested) {
focusRequested = true
sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong()))
GearslipLog.i("audio: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over")
GearslipLog.i("$tag: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over")
}
// Some head units grant the sink silently and never answer. Do not let that be the end of
// it: start anyway, and let the acks say whether anything is being taken.
@@ -170,12 +177,12 @@ class AudioLink(
if (!ready || started) return
sendOnChannel(
MSG_START,
Protobuf.varintField(1, SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()),
Protobuf.varintField(1, sessionId.toLong()) + Protobuf.varintField(2, configIndex.toLong()),
)
inFlight = 0
lastAckAt = System.currentTimeMillis()
started = true
GearslipLog.i("audio: -> Start(session=$SESSION_ID, config=$configIndex) - streaming ${sampleRate}Hz x$channels")
GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels")
}
override fun stop() {
@@ -183,12 +190,12 @@ class AudioLink(
started = false
synchronized(ackLock) { ackLock.notifyAll() }
runCatching {
sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong()))
sendOnChannel(MSG_STOP, Protobuf.varintField(1, sessionId.toLong()))
if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong()))
}
focusRequested = false
focusGranted = false
GearslipLog.i("audio: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes")
GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes")
chunks = 0
bytes = 0
}
@@ -205,7 +212,7 @@ class AudioLink(
// A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so
// once, then now and then, so the rest of the log stays readable.
if (stalls++ % 500 == 0) {
GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)")
GearslipLog.w("$tag: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)")
}
inFlight = 0
break
@@ -223,7 +230,7 @@ class AudioLink(
try {
sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size))
} catch (e: java.io.IOException) {
GearslipLog.e("audio: the link died mid-stream", e)
GearslipLog.e("$tag: the link died mid-stream", e)
started = false
return
}
@@ -234,7 +241,7 @@ class AudioLink(
// the car is still taking audio and to catch a window that has quietly closed.
if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) {
GearslipLog.i(
"audio: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " +
"$tag: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " +
"dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}",
)
}
@@ -245,7 +252,7 @@ class AudioLink(
configTimer = null
stop()
ready = false
CarAudio.detach(this)
onClose(this)
}
companion object {
@@ -65,6 +65,7 @@ object CarServices {
app.seb3thehacker.gearslip.notify.CarMotion.start(appContext)
app.seb3thehacker.gearslip.call.CarCalls.start(appContext)
CarAssistant.init(appContext)
DirectionsVoice.init(appContext)
CarKeys.init(appContext)
VoiceReply.init(appContext)
// Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest.
@@ -28,6 +28,7 @@ object CarSettings {
private const val KEY_OPEN = "car_open_on_connect"
private const val KEY_NIGHT = "car_night_mode"
private const val KEY_PIPE_AUDIO = "car_pipe_audio"
private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions"
private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked"
private const val KEY_LAST_NAV = "car_last_nav"
private const val KEY_LAST_MEDIA = "car_last_media"
@@ -58,6 +59,7 @@ object CarSettings {
private val openFlow = MutableStateFlow(HOME)
private val nightFlow = MutableStateFlow(NightMode.AUTO)
private val pipeAudioFlow = MutableStateFlow(true)
private val speakDirectionsFlow = MutableStateFlow(true)
private val lastNavFlow = MutableStateFlow<String?>(null)
private val lastMediaFlow = MutableStateFlow<String?>(null)
private val autoplayFlow = MutableStateFlow(false)
@@ -153,6 +155,9 @@ object CarSettings {
*/
val pipeAudio: StateFlow<Boolean> = pipeAudioFlow
/** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */
val speakDirections: StateFlow<Boolean> = speakDirectionsFlow
/**
* Whether the driver has already been offered Bluetooth after turning down the capture
* dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged.
@@ -205,6 +210,7 @@ object CarSettings {
autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true"
playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true"
pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true"
speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, "true") == "true"
nightFlow.value = runCatching {
NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name))
}.getOrDefault(NightMode.AUTO)
@@ -274,6 +280,11 @@ object CarSettings {
if (on) CarAudio.request() else CarAudio.release()
}
fun setSpeakDirections(on: Boolean) {
speakDirectionsFlow.value = on
AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString())
}
fun setExperimentalFeaturesEnabled(on: Boolean) {
experimentalFeaturesFlow.value = on
AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
@@ -53,6 +53,7 @@ fun CarSettingsScreen() {
val units by CarSettings.units.collectAsState()
val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState()
val pipeAudio by CarSettings.pipeAudio.collectAsState()
val speakDirections by CarSettings.speakDirections.collectAsState()
val autoplay by CarSettings.autoplay.collectAsState()
val playerTitle by CarSettings.playerTitle.collectAsState()
val theme by CarSettings.appTheme.collectAsState()
@@ -137,6 +138,18 @@ fun CarSettingsScreen() {
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
ChoiceRow(
"Spoken directions",
listOf(false to "Off", true to "On"),
speakDirections,
) { CarSettings.setSpeakDirections(it) }
Text(
"Gearslip reads each turn aloud through the car's speakers, and the music dips while it " +
"speaks. Turn it off if your map app's own voice already plays in the car.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
ExperimentalFeatureRow(
title = "Use the phone's speech recognition",
description = "Listens through the phone's own speech service instead of Gearslip's. On " +
@@ -60,23 +60,8 @@ class ClusterNav(
running = false
}
/** One turn as the car wants it, from whichever source the map app fills. */
private class Turn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean)
private fun current(): Turn? {
val nav = CarServices.navOrNull() ?: return null
nav.trip.value?.let { trip ->
val step = trip.steps.firstOrNull() ?: return@let
val estimate = trip.stepTravelEstimates.firstOrNull()
return Turn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading)
}
val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null
val step = info.currentStep ?: return null
return Turn(step, info.currentDistance, null, info.isLoading)
}
private fun tick() {
val turn = current()
val turn = NavTurn.current()
val status = when {
turn == null -> STATUS_INACTIVE
turn.rerouting -> STATUS_REROUTING
@@ -114,7 +99,7 @@ class ClusterNav(
* 7 yards), so the app's own display choice passes straight through.
*/
private fun sendDistance(distance: Distance, seconds: Long?) {
val meters = toMeters(distance)
val meters = NavTurn.toMeters(distance)
val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}"
if (key == sentDistance) return
sentDistance = key
@@ -127,14 +112,6 @@ class ClusterNav(
)
}
private fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344
Distance.UNIT_FEET -> 0.3048
Distance.UNIT_YARDS -> 0.9144
else -> 1.0
}).toInt()
private fun CarText?.text() = this?.toString().orEmpty()
companion object {
@@ -194,3 +171,29 @@ class ClusterNav(
}
}
}
/** One turn as the car wants it, from whichever source the map app fills. */
internal class NavTurn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) {
companion object {
/** The map app's next turn: from its trip (what clusters get), or its routing card. */
fun current(): NavTurn? {
val nav = CarServices.navOrNull() ?: return null
nav.trip.value?.let { trip ->
val step = trip.steps.firstOrNull() ?: return@let
val estimate = trip.stepTravelEstimates.firstOrNull()
return NavTurn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading)
}
val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null
val step = info.currentStep ?: return null
return NavTurn(step, info.currentDistance, null, info.isLoading)
}
fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344
Distance.UNIT_FEET -> 0.3048
Distance.UNIT_YARDS -> 0.9144
else -> 1.0
}).toInt()
}
}
@@ -0,0 +1,326 @@
package app.seb3thehacker.gearslip.car
import android.content.Context
import android.os.Bundle
import android.speech.tts.TextToSpeech
import android.speech.tts.UtteranceProgressListener
import androidx.car.app.model.CarText
import androidx.car.app.model.Distance
import androidx.car.app.navigation.model.Maneuver
import androidx.car.app.navigation.model.Step
import app.seb3thehacker.gearslip.GearslipLog
import app.seb3thehacker.gearslip.audio.AudioLink
import java.io.File
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import kotlin.math.roundToInt
/**
* Spoken turn-by-turn on the car's directions channel (the GUIDANCE audio sink).
*
* A map app on the phone speaks its own prompts as navigation audio, which Android can't hand to
* Gearslip without root, and which the phone mutes while USB audio runs. So Gearslip reads the
* same turn the instrument cluster gets ([NavTurn]) and says it itself: once when the turn comes
* up, again a kilometre or so out on a long stretch, and once more just before it. The words are
* rendered with the phone's text-to-speech to a file, never played on the phone, and streamed to
* the car, which ducks the music under them the way it does for real Android Auto.
*
* Nothing spoken is logged: road names are where the driver is going.
*/
object DirectionsVoice {
private enum class Stage { FAR, PREPARED, DONE }
private var appContext: Context? = null
private var tts: TextToSpeech? = null
@Volatile private var ttsReady = false
private val pending = ConcurrentHashMap<String, CountDownLatch>()
@Volatile private var link: AudioLink? = null
@Volatile private var running = false
private val speaker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } }
@Volatile private var latest = 0
private var turnKey: String? = null
private var stage = Stage.DONE
fun init(context: Context) {
if (tts != null) return
appContext = context.applicationContext
tts = TextToSpeech(appContext) { status -> ttsReady = status == TextToSpeech.SUCCESS }.apply {
setOnUtteranceProgressListener(object : UtteranceProgressListener() {
override fun onStart(utteranceId: String?) {}
override fun onDone(utteranceId: String?) { pending.remove(utteranceId)?.countDown() }
@Deprecated("Deprecated in Java, no replacement on this API level")
override fun onError(utteranceId: String?) { pending.remove(utteranceId)?.countDown() }
})
}
}
/** The car opened its directions sink: start following the map app's turns. */
fun attach(link: AudioLink) {
this.link = link
GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}")
if (running) return
running = true
turnKey = null
Thread({
while (running) {
runCatching { tick() }.onFailure { GearslipLog.w("guidance: ${it.message}") }
Thread.sleep(TICK_MS)
}
}, "gearslip-directions-watch").apply { isDaemon = true }.start()
}
fun detach(link: AudioLink) {
if (this.link !== link) return
this.link = null
running = false
}
private fun tick() {
if (!CarSettings.speakDirections.value) return
val turn = NavTurn.current()
if (turn == null) {
turnKey = null
return
}
if (turn.rerouting) return
val step = turn.step
val instruction = instruction(step) ?: return
val meters = turn.distance?.let(NavTurn::toMeters)
val close = (turn.seconds != null && turn.seconds <= NEAR_SECONDS) || (meters != null && meters <= NEAR_METERS)
val key = "${step.road.text()}|${step.maneuver?.type}|${step.maneuver?.roundaboutExitNumber}|${step.cue.text()}"
if (key != turnKey) {
turnKey = key
if (step.maneuver?.type == Maneuver.TYPE_DEPART) {
stage = Stage.DONE
return
}
if (close || turn.distance == null) {
say(instruction)
stage = Stage.DONE
} else {
say("In ${spoken(turn.distance)}, $instruction")
stage = if (meters != null && meters > PREPARE_FROM_METERS) Stage.FAR else Stage.PREPARED
}
return
}
when {
stage != Stage.DONE && close -> {
say(instruction)
stage = Stage.DONE
}
stage == Stage.FAR && meters != null && meters <= PREPARE_AT_METERS -> {
say("In ${spoken(turn.distance)}, $instruction")
stage = Stage.PREPARED
}
}
}
/** What to do, in words: built from the manoeuvre when the app names one, else the app's own text. */
private fun instruction(step: Step): String? {
val road = step.road.text()
val onto = if (road.isBlank()) "" else " onto $road"
val toward = if (road.isBlank()) "" else " toward $road"
val maneuver = step.maneuver
val built = when (maneuver?.type) {
Maneuver.TYPE_TURN_SLIGHT_LEFT -> "Turn slightly left$onto"
Maneuver.TYPE_TURN_SLIGHT_RIGHT -> "Turn slightly right$onto"
Maneuver.TYPE_TURN_NORMAL_LEFT -> "Turn left$onto"
Maneuver.TYPE_TURN_NORMAL_RIGHT -> "Turn right$onto"
Maneuver.TYPE_TURN_SHARP_LEFT -> "Turn sharply left$onto"
Maneuver.TYPE_TURN_SHARP_RIGHT -> "Turn sharply right$onto"
Maneuver.TYPE_U_TURN_LEFT, Maneuver.TYPE_U_TURN_RIGHT -> "Make a U-turn"
Maneuver.TYPE_KEEP_LEFT -> "Keep left$toward"
Maneuver.TYPE_KEEP_RIGHT -> "Keep right$toward"
Maneuver.TYPE_FORK_LEFT -> "Keep left at the fork$toward"
Maneuver.TYPE_FORK_RIGHT -> "Keep right at the fork$toward"
Maneuver.TYPE_ON_RAMP_SLIGHT_LEFT, Maneuver.TYPE_ON_RAMP_NORMAL_LEFT, Maneuver.TYPE_ON_RAMP_SHARP_LEFT,
Maneuver.TYPE_ON_RAMP_U_TURN_LEFT -> "Take the ramp on the left$onto"
Maneuver.TYPE_ON_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_ON_RAMP_NORMAL_RIGHT, Maneuver.TYPE_ON_RAMP_SHARP_RIGHT,
Maneuver.TYPE_ON_RAMP_U_TURN_RIGHT -> "Take the ramp on the right$onto"
Maneuver.TYPE_OFF_RAMP_SLIGHT_LEFT, Maneuver.TYPE_OFF_RAMP_NORMAL_LEFT -> "Take the exit on the left$toward"
Maneuver.TYPE_OFF_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_OFF_RAMP_NORMAL_RIGHT -> "Take the exit on the right$toward"
Maneuver.TYPE_MERGE_LEFT -> "Merge left$onto"
Maneuver.TYPE_MERGE_RIGHT -> "Merge right$onto"
Maneuver.TYPE_MERGE_SIDE_UNSPECIFIED -> "Merge$onto"
Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW_WITH_ANGLE,
Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW_WITH_ANGLE ->
maneuver.roundaboutExitNumber.takeIf { it in 1..ORDINALS.size }
?.let { "At the roundabout, take the ${ORDINALS[it - 1]} exit$onto" }
?: "Go through the roundabout$onto"
Maneuver.TYPE_ROUNDABOUT_ENTER_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_CCW -> "Enter the roundabout"
Maneuver.TYPE_ROUNDABOUT_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_EXIT_CCW -> "Exit the roundabout$onto"
Maneuver.TYPE_STRAIGHT -> "Continue straight$onto"
Maneuver.TYPE_NAME_CHANGE -> "Continue$onto"
Maneuver.TYPE_FERRY_BOAT, Maneuver.TYPE_FERRY_BOAT_LEFT, Maneuver.TYPE_FERRY_BOAT_RIGHT -> "Take the ferry"
Maneuver.TYPE_FERRY_TRAIN, Maneuver.TYPE_FERRY_TRAIN_LEFT, Maneuver.TYPE_FERRY_TRAIN_RIGHT -> "Take the train"
Maneuver.TYPE_DESTINATION, Maneuver.TYPE_DESTINATION_STRAIGHT -> "Your destination is ahead"
Maneuver.TYPE_DESTINATION_LEFT -> "Your destination is on the left"
Maneuver.TYPE_DESTINATION_RIGHT -> "Your destination is on the right"
else -> null
}
return built ?: step.cue.text().ifBlank { null }
}
/** The app's own rounded distance, read the way a person would say it. */
private fun spoken(d: Distance): String {
val n = d.displayDistance
val number = if (n % 1.0 == 0.0) n.toInt().toString() else "%.1f".format(n)
val one = n == 1.0
return when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> if (one) "1 kilometer" else "$number kilometers"
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> when (n) {
0.25 -> "a quarter mile"
0.5 -> "half a mile"
0.75 -> "three quarters of a mile"
1.0 -> "1 mile"
else -> "$number miles"
}
Distance.UNIT_FEET -> "$number feet"
Distance.UNIT_YARDS -> "$number yards"
else -> "${n.roundToInt()} meters"
}
}
/** Newest wins: a prompt still waiting when the next one arrives is already out of date. */
private fun say(text: String) {
val id = ++latest
speaker.execute {
if (id != latest) return@execute
runCatching { play(text, id) }.onFailure { GearslipLog.w("guidance: could not speak a turn: ${it.message}") }
}
}
private fun play(text: String, id: Int) {
val engine = tts ?: return
val context = appContext ?: return
val sink = link ?: return
if (!ttsReady) {
GearslipLog.w("guidance: text-to-speech isn't ready, a turn went unspoken")
return
}
val file = File(context.cacheDir, "directions.wav")
val utterance = "gearslip-directions-$id"
val done = CountDownLatch(1)
pending[utterance] = done
if (engine.synthesizeToFile(text, Bundle(), file, utterance) != TextToSpeech.SUCCESS) {
pending.remove(utterance)
return
}
if (!done.await(SYNTH_TIMEOUT_S, TimeUnit.SECONDS)) {
pending.remove(utterance)
GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken")
return
}
val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run {
GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read")
return
}
stream(sink, pcm)
GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)")
}
/** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */
private fun stream(sink: AudioLink, pcm: ByteArray) {
val frame = 2 * sink.channels
val chunk = sink.sampleRate / 50 * frame
val chunkMs = 20L
val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped
sink.start()
try {
val began = System.currentTimeMillis()
var offset = 0
var sent = 0
while (offset < padded.size && link === sink) {
val size = minOf(chunk, padded.size - offset)
sink.write(padded.copyOfRange(offset, offset + size), size)
offset += size
sent++
val ahead = sent * chunkMs - (System.currentTimeMillis() - began)
if (ahead > LEAD_MS) Thread.sleep(ahead - LEAD_MS)
}
// Let the car play what it has before closing the stream on it.
Thread.sleep(LEAD_MS)
} finally {
sink.stop()
}
}
private fun CarText?.text() = this?.toString().orEmpty()
/** 16-bit PCM from the WAV text-to-speech writes, reshaped for the car's sink. */
private object Wav {
class Pcm(val samples: ShortArray, val rate: Int, val channels: Int)
fun read(file: File): Pcm? {
val bytes = file.readBytes()
if (bytes.size < 12 || String(bytes, 0, 4) != "RIFF" || String(bytes, 8, 4) != "WAVE") return null
val buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
var pos = 12
var rate = 0
var channels = 0
var bits = 0
while (pos + 8 <= bytes.size) {
val name = String(bytes, pos, 4)
val size = buf.getInt(pos + 4)
val body = pos + 8
// Some engines write a streaming WAV with a zero or oversized data length.
val end = if (size <= 0 || body + size > bytes.size) bytes.size else body + size
when (name) {
"fmt " -> {
channels = buf.getShort(body + 2).toInt()
rate = buf.getInt(body + 4)
bits = buf.getShort(body + 14).toInt()
}
"data" -> {
if (bits != 16 || rate <= 0 || channels <= 0) return null
val samples = ShortArray((end - body) / 2)
buf.position(body)
buf.asShortBuffer().get(samples, 0, samples.size)
return Pcm(samples, rate, channels)
}
}
pos = end + (end and 1)
}
return null
}
/** Mixes to mono, resamples linearly, then copies to each of the car's channels. */
fun convert(pcm: Pcm, rate: Int, channels: Int): ByteArray {
val frames = pcm.samples.size / pcm.channels
val mono = FloatArray(frames) { f ->
var sum = 0f
for (c in 0 until pcm.channels) sum += pcm.samples[f * pcm.channels + c]
sum / pcm.channels
}
val outFrames = (frames.toLong() * rate / pcm.rate).toInt()
val out = ByteBuffer.allocate(outFrames * channels * 2).order(ByteOrder.LITTLE_ENDIAN)
val step = pcm.rate.toDouble() / rate
for (i in 0 until outFrames) {
val at = i * step
val j = at.toInt().coerceAtMost(frames - 1)
val next = (j + 1).coerceAtMost(frames - 1)
val v = mono[j] + (mono[next] - mono[j]) * (at - j).toFloat()
val s = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort()
repeat(channels) { out.putShort(s) }
}
return out.array()
}
}
private const val TICK_MS = 500L
private const val NEAR_SECONDS = 12L
private const val NEAR_METERS = 120
private const val PREPARE_FROM_METERS = 2_500
private const val PREPARE_AT_METERS = 1_000
private const val LEAD_MS = 100L
private const val SYNTH_TIMEOUT_S = 10L
private val ORDINALS = listOf("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth")
}