diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index 13c9cfc..c48df78 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -5,6 +5,7 @@ import app.seb3thehacker.gearslip.audio.AudioLink import app.seb3thehacker.gearslip.audio.CarMic import app.seb3thehacker.gearslip.media.CarMediaStatus import app.seb3thehacker.gearslip.car.ClusterNav +import app.seb3thehacker.gearslip.car.DirectionsVoice import app.seb3thehacker.gearslip.car.CarAssistant import app.seb3thehacker.gearslip.car.CarEnvironment import app.seb3thehacker.gearslip.car.CarKeys @@ -79,6 +80,7 @@ class GearslipRunner( @Volatile private var clusterNav: ClusterNav? = null private var sensorTypes: List = emptyList() @Volatile private var audioLink: AudioLink? = null + @Volatile private var guidanceLink: AudioLink? = null private var audioMessagesSeen = 0 private var touchWidth = 0 private var touchHeight = 0 @@ -154,6 +156,8 @@ class GearslipRunner( videoSource = null audioLink?.close() audioLink = null + guidanceLink?.close() + guidanceLink = null CarSensors.clear() CarMic.current = null mediaStatus?.stop() @@ -201,6 +205,8 @@ class GearslipRunner( clusterNav?.onMessage(messageId, body) } else if (frame.channel == mediaStatus?.channelId) { mediaStatus?.onMessage(messageId, body) + } else if (frame.channel == guidanceLink?.channelId) { + guidanceLink?.onMessage(messageId, body) } else if (frame.channel == CarMic.current?.channelId) { CarMic.current?.onMessage(messageId, body) } else if (frame.channel == audioLink?.channelId) { @@ -452,6 +458,7 @@ class GearslipRunner( openChannel(video.serviceId) startAudioChannel(serviceDiscoveryResponse) + startGuidanceChannel(serviceDiscoveryResponse) startSensorChannel(serviceDiscoveryResponse) startMicChannel(serviceDiscoveryResponse) startMediaStatusChannel(serviceDiscoveryResponse) @@ -473,7 +480,7 @@ class GearslipRunner( } } - /** Media audio rides its own sink; guidance, system and telephony sinks are left closed. */ + /** Media audio rides its own sink; spoken directions take the guidance sink, and system and telephony stay closed. */ private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) { val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) sinks.forEach { sink -> @@ -505,6 +512,27 @@ class GearslipRunner( openChannel(media.serviceId) } + /** The car's directions channel, where [DirectionsVoice] reads out each turn. */ + private fun startGuidanceChannel(serviceDiscoveryResponse: ByteArray) { + val guidance = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) + .filter { it.streamType == ServiceDiscovery.STREAM_GUIDANCE } + .firstOrNull { it.codecType == ServiceDiscovery.CODEC_PCM } ?: run { + log.i("no PCM directions sink - turns won't be spoken") + return + } + guidanceLink = AudioLink( + guidance, + sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = guidance.serviceId) }, + sendControl = { id, body -> send(id, body, encrypted = true) }, + tag = "guidance", + sessionId = GUIDANCE_SESSION_ID, + asksFocus = false, + onReady = { DirectionsVoice.attach(it) }, + onClose = { DirectionsVoice.detach(it) }, + ) + openChannel(guidance.serviceId) + } + /** The car's microphone, for voice: opened now, recording only while something listens. */ private fun startMicChannel(serviceDiscoveryResponse: ByteArray) { val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run { @@ -1292,6 +1320,7 @@ class GearslipRunner( const val ACK_STALL_MS = 2_000L const val AUDIO_TRACE_MESSAGES = 6 + const val GUIDANCE_SESSION_ID = 1 const val PERIODIC_KEYFRAME_MS = 5_000L } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt index 2aad847..529d99f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt @@ -153,6 +153,7 @@ object ServiceDiscovery { return null } + const val STREAM_GUIDANCE = 1 const val STREAM_MEDIA = 3 const val CODEC_PCM = 1 diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index 0ed7e80..9525bbc 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -22,6 +22,13 @@ class AudioLink( private val service: ServiceDiscovery.AudioService, private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit, private val sendControl: (messageId: Int, body: ByteArray) -> Unit, + /** Log name: "audio" for the media sink, "guidance" for the directions sink. */ + private val tag: String = "audio", + private val sessionId: Int = SESSION_ID, + /** Media asks the car for focus; directions just start, and the car ducks music under them. */ + private val asksFocus: Boolean = true, + private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) }, + private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) }, ) : CarAudio.Link { private var configIndex = 0 @@ -47,9 +54,9 @@ class AudioLink( val channelId get() = service.serviceId fun opened(status: Int) { - GearslipLog.i("audio: <- ChannelOpenResponse (media audio): status=$status") + GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status") if (status != 0) { - GearslipLog.w("audio: the head unit refused the media audio channel (status $status)") + GearslipLog.w("$tag: the head unit refused the $tag channel (status $status)") return } // Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink @@ -57,7 +64,7 @@ class AudioLink( // read from an older proto as a config index; 0 is no codec at all, and the 2024 VW // answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing. sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong())) - GearslipLog.i("audio: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}") + GearslipLog.i("$tag: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}") armConfigFallback() } @@ -89,12 +96,12 @@ class AudioLink( if (ready) return configTimer?.interrupt() GearslipLog.i( - "audio: $why - using the sink the head unit advertised " + + "$tag: $why - using the sink the head unit advertised " + "(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)", ) configIndex = 0 ready = true - CarAudio.attach(this) + onReady(this) } fun onMessage(messageId: Int, body: ByteArray) { @@ -106,17 +113,17 @@ class AudioLink( val status = Wire.varint(fields, 1)?.toInt() val window = Wire.varint(fields, 2)?.toInt() val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() } - GearslipLog.i("audio: <- Config status=$status max_unacked=$window indices=$indices") + GearslipLog.i("$tag: <- Config status=$status max_unacked=$window indices=$indices") if (window != null) maxUnacked = maxOf(1, window) configIndex = indices.firstOrNull() ?: 0 if (status == STATUS_READY) { configTimer?.interrupt() ready = true - GearslipLog.i("audio: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") - CarAudio.attach(this) + GearslipLog.i("$tag: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") + onReady(this) } else { GearslipLog.w( - "audio: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + + "$tag: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + "nothing will be sent until it is", ) } @@ -135,7 +142,7 @@ class AudioLink( else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) { useAdvertisedSink("<- empty Setup answer (id $messageId)") } else { - GearslipLog.i("audio: <- unhandled message id=$messageId (${body.size} bytes)") + GearslipLog.i("$tag: <- unhandled message id=$messageId (${body.size} bytes)") GearslipLog.hex(" audio msg", body, limit = 64) } } @@ -148,7 +155,7 @@ class AudioLink( */ fun onFocus(state: Int) { val name = FOCUS_STATES[state] ?: "unknown($state)" - GearslipLog.i("audio: <- AudioFocusResponse state=$name") + GearslipLog.i("$tag: <- AudioFocusResponse state=$name") focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT || state == FOCUS_STATE_GAIN_MEDIA_ONLY if (focusGranted) startStream() else stop() @@ -156,10 +163,10 @@ class AudioLink( override fun start() { if (!ready || started) return - if (!focusRequested) { + if (asksFocus && !focusRequested) { focusRequested = true sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong())) - GearslipLog.i("audio: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") + GearslipLog.i("$tag: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") } // Some head units grant the sink silently and never answer. Do not let that be the end of // it: start anyway, and let the acks say whether anything is being taken. @@ -170,12 +177,12 @@ class AudioLink( if (!ready || started) return sendOnChannel( MSG_START, - Protobuf.varintField(1, SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()), + Protobuf.varintField(1, sessionId.toLong()) + Protobuf.varintField(2, configIndex.toLong()), ) inFlight = 0 lastAckAt = System.currentTimeMillis() started = true - GearslipLog.i("audio: -> Start(session=$SESSION_ID, config=$configIndex) - streaming ${sampleRate}Hz x$channels") + GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels") } override fun stop() { @@ -183,12 +190,12 @@ class AudioLink( started = false synchronized(ackLock) { ackLock.notifyAll() } runCatching { - sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong())) + sendOnChannel(MSG_STOP, Protobuf.varintField(1, sessionId.toLong())) if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong())) } focusRequested = false focusGranted = false - GearslipLog.i("audio: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") + GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") chunks = 0 bytes = 0 } @@ -205,7 +212,7 @@ class AudioLink( // A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so // once, then now and then, so the rest of the log stays readable. if (stalls++ % 500 == 0) { - GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)") + GearslipLog.w("$tag: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)") } inFlight = 0 break @@ -223,7 +230,7 @@ class AudioLink( try { sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size)) } catch (e: java.io.IOException) { - GearslipLog.e("audio: the link died mid-stream", e) + GearslipLog.e("$tag: the link died mid-stream", e) started = false return } @@ -234,7 +241,7 @@ class AudioLink( // the car is still taking audio and to catch a window that has quietly closed. if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) { GearslipLog.i( - "audio: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + + "$tag: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + "dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}", ) } @@ -245,7 +252,7 @@ class AudioLink( configTimer = null stop() ready = false - CarAudio.detach(this) + onClose(this) } companion object { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt index a4992dc..587c4ea 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt @@ -65,6 +65,7 @@ object CarServices { app.seb3thehacker.gearslip.notify.CarMotion.start(appContext) app.seb3thehacker.gearslip.call.CarCalls.start(appContext) CarAssistant.init(appContext) + DirectionsVoice.init(appContext) CarKeys.init(appContext) VoiceReply.init(appContext) // Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest. diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index a0b3024..43c505c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -28,6 +28,7 @@ object CarSettings { private const val KEY_OPEN = "car_open_on_connect" private const val KEY_NIGHT = "car_night_mode" private const val KEY_PIPE_AUDIO = "car_pipe_audio" +private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions" private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked" private const val KEY_LAST_NAV = "car_last_nav" private const val KEY_LAST_MEDIA = "car_last_media" @@ -58,6 +59,7 @@ object CarSettings { private val openFlow = MutableStateFlow(HOME) private val nightFlow = MutableStateFlow(NightMode.AUTO) private val pipeAudioFlow = MutableStateFlow(true) + private val speakDirectionsFlow = MutableStateFlow(true) private val lastNavFlow = MutableStateFlow(null) private val lastMediaFlow = MutableStateFlow(null) private val autoplayFlow = MutableStateFlow(false) @@ -153,6 +155,9 @@ object CarSettings { */ val pipeAudio: StateFlow = pipeAudioFlow + /** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */ + val speakDirections: StateFlow = speakDirectionsFlow + /** * Whether the driver has already been offered Bluetooth after turning down the capture * dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged. @@ -205,6 +210,7 @@ object CarSettings { autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true" playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true" pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true" + speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, "true") == "true" nightFlow.value = runCatching { NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name)) }.getOrDefault(NightMode.AUTO) @@ -274,6 +280,11 @@ object CarSettings { if (on) CarAudio.request() else CarAudio.release() } + fun setSpeakDirections(on: Boolean) { + speakDirectionsFlow.value = on + AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString()) + } + fun setExperimentalFeaturesEnabled(on: Boolean) { experimentalFeaturesFlow.value = on AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString()) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index 2fec3e0..e14ec7a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -53,6 +53,7 @@ fun CarSettingsScreen() { val units by CarSettings.units.collectAsState() val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState() + val speakDirections by CarSettings.speakDirections.collectAsState() val autoplay by CarSettings.autoplay.collectAsState() val playerTitle by CarSettings.playerTitle.collectAsState() val theme by CarSettings.appTheme.collectAsState() @@ -137,6 +138,18 @@ fun CarSettingsScreen() { color = MaterialTheme.colorScheme.onSurfaceVariant, ) + ChoiceRow( + "Spoken directions", + listOf(false to "Off", true to "On"), + speakDirections, + ) { CarSettings.setSpeakDirections(it) } + Text( + "Gearslip reads each turn aloud through the car's speakers, and the music dips while it " + + "speaks. Turn it off if your map app's own voice already plays in the car.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + ExperimentalFeatureRow( title = "Use the phone's speech recognition", description = "Listens through the phone's own speech service instead of Gearslip's. On " + diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt index fda6c4c..8a4f8f6 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt @@ -60,23 +60,8 @@ class ClusterNav( running = false } - /** One turn as the car wants it, from whichever source the map app fills. */ - private class Turn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) - - private fun current(): Turn? { - val nav = CarServices.navOrNull() ?: return null - nav.trip.value?.let { trip -> - val step = trip.steps.firstOrNull() ?: return@let - val estimate = trip.stepTravelEstimates.firstOrNull() - return Turn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading) - } - val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null - val step = info.currentStep ?: return null - return Turn(step, info.currentDistance, null, info.isLoading) - } - private fun tick() { - val turn = current() + val turn = NavTurn.current() val status = when { turn == null -> STATUS_INACTIVE turn.rerouting -> STATUS_REROUTING @@ -114,7 +99,7 @@ class ClusterNav( * 7 yards), so the app's own display choice passes straight through. */ private fun sendDistance(distance: Distance, seconds: Long?) { - val meters = toMeters(distance) + val meters = NavTurn.toMeters(distance) val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}" if (key == sentDistance) return sentDistance = key @@ -127,14 +112,6 @@ class ClusterNav( ) } - private fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) { - Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0 - Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344 - Distance.UNIT_FEET -> 0.3048 - Distance.UNIT_YARDS -> 0.9144 - else -> 1.0 - }).toInt() - private fun CarText?.text() = this?.toString().orEmpty() companion object { @@ -194,3 +171,29 @@ class ClusterNav( } } } + +/** One turn as the car wants it, from whichever source the map app fills. */ +internal class NavTurn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) { + companion object { + /** The map app's next turn: from its trip (what clusters get), or its routing card. */ + fun current(): NavTurn? { + val nav = CarServices.navOrNull() ?: return null + nav.trip.value?.let { trip -> + val step = trip.steps.firstOrNull() ?: return@let + val estimate = trip.stepTravelEstimates.firstOrNull() + return NavTurn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading) + } + val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null + val step = info.currentStep ?: return null + return NavTurn(step, info.currentDistance, null, info.isLoading) + } + + fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) { + Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0 + Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344 + Distance.UNIT_FEET -> 0.3048 + Distance.UNIT_YARDS -> 0.9144 + else -> 1.0 + }).toInt() + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt new file mode 100644 index 0000000..6b9abf3 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt @@ -0,0 +1,326 @@ +package app.seb3thehacker.gearslip.car + +import android.content.Context +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import androidx.car.app.model.CarText +import androidx.car.app.model.Distance +import androidx.car.app.navigation.model.Maneuver +import androidx.car.app.navigation.model.Step +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.audio.AudioLink +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.math.roundToInt + +/** + * Spoken turn-by-turn on the car's directions channel (the GUIDANCE audio sink). + * + * A map app on the phone speaks its own prompts as navigation audio, which Android can't hand to + * Gearslip without root, and which the phone mutes while USB audio runs. So Gearslip reads the + * same turn the instrument cluster gets ([NavTurn]) and says it itself: once when the turn comes + * up, again a kilometre or so out on a long stretch, and once more just before it. The words are + * rendered with the phone's text-to-speech to a file, never played on the phone, and streamed to + * the car, which ducks the music under them the way it does for real Android Auto. + * + * Nothing spoken is logged: road names are where the driver is going. + */ +object DirectionsVoice { + + private enum class Stage { FAR, PREPARED, DONE } + + private var appContext: Context? = null + private var tts: TextToSpeech? = null + @Volatile private var ttsReady = false + private val pending = ConcurrentHashMap() + + @Volatile private var link: AudioLink? = null + @Volatile private var running = false + private val speaker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } } + @Volatile private var latest = 0 + + private var turnKey: String? = null + private var stage = Stage.DONE + + fun init(context: Context) { + if (tts != null) return + appContext = context.applicationContext + tts = TextToSpeech(appContext) { status -> ttsReady = status == TextToSpeech.SUCCESS }.apply { + setOnUtteranceProgressListener(object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) {} + override fun onDone(utteranceId: String?) { pending.remove(utteranceId)?.countDown() } + @Deprecated("Deprecated in Java, no replacement on this API level") + override fun onError(utteranceId: String?) { pending.remove(utteranceId)?.countDown() } + }) + } + } + + /** The car opened its directions sink: start following the map app's turns. */ + fun attach(link: AudioLink) { + this.link = link + GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}") + if (running) return + running = true + turnKey = null + Thread({ + while (running) { + runCatching { tick() }.onFailure { GearslipLog.w("guidance: ${it.message}") } + Thread.sleep(TICK_MS) + } + }, "gearslip-directions-watch").apply { isDaemon = true }.start() + } + + fun detach(link: AudioLink) { + if (this.link !== link) return + this.link = null + running = false + } + + private fun tick() { + if (!CarSettings.speakDirections.value) return + val turn = NavTurn.current() + if (turn == null) { + turnKey = null + return + } + if (turn.rerouting) return + val step = turn.step + val instruction = instruction(step) ?: return + val meters = turn.distance?.let(NavTurn::toMeters) + val close = (turn.seconds != null && turn.seconds <= NEAR_SECONDS) || (meters != null && meters <= NEAR_METERS) + + val key = "${step.road.text()}|${step.maneuver?.type}|${step.maneuver?.roundaboutExitNumber}|${step.cue.text()}" + if (key != turnKey) { + turnKey = key + if (step.maneuver?.type == Maneuver.TYPE_DEPART) { + stage = Stage.DONE + return + } + if (close || turn.distance == null) { + say(instruction) + stage = Stage.DONE + } else { + say("In ${spoken(turn.distance)}, $instruction") + stage = if (meters != null && meters > PREPARE_FROM_METERS) Stage.FAR else Stage.PREPARED + } + return + } + when { + stage != Stage.DONE && close -> { + say(instruction) + stage = Stage.DONE + } + stage == Stage.FAR && meters != null && meters <= PREPARE_AT_METERS -> { + say("In ${spoken(turn.distance)}, $instruction") + stage = Stage.PREPARED + } + } + } + + /** What to do, in words: built from the manoeuvre when the app names one, else the app's own text. */ + private fun instruction(step: Step): String? { + val road = step.road.text() + val onto = if (road.isBlank()) "" else " onto $road" + val toward = if (road.isBlank()) "" else " toward $road" + val maneuver = step.maneuver + val built = when (maneuver?.type) { + Maneuver.TYPE_TURN_SLIGHT_LEFT -> "Turn slightly left$onto" + Maneuver.TYPE_TURN_SLIGHT_RIGHT -> "Turn slightly right$onto" + Maneuver.TYPE_TURN_NORMAL_LEFT -> "Turn left$onto" + Maneuver.TYPE_TURN_NORMAL_RIGHT -> "Turn right$onto" + Maneuver.TYPE_TURN_SHARP_LEFT -> "Turn sharply left$onto" + Maneuver.TYPE_TURN_SHARP_RIGHT -> "Turn sharply right$onto" + Maneuver.TYPE_U_TURN_LEFT, Maneuver.TYPE_U_TURN_RIGHT -> "Make a U-turn" + Maneuver.TYPE_KEEP_LEFT -> "Keep left$toward" + Maneuver.TYPE_KEEP_RIGHT -> "Keep right$toward" + Maneuver.TYPE_FORK_LEFT -> "Keep left at the fork$toward" + Maneuver.TYPE_FORK_RIGHT -> "Keep right at the fork$toward" + Maneuver.TYPE_ON_RAMP_SLIGHT_LEFT, Maneuver.TYPE_ON_RAMP_NORMAL_LEFT, Maneuver.TYPE_ON_RAMP_SHARP_LEFT, + Maneuver.TYPE_ON_RAMP_U_TURN_LEFT -> "Take the ramp on the left$onto" + Maneuver.TYPE_ON_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_ON_RAMP_NORMAL_RIGHT, Maneuver.TYPE_ON_RAMP_SHARP_RIGHT, + Maneuver.TYPE_ON_RAMP_U_TURN_RIGHT -> "Take the ramp on the right$onto" + Maneuver.TYPE_OFF_RAMP_SLIGHT_LEFT, Maneuver.TYPE_OFF_RAMP_NORMAL_LEFT -> "Take the exit on the left$toward" + Maneuver.TYPE_OFF_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_OFF_RAMP_NORMAL_RIGHT -> "Take the exit on the right$toward" + Maneuver.TYPE_MERGE_LEFT -> "Merge left$onto" + Maneuver.TYPE_MERGE_RIGHT -> "Merge right$onto" + Maneuver.TYPE_MERGE_SIDE_UNSPECIFIED -> "Merge$onto" + Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW_WITH_ANGLE, + Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW_WITH_ANGLE -> + maneuver.roundaboutExitNumber.takeIf { it in 1..ORDINALS.size } + ?.let { "At the roundabout, take the ${ORDINALS[it - 1]} exit$onto" } + ?: "Go through the roundabout$onto" + Maneuver.TYPE_ROUNDABOUT_ENTER_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_CCW -> "Enter the roundabout" + Maneuver.TYPE_ROUNDABOUT_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_EXIT_CCW -> "Exit the roundabout$onto" + Maneuver.TYPE_STRAIGHT -> "Continue straight$onto" + Maneuver.TYPE_NAME_CHANGE -> "Continue$onto" + Maneuver.TYPE_FERRY_BOAT, Maneuver.TYPE_FERRY_BOAT_LEFT, Maneuver.TYPE_FERRY_BOAT_RIGHT -> "Take the ferry" + Maneuver.TYPE_FERRY_TRAIN, Maneuver.TYPE_FERRY_TRAIN_LEFT, Maneuver.TYPE_FERRY_TRAIN_RIGHT -> "Take the train" + Maneuver.TYPE_DESTINATION, Maneuver.TYPE_DESTINATION_STRAIGHT -> "Your destination is ahead" + Maneuver.TYPE_DESTINATION_LEFT -> "Your destination is on the left" + Maneuver.TYPE_DESTINATION_RIGHT -> "Your destination is on the right" + else -> null + } + return built ?: step.cue.text().ifBlank { null } + } + + /** The app's own rounded distance, read the way a person would say it. */ + private fun spoken(d: Distance): String { + val n = d.displayDistance + val number = if (n % 1.0 == 0.0) n.toInt().toString() else "%.1f".format(n) + val one = n == 1.0 + return when (d.displayUnit) { + Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> if (one) "1 kilometer" else "$number kilometers" + Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> when (n) { + 0.25 -> "a quarter mile" + 0.5 -> "half a mile" + 0.75 -> "three quarters of a mile" + 1.0 -> "1 mile" + else -> "$number miles" + } + Distance.UNIT_FEET -> "$number feet" + Distance.UNIT_YARDS -> "$number yards" + else -> "${n.roundToInt()} meters" + } + } + + /** Newest wins: a prompt still waiting when the next one arrives is already out of date. */ + private fun say(text: String) { + val id = ++latest + speaker.execute { + if (id != latest) return@execute + runCatching { play(text, id) }.onFailure { GearslipLog.w("guidance: could not speak a turn: ${it.message}") } + } + } + + private fun play(text: String, id: Int) { + val engine = tts ?: return + val context = appContext ?: return + val sink = link ?: return + if (!ttsReady) { + GearslipLog.w("guidance: text-to-speech isn't ready, a turn went unspoken") + return + } + val file = File(context.cacheDir, "directions.wav") + val utterance = "gearslip-directions-$id" + val done = CountDownLatch(1) + pending[utterance] = done + if (engine.synthesizeToFile(text, Bundle(), file, utterance) != TextToSpeech.SUCCESS) { + pending.remove(utterance) + return + } + if (!done.await(SYNTH_TIMEOUT_S, TimeUnit.SECONDS)) { + pending.remove(utterance) + GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken") + return + } + val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run { + GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read") + return + } + stream(sink, pcm) + GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)") + } + + /** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */ + private fun stream(sink: AudioLink, pcm: ByteArray) { + val frame = 2 * sink.channels + val chunk = sink.sampleRate / 50 * frame + val chunkMs = 20L + val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped + sink.start() + try { + val began = System.currentTimeMillis() + var offset = 0 + var sent = 0 + while (offset < padded.size && link === sink) { + val size = minOf(chunk, padded.size - offset) + sink.write(padded.copyOfRange(offset, offset + size), size) + offset += size + sent++ + val ahead = sent * chunkMs - (System.currentTimeMillis() - began) + if (ahead > LEAD_MS) Thread.sleep(ahead - LEAD_MS) + } + // Let the car play what it has before closing the stream on it. + Thread.sleep(LEAD_MS) + } finally { + sink.stop() + } + } + + private fun CarText?.text() = this?.toString().orEmpty() + + /** 16-bit PCM from the WAV text-to-speech writes, reshaped for the car's sink. */ + private object Wav { + class Pcm(val samples: ShortArray, val rate: Int, val channels: Int) + + fun read(file: File): Pcm? { + val bytes = file.readBytes() + if (bytes.size < 12 || String(bytes, 0, 4) != "RIFF" || String(bytes, 8, 4) != "WAVE") return null + val buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) + var pos = 12 + var rate = 0 + var channels = 0 + var bits = 0 + while (pos + 8 <= bytes.size) { + val name = String(bytes, pos, 4) + val size = buf.getInt(pos + 4) + val body = pos + 8 + // Some engines write a streaming WAV with a zero or oversized data length. + val end = if (size <= 0 || body + size > bytes.size) bytes.size else body + size + when (name) { + "fmt " -> { + channels = buf.getShort(body + 2).toInt() + rate = buf.getInt(body + 4) + bits = buf.getShort(body + 14).toInt() + } + "data" -> { + if (bits != 16 || rate <= 0 || channels <= 0) return null + val samples = ShortArray((end - body) / 2) + buf.position(body) + buf.asShortBuffer().get(samples, 0, samples.size) + return Pcm(samples, rate, channels) + } + } + pos = end + (end and 1) + } + return null + } + + /** Mixes to mono, resamples linearly, then copies to each of the car's channels. */ + fun convert(pcm: Pcm, rate: Int, channels: Int): ByteArray { + val frames = pcm.samples.size / pcm.channels + val mono = FloatArray(frames) { f -> + var sum = 0f + for (c in 0 until pcm.channels) sum += pcm.samples[f * pcm.channels + c] + sum / pcm.channels + } + val outFrames = (frames.toLong() * rate / pcm.rate).toInt() + val out = ByteBuffer.allocate(outFrames * channels * 2).order(ByteOrder.LITTLE_ENDIAN) + val step = pcm.rate.toDouble() / rate + for (i in 0 until outFrames) { + val at = i * step + val j = at.toInt().coerceAtMost(frames - 1) + val next = (j + 1).coerceAtMost(frames - 1) + val v = mono[j] + (mono[next] - mono[j]) * (at - j).toFloat() + val s = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() + repeat(channels) { out.putShort(s) } + } + return out.array() + } + } + + private const val TICK_MS = 500L + private const val NEAR_SECONDS = 12L + private const val NEAR_METERS = 120 + private const val PREPARE_FROM_METERS = 2_500 + private const val PREPARE_AT_METERS = 1_000 + private const val LEAD_MS = 100L + private const val SYNTH_TIMEOUT_S = 10L + private val ORDINALS = listOf("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth") +}