diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index d39b1bc..c48df78 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -5,6 +5,7 @@ import app.seb3thehacker.gearslip.audio.AudioLink import app.seb3thehacker.gearslip.audio.CarMic import app.seb3thehacker.gearslip.media.CarMediaStatus import app.seb3thehacker.gearslip.car.ClusterNav +import app.seb3thehacker.gearslip.car.DirectionsVoice import app.seb3thehacker.gearslip.car.CarAssistant import app.seb3thehacker.gearslip.car.CarEnvironment import app.seb3thehacker.gearslip.car.CarKeys @@ -79,6 +80,7 @@ class GearslipRunner( @Volatile private var clusterNav: ClusterNav? = null private var sensorTypes: List = emptyList() @Volatile private var audioLink: AudioLink? = null + @Volatile private var guidanceLink: AudioLink? = null private var audioMessagesSeen = 0 private var touchWidth = 0 private var touchHeight = 0 @@ -154,6 +156,8 @@ class GearslipRunner( videoSource = null audioLink?.close() audioLink = null + guidanceLink?.close() + guidanceLink = null CarSensors.clear() CarMic.current = null mediaStatus?.stop() @@ -201,6 +205,8 @@ class GearslipRunner( clusterNav?.onMessage(messageId, body) } else if (frame.channel == mediaStatus?.channelId) { mediaStatus?.onMessage(messageId, body) + } else if (frame.channel == guidanceLink?.channelId) { + guidanceLink?.onMessage(messageId, body) } else if (frame.channel == CarMic.current?.channelId) { CarMic.current?.onMessage(messageId, body) } else if (frame.channel == audioLink?.channelId) { @@ -452,6 +458,7 @@ class GearslipRunner( openChannel(video.serviceId) startAudioChannel(serviceDiscoveryResponse) + startGuidanceChannel(serviceDiscoveryResponse) startSensorChannel(serviceDiscoveryResponse) startMicChannel(serviceDiscoveryResponse) startMediaStatusChannel(serviceDiscoveryResponse) @@ -473,7 +480,7 @@ class GearslipRunner( } } - /** Media audio rides its own sink; guidance, system and telephony sinks are left closed. */ + /** Media audio rides its own sink; spoken directions take the guidance sink, and system and telephony stay closed. */ private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) { val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) sinks.forEach { sink -> @@ -505,6 +512,27 @@ class GearslipRunner( openChannel(media.serviceId) } + /** The car's directions channel, where [DirectionsVoice] reads out each turn. */ + private fun startGuidanceChannel(serviceDiscoveryResponse: ByteArray) { + val guidance = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse) + .filter { it.streamType == ServiceDiscovery.STREAM_GUIDANCE } + .firstOrNull { it.codecType == ServiceDiscovery.CODEC_PCM } ?: run { + log.i("no PCM directions sink - turns won't be spoken") + return + } + guidanceLink = AudioLink( + guidance, + sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = guidance.serviceId) }, + sendControl = { id, body -> send(id, body, encrypted = true) }, + tag = "guidance", + sessionId = GUIDANCE_SESSION_ID, + asksFocus = false, + onReady = { DirectionsVoice.attach(it) }, + onClose = { DirectionsVoice.detach(it) }, + ) + openChannel(guidance.serviceId) + } + /** The car's microphone, for voice: opened now, recording only while something listens. */ private fun startMicChannel(serviceDiscoveryResponse: ByteArray) { val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run { @@ -513,7 +541,14 @@ class GearslipRunner( } val config = mic.configs.first() log.i("car microphone: channel=${mic.serviceId} ${mic.codecName} ${config.sampleRate}Hz/${config.bits}bit/x${config.channels}") - CarMic.current = CarMic(mic) { id, body -> send(id, body, encrypted = true, channel = mic.serviceId) } + CarMic.current = CarMic( + mic, + sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = mic.serviceId) }, + // VoiceSessionNotification { status = 1 START, 2 END }, on the control channel. + voiceSession = { start -> + send(MSG_VOICE_SESSION_NOTIFICATION, Protobuf.varintField(1, if (start) 1L else 2L), encrypted = true) + }, + ) openChannel(mic.serviceId) } @@ -660,8 +695,8 @@ class GearslipRunner( * right-hand third of the screen away entirely - taps on a right-edge button landed on * the middle one. The head unit maps its own panel onto our video before reporting. * - * So: inject as-is, clamped to the video bounds. The advertised touchscreen size is kept - * only for diagnostics. + * So: inject them unscaled, shifted past the cropped margin (see below) and clamped to the + * video. The advertised touchscreen size is kept only for diagnostics. */ private fun onInputMessage(messageId: Int, body: ByteArray) { when (messageId) { @@ -692,13 +727,20 @@ class GearslipRunner( val action = Wire.varint(touchFields, 3)?.toInt() ?: 0 val pointers = Wire.allBytes(touchFields, 1) if (pointers.isEmpty()) return - val size = videoConfigs.getOrNull(selectedConfigIndex ?: 0)?.pixelSize() + val config = videoConfigs.getOrNull(selectedConfigIndex ?: 0) + val size = config?.pixelSize() + // The car crops its margin off the frame and measures taps from the corner of what + // it shows, so they come in short by half the margin: the 2024 VW (margin 240) sent + // y=6, which is inside the cropped band, and the Uconnect's never passed 351 on a + // 480-high frame with a 130 margin. Without this every tap landed 120 px too high + // and the VW couldn't press past its first screen. + val shift = config?.marginInsets() ?: Insets.NONE val points = pointers.mapIndexedNotNull { index, pointer -> val fields = Wire.fields(pointer) val rawX = Wire.varint(fields, 1)?.toInt() ?: return@mapIndexedNotNull null val rawY = Wire.varint(fields, 2)?.toInt() ?: return@mapIndexedNotNull null val id = Wire.varint(fields, 3)?.toInt() ?: index - TouchPoint(id, clamp(rawX, size?.first), clamp(rawY, size?.second)) + TouchPoint(id, clamp(rawX + shift.left, size?.first), clamp(rawY + shift.top, size?.second)) } if (points.isEmpty()) return val actionIndex = (Wire.varint(touchFields, 2)?.toInt() ?: 0).coerceIn(0, points.size - 1) @@ -856,13 +898,17 @@ class GearslipRunner( /** * The offered config that best fills the car's screen: the smallest whose visible width * reaches most of the touchscreen's, else the largest there is. Always taking the smallest - * put 800x480 on a 1540x720 Audi screen and stretched it. With no touchscreen size to go - * by, the smallest is still the safe choice. + * put 800x480 on a 1540x720 Audi screen and stretched it. A car that gives no touchscreen + * size gets the largest up to 1080p: a car only offers sizes it can show, and the 2024 VW + * (touchscreen 0x0) blurred 800x480 across a 1080p screen. */ private fun fitToScreen(offered: List): Int? { val known = offered.filter { videoConfigs.getOrNull(it)?.pixelSize() != null } fun pixels(idx: Int) = videoConfigs[idx].pixelCount() - if (touchWidth <= 0) return known.minByOrNull(::pixels) + if (touchWidth <= 0) { + return known.filter { pixels(it) <= 1920 * 1080 }.maxByOrNull(::pixels) + ?: known.minByOrNull(::pixels) + } val wide = known.filter { (videoConfigs[it].visibleSize()?.first ?: 0) >= touchWidth * 8 / 10 } return wide.minByOrNull(::pixels) ?: known.maxByOrNull(::pixels) } @@ -1233,6 +1279,7 @@ class GearslipRunner( const val MSG_PING_RESPONSE = 12 const val MSG_BYEBYE_REQUEST = 15 const val MSG_BYEBYE_RESPONSE = 16 + const val MSG_VOICE_SESSION_NOTIFICATION = 17 const val MSG_CHANNEL_OPEN_REQUEST = 7 const val MSG_CHANNEL_OPEN_RESPONSE = 8 @@ -1273,6 +1320,7 @@ class GearslipRunner( const val ACK_STALL_MS = 2_000L const val AUDIO_TRACE_MESSAGES = 6 + const val GUIDANCE_SESSION_ID = 1 const val PERIODIC_KEYFRAME_MS = 5_000L } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt index 2aad847..529d99f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt @@ -153,6 +153,7 @@ object ServiceDiscovery { return null } + const val STREAM_GUIDANCE = 1 const val STREAM_MEDIA = 3 const val CODEC_PCM = 1 diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index f250b1c..9525bbc 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -22,6 +22,13 @@ class AudioLink( private val service: ServiceDiscovery.AudioService, private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit, private val sendControl: (messageId: Int, body: ByteArray) -> Unit, + /** Log name: "audio" for the media sink, "guidance" for the directions sink. */ + private val tag: String = "audio", + private val sessionId: Int = SESSION_ID, + /** Media asks the car for focus; directions just start, and the car ducks music under them. */ + private val asksFocus: Boolean = true, + private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) }, + private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) }, ) : CarAudio.Link { private var configIndex = 0 @@ -33,6 +40,7 @@ class AudioLink( // The Uconnect asks for 4 on its video channel; assume the same until its Config says otherwise. @Volatile private var maxUnacked = 4 @Volatile private var lastAckAt = 0L + private var stalls = 0 private var chunks = 0 private var bytes = 0L private var dropped = 0 @@ -46,23 +54,24 @@ class AudioLink( val channelId get() = service.serviceId fun opened(status: Int) { - GearslipLog.i("audio: <- ChannelOpenResponse (media audio): status=$status") + GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status") if (status != 0) { - GearslipLog.w("audio: the head unit refused the media audio channel (status $status)") + GearslipLog.w("$tag: the head unit refused the $tag channel (status $status)") return } - // AVChannelSetupRequest { required uint32 config_index = 1 }. This field is an index into - // the sink's own audio_configs, not a codec id - sending the codec type asks for a config - // the sink does not have, and it answers with a status that never reaches READY. - sendOnChannel(MSG_SETUP, Protobuf.varintField(1, 0L)) - GearslipLog.i("audio: -> Setup(config_index=0) on channel ${service.serviceId}, codec ${service.codecName}") + // Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink + // (aasdk's current protos, and LIVI reads it as the codec). Gearslip used to send 0 here, + // read from an older proto as a config index; 0 is no codec at all, and the 2024 VW + // answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing. + sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong())) + GearslipLog.i("$tag: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}") armConfigFallback() } /** - * The 2018 Uconnect and a Dacia ULC 4.5 both answer our Setup with a two-byte message we - * cannot identify (id 255, no body) rather than the AVChannelSetupResponse the video channel - * gets, so the sink never reaches READY. [MSG_SETUP_ANSWER] takes that as the go-ahead. + * The 2018 Uconnect, a Dacia ULC 4.5 and the 2024 VW answered the old Setup (codec 0) with + * UNEXPECTED_MESSAGE (id 255, no body) rather than a Config, so the sink never reached READY. + * [MSG_SETUP_ANSWER] still takes that as the go-ahead, in case a car rejects the real codec too. * * Service discovery already told us everything a Config would have: the sink's rate, channels * and its one configuration. So for a car that answers nothing at all, wait a moment and then @@ -87,12 +96,12 @@ class AudioLink( if (ready) return configTimer?.interrupt() GearslipLog.i( - "audio: $why - using the sink the head unit advertised " + + "$tag: $why - using the sink the head unit advertised " + "(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)", ) configIndex = 0 ready = true - CarAudio.attach(this) + onReady(this) } fun onMessage(messageId: Int, body: ByteArray) { @@ -104,17 +113,17 @@ class AudioLink( val status = Wire.varint(fields, 1)?.toInt() val window = Wire.varint(fields, 2)?.toInt() val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() } - GearslipLog.i("audio: <- Config status=$status max_unacked=$window indices=$indices") + GearslipLog.i("$tag: <- Config status=$status max_unacked=$window indices=$indices") if (window != null) maxUnacked = maxOf(1, window) configIndex = indices.firstOrNull() ?: 0 if (status == STATUS_READY) { configTimer?.interrupt() ready = true - GearslipLog.i("audio: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") - CarAudio.attach(this) + GearslipLog.i("$tag: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit") + onReady(this) } else { GearslipLog.w( - "audio: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + + "$tag: the sink is not ready (status=$status, 1=WAIT 2=READY) - " + "nothing will be sent until it is", ) } @@ -133,7 +142,7 @@ class AudioLink( else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) { useAdvertisedSink("<- empty Setup answer (id $messageId)") } else { - GearslipLog.i("audio: <- unhandled message id=$messageId (${body.size} bytes)") + GearslipLog.i("$tag: <- unhandled message id=$messageId (${body.size} bytes)") GearslipLog.hex(" audio msg", body, limit = 64) } } @@ -146,7 +155,7 @@ class AudioLink( */ fun onFocus(state: Int) { val name = FOCUS_STATES[state] ?: "unknown($state)" - GearslipLog.i("audio: <- AudioFocusResponse state=$name") + GearslipLog.i("$tag: <- AudioFocusResponse state=$name") focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT || state == FOCUS_STATE_GAIN_MEDIA_ONLY if (focusGranted) startStream() else stop() @@ -154,10 +163,10 @@ class AudioLink( override fun start() { if (!ready || started) return - if (!focusRequested) { + if (asksFocus && !focusRequested) { focusRequested = true sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong())) - GearslipLog.i("audio: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") + GearslipLog.i("$tag: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over") } // Some head units grant the sink silently and never answer. Do not let that be the end of // it: start anyway, and let the acks say whether anything is being taken. @@ -168,12 +177,12 @@ class AudioLink( if (!ready || started) return sendOnChannel( MSG_START, - Protobuf.varintField(1, SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()), + Protobuf.varintField(1, sessionId.toLong()) + Protobuf.varintField(2, configIndex.toLong()), ) inFlight = 0 lastAckAt = System.currentTimeMillis() started = true - GearslipLog.i("audio: -> Start(session=$SESSION_ID, config=$configIndex) - streaming ${sampleRate}Hz x$channels") + GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels") } override fun stop() { @@ -181,12 +190,12 @@ class AudioLink( started = false synchronized(ackLock) { ackLock.notifyAll() } runCatching { - sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong())) + sendOnChannel(MSG_STOP, Protobuf.varintField(1, sessionId.toLong())) if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong())) } focusRequested = false focusGranted = false - GearslipLog.i("audio: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") + GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes") chunks = 0 bytes = 0 } @@ -200,7 +209,11 @@ class AudioLink( val waited = System.currentTimeMillis() - lastAckAt if (waited > STALL_MS) { // A lost ack must not silence the stream for good; after a stall assume the window cleared. - GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window") + // A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so + // once, then now and then, so the rest of the log stays readable. + if (stalls++ % 500 == 0) { + GearslipLog.w("$tag: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)") + } inFlight = 0 break } @@ -217,7 +230,7 @@ class AudioLink( try { sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size)) } catch (e: java.io.IOException) { - GearslipLog.e("audio: the link died mid-stream", e) + GearslipLog.e("$tag: the link died mid-stream", e) started = false return } @@ -228,7 +241,7 @@ class AudioLink( // the car is still taking audio and to catch a window that has quietly closed. if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) { GearslipLog.i( - "audio: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + + "$tag: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " + "dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}", ) } @@ -239,7 +252,7 @@ class AudioLink( configTimer = null stop() ready = false - CarAudio.detach(this) + onClose(this) } companion object { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt index 1236601..c3ee465 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarMic.kt @@ -12,17 +12,21 @@ import java.util.concurrent.TimeUnit * * -> CHANNEL_OPEN_REQUEST <- CHANNEL_OPEN_RESPONSE * -> Setup { config_index } <- Config + * -> VoiceSessionNotification { START } (control channel) * -> MicrophoneRequest { open } <- MicrophoneResponse { status, session_id } * <- Data { 8-byte timestamp, PCM }... -> Ack { session_id, ack } * -> MicrophoneRequest { open = false } + * -> VoiceSessionNotification { END } * - * The channel opens with the session, but the car only records between [start] and [stop]. + * The channel opens with the session, but the car only records between [start] and [stop], and + * only inside a voice session: the 2024 VW answered a bare MicrophoneRequest with status -7. * Speech wants 16 kHz mono; a car that records at a multiple of that is averaged down to it, and * one that doesn't isn't used, so voice falls back to the phone's microphone. */ class CarMic( private val service: ServiceDiscovery.AudioService, private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit, + private val voiceSession: (start: Boolean) -> Unit, ) { val channelId get() = service.serviceId @@ -43,7 +47,8 @@ class CarMic( val status = Protobuf.readInt32Field(body, 1) ?: 0 GearslipLog.i("mic: <- ChannelOpenResponse: status=$status") if (status != 0) return - sendOnChannel(MSG_SETUP, Protobuf.varintField(1, 0L)) + // Setup { type = the codec }, as for audio: 0 got UNEXPECTED_MESSAGE from the VW. + sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong())) ready = usable if (!usable) { GearslipLog.w( @@ -97,6 +102,7 @@ class CarMic( chunks.clear() packets = 0 listening = true + voiceSession(true) sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 1L) + Protobuf.varintField(4, 1L)) GearslipLog.i("mic: -> MicrophoneRequest(open)") return true @@ -109,7 +115,8 @@ class CarMic( if (!listening) return listening = false sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 0L)) - GearslipLog.i("mic: -> MicrophoneRequest(close) after $packets packets") + voiceSession(false) + GearslipLog.i("mic: -> MicrophoneRequest(close) after $packets packets, voice session ended") } companion object { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt index aaa448c..43e6ca2 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt @@ -2,6 +2,7 @@ package app.seb3thehacker.gearslip.car import android.content.Context import app.seb3thehacker.gearslip.media.CarMedia +import app.seb3thehacker.gearslip.speech.VoiceFocus import java.text.DateFormat import java.util.Date import kotlinx.coroutines.Dispatchers @@ -26,6 +27,7 @@ object AssistantCommands { VoiceHelp.show() return "Here's what you can say." } + if (heard.matches(Regex("(fuck|f\\*+) (you|off)( too)?"))) return "Fuck you too." navigateCommand(context, navigator, frame, heard)?.let { return it } playSearchCommand(context, navigator, heard)?.let { return it } @@ -123,6 +125,10 @@ object AssistantCommands { break } } + // "the nearest gas station" -> "gas station": the map app already searches around the car + // (see CarMotion.nearHere), and Organic Maps took "nearest" as part of the name. + val (place, nearby) = plainPlace(destination) + destination = place if (destination.isBlank()) return "Where to?" val chosen = chosenApp ?: when { @@ -133,7 +139,23 @@ object AssistantCommands { CarServices.connectNav(template, frame, destination) navigator.home() - return "Navigating to $destination with ${chosen.label}." + return if (nearby) "Finding the nearest $destination with ${chosen.label}." else "Navigating to $destination with ${chosen.label}." + } + + /** [destination] without "a", "the nearest", "near me" and the like, and whether any of that said "close by". */ + private fun plainPlace(destination: String): Pair { + var t = destination + var nearby = false + var changed = true + while (changed) { + changed = false + for (lead in PLACE_LEADS) if (t.startsWith(lead)) { + t = t.removePrefix(lead).trim(); changed = true + if (lead.trim() !in ARTICLES) nearby = true + } + for (tail in PLACE_TAILS) if (t.endsWith(tail)) { t = t.removeSuffix(tail).trim(); changed = true; nearby = true } + } + return t to nearby } /** @@ -194,7 +216,8 @@ object AssistantCommands { val media = CarServices.media return when { "pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> { - if (media.now.value.playing) media.togglePlay() + media.pause() + VoiceFocus.afterRelease { media.pause() } "Paused." } "resume" in heard || "unpause" in heard || heard in RESUME_PHRASES || @@ -315,8 +338,18 @@ object AssistantCommands { "navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ", "take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ", "how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ", + // Last, so "navigate to" wins: the recognizer sometimes drops the "to". + "navigate me ", "navigate ", "find me the nearest ", "find the nearest ", "find the closest ", ) + private val ARTICLES = setOf("a", "an", "the", "some") + // Longest first, so "the nearest " goes before "the ". + private val PLACE_LEADS = listOf( + "the nearest ", "the closest ", "a nearby ", "the nearby ", "nearest ", "closest ", "nearby ", + "a ", "an ", "some ", + ) + private val PLACE_TAILS = listOf(" near me", " nearby", " near here", " around here", " close by", " closest to me") + private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ") private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ") diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt index a4992dc..587c4ea 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt @@ -65,6 +65,7 @@ object CarServices { app.seb3thehacker.gearslip.notify.CarMotion.start(appContext) app.seb3thehacker.gearslip.call.CarCalls.start(appContext) CarAssistant.init(appContext) + DirectionsVoice.init(appContext) CarKeys.init(appContext) VoiceReply.init(appContext) // Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest. diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index a0b3024..43c505c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -28,6 +28,7 @@ object CarSettings { private const val KEY_OPEN = "car_open_on_connect" private const val KEY_NIGHT = "car_night_mode" private const val KEY_PIPE_AUDIO = "car_pipe_audio" +private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions" private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked" private const val KEY_LAST_NAV = "car_last_nav" private const val KEY_LAST_MEDIA = "car_last_media" @@ -58,6 +59,7 @@ object CarSettings { private val openFlow = MutableStateFlow(HOME) private val nightFlow = MutableStateFlow(NightMode.AUTO) private val pipeAudioFlow = MutableStateFlow(true) + private val speakDirectionsFlow = MutableStateFlow(true) private val lastNavFlow = MutableStateFlow(null) private val lastMediaFlow = MutableStateFlow(null) private val autoplayFlow = MutableStateFlow(false) @@ -153,6 +155,9 @@ object CarSettings { */ val pipeAudio: StateFlow = pipeAudioFlow + /** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */ + val speakDirections: StateFlow = speakDirectionsFlow + /** * Whether the driver has already been offered Bluetooth after turning down the capture * dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged. @@ -205,6 +210,7 @@ object CarSettings { autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true" playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true" pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true" + speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, "true") == "true" nightFlow.value = runCatching { NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name)) }.getOrDefault(NightMode.AUTO) @@ -274,6 +280,11 @@ object CarSettings { if (on) CarAudio.request() else CarAudio.release() } + fun setSpeakDirections(on: Boolean) { + speakDirectionsFlow.value = on + AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString()) + } + fun setExperimentalFeaturesEnabled(on: Boolean) { experimentalFeaturesFlow.value = on AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString()) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index 2fec3e0..e14ec7a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -53,6 +53,7 @@ fun CarSettingsScreen() { val units by CarSettings.units.collectAsState() val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState() + val speakDirections by CarSettings.speakDirections.collectAsState() val autoplay by CarSettings.autoplay.collectAsState() val playerTitle by CarSettings.playerTitle.collectAsState() val theme by CarSettings.appTheme.collectAsState() @@ -137,6 +138,18 @@ fun CarSettingsScreen() { color = MaterialTheme.colorScheme.onSurfaceVariant, ) + ChoiceRow( + "Spoken directions", + listOf(false to "Off", true to "On"), + speakDirections, + ) { CarSettings.setSpeakDirections(it) } + Text( + "Gearslip reads each turn aloud through the car's speakers, and the music dips while it " + + "speaks. Turn it off if your map app's own voice already plays in the car.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + ExperimentalFeatureRow( title = "Use the phone's speech recognition", description = "Listens through the phone's own speech service instead of Gearslip's. On " + diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt index fda6c4c..8a4f8f6 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/ClusterNav.kt @@ -60,23 +60,8 @@ class ClusterNav( running = false } - /** One turn as the car wants it, from whichever source the map app fills. */ - private class Turn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) - - private fun current(): Turn? { - val nav = CarServices.navOrNull() ?: return null - nav.trip.value?.let { trip -> - val step = trip.steps.firstOrNull() ?: return@let - val estimate = trip.stepTravelEstimates.firstOrNull() - return Turn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading) - } - val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null - val step = info.currentStep ?: return null - return Turn(step, info.currentDistance, null, info.isLoading) - } - private fun tick() { - val turn = current() + val turn = NavTurn.current() val status = when { turn == null -> STATUS_INACTIVE turn.rerouting -> STATUS_REROUTING @@ -114,7 +99,7 @@ class ClusterNav( * 7 yards), so the app's own display choice passes straight through. */ private fun sendDistance(distance: Distance, seconds: Long?) { - val meters = toMeters(distance) + val meters = NavTurn.toMeters(distance) val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}" if (key == sentDistance) return sentDistance = key @@ -127,14 +112,6 @@ class ClusterNav( ) } - private fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) { - Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0 - Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344 - Distance.UNIT_FEET -> 0.3048 - Distance.UNIT_YARDS -> 0.9144 - else -> 1.0 - }).toInt() - private fun CarText?.text() = this?.toString().orEmpty() companion object { @@ -194,3 +171,29 @@ class ClusterNav( } } } + +/** One turn as the car wants it, from whichever source the map app fills. */ +internal class NavTurn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) { + companion object { + /** The map app's next turn: from its trip (what clusters get), or its routing card. */ + fun current(): NavTurn? { + val nav = CarServices.navOrNull() ?: return null + nav.trip.value?.let { trip -> + val step = trip.steps.firstOrNull() ?: return@let + val estimate = trip.stepTravelEstimates.firstOrNull() + return NavTurn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading) + } + val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null + val step = info.currentStep ?: return null + return NavTurn(step, info.currentDistance, null, info.isLoading) + } + + fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) { + Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0 + Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344 + Distance.UNIT_FEET -> 0.3048 + Distance.UNIT_YARDS -> 0.9144 + else -> 1.0 + }).toInt() + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt new file mode 100644 index 0000000..6b9abf3 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/DirectionsVoice.kt @@ -0,0 +1,326 @@ +package app.seb3thehacker.gearslip.car + +import android.content.Context +import android.os.Bundle +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import androidx.car.app.model.CarText +import androidx.car.app.model.Distance +import androidx.car.app.navigation.model.Maneuver +import androidx.car.app.navigation.model.Step +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.audio.AudioLink +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import kotlin.math.roundToInt + +/** + * Spoken turn-by-turn on the car's directions channel (the GUIDANCE audio sink). + * + * A map app on the phone speaks its own prompts as navigation audio, which Android can't hand to + * Gearslip without root, and which the phone mutes while USB audio runs. So Gearslip reads the + * same turn the instrument cluster gets ([NavTurn]) and says it itself: once when the turn comes + * up, again a kilometre or so out on a long stretch, and once more just before it. The words are + * rendered with the phone's text-to-speech to a file, never played on the phone, and streamed to + * the car, which ducks the music under them the way it does for real Android Auto. + * + * Nothing spoken is logged: road names are where the driver is going. + */ +object DirectionsVoice { + + private enum class Stage { FAR, PREPARED, DONE } + + private var appContext: Context? = null + private var tts: TextToSpeech? = null + @Volatile private var ttsReady = false + private val pending = ConcurrentHashMap() + + @Volatile private var link: AudioLink? = null + @Volatile private var running = false + private val speaker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } } + @Volatile private var latest = 0 + + private var turnKey: String? = null + private var stage = Stage.DONE + + fun init(context: Context) { + if (tts != null) return + appContext = context.applicationContext + tts = TextToSpeech(appContext) { status -> ttsReady = status == TextToSpeech.SUCCESS }.apply { + setOnUtteranceProgressListener(object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) {} + override fun onDone(utteranceId: String?) { pending.remove(utteranceId)?.countDown() } + @Deprecated("Deprecated in Java, no replacement on this API level") + override fun onError(utteranceId: String?) { pending.remove(utteranceId)?.countDown() } + }) + } + } + + /** The car opened its directions sink: start following the map app's turns. */ + fun attach(link: AudioLink) { + this.link = link + GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}") + if (running) return + running = true + turnKey = null + Thread({ + while (running) { + runCatching { tick() }.onFailure { GearslipLog.w("guidance: ${it.message}") } + Thread.sleep(TICK_MS) + } + }, "gearslip-directions-watch").apply { isDaemon = true }.start() + } + + fun detach(link: AudioLink) { + if (this.link !== link) return + this.link = null + running = false + } + + private fun tick() { + if (!CarSettings.speakDirections.value) return + val turn = NavTurn.current() + if (turn == null) { + turnKey = null + return + } + if (turn.rerouting) return + val step = turn.step + val instruction = instruction(step) ?: return + val meters = turn.distance?.let(NavTurn::toMeters) + val close = (turn.seconds != null && turn.seconds <= NEAR_SECONDS) || (meters != null && meters <= NEAR_METERS) + + val key = "${step.road.text()}|${step.maneuver?.type}|${step.maneuver?.roundaboutExitNumber}|${step.cue.text()}" + if (key != turnKey) { + turnKey = key + if (step.maneuver?.type == Maneuver.TYPE_DEPART) { + stage = Stage.DONE + return + } + if (close || turn.distance == null) { + say(instruction) + stage = Stage.DONE + } else { + say("In ${spoken(turn.distance)}, $instruction") + stage = if (meters != null && meters > PREPARE_FROM_METERS) Stage.FAR else Stage.PREPARED + } + return + } + when { + stage != Stage.DONE && close -> { + say(instruction) + stage = Stage.DONE + } + stage == Stage.FAR && meters != null && meters <= PREPARE_AT_METERS -> { + say("In ${spoken(turn.distance)}, $instruction") + stage = Stage.PREPARED + } + } + } + + /** What to do, in words: built from the manoeuvre when the app names one, else the app's own text. */ + private fun instruction(step: Step): String? { + val road = step.road.text() + val onto = if (road.isBlank()) "" else " onto $road" + val toward = if (road.isBlank()) "" else " toward $road" + val maneuver = step.maneuver + val built = when (maneuver?.type) { + Maneuver.TYPE_TURN_SLIGHT_LEFT -> "Turn slightly left$onto" + Maneuver.TYPE_TURN_SLIGHT_RIGHT -> "Turn slightly right$onto" + Maneuver.TYPE_TURN_NORMAL_LEFT -> "Turn left$onto" + Maneuver.TYPE_TURN_NORMAL_RIGHT -> "Turn right$onto" + Maneuver.TYPE_TURN_SHARP_LEFT -> "Turn sharply left$onto" + Maneuver.TYPE_TURN_SHARP_RIGHT -> "Turn sharply right$onto" + Maneuver.TYPE_U_TURN_LEFT, Maneuver.TYPE_U_TURN_RIGHT -> "Make a U-turn" + Maneuver.TYPE_KEEP_LEFT -> "Keep left$toward" + Maneuver.TYPE_KEEP_RIGHT -> "Keep right$toward" + Maneuver.TYPE_FORK_LEFT -> "Keep left at the fork$toward" + Maneuver.TYPE_FORK_RIGHT -> "Keep right at the fork$toward" + Maneuver.TYPE_ON_RAMP_SLIGHT_LEFT, Maneuver.TYPE_ON_RAMP_NORMAL_LEFT, Maneuver.TYPE_ON_RAMP_SHARP_LEFT, + Maneuver.TYPE_ON_RAMP_U_TURN_LEFT -> "Take the ramp on the left$onto" + Maneuver.TYPE_ON_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_ON_RAMP_NORMAL_RIGHT, Maneuver.TYPE_ON_RAMP_SHARP_RIGHT, + Maneuver.TYPE_ON_RAMP_U_TURN_RIGHT -> "Take the ramp on the right$onto" + Maneuver.TYPE_OFF_RAMP_SLIGHT_LEFT, Maneuver.TYPE_OFF_RAMP_NORMAL_LEFT -> "Take the exit on the left$toward" + Maneuver.TYPE_OFF_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_OFF_RAMP_NORMAL_RIGHT -> "Take the exit on the right$toward" + Maneuver.TYPE_MERGE_LEFT -> "Merge left$onto" + Maneuver.TYPE_MERGE_RIGHT -> "Merge right$onto" + Maneuver.TYPE_MERGE_SIDE_UNSPECIFIED -> "Merge$onto" + Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW_WITH_ANGLE, + Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW_WITH_ANGLE -> + maneuver.roundaboutExitNumber.takeIf { it in 1..ORDINALS.size } + ?.let { "At the roundabout, take the ${ORDINALS[it - 1]} exit$onto" } + ?: "Go through the roundabout$onto" + Maneuver.TYPE_ROUNDABOUT_ENTER_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_CCW -> "Enter the roundabout" + Maneuver.TYPE_ROUNDABOUT_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_EXIT_CCW -> "Exit the roundabout$onto" + Maneuver.TYPE_STRAIGHT -> "Continue straight$onto" + Maneuver.TYPE_NAME_CHANGE -> "Continue$onto" + Maneuver.TYPE_FERRY_BOAT, Maneuver.TYPE_FERRY_BOAT_LEFT, Maneuver.TYPE_FERRY_BOAT_RIGHT -> "Take the ferry" + Maneuver.TYPE_FERRY_TRAIN, Maneuver.TYPE_FERRY_TRAIN_LEFT, Maneuver.TYPE_FERRY_TRAIN_RIGHT -> "Take the train" + Maneuver.TYPE_DESTINATION, Maneuver.TYPE_DESTINATION_STRAIGHT -> "Your destination is ahead" + Maneuver.TYPE_DESTINATION_LEFT -> "Your destination is on the left" + Maneuver.TYPE_DESTINATION_RIGHT -> "Your destination is on the right" + else -> null + } + return built ?: step.cue.text().ifBlank { null } + } + + /** The app's own rounded distance, read the way a person would say it. */ + private fun spoken(d: Distance): String { + val n = d.displayDistance + val number = if (n % 1.0 == 0.0) n.toInt().toString() else "%.1f".format(n) + val one = n == 1.0 + return when (d.displayUnit) { + Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> if (one) "1 kilometer" else "$number kilometers" + Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> when (n) { + 0.25 -> "a quarter mile" + 0.5 -> "half a mile" + 0.75 -> "three quarters of a mile" + 1.0 -> "1 mile" + else -> "$number miles" + } + Distance.UNIT_FEET -> "$number feet" + Distance.UNIT_YARDS -> "$number yards" + else -> "${n.roundToInt()} meters" + } + } + + /** Newest wins: a prompt still waiting when the next one arrives is already out of date. */ + private fun say(text: String) { + val id = ++latest + speaker.execute { + if (id != latest) return@execute + runCatching { play(text, id) }.onFailure { GearslipLog.w("guidance: could not speak a turn: ${it.message}") } + } + } + + private fun play(text: String, id: Int) { + val engine = tts ?: return + val context = appContext ?: return + val sink = link ?: return + if (!ttsReady) { + GearslipLog.w("guidance: text-to-speech isn't ready, a turn went unspoken") + return + } + val file = File(context.cacheDir, "directions.wav") + val utterance = "gearslip-directions-$id" + val done = CountDownLatch(1) + pending[utterance] = done + if (engine.synthesizeToFile(text, Bundle(), file, utterance) != TextToSpeech.SUCCESS) { + pending.remove(utterance) + return + } + if (!done.await(SYNTH_TIMEOUT_S, TimeUnit.SECONDS)) { + pending.remove(utterance) + GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken") + return + } + val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run { + GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read") + return + } + stream(sink, pcm) + GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)") + } + + /** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */ + private fun stream(sink: AudioLink, pcm: ByteArray) { + val frame = 2 * sink.channels + val chunk = sink.sampleRate / 50 * frame + val chunkMs = 20L + val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped + sink.start() + try { + val began = System.currentTimeMillis() + var offset = 0 + var sent = 0 + while (offset < padded.size && link === sink) { + val size = minOf(chunk, padded.size - offset) + sink.write(padded.copyOfRange(offset, offset + size), size) + offset += size + sent++ + val ahead = sent * chunkMs - (System.currentTimeMillis() - began) + if (ahead > LEAD_MS) Thread.sleep(ahead - LEAD_MS) + } + // Let the car play what it has before closing the stream on it. + Thread.sleep(LEAD_MS) + } finally { + sink.stop() + } + } + + private fun CarText?.text() = this?.toString().orEmpty() + + /** 16-bit PCM from the WAV text-to-speech writes, reshaped for the car's sink. */ + private object Wav { + class Pcm(val samples: ShortArray, val rate: Int, val channels: Int) + + fun read(file: File): Pcm? { + val bytes = file.readBytes() + if (bytes.size < 12 || String(bytes, 0, 4) != "RIFF" || String(bytes, 8, 4) != "WAVE") return null + val buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN) + var pos = 12 + var rate = 0 + var channels = 0 + var bits = 0 + while (pos + 8 <= bytes.size) { + val name = String(bytes, pos, 4) + val size = buf.getInt(pos + 4) + val body = pos + 8 + // Some engines write a streaming WAV with a zero or oversized data length. + val end = if (size <= 0 || body + size > bytes.size) bytes.size else body + size + when (name) { + "fmt " -> { + channels = buf.getShort(body + 2).toInt() + rate = buf.getInt(body + 4) + bits = buf.getShort(body + 14).toInt() + } + "data" -> { + if (bits != 16 || rate <= 0 || channels <= 0) return null + val samples = ShortArray((end - body) / 2) + buf.position(body) + buf.asShortBuffer().get(samples, 0, samples.size) + return Pcm(samples, rate, channels) + } + } + pos = end + (end and 1) + } + return null + } + + /** Mixes to mono, resamples linearly, then copies to each of the car's channels. */ + fun convert(pcm: Pcm, rate: Int, channels: Int): ByteArray { + val frames = pcm.samples.size / pcm.channels + val mono = FloatArray(frames) { f -> + var sum = 0f + for (c in 0 until pcm.channels) sum += pcm.samples[f * pcm.channels + c] + sum / pcm.channels + } + val outFrames = (frames.toLong() * rate / pcm.rate).toInt() + val out = ByteBuffer.allocate(outFrames * channels * 2).order(ByteOrder.LITTLE_ENDIAN) + val step = pcm.rate.toDouble() / rate + for (i in 0 until outFrames) { + val at = i * step + val j = at.toInt().coerceAtMost(frames - 1) + val next = (j + 1).coerceAtMost(frames - 1) + val v = mono[j] + (mono[next] - mono[j]) * (at - j).toFloat() + val s = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort() + repeat(channels) { out.putShort(s) } + } + return out.array() + } + } + + private const val TICK_MS = 500L + private const val NEAR_SECONDS = 12L + private const val NEAR_METERS = 120 + private const val PREPARE_FROM_METERS = 2_500 + private const val PREPARE_AT_METERS = 1_000 + private const val LEAD_MS = 100L + private const val SYNTH_TIMEOUT_S = 10L + private val ORDINALS = listOf("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth") +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsTheme.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsTheme.kt index e16aeab..2f284fe 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsTheme.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsTheme.kt @@ -112,7 +112,7 @@ object GsThemes { control = ControlLook(holdMarker = HoldMarkerStyle.DOT), ) - val default = Tactile + val default = Flat val builtIn = listOf(Tactile, Keycap, Flat) fun byId(id: String?): GsTheme = builtIn.firstOrNull { it.id == id } ?: default diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt index eaa9006..4f77dca 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt @@ -433,6 +433,12 @@ class CarMedia(private val context: Context) { val controls = controller?.transportControls ?: return if (_now.value.playing) controls.pause() else controls.play() } + /** + * Pauses whatever the app thinks it's doing. Voice pauses the music to listen, so by the time + * "pause" is heard the app already looks stopped; a toggle did nothing and the song came back + * as soon as the microphone let go. A real pause tells the app to stay stopped. + */ + fun pause() = controller?.transportControls?.pause() fun next() = controller?.transportControls?.skipToNext() fun previous() = controller?.transportControls?.skipToPrevious() fun seek(ms: Long) = controller?.transportControls?.seekTo(ms) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt index 74437fb..11a2957 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt @@ -15,8 +15,10 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream /** * The speech models Gearslip listens with, in two roles: a streaming one that writes the words * while the driver talks, and an optional final one that reads the whole phrase again once they - * stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's - * GitHub releases, where its authors publish it, and kept in the app's own storage. + * stop, for a more accurate result. None ships in the APK: each is fetched once and kept in the + * app's own storage. sherpa-onnx's author publishes them twice: as .tar.bz2 archives on GitHub and + * as plain files on Hugging Face. The plain files come first where they exist. Unpacking bzip2 on + * the phone pinned a core and held a download to 150 KB/s on Wi-Fi that does 19 MB/s. */ object SpeechModels { @@ -34,9 +36,11 @@ object SpeechModels { /** Empty for Canary, which decodes with attention and has no joiner. */ val joiner: String, val modelType: String = "", + /** Whether Hugging Face has the files unpacked under the archive's name. */ + val plainFiles: Boolean = true, ) { KROKO( - Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54, + Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 68, "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06", "encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2", ), @@ -44,20 +48,22 @@ object SpeechModels { Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103, "sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8", "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer", + plainFiles = false, ), CANARY( - Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 154, + Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 198, "sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8", "encoder.int8.onnx", "decoder.int8.onnx", "", modelType = "canary", ), PARAKEET_LARGE( - Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460, + Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 631, "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer", ), ; val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2" + fun fileUrl(name: String) = "https://huggingface.co/csukuangfj/$archive/resolve/main/$name" val files get() = listOf(encoder, decoder, joiner, "tokens.txt").filter { it.isNotEmpty() } } @@ -123,7 +129,7 @@ object SpeechModels { if (model == _selected.value || model == _final.value) SpeechEngine.unload() } - /** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */ + /** Fetches just the files the recognizer reads, plain where it can, else out of the archive. */ fun download(context: Context, model: Model) { val app = context.applicationContext val state = states.getValue(model) @@ -134,23 +140,33 @@ object SpeechModels { val target = dir(app, model) val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() } try { - val conn = (URL(model.url).openConnection() as HttpURLConnection).apply { - connectTimeout = 15_000 - readTimeout = 30_000 - instanceFollowRedirects = true - } - check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" } - val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L) - val counting = CountingStream(conn.inputStream) { read -> + val total = model.downloadMb * 1_048_576L + var done = 0L + fun progress(read: Long) { if (cancelled == model) throw InterruptedException("cancelled") - state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99)) + state.value = State.Downloading((((done + read) * 100) / total).toInt().coerceIn(0, 99)) } - TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar -> - while (true) { - val entry = tar.nextEntry ?: break - val name = entry.name.substringAfterLast('/') - if (entry.isDirectory || name !in model.files) continue - File(partial, name).outputStream().use { tar.copyTo(it) } + if (model.plainFiles) { + for (name in model.files) { + val conn = open(model.fileUrl(name)) + val length = conn.contentLengthLong + val file = File(partial, name) + CountingStream(conn.inputStream, ::progress).use { input -> + file.outputStream().use { input.copyTo(it, 256 * 1024) } + } + check(length <= 0 || file.length() == length) { "$name arrived cut short" } + done += file.length() + } + } else { + val conn = open(model.url) + val counting = CountingStream(conn.inputStream, ::progress) + TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar -> + while (true) { + val entry = tar.nextEntry ?: break + val name = entry.name.substringAfterLast('/') + if (entry.isDirectory || name !in model.files) continue + File(partial, name).outputStream().use { tar.copyTo(it) } + } } } check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" } @@ -170,6 +186,13 @@ object SpeechModels { } } + private fun open(url: String): HttpURLConnection = + (URL(url).openConnection() as HttpURLConnection).apply { + connectTimeout = 15_000 + readTimeout = 30_000 + instanceFollowRedirects = true + }.also { check(it.responseCode == 200) { "the server answered ${it.responseCode}" } } + /** Counts bytes as they pass, reporting about every megabyte. */ private class CountingStream( private val inner: java.io.InputStream, diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/VoiceFocus.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/VoiceFocus.kt index e79f2e5..1b16b8f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/VoiceFocus.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/VoiceFocus.kt @@ -19,17 +19,35 @@ import app.seb3thehacker.gearslip.GearslipLog object VoiceFocus { private const val RELEASE_DELAY_MS = 1_500L + private const val AFTER_RELEASE_MS = 300L private val main = Handler(Looper.getMainLooper()) private var request: AudioFocusRequest? = null private var audio: AudioManager? = null private var holders = 0 + private val afterRelease = mutableListOf<() -> Unit>() private val releaseNow = Runnable { - val r = request ?: return@Runnable - audio?.abandonAudioFocusRequest(r) - request = null - GearslipLog.i("voice: gave audio focus back") + request?.let { r -> + audio?.abandonAudioFocusRequest(r) + request = null + GearslipLog.i("voice: gave audio focus back") + } + val pending = afterRelease.toList() + afterRelease.clear() + // Just after the music app has had its focus back and acted on it. + if (pending.isNotEmpty()) main.postDelayed({ pending.forEach { it() } }, AFTER_RELEASE_MS) + } + + /** + * Runs [action] once the music has its focus back, or now if Gearslip isn't holding it. + * Metrolist takes a pause sent while it's held as nothing and resumes when focus returns, + * so "pause" sends it again from here. + */ + fun afterRelease(action: () -> Unit) { + main.post { + if (request == null && holders == 0) action() else afterRelease += action + } } /** Pauses the music until a matching [release]. Safe from any thread. */ diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt index 84b94be..a29d94b 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt @@ -75,6 +75,17 @@ import androidx.compose.ui.graphics.BlendMode import androidx.compose.ui.graphics.Brush import androidx.compose.ui.graphics.CompositingStrategy import androidx.compose.foundation.layout.wrapContentHeight +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.PlayArrow +import androidx.compose.material.icons.filled.Search +import androidx.compose.runtime.CompositionLocalProvider +import app.seb3thehacker.gearslip.car.CarSettings +import app.seb3thehacker.gearslip.car.theme.GsButton +import app.seb3thehacker.gearslip.car.theme.GsIconButton +import app.seb3thehacker.gearslip.car.theme.GsThemes +import app.seb3thehacker.gearslip.car.theme.GsTone +import app.seb3thehacker.gearslip.car.theme.LocalGsTheme import app.seb3thehacker.gearslip.speech.SpeechModels import app.seb3thehacker.gearslip.stats.UsageStats import kotlin.math.PI @@ -106,14 +117,14 @@ object Intro { } /** The intro's tones, taken from the phone's own Material You colours like the rest of the app. */ -private class IntroColors( +internal class IntroColors( val surface: Color, val onSurface: Color, val onVariant: Color, val outline: Color, val primary: Color, val onPrimary: Color, val primaryC: Color, val onPrimaryC: Color, val tertiaryC: Color, val onTertiaryC: Color, val secondaryC: Color, val onSecondaryC: Color, ) @Composable -private fun introColors(): IntroColors { +internal fun introColors(): IntroColors { val m = MaterialTheme.colorScheme return IntroColors( surface = m.surface, onSurface = m.onSurface, onVariant = m.onSurfaceVariant, outline = m.outline, @@ -123,7 +134,7 @@ private fun introColors(): IntroColors { ) } -private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, DRIVE } +private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, THEME, DRIVE } private val Pop = CubicBezierEasing(0.3f, 1.6f, 0.5f, 1f) private val Rise = CubicBezierEasing(0.2f, 0.8f, 0.2f, 1f) @@ -183,6 +194,9 @@ fun IntroScreen(onDone: () -> Unit) { IntroStep.APPS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Good company.", "Free, open-source apps that work well in the car. Tap one to see it on GitHub.", step, extra = { AppPicks(c, ::cq, ::cqSp, step) }) { StarIcon(c.onSecondaryC) } + IntroStep.THEME -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Your buttons.", + "Pick how the car's buttons look. Change it any time in the car's settings.", step, + extra = { ThemePicker(c, ::cq, ::cqSp, step) }) { ButtonsIcon(c.onTertiaryC) } IntroStep.DRIVE -> BadgeStep(c, ::cq, ::cqSp, c.primaryC, "Let's drive.", "That's ${Intro.release}. Plug in, or just get in.", step, action = "Start" to finish) { CheckIcon(c.onPrimaryC) } @@ -195,7 +209,7 @@ fun IntroScreen(onDone: () -> Unit) { /** 0 to 1 once each time [key] changes, after [delayMs]: drives the entrance of one element. */ @Composable -private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float { +internal fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float { val a = remember(key) { Animatable(0f) } LaunchedEffect(key) { delay(delayMs.toLong()) @@ -205,14 +219,14 @@ private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float { } /** Slides up and fades in, like the mockup's `up`. */ -private fun Modifier.up(p: Float, distance: Dp) = graphicsLayer { +internal fun Modifier.up(p: Float, distance: Dp) = graphicsLayer { val e = Rise.transform(p) alpha = e translationY = (1 - e) * distance.toPx() } /** Grows in with a bounce, like the mockup's `bounce`. */ -private fun Modifier.pop(p: Float) = graphicsLayer { +internal fun Modifier.pop(p: Float) = graphicsLayer { val e = Pop.transform(p) alpha = p.coerceAtMost(1f) scaleX = 0.5f + 0.5f * e @@ -501,9 +515,79 @@ private fun AppPicks(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUni } } +/** + * The car's button styles side by side, each drawn in its own style, and a row of real car + * controls below that redraws in whichever is picked. Picking saves it, so the car shows it too. + */ +@Composable +private fun ThemePicker(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) { + val chosenId by CarSettings.uiTheme.collectAsState() + val chosen = GsThemes.byId(chosenId) + // Flat first: it's the default. + val themes = listOf(GsThemes.Flat) + GsThemes.builtIn.filter { it != GsThemes.Flat } + Column( + Modifier.padding(top = cq(7f)).fillMaxWidth().up(entrance(key, 500, 700), cq(7f)), + verticalArrangement = Arrangement.spacedBy(cq(6f)), + ) { + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(cq(3f))) { + themes.forEach { theme -> + CompositionLocalProvider(LocalGsTheme provides theme) { + GsButton( + onClick = { CarSettings.setUiTheme(theme.id) }, + modifier = Modifier.weight(1f), + tone = if (theme == chosen) GsTone.Primary else GsTone.Neutral, + contentPadding = PaddingValues(horizontal = cq(1f), vertical = cq(3f)), + ) { + Text( + if (theme == chosen) "✓ ${theme.name}" else theme.name, + style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.Bold), + maxLines = 1, + ) + } + } + } + } + // What the car shows: the player's controls and a full-width action. + CompositionLocalProvider(LocalGsTheme provides chosen) { + Row( + Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(cq(3f)), + verticalAlignment = Alignment.CenterVertically, + ) { + GsIconButton(Icons.Filled.Search, "Search", onClick = {}, size = cq(15f), iconSize = cq(7f)) + GsIconButton(Icons.Filled.PlayArrow, "Play", onClick = {}, tone = GsTone.Primary, size = cq(15f), iconSize = cq(8f)) + GsButton( + onClick = {}, + modifier = Modifier.weight(1f).height(cq(15f)), + tone = GsTone.Tonal, + ) { Text("Navigate", style = TextStyle(fontSize = cqSp(4.6f), fontWeight = FontWeight.Bold)) } + } + } + } +} + +/** Three keys of a keypad, pressing one after another. */ +@Composable +private fun ButtonsIcon(color: Color) { + val t by rememberInfiniteTransition(label = "keys").animateFloat( + 0f, 3f, infiniteRepeatable(tween(2400, easing = LinearEasing)), label = "keys", + ) + Canvas(Modifier.fillMaxSize()) { + inIconBox { + listOf(6f, 19f, 32f).forEachIndexed { i, x -> + // Each key dips for a moment while its turn comes round. + val dip = (1f - kotlin.math.abs(t - i - 0.5f) * 2f).coerceIn(0f, 1f) * 4f + drawRoundRect(color.copy(alpha = 0.45f), Offset(x, 16f + 4f), Size(10f, 16f), + androidx.compose.ui.geometry.CornerRadius(3f)) + drawRoundRect(color, Offset(x, 16f + dip), Size(10f, 16f), androidx.compose.ui.geometry.CornerRadius(3f)) + } + } + } +} + /** Material's scalloped "cookie": a circle whose radius waves around its edge. */ @Composable -private fun Cookie(color: Color, modifier: Modifier) { +internal fun Cookie(color: Color, modifier: Modifier) { Canvas(modifier) { val unit = size.minDimension / 100f val path = Path() @@ -578,7 +662,7 @@ private fun DialIcon(color: Color) { } @Composable -private fun MicIcon(color: Color) { +internal fun MicIcon(color: Color) { val t = rememberInfiniteTransition(label = "mic") val waves = (0..1).map { i -> t.animateFloat( @@ -605,7 +689,7 @@ private fun MicIcon(color: Color) { } @Composable -private fun StarIcon(color: Color) { +internal fun StarIcon(color: Color) { val turn by rememberInfiniteTransition(label = "star").animateFloat( -8f, 8f, infiniteRepeatable(tween(1600, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "star", ) @@ -626,7 +710,7 @@ private fun StarIcon(color: Color) { } @Composable -private fun ShieldIcon(color: Color, tick: Color) { +internal fun ShieldIcon(color: Color, tick: Color) { val p = entrance(IntroStep.GRAPHENE, 800, 500) Canvas(Modifier.fillMaxSize()) { inIconBox { @@ -643,7 +727,7 @@ private fun ShieldIcon(color: Color, tick: Color) { } @Composable -private fun HeartIcon(color: Color) { +internal fun HeartIcon(color: Color) { val beat by rememberInfiniteTransition(label = "beat").animateFloat( 1f, 1f, infiniteRepeatable(keyframes { @@ -696,7 +780,7 @@ private fun DrawScope.drawPartialStroke(points: List, progress: Float, c } @Composable -private fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) { +internal fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) { Row(modifier, horizontalArrangement = Arrangement.spacedBy(cq(2f))) { repeat(count) { i -> val on = i == current diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt index 5de6a36..10ce412 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt @@ -1,6 +1,38 @@ package app.seb3thehacker.gearslip.ui import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.systemBarsPadding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Build +import androidx.compose.material.icons.filled.Call +import androidx.compose.material.icons.filled.Check +import androidx.compose.material.icons.filled.Close +import androidx.compose.material.icons.filled.DateRange +import androidx.compose.material.icons.filled.Email +import androidx.compose.material.icons.filled.LocationOn +import androidx.compose.material.icons.filled.Notifications +import androidx.compose.material.icons.filled.Phone +import androidx.compose.material.icons.filled.Warning +import androidx.compose.material3.Icon +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.graphicsLayer +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.unit.Dp +import androidx.compose.ui.unit.TextUnit import androidx.compose.foundation.layout.heightIn import androidx.compose.foundation.rememberScrollState import androidx.compose.foundation.verticalScroll @@ -27,15 +59,7 @@ import androidx.compose.foundation.layout.fillMaxSize import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.padding -import androidx.compose.material3.Button -import androidx.compose.material3.FilledTonalButton -import androidx.compose.material3.LinearProgressIndicator -import androidx.compose.material3.MaterialTheme -import androidx.compose.material3.OutlinedButton -import androidx.compose.material3.Scaffold -import androidx.compose.material3.Surface import androidx.compose.material3.Text -import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableIntStateOf @@ -66,14 +90,23 @@ private class GuidePage( val title: String, /** Plain text, or an [AnnotatedString] when part of it needs to stand out. */ val explanation: CharSequence, + /** Drawn inside the page's badge, in the badge's content colour. */ + val icon: @Composable (Color) -> Unit, + /** Marked "Optional" above the title: everything works without it. */ + val optional: Boolean = false, val permissions: Array? = null, val command: String? = null, val actionLabel: String? = null, val onAction: (() -> Unit)? = null, - /** Drawn under [explanation], inside the card. */ + /** Drawn under [explanation]. */ val content: (@Composable () -> Unit)? = null, ) +/** A plain Material icon, sized to fill the badge's icon box. */ +private fun glyph(icon: ImageVector): @Composable (Color) -> Unit = { tint -> + Icon(icon, contentDescription = null, tint = tint, modifier = Modifier.fillMaxSize()) +} + /** True once every permission in [permissions] is already granted - nothing left for this page to do. */ private fun granted(context: Context, permissions: Array): Boolean = permissions.all { context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED } @@ -105,9 +138,11 @@ private fun hasNotificationAccess(context: Context): Boolean { * who set up before the notes existed see once. Leaving it without turning them on is a no. */ private fun usageNotesPage(context: Context) = GuidePage( - title = USAGE_STATS_TITLE, - explanation = USAGE_STATS_INTRO, + title = "Help other drivers.", + explanation = USAGE_STATS_SHORT, + icon = { HeartIcon(it) }, content = { + UsageStatsReadMore() UsageStatsChoices() DisposableEffect(Unit) { onDispose { if (!UsageStats.asked(context)) UsageStats.set(context, false) } @@ -119,11 +154,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit): List Unit, onDone: () -> Unit, onl context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED } ?: false - Scaffold(containerColor = MaterialTheme.colorScheme.background) { padding -> - Column( - Modifier - .fillMaxSize() - .padding(padding) - .padding(horizontal = 24.dp), - horizontalAlignment = Alignment.CenterHorizontally, - ) { - Spacer(Modifier.height(24.dp)) - // A single page has no progress to show. - if (pages.size > 1) { - LinearProgressIndicator( - progress = { (index + 1f) / pages.size }, - modifier = Modifier.fillMaxWidth().height(10.dp).clip(MaterialTheme.shapes.extraLarge), - ) - Spacer(Modifier.height(12.dp)) - Text( - "${index + 1} of ${pages.size}", - style = MaterialTheme.typography.titleMedium, - fontWeight = FontWeight.SemiBold, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - } - // The page scrolls between the progress bar and the buttons, so a tall page (the usage - // note's choices, say) never pushes Next off the screen. Short pages stay centred. + val c = introColors() + // The badges take turns through the phone's three accent colours, as the release intro does. + val (badge, onBadge) = listOf(c.primaryC to c.onPrimaryC, c.secondaryC to c.onSecondaryC, c.tertiaryC to c.onTertiaryC)[index % 3] + val spin by rememberInfiniteTransition(label = "spin").animateFloat( + 0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin", + ) + + BoxWithConstraints(Modifier.fillMaxSize().background(c.surface).systemBarsPadding()) { + val w = maxWidth + // One percent of the screen's width, as in the intro, so it scales the same way. + fun cq(n: Float): Dp = w * (n / 100f) + val density = LocalDensity.current + fun cqSp(n: Float): TextUnit = with(density) { cq(n).toSp() } + + Column(Modifier.fillMaxSize().padding(horizontal = cq(8f))) { + // The page scrolls above the buttons, so a tall one (the usage notes' choices) never + // pushes them off the screen. Short pages sit in the middle. BoxWithConstraints(Modifier.weight(1f).fillMaxWidth()) { key(index) { Column( @@ -357,88 +387,114 @@ fun SetupGuideScreen(onRequestCallScreening: () -> Unit, onDone: () -> Unit, onl .fillMaxWidth() .verticalScroll(rememberScrollState()) .heightIn(min = maxHeight) - .padding(vertical = 24.dp), + .padding(vertical = cq(6f)), verticalArrangement = Arrangement.Center, ) { - Surface(color = MaterialTheme.colorScheme.surfaceContainerLow, shape = MaterialTheme.shapes.extraLarge) { - Column(Modifier.fillMaxWidth().padding(24.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) { + Box(Modifier.size(cq(28f)).pop(entrance(index, 50, 800)), contentAlignment = Alignment.Center) { + Cookie(badge, Modifier.fillMaxSize().graphicsLayer { rotationZ = spin }) + Box(Modifier.size(cq(12f))) { page.icon(onBadge) } + } + Spacer(Modifier.height(cq(7f))) + if (page.optional) { + Text( + "OPTIONAL", color = c.primary, + style = TextStyle(fontSize = cqSp(3.4f), fontWeight = FontWeight.Bold, letterSpacing = cqSp(0.4f)), + modifier = Modifier.up(entrance(index, 150, 700), cq(7f)), + ) + Spacer(Modifier.height(cq(1.5f))) + } + Text( + page.title, color = c.onSurface, + // Long titles step down a size so they stay on two lines. + style = TextStyle( + fontSize = cqSp(if (page.title.length > 16) 8f else 10f), fontWeight = FontWeight.Black, + letterSpacing = cqSp(-0.4f), lineHeight = cqSp(if (page.title.length > 16) 8.6f else 10.5f), + ), + modifier = Modifier.up(entrance(index, 200, 700), cq(7f)), + ) + Spacer(Modifier.height(cq(4f))) + Column( + Modifier.up(entrance(index, 350, 700), cq(7f)), + verticalArrangement = Arrangement.spacedBy(cq(3f)), + ) { + Text( + page.explanation as? AnnotatedString ?: AnnotatedString(page.explanation.toString()), + color = c.onVariant, + style = TextStyle(fontSize = cqSp(4.5f), lineHeight = cqSp(6.5f)), + ) + page.command?.let { CopyableCommand(it) } + page.content?.invoke() + if (page.permissions != null && alreadyGranted) { Text( - page.title, - style = MaterialTheme.typography.headlineSmall, - fontWeight = FontWeight.Bold, - color = MaterialTheme.colorScheme.onSurface, + "Already allowed.", color = c.primary, + style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.SemiBold), ) - Text( - page.explanation as? AnnotatedString ?: AnnotatedString(page.explanation.toString()), - style = MaterialTheme.typography.bodyLarge, - color = MaterialTheme.colorScheme.onSurfaceVariant, - ) - page.command?.let { CopyableCommand(it) } - page.content?.invoke() } } } } } + // A single page has nothing to count. + if (pages.size > 1) { + Dots(c, pages.size, index, ::cq, Modifier.align(Alignment.CenterHorizontally)) + Spacer(Modifier.height(cq(5f))) + } + val back = if (index > 0) ({ index -= 1 }) else null + val next = { index += 1 } when { - page.permissions != null && alreadyGranted -> { - Text( - "Already allowed.", - style = MaterialTheme.typography.bodyMedium, - color = MaterialTheme.colorScheme.primary, - ) - Spacer(Modifier.height(16.dp)) - PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) { - Button(onClick = { index += 1 }, modifier = it) { Text("Next") } - } - } - page.permissions != null -> { - PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) { - Button(onClick = { launcher.launch(page.permissions) }, modifier = it) { Text("Allow") } - } - Spacer(Modifier.height(8.dp)) - TextButton(onClick = { index += 1 }) { Text("Skip for now") } + page.permissions != null && !alreadyGranted -> { + PillRow(c, ::cq, ::cqSp, back, "Allow") { launcher.launch(page.permissions) } + Pill(c, ::cqSp, "Skip for now", PillStyle.TEXT, Modifier.align(Alignment.CenterHorizontally), next) } page.onAction != null -> { - FilledTonalButton( - onClick = page.onAction, - modifier = Modifier.fillMaxWidth().height(52.dp), - ) { Text(page.actionLabel ?: "Open") } - Spacer(Modifier.height(8.dp)) - PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) { - Button(onClick = { index += 1 }, modifier = it) { Text("Next") } - } - } - else -> { - PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) { - Button(onClick = { index += 1 }, modifier = it) { - Text(if (index == pages.lastIndex) "Done" else "Next") - } - } + Pill(c, ::cqSp, page.actionLabel ?: "Open", PillStyle.OUTLINED, Modifier.fillMaxWidth().padding(bottom = cq(3f)), page.onAction) + PillRow(c, ::cq, ::cqSp, back, "Next", next) } + else -> PillRow(c, ::cq, ::cqSp, back, if (index == pages.lastIndex) "Done" else "Next", next) } - Spacer(Modifier.height(24.dp)) + Spacer(Modifier.height(cq(6f))) } } } -/** - * The bottom action row: a lower-emphasis Back on the left once there is a previous page, and the - * page's own primary button - passed [content] so the caller keeps the button's own label and - * click behavior - filling the rest of the width. - */ +private enum class PillStyle { FILLED, OUTLINED, TEXT } + +/** The intro's rounded buttons: filled for the step's main action, outlined beside it, bare text to skip. */ @Composable -private fun PrimaryRow(showBack: Boolean, onBack: () -> Unit, content: @Composable (Modifier) -> Unit) { - if (showBack) { - Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(12.dp)) { - OutlinedButton( - onClick = onBack, - modifier = Modifier.weight(1f).height(52.dp), - ) { Text("Back") } - content(Modifier.weight(1f).height(52.dp)) - } - } else { - content(Modifier.fillMaxWidth().height(52.dp)) +private fun Pill(c: IntroColors, cqSp: (Float) -> TextUnit, label: String, style: PillStyle, modifier: Modifier = Modifier, onClick: () -> Unit) { + val shape = RoundedCornerShape(50) + Box( + modifier + .clip(shape) + .then( + when (style) { + PillStyle.FILLED -> Modifier.background(c.primary) + PillStyle.OUTLINED -> Modifier.border(BorderStroke(2.dp, c.outline), shape) + PillStyle.TEXT -> Modifier + }, + ) + .clickable(onClick = onClick) + .heightIn(min = 52.dp) + .padding(horizontal = 20.dp, vertical = 12.dp), + contentAlignment = Alignment.Center, + ) { + Text( + label, + color = if (style == PillStyle.FILLED) c.onPrimary else c.primary, + style = TextStyle(fontSize = cqSp(4.4f), fontWeight = FontWeight.Bold), + ) + } +} + +/** Back on the left once there's a page to go back to, and the page's main action filling the rest. */ +@Composable +private fun PillRow( + c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, + onBack: (() -> Unit)?, label: String, onClick: () -> Unit, +) { + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(cq(3f))) { + if (onBack != null) Pill(c, cqSp, "Back", PillStyle.OUTLINED, Modifier.weight(1f), onBack) + Pill(c, cqSp, label, PillStyle.FILLED, Modifier.weight(if (onBack != null) 1.4f else 1f), onClick) } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt index a6a5469..66e6180 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt @@ -37,6 +37,10 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.AlertDialog +import androidx.compose.material3.TextButton import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.semantics.Role import androidx.compose.ui.text.AnnotatedString @@ -70,6 +74,31 @@ internal val USAGE_STATS_INTRO: AnnotatedString = buildAnnotatedString { append(INTRO_END) } +/** + * The setup guide's version of [USAGE_STATS_INTRO]: what the notes are and why, in three + * sentences. The full text is one tap away in [UsageStatsReadMore]. + */ +internal val USAGE_STATS_SHORT: AnnotatedString = buildAnnotatedString { + append("Gearslip can send a short note after each drive: its version and the car details you pick below. ") + append("The notes show which cars work and where the others fail. ") + withStyle(SpanStyle(fontWeight = FontWeight.Bold)) { append("No name, account, or location goes with them.") } +} + +/** "Read more", opening the whole pitch and privacy promise in a dialog. */ +@Composable +internal fun UsageStatsReadMore() { + var open by remember { mutableStateOf(false) } + TextButton(onClick = { open = true }, contentPadding = PaddingValues(0.dp)) { Text("Read more") } + if (open) { + AlertDialog( + onDismissRequest = { open = false }, + title = { Text("About usage notes") }, + text = { Text(USAGE_STATS_INTRO, modifier = Modifier.verticalScroll(rememberScrollState())) }, + confirmButton = { TextButton(onClick = { open = false }) { Text("Close") } }, + ) + } +} + /** The usage notes card for the setup guide: the same rows Settings shows. */ @Composable internal fun UsageStatsChoices(modifier: Modifier = Modifier) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt index c11665d..d33033d 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt @@ -27,7 +27,7 @@ import app.seb3thehacker.gearslip.speech.SpeechModels.State /** * Which speech model the voice assistant listens with, and downloading it. Each is fetched once - * from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening. + * from sherpa-onnx's author ([SpeechModels] says where); nothing is sent anywhere while listening. */ @Composable internal fun VoiceModelSection() { @@ -38,7 +38,7 @@ internal fun VoiceModelSection() { SettingsSection(title = "Voice: live words") { Text( "The voice assistant turns speech into text on this phone, and nothing you say leaves it. " + - "It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " + + "It needs two models, about 700 MB together, downloaded once. Use Wi-Fi if you can. " + "This one writes your words as you speak.", style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant,