Merge branch 'experimental' into wireless

This commit is contained in:
Seb3thehacker committed 2026-10-13 12:41:09 -07:00
commit 02d6cb546e
18 files changed
+928 -256

No files matched your search

@@ -5,6 +5,7 @@ import app.seb3thehacker.gearslip.audio.AudioLink
import app.seb3thehacker.gearslip.audio.CarMic
import app.seb3thehacker.gearslip.media.CarMediaStatus
import app.seb3thehacker.gearslip.car.ClusterNav
import app.seb3thehacker.gearslip.car.DirectionsVoice
import app.seb3thehacker.gearslip.car.CarAssistant
import app.seb3thehacker.gearslip.car.CarEnvironment
import app.seb3thehacker.gearslip.car.CarKeys
@@ -79,6 +80,7 @@ class GearslipRunner(
@Volatile private var clusterNav: ClusterNav? = null
private var sensorTypes: List<Int> = emptyList()
@Volatile private var audioLink: AudioLink? = null
@Volatile private var guidanceLink: AudioLink? = null
private var audioMessagesSeen = 0
private var touchWidth = 0
private var touchHeight = 0
@@ -154,6 +156,8 @@ class GearslipRunner(
videoSource = null
audioLink?.close()
audioLink = null
guidanceLink?.close()
guidanceLink = null
CarSensors.clear()
CarMic.current = null
mediaStatus?.stop()
@@ -201,6 +205,8 @@ class GearslipRunner(
clusterNav?.onMessage(messageId, body)
} else if (frame.channel == mediaStatus?.channelId) {
mediaStatus?.onMessage(messageId, body)
} else if (frame.channel == guidanceLink?.channelId) {
guidanceLink?.onMessage(messageId, body)
} else if (frame.channel == CarMic.current?.channelId) {
CarMic.current?.onMessage(messageId, body)
} else if (frame.channel == audioLink?.channelId) {
@@ -452,6 +458,7 @@ class GearslipRunner(
openChannel(video.serviceId)
startAudioChannel(serviceDiscoveryResponse)
startGuidanceChannel(serviceDiscoveryResponse)
startSensorChannel(serviceDiscoveryResponse)
startMicChannel(serviceDiscoveryResponse)
startMediaStatusChannel(serviceDiscoveryResponse)
@@ -473,7 +480,7 @@ class GearslipRunner(
}
}
/** Media audio rides its own sink; guidance, system and telephony sinks are left closed. */
/** Media audio rides its own sink; spoken directions take the guidance sink, and system and telephony stay closed. */
private fun startAudioChannel(serviceDiscoveryResponse: ByteArray) {
val sinks = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse)
sinks.forEach { sink ->
@@ -505,6 +512,27 @@ class GearslipRunner(
openChannel(media.serviceId)
}
/** The car's directions channel, where [DirectionsVoice] reads out each turn. */
private fun startGuidanceChannel(serviceDiscoveryResponse: ByteArray) {
val guidance = ServiceDiscovery.findAudioServices(serviceDiscoveryResponse)
.filter { it.streamType == ServiceDiscovery.STREAM_GUIDANCE }
.firstOrNull { it.codecType == ServiceDiscovery.CODEC_PCM } ?: run {
log.i("no PCM directions sink - turns won't be spoken")
return
}
guidanceLink = AudioLink(
guidance,
sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = guidance.serviceId) },
sendControl = { id, body -> send(id, body, encrypted = true) },
tag = "guidance",
sessionId = GUIDANCE_SESSION_ID,
asksFocus = false,
onReady = { DirectionsVoice.attach(it) },
onClose = { DirectionsVoice.detach(it) },
)
openChannel(guidance.serviceId)
}
/** The car's microphone, for voice: opened now, recording only while something listens. */
private fun startMicChannel(serviceDiscoveryResponse: ByteArray) {
val mic = ServiceDiscovery.findMicService(serviceDiscoveryResponse) ?: run {
@@ -513,7 +541,14 @@ class GearslipRunner(
}
val config = mic.configs.first()
log.i("car microphone: channel=${mic.serviceId} ${mic.codecName} ${config.sampleRate}Hz/${config.bits}bit/x${config.channels}")
CarMic.current = CarMic(mic) { id, body -> send(id, body, encrypted = true, channel = mic.serviceId) }
CarMic.current = CarMic(
mic,
sendOnChannel = { id, body -> send(id, body, encrypted = true, channel = mic.serviceId) },
// VoiceSessionNotification { status = 1 START, 2 END }, on the control channel.
voiceSession = { start ->
send(MSG_VOICE_SESSION_NOTIFICATION, Protobuf.varintField(1, if (start) 1L else 2L), encrypted = true)
},
)
openChannel(mic.serviceId)
}
@@ -660,8 +695,8 @@ class GearslipRunner(
* right-hand third of the screen away entirely - taps on a right-edge button landed on
* the middle one. The head unit maps its own panel onto our video before reporting.
*
* So: inject as-is, clamped to the video bounds. The advertised touchscreen size is kept
* only for diagnostics.
* So: inject them unscaled, shifted past the cropped margin (see below) and clamped to the
* video. The advertised touchscreen size is kept only for diagnostics.
*/
private fun onInputMessage(messageId: Int, body: ByteArray) {
when (messageId) {
@@ -692,13 +727,20 @@ class GearslipRunner(
val action = Wire.varint(touchFields, 3)?.toInt() ?: 0
val pointers = Wire.allBytes(touchFields, 1)
if (pointers.isEmpty()) return
val size = videoConfigs.getOrNull(selectedConfigIndex ?: 0)?.pixelSize()
val config = videoConfigs.getOrNull(selectedConfigIndex ?: 0)
val size = config?.pixelSize()
// The car crops its margin off the frame and measures taps from the corner of what
// it shows, so they come in short by half the margin: the 2024 VW (margin 240) sent
// y=6, which is inside the cropped band, and the Uconnect's never passed 351 on a
// 480-high frame with a 130 margin. Without this every tap landed 120 px too high
// and the VW couldn't press past its first screen.
val shift = config?.marginInsets() ?: Insets.NONE
val points = pointers.mapIndexedNotNull { index, pointer ->
val fields = Wire.fields(pointer)
val rawX = Wire.varint(fields, 1)?.toInt() ?: return@mapIndexedNotNull null
val rawY = Wire.varint(fields, 2)?.toInt() ?: return@mapIndexedNotNull null
val id = Wire.varint(fields, 3)?.toInt() ?: index
TouchPoint(id, clamp(rawX, size?.first), clamp(rawY, size?.second))
TouchPoint(id, clamp(rawX + shift.left, size?.first), clamp(rawY + shift.top, size?.second))
}
if (points.isEmpty()) return
val actionIndex = (Wire.varint(touchFields, 2)?.toInt() ?: 0).coerceIn(0, points.size - 1)
@@ -856,13 +898,17 @@ class GearslipRunner(
/**
* The offered config that best fills the car's screen: the smallest whose visible width
* reaches most of the touchscreen's, else the largest there is. Always taking the smallest
* put 800x480 on a 1540x720 Audi screen and stretched it. With no touchscreen size to go
* by, the smallest is still the safe choice.
* put 800x480 on a 1540x720 Audi screen and stretched it. A car that gives no touchscreen
* size gets the largest up to 1080p: a car only offers sizes it can show, and the 2024 VW
* (touchscreen 0x0) blurred 800x480 across a 1080p screen.
*/
private fun fitToScreen(offered: List<Int>): Int? {
val known = offered.filter { videoConfigs.getOrNull(it)?.pixelSize() != null }
fun pixels(idx: Int) = videoConfigs[idx].pixelCount()
if (touchWidth <= 0) return known.minByOrNull(::pixels)
if (touchWidth <= 0) {
return known.filter { pixels(it) <= 1920 * 1080 }.maxByOrNull(::pixels)
?: known.minByOrNull(::pixels)
}
val wide = known.filter { (videoConfigs[it].visibleSize()?.first ?: 0) >= touchWidth * 8 / 10 }
return wide.minByOrNull(::pixels) ?: known.maxByOrNull(::pixels)
}
@@ -1233,6 +1279,7 @@ class GearslipRunner(
const val MSG_PING_RESPONSE = 12
const val MSG_BYEBYE_REQUEST = 15
const val MSG_BYEBYE_RESPONSE = 16
const val MSG_VOICE_SESSION_NOTIFICATION = 17
const val MSG_CHANNEL_OPEN_REQUEST = 7
const val MSG_CHANNEL_OPEN_RESPONSE = 8
@@ -1273,6 +1320,7 @@ class GearslipRunner(
const val ACK_STALL_MS = 2_000L
const val AUDIO_TRACE_MESSAGES = 6
const val GUIDANCE_SESSION_ID = 1
const val PERIODIC_KEYFRAME_MS = 5_000L
}
}
@@ -153,6 +153,7 @@ object ServiceDiscovery {
return null
}
const val STREAM_GUIDANCE = 1
const val STREAM_MEDIA = 3
const val CODEC_PCM = 1
@@ -22,6 +22,13 @@ class AudioLink(
private val service: ServiceDiscovery.AudioService,
private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit,
private val sendControl: (messageId: Int, body: ByteArray) -> Unit,
/** Log name: "audio" for the media sink, "guidance" for the directions sink. */
private val tag: String = "audio",
private val sessionId: Int = SESSION_ID,
/** Media asks the car for focus; directions just start, and the car ducks music under them. */
private val asksFocus: Boolean = true,
private val onReady: (AudioLink) -> Unit = { CarAudio.attach(it) },
private val onClose: (AudioLink) -> Unit = { CarAudio.detach(it) },
) : CarAudio.Link {
private var configIndex = 0
@@ -33,6 +40,7 @@ class AudioLink(
// The Uconnect asks for 4 on its video channel; assume the same until its Config says otherwise.
@Volatile private var maxUnacked = 4
@Volatile private var lastAckAt = 0L
private var stalls = 0
private var chunks = 0
private var bytes = 0L
private var dropped = 0
@@ -46,23 +54,24 @@ class AudioLink(
val channelId get() = service.serviceId
fun opened(status: Int) {
GearslipLog.i("audio: <- ChannelOpenResponse (media audio): status=$status")
GearslipLog.i("$tag: <- ChannelOpenResponse: status=$status")
if (status != 0) {
GearslipLog.w("audio: the head unit refused the media audio channel (status $status)")
GearslipLog.w("$tag: the head unit refused the $tag channel (status $status)")
return
}
// AVChannelSetupRequest { required uint32 config_index = 1 }. This field is an index into
// the sink's own audio_configs, not a codec id - sending the codec type asks for a config
// the sink does not have, and it answers with a status that never reaches READY.
sendOnChannel(MSG_SETUP, Protobuf.varintField(1, 0L))
GearslipLog.i("audio: -> Setup(config_index=0) on channel ${service.serviceId}, codec ${service.codecName}")
// Setup { required MediaCodecType type = 1 }: the codec the car advertised for this sink
// (aasdk's current protos, and LIVI reads it as the codec). Gearslip used to send 0 here,
// read from an older proto as a config index; 0 is no codec at all, and the 2024 VW
// answered UNEXPECTED_MESSAGE (id 255), never acked a chunk and played nothing.
sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong()))
GearslipLog.i("$tag: -> Setup(codec=${service.codecName}) on channel ${service.serviceId}")
armConfigFallback()
}
/**
* The 2018 Uconnect and a Dacia ULC 4.5 both answer our Setup with a two-byte message we
* cannot identify (id 255, no body) rather than the AVChannelSetupResponse the video channel
* gets, so the sink never reaches READY. [MSG_SETUP_ANSWER] takes that as the go-ahead.
* The 2018 Uconnect, a Dacia ULC 4.5 and the 2024 VW answered the old Setup (codec 0) with
* UNEXPECTED_MESSAGE (id 255, no body) rather than a Config, so the sink never reached READY.
* [MSG_SETUP_ANSWER] still takes that as the go-ahead, in case a car rejects the real codec too.
*
* Service discovery already told us everything a Config would have: the sink's rate, channels
* and its one configuration. So for a car that answers nothing at all, wait a moment and then
@@ -87,12 +96,12 @@ class AudioLink(
if (ready) return
configTimer?.interrupt()
GearslipLog.i(
"audio: $why - using the sink the head unit advertised " +
"$tag: $why - using the sink the head unit advertised " +
"(${sampleRate}Hz x$channels, config_index=0, max_unacked=$maxUnacked)",
)
configIndex = 0
ready = true
CarAudio.attach(this)
onReady(this)
}
fun onMessage(messageId: Int, body: ByteArray) {
@@ -104,17 +113,17 @@ class AudioLink(
val status = Wire.varint(fields, 1)?.toInt()
val window = Wire.varint(fields, 2)?.toInt()
val indices = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() }
GearslipLog.i("audio: <- Config status=$status max_unacked=$window indices=$indices")
GearslipLog.i("$tag: <- Config status=$status max_unacked=$window indices=$indices")
if (window != null) maxUnacked = maxOf(1, window)
configIndex = indices.firstOrNull() ?: 0
if (status == STATUS_READY) {
configTimer?.interrupt()
ready = true
GearslipLog.i("audio: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit")
CarAudio.attach(this)
GearslipLog.i("$tag: sink ready - ${sampleRate}Hz x$channels ${config?.bits}bit")
onReady(this)
} else {
GearslipLog.w(
"audio: the sink is not ready (status=$status, 1=WAIT 2=READY) - " +
"$tag: the sink is not ready (status=$status, 1=WAIT 2=READY) - " +
"nothing will be sent until it is",
)
}
@@ -133,7 +142,7 @@ class AudioLink(
else -> if (messageId == MSG_SETUP_ANSWER && body.isEmpty()) {
useAdvertisedSink("<- empty Setup answer (id $messageId)")
} else {
GearslipLog.i("audio: <- unhandled message id=$messageId (${body.size} bytes)")
GearslipLog.i("$tag: <- unhandled message id=$messageId (${body.size} bytes)")
GearslipLog.hex(" audio msg", body, limit = 64)
}
}
@@ -146,7 +155,7 @@ class AudioLink(
*/
fun onFocus(state: Int) {
val name = FOCUS_STATES[state] ?: "unknown($state)"
GearslipLog.i("audio: <- AudioFocusResponse state=$name")
GearslipLog.i("$tag: <- AudioFocusResponse state=$name")
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT ||
state == FOCUS_STATE_GAIN_MEDIA_ONLY
if (focusGranted) startStream() else stop()
@@ -154,10 +163,10 @@ class AudioLink(
override fun start() {
if (!ready || started) return
if (!focusRequested) {
if (asksFocus && !focusRequested) {
focusRequested = true
sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_GAIN.toLong()))
GearslipLog.i("audio: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over")
GearslipLog.i("$tag: -> AudioFocusRequest(GAIN); waiting for the head unit to hand the sink over")
}
// Some head units grant the sink silently and never answer. Do not let that be the end of
// it: start anyway, and let the acks say whether anything is being taken.
@@ -168,12 +177,12 @@ class AudioLink(
if (!ready || started) return
sendOnChannel(
MSG_START,
Protobuf.varintField(1, SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()),
Protobuf.varintField(1, sessionId.toLong()) + Protobuf.varintField(2, configIndex.toLong()),
)
inFlight = 0
lastAckAt = System.currentTimeMillis()
started = true
GearslipLog.i("audio: -> Start(session=$SESSION_ID, config=$configIndex) - streaming ${sampleRate}Hz x$channels")
GearslipLog.i("$tag: -> Start(session=$sessionId, config=$configIndex) - streaming ${sampleRate}Hz x$channels")
}
override fun stop() {
@@ -181,12 +190,12 @@ class AudioLink(
started = false
synchronized(ackLock) { ackLock.notifyAll() }
runCatching {
sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong()))
sendOnChannel(MSG_STOP, Protobuf.varintField(1, sessionId.toLong()))
if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong()))
}
focusRequested = false
focusGranted = false
GearslipLog.i("audio: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes")
GearslipLog.i("$tag: -> Stop, AudioFocusRequest(RELEASE) after $chunks chunks / $bytes bytes")
chunks = 0
bytes = 0
}
@@ -200,7 +209,11 @@ class AudioLink(
val waited = System.currentTimeMillis() - lastAckAt
if (waited > STALL_MS) {
// A lost ack must not silence the stream for good; after a stall assume the window cleared.
GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window")
// A car that never acks (the 2024 VW, the Bolt) hits this every chunk: say so
// once, then now and then, so the rest of the log stays readable.
if (stalls++ % 500 == 0) {
GearslipLog.w("$tag: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window ($stalls times)")
}
inFlight = 0
break
}
@@ -217,7 +230,7 @@ class AudioLink(
try {
sendOnChannel(MSG_DATA, timestamp + pcm.copyOf(size))
} catch (e: java.io.IOException) {
GearslipLog.e("audio: the link died mid-stream", e)
GearslipLog.e("$tag: the link died mid-stream", e)
started = false
return
}
@@ -228,7 +241,7 @@ class AudioLink(
// the car is still taking audio and to catch a window that has quietly closed.
if (chunks == 1 || chunks % CHUNK_LOG_EVERY == 0) {
GearslipLog.i(
"audio: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " +
"$tag: -> $chunks chunks / $bytes bytes, inFlight=$inFlight/$maxUnacked, " +
"dropped=$dropped, focus=${if (focusGranted) "granted" else "not granted"}",
)
}
@@ -239,7 +252,7 @@ class AudioLink(
configTimer = null
stop()
ready = false
CarAudio.detach(this)
onClose(this)
}
companion object {
@@ -12,17 +12,21 @@ import java.util.concurrent.TimeUnit
*
* -> CHANNEL_OPEN_REQUEST <- CHANNEL_OPEN_RESPONSE
* -> Setup { config_index } <- Config
* -> VoiceSessionNotification { START } (control channel)
* -> MicrophoneRequest { open } <- MicrophoneResponse { status, session_id }
* <- Data { 8-byte timestamp, PCM }... -> Ack { session_id, ack }
* -> MicrophoneRequest { open = false }
* -> VoiceSessionNotification { END }
*
* The channel opens with the session, but the car only records between [start] and [stop].
* The channel opens with the session, but the car only records between [start] and [stop], and
* only inside a voice session: the 2024 VW answered a bare MicrophoneRequest with status -7.
* Speech wants 16 kHz mono; a car that records at a multiple of that is averaged down to it, and
* one that doesn't isn't used, so voice falls back to the phone's microphone.
*/
class CarMic(
private val service: ServiceDiscovery.AudioService,
private val sendOnChannel: (messageId: Int, body: ByteArray) -> Unit,
private val voiceSession: (start: Boolean) -> Unit,
) {
val channelId get() = service.serviceId
@@ -43,7 +47,8 @@ class CarMic(
val status = Protobuf.readInt32Field(body, 1) ?: 0
GearslipLog.i("mic: <- ChannelOpenResponse: status=$status")
if (status != 0) return
sendOnChannel(MSG_SETUP, Protobuf.varintField(1, 0L))
// Setup { type = the codec }, as for audio: 0 got UNEXPECTED_MESSAGE from the VW.
sendOnChannel(MSG_SETUP, Protobuf.varintField(1, service.codecType.toLong()))
ready = usable
if (!usable) {
GearslipLog.w(
@@ -97,6 +102,7 @@ class CarMic(
chunks.clear()
packets = 0
listening = true
voiceSession(true)
sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 1L) + Protobuf.varintField(4, 1L))
GearslipLog.i("mic: -> MicrophoneRequest(open)")
return true
@@ -109,7 +115,8 @@ class CarMic(
if (!listening) return
listening = false
sendOnChannel(MSG_MIC_REQUEST, Protobuf.varintField(1, 0L))
GearslipLog.i("mic: -> MicrophoneRequest(close) after $packets packets")
voiceSession(false)
GearslipLog.i("mic: -> MicrophoneRequest(close) after $packets packets, voice session ended")
}
companion object {
@@ -2,6 +2,7 @@ package app.seb3thehacker.gearslip.car
import android.content.Context
import app.seb3thehacker.gearslip.media.CarMedia
import app.seb3thehacker.gearslip.speech.VoiceFocus
import java.text.DateFormat
import java.util.Date
import kotlinx.coroutines.Dispatchers
@@ -26,6 +27,7 @@ object AssistantCommands {
VoiceHelp.show()
return "Here's what you can say."
}
if (heard.matches(Regex("(fuck|f\\*+) (you|off)( too)?"))) return "Fuck you too."
navigateCommand(context, navigator, frame, heard)?.let { return it }
playSearchCommand(context, navigator, heard)?.let { return it }
@@ -123,6 +125,10 @@ object AssistantCommands {
break
}
}
// "the nearest gas station" -> "gas station": the map app already searches around the car
// (see CarMotion.nearHere), and Organic Maps took "nearest" as part of the name.
val (place, nearby) = plainPlace(destination)
destination = place
if (destination.isBlank()) return "Where to?"
val chosen = chosenApp ?: when {
@@ -133,7 +139,23 @@ object AssistantCommands {
CarServices.connectNav(template, frame, destination)
navigator.home()
return "Navigating to $destination with ${chosen.label}."
return if (nearby) "Finding the nearest $destination with ${chosen.label}." else "Navigating to $destination with ${chosen.label}."
}
/** [destination] without "a", "the nearest", "near me" and the like, and whether any of that said "close by". */
private fun plainPlace(destination: String): Pair<String, Boolean> {
var t = destination
var nearby = false
var changed = true
while (changed) {
changed = false
for (lead in PLACE_LEADS) if (t.startsWith(lead)) {
t = t.removePrefix(lead).trim(); changed = true
if (lead.trim() !in ARTICLES) nearby = true
}
for (tail in PLACE_TAILS) if (t.endsWith(tail)) { t = t.removeSuffix(tail).trim(); changed = true; nearby = true }
}
return t to nearby
}
/**
@@ -194,7 +216,8 @@ object AssistantCommands {
val media = CarServices.media
return when {
"pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> {
if (media.now.value.playing) media.togglePlay()
media.pause()
VoiceFocus.afterRelease { media.pause() }
"Paused."
}
"resume" in heard || "unpause" in heard || heard in RESUME_PHRASES ||
@@ -315,8 +338,18 @@ object AssistantCommands {
"navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ",
"take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ",
"how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ",
// Last, so "navigate to" wins: the recognizer sometimes drops the "to".
"navigate me ", "navigate ", "find me the nearest ", "find the nearest ", "find the closest ",
)
private val ARTICLES = setOf("a", "an", "the", "some")
// Longest first, so "the nearest " goes before "the ".
private val PLACE_LEADS = listOf(
"the nearest ", "the closest ", "a nearby ", "the nearby ", "nearest ", "closest ", "nearby ",
"a ", "an ", "some ",
)
private val PLACE_TAILS = listOf(" near me", " nearby", " near here", " around here", " close by", " closest to me")
private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ")
private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ")
@@ -65,6 +65,7 @@ object CarServices {
app.seb3thehacker.gearslip.notify.CarMotion.start(appContext)
app.seb3thehacker.gearslip.call.CarCalls.start(appContext)
CarAssistant.init(appContext)
DirectionsVoice.init(appContext)
CarKeys.init(appContext)
VoiceReply.init(appContext)
// Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest.
@@ -28,6 +28,7 @@ object CarSettings {
private const val KEY_OPEN = "car_open_on_connect"
private const val KEY_NIGHT = "car_night_mode"
private const val KEY_PIPE_AUDIO = "car_pipe_audio"
private const val KEY_SPEAK_DIRECTIONS = "car_speak_directions"
private const val KEY_AUDIO_CHOICE_ASKED = "car_audio_choice_asked"
private const val KEY_LAST_NAV = "car_last_nav"
private const val KEY_LAST_MEDIA = "car_last_media"
@@ -58,6 +59,7 @@ object CarSettings {
private val openFlow = MutableStateFlow(HOME)
private val nightFlow = MutableStateFlow(NightMode.AUTO)
private val pipeAudioFlow = MutableStateFlow(true)
private val speakDirectionsFlow = MutableStateFlow(true)
private val lastNavFlow = MutableStateFlow<String?>(null)
private val lastMediaFlow = MutableStateFlow<String?>(null)
private val autoplayFlow = MutableStateFlow(false)
@@ -153,6 +155,9 @@ object CarSettings {
*/
val pipeAudio: StateFlow<Boolean> = pipeAudioFlow
/** Gearslip reads the map app's turns aloud on the car's directions channel; see DirectionsVoice. */
val speakDirections: StateFlow<Boolean> = speakDirectionsFlow
/**
* Whether the driver has already been offered Bluetooth after turning down the capture
* dialog. Offered once only: someone who chose to keep the USB route shouldn't be nagged.
@@ -205,6 +210,7 @@ object CarSettings {
autoplayFlow.value = AppSettings.getString(app, KEY_AUTOPLAY, "false") == "true"
playerTitleFlow.value = AppSettings.getString(app, KEY_PLAYER_TITLE, "false") == "true"
pipeAudioFlow.value = AppSettings.getString(app, KEY_PIPE_AUDIO, "true") == "true"
speakDirectionsFlow.value = AppSettings.getString(app, KEY_SPEAK_DIRECTIONS, "true") == "true"
nightFlow.value = runCatching {
NightMode.valueOf(AppSettings.getString(app, KEY_NIGHT, NightMode.AUTO.name))
}.getOrDefault(NightMode.AUTO)
@@ -274,6 +280,11 @@ object CarSettings {
if (on) CarAudio.request() else CarAudio.release()
}
fun setSpeakDirections(on: Boolean) {
speakDirectionsFlow.value = on
AppSettings.putString(app, KEY_SPEAK_DIRECTIONS, on.toString())
}
fun setExperimentalFeaturesEnabled(on: Boolean) {
experimentalFeaturesFlow.value = on
AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
@@ -53,6 +53,7 @@ fun CarSettingsScreen() {
val units by CarSettings.units.collectAsState()
val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState()
val pipeAudio by CarSettings.pipeAudio.collectAsState()
val speakDirections by CarSettings.speakDirections.collectAsState()
val autoplay by CarSettings.autoplay.collectAsState()
val playerTitle by CarSettings.playerTitle.collectAsState()
val theme by CarSettings.appTheme.collectAsState()
@@ -137,6 +138,18 @@ fun CarSettingsScreen() {
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
ChoiceRow(
"Spoken directions",
listOf(false to "Off", true to "On"),
speakDirections,
) { CarSettings.setSpeakDirections(it) }
Text(
"Gearslip reads each turn aloud through the car's speakers, and the music dips while it " +
"speaks. Turn it off if your map app's own voice already plays in the car.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
ExperimentalFeatureRow(
title = "Use the phone's speech recognition",
description = "Listens through the phone's own speech service instead of Gearslip's. On " +
@@ -60,23 +60,8 @@ class ClusterNav(
running = false
}
/** One turn as the car wants it, from whichever source the map app fills. */
private class Turn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean)
private fun current(): Turn? {
val nav = CarServices.navOrNull() ?: return null
nav.trip.value?.let { trip ->
val step = trip.steps.firstOrNull() ?: return@let
val estimate = trip.stepTravelEstimates.firstOrNull()
return Turn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading)
}
val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null
val step = info.currentStep ?: return null
return Turn(step, info.currentDistance, null, info.isLoading)
}
private fun tick() {
val turn = current()
val turn = NavTurn.current()
val status = when {
turn == null -> STATUS_INACTIVE
turn.rerouting -> STATUS_REROUTING
@@ -114,7 +99,7 @@ class ClusterNav(
* 7 yards), so the app's own display choice passes straight through.
*/
private fun sendDistance(distance: Distance, seconds: Long?) {
val meters = toMeters(distance)
val meters = NavTurn.toMeters(distance)
val key = "${distance.displayDistance}|${distance.displayUnit}|${seconds ?: -1}"
if (key == sentDistance) return
sentDistance = key
@@ -127,14 +112,6 @@ class ClusterNav(
)
}
private fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344
Distance.UNIT_FEET -> 0.3048
Distance.UNIT_YARDS -> 0.9144
else -> 1.0
}).toInt()
private fun CarText?.text() = this?.toString().orEmpty()
companion object {
@@ -194,3 +171,29 @@ class ClusterNav(
}
}
}
/** One turn as the car wants it, from whichever source the map app fills. */
internal class NavTurn(val step: Step, val distance: Distance?, val seconds: Long?, val rerouting: Boolean) {
companion object {
/** The map app's next turn: from its trip (what clusters get), or its routing card. */
fun current(): NavTurn? {
val nav = CarServices.navOrNull() ?: return null
nav.trip.value?.let { trip ->
val step = trip.steps.firstOrNull() ?: return@let
val estimate = trip.stepTravelEstimates.firstOrNull()
return NavTurn(step, estimate?.remainingDistance, estimate?.remainingTimeSeconds?.takeIf { it >= 0 }, trip.isLoading)
}
val info = (nav.template.value as? NavigationTemplate)?.navigationInfo as? RoutingInfo ?: return null
val step = info.currentStep ?: return null
return NavTurn(step, info.currentDistance, null, info.isLoading)
}
fun toMeters(d: Distance): Int = (d.displayDistance * when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> 1000.0
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> 1609.344
Distance.UNIT_FEET -> 0.3048
Distance.UNIT_YARDS -> 0.9144
else -> 1.0
}).toInt()
}
}
@@ -0,0 +1,326 @@
package app.seb3thehacker.gearslip.car
import android.content.Context
import android.os.Bundle
import android.speech.tts.TextToSpeech
import android.speech.tts.UtteranceProgressListener
import androidx.car.app.model.CarText
import androidx.car.app.model.Distance
import androidx.car.app.navigation.model.Maneuver
import androidx.car.app.navigation.model.Step
import app.seb3thehacker.gearslip.GearslipLog
import app.seb3thehacker.gearslip.audio.AudioLink
import java.io.File
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import kotlin.math.roundToInt
/**
* Spoken turn-by-turn on the car's directions channel (the GUIDANCE audio sink).
*
* A map app on the phone speaks its own prompts as navigation audio, which Android can't hand to
* Gearslip without root, and which the phone mutes while USB audio runs. So Gearslip reads the
* same turn the instrument cluster gets ([NavTurn]) and says it itself: once when the turn comes
* up, again a kilometre or so out on a long stretch, and once more just before it. The words are
* rendered with the phone's text-to-speech to a file, never played on the phone, and streamed to
* the car, which ducks the music under them the way it does for real Android Auto.
*
* Nothing spoken is logged: road names are where the driver is going.
*/
object DirectionsVoice {
private enum class Stage { FAR, PREPARED, DONE }
private var appContext: Context? = null
private var tts: TextToSpeech? = null
@Volatile private var ttsReady = false
private val pending = ConcurrentHashMap<String, CountDownLatch>()
@Volatile private var link: AudioLink? = null
@Volatile private var running = false
private val speaker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-directions").apply { isDaemon = true } }
@Volatile private var latest = 0
private var turnKey: String? = null
private var stage = Stage.DONE
fun init(context: Context) {
if (tts != null) return
appContext = context.applicationContext
tts = TextToSpeech(appContext) { status -> ttsReady = status == TextToSpeech.SUCCESS }.apply {
setOnUtteranceProgressListener(object : UtteranceProgressListener() {
override fun onStart(utteranceId: String?) {}
override fun onDone(utteranceId: String?) { pending.remove(utteranceId)?.countDown() }
@Deprecated("Deprecated in Java, no replacement on this API level")
override fun onError(utteranceId: String?) { pending.remove(utteranceId)?.countDown() }
})
}
}
/** The car opened its directions sink: start following the map app's turns. */
fun attach(link: AudioLink) {
this.link = link
GearslipLog.i("guidance: the car offered a directions sink at ${link.sampleRate}Hz x${link.channels}")
if (running) return
running = true
turnKey = null
Thread({
while (running) {
runCatching { tick() }.onFailure { GearslipLog.w("guidance: ${it.message}") }
Thread.sleep(TICK_MS)
}
}, "gearslip-directions-watch").apply { isDaemon = true }.start()
}
fun detach(link: AudioLink) {
if (this.link !== link) return
this.link = null
running = false
}
private fun tick() {
if (!CarSettings.speakDirections.value) return
val turn = NavTurn.current()
if (turn == null) {
turnKey = null
return
}
if (turn.rerouting) return
val step = turn.step
val instruction = instruction(step) ?: return
val meters = turn.distance?.let(NavTurn::toMeters)
val close = (turn.seconds != null && turn.seconds <= NEAR_SECONDS) || (meters != null && meters <= NEAR_METERS)
val key = "${step.road.text()}|${step.maneuver?.type}|${step.maneuver?.roundaboutExitNumber}|${step.cue.text()}"
if (key != turnKey) {
turnKey = key
if (step.maneuver?.type == Maneuver.TYPE_DEPART) {
stage = Stage.DONE
return
}
if (close || turn.distance == null) {
say(instruction)
stage = Stage.DONE
} else {
say("In ${spoken(turn.distance)}, $instruction")
stage = if (meters != null && meters > PREPARE_FROM_METERS) Stage.FAR else Stage.PREPARED
}
return
}
when {
stage != Stage.DONE && close -> {
say(instruction)
stage = Stage.DONE
}
stage == Stage.FAR && meters != null && meters <= PREPARE_AT_METERS -> {
say("In ${spoken(turn.distance)}, $instruction")
stage = Stage.PREPARED
}
}
}
/** What to do, in words: built from the manoeuvre when the app names one, else the app's own text. */
private fun instruction(step: Step): String? {
val road = step.road.text()
val onto = if (road.isBlank()) "" else " onto $road"
val toward = if (road.isBlank()) "" else " toward $road"
val maneuver = step.maneuver
val built = when (maneuver?.type) {
Maneuver.TYPE_TURN_SLIGHT_LEFT -> "Turn slightly left$onto"
Maneuver.TYPE_TURN_SLIGHT_RIGHT -> "Turn slightly right$onto"
Maneuver.TYPE_TURN_NORMAL_LEFT -> "Turn left$onto"
Maneuver.TYPE_TURN_NORMAL_RIGHT -> "Turn right$onto"
Maneuver.TYPE_TURN_SHARP_LEFT -> "Turn sharply left$onto"
Maneuver.TYPE_TURN_SHARP_RIGHT -> "Turn sharply right$onto"
Maneuver.TYPE_U_TURN_LEFT, Maneuver.TYPE_U_TURN_RIGHT -> "Make a U-turn"
Maneuver.TYPE_KEEP_LEFT -> "Keep left$toward"
Maneuver.TYPE_KEEP_RIGHT -> "Keep right$toward"
Maneuver.TYPE_FORK_LEFT -> "Keep left at the fork$toward"
Maneuver.TYPE_FORK_RIGHT -> "Keep right at the fork$toward"
Maneuver.TYPE_ON_RAMP_SLIGHT_LEFT, Maneuver.TYPE_ON_RAMP_NORMAL_LEFT, Maneuver.TYPE_ON_RAMP_SHARP_LEFT,
Maneuver.TYPE_ON_RAMP_U_TURN_LEFT -> "Take the ramp on the left$onto"
Maneuver.TYPE_ON_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_ON_RAMP_NORMAL_RIGHT, Maneuver.TYPE_ON_RAMP_SHARP_RIGHT,
Maneuver.TYPE_ON_RAMP_U_TURN_RIGHT -> "Take the ramp on the right$onto"
Maneuver.TYPE_OFF_RAMP_SLIGHT_LEFT, Maneuver.TYPE_OFF_RAMP_NORMAL_LEFT -> "Take the exit on the left$toward"
Maneuver.TYPE_OFF_RAMP_SLIGHT_RIGHT, Maneuver.TYPE_OFF_RAMP_NORMAL_RIGHT -> "Take the exit on the right$toward"
Maneuver.TYPE_MERGE_LEFT -> "Merge left$onto"
Maneuver.TYPE_MERGE_RIGHT -> "Merge right$onto"
Maneuver.TYPE_MERGE_SIDE_UNSPECIFIED -> "Merge$onto"
Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CW_WITH_ANGLE,
Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW, Maneuver.TYPE_ROUNDABOUT_ENTER_AND_EXIT_CCW_WITH_ANGLE ->
maneuver.roundaboutExitNumber.takeIf { it in 1..ORDINALS.size }
?.let { "At the roundabout, take the ${ORDINALS[it - 1]} exit$onto" }
?: "Go through the roundabout$onto"
Maneuver.TYPE_ROUNDABOUT_ENTER_CW, Maneuver.TYPE_ROUNDABOUT_ENTER_CCW -> "Enter the roundabout"
Maneuver.TYPE_ROUNDABOUT_EXIT_CW, Maneuver.TYPE_ROUNDABOUT_EXIT_CCW -> "Exit the roundabout$onto"
Maneuver.TYPE_STRAIGHT -> "Continue straight$onto"
Maneuver.TYPE_NAME_CHANGE -> "Continue$onto"
Maneuver.TYPE_FERRY_BOAT, Maneuver.TYPE_FERRY_BOAT_LEFT, Maneuver.TYPE_FERRY_BOAT_RIGHT -> "Take the ferry"
Maneuver.TYPE_FERRY_TRAIN, Maneuver.TYPE_FERRY_TRAIN_LEFT, Maneuver.TYPE_FERRY_TRAIN_RIGHT -> "Take the train"
Maneuver.TYPE_DESTINATION, Maneuver.TYPE_DESTINATION_STRAIGHT -> "Your destination is ahead"
Maneuver.TYPE_DESTINATION_LEFT -> "Your destination is on the left"
Maneuver.TYPE_DESTINATION_RIGHT -> "Your destination is on the right"
else -> null
}
return built ?: step.cue.text().ifBlank { null }
}
/** The app's own rounded distance, read the way a person would say it. */
private fun spoken(d: Distance): String {
val n = d.displayDistance
val number = if (n % 1.0 == 0.0) n.toInt().toString() else "%.1f".format(n)
val one = n == 1.0
return when (d.displayUnit) {
Distance.UNIT_KILOMETERS, Distance.UNIT_KILOMETERS_P1 -> if (one) "1 kilometer" else "$number kilometers"
Distance.UNIT_MILES, Distance.UNIT_MILES_P1 -> when (n) {
0.25 -> "a quarter mile"
0.5 -> "half a mile"
0.75 -> "three quarters of a mile"
1.0 -> "1 mile"
else -> "$number miles"
}
Distance.UNIT_FEET -> "$number feet"
Distance.UNIT_YARDS -> "$number yards"
else -> "${n.roundToInt()} meters"
}
}
/** Newest wins: a prompt still waiting when the next one arrives is already out of date. */
private fun say(text: String) {
val id = ++latest
speaker.execute {
if (id != latest) return@execute
runCatching { play(text, id) }.onFailure { GearslipLog.w("guidance: could not speak a turn: ${it.message}") }
}
}
private fun play(text: String, id: Int) {
val engine = tts ?: return
val context = appContext ?: return
val sink = link ?: return
if (!ttsReady) {
GearslipLog.w("guidance: text-to-speech isn't ready, a turn went unspoken")
return
}
val file = File(context.cacheDir, "directions.wav")
val utterance = "gearslip-directions-$id"
val done = CountDownLatch(1)
pending[utterance] = done
if (engine.synthesizeToFile(text, Bundle(), file, utterance) != TextToSpeech.SUCCESS) {
pending.remove(utterance)
return
}
if (!done.await(SYNTH_TIMEOUT_S, TimeUnit.SECONDS)) {
pending.remove(utterance)
GearslipLog.w("guidance: text-to-speech took too long, a turn went unspoken")
return
}
val pcm = Wav.read(file)?.let { Wav.convert(it, sink.sampleRate, sink.channels) } ?: run {
GearslipLog.w("guidance: text-to-speech gave audio Gearslip can't read")
return
}
stream(sink, pcm)
GearslipLog.i("guidance: spoke a turn (${pcm.size / (2 * sink.channels) * 1000 / sink.sampleRate} ms)")
}
/** Start, the words in 20 ms chunks at about real time, a little silence, Stop. */
private fun stream(sink: AudioLink, pcm: ByteArray) {
val frame = 2 * sink.channels
val chunk = sink.sampleRate / 50 * frame
val chunkMs = 20L
val padded = pcm + ByteArray(sink.sampleRate / 5 * frame) // 200 ms tail so the last word isn't clipped
sink.start()
try {
val began = System.currentTimeMillis()
var offset = 0
var sent = 0
while (offset < padded.size && link === sink) {
val size = minOf(chunk, padded.size - offset)
sink.write(padded.copyOfRange(offset, offset + size), size)
offset += size
sent++
val ahead = sent * chunkMs - (System.currentTimeMillis() - began)
if (ahead > LEAD_MS) Thread.sleep(ahead - LEAD_MS)
}
// Let the car play what it has before closing the stream on it.
Thread.sleep(LEAD_MS)
} finally {
sink.stop()
}
}
private fun CarText?.text() = this?.toString().orEmpty()
/** 16-bit PCM from the WAV text-to-speech writes, reshaped for the car's sink. */
private object Wav {
class Pcm(val samples: ShortArray, val rate: Int, val channels: Int)
fun read(file: File): Pcm? {
val bytes = file.readBytes()
if (bytes.size < 12 || String(bytes, 0, 4) != "RIFF" || String(bytes, 8, 4) != "WAVE") return null
val buf = ByteBuffer.wrap(bytes).order(ByteOrder.LITTLE_ENDIAN)
var pos = 12
var rate = 0
var channels = 0
var bits = 0
while (pos + 8 <= bytes.size) {
val name = String(bytes, pos, 4)
val size = buf.getInt(pos + 4)
val body = pos + 8
// Some engines write a streaming WAV with a zero or oversized data length.
val end = if (size <= 0 || body + size > bytes.size) bytes.size else body + size
when (name) {
"fmt " -> {
channels = buf.getShort(body + 2).toInt()
rate = buf.getInt(body + 4)
bits = buf.getShort(body + 14).toInt()
}
"data" -> {
if (bits != 16 || rate <= 0 || channels <= 0) return null
val samples = ShortArray((end - body) / 2)
buf.position(body)
buf.asShortBuffer().get(samples, 0, samples.size)
return Pcm(samples, rate, channels)
}
}
pos = end + (end and 1)
}
return null
}
/** Mixes to mono, resamples linearly, then copies to each of the car's channels. */
fun convert(pcm: Pcm, rate: Int, channels: Int): ByteArray {
val frames = pcm.samples.size / pcm.channels
val mono = FloatArray(frames) { f ->
var sum = 0f
for (c in 0 until pcm.channels) sum += pcm.samples[f * pcm.channels + c]
sum / pcm.channels
}
val outFrames = (frames.toLong() * rate / pcm.rate).toInt()
val out = ByteBuffer.allocate(outFrames * channels * 2).order(ByteOrder.LITTLE_ENDIAN)
val step = pcm.rate.toDouble() / rate
for (i in 0 until outFrames) {
val at = i * step
val j = at.toInt().coerceAtMost(frames - 1)
val next = (j + 1).coerceAtMost(frames - 1)
val v = mono[j] + (mono[next] - mono[j]) * (at - j).toFloat()
val s = v.roundToInt().coerceIn(Short.MIN_VALUE.toInt(), Short.MAX_VALUE.toInt()).toShort()
repeat(channels) { out.putShort(s) }
}
return out.array()
}
}
private const val TICK_MS = 500L
private const val NEAR_SECONDS = 12L
private const val NEAR_METERS = 120
private const val PREPARE_FROM_METERS = 2_500
private const val PREPARE_AT_METERS = 1_000
private const val LEAD_MS = 100L
private const val SYNTH_TIMEOUT_S = 10L
private val ORDINALS = listOf("first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth")
}
@@ -112,7 +112,7 @@ object GsThemes {
control = ControlLook(holdMarker = HoldMarkerStyle.DOT),
)
val default = Tactile
val default = Flat
val builtIn = listOf(Tactile, Keycap, Flat)
fun byId(id: String?): GsTheme = builtIn.firstOrNull { it.id == id } ?: default
@@ -433,6 +433,12 @@ class CarMedia(private val context: Context) {
val controls = controller?.transportControls ?: return
if (_now.value.playing) controls.pause() else controls.play()
}
/**
* Pauses whatever the app thinks it's doing. Voice pauses the music to listen, so by the time
* "pause" is heard the app already looks stopped; a toggle did nothing and the song came back
* as soon as the microphone let go. A real pause tells the app to stay stopped.
*/
fun pause() = controller?.transportControls?.pause()
fun next() = controller?.transportControls?.skipToNext()
fun previous() = controller?.transportControls?.skipToPrevious()
fun seek(ms: Long) = controller?.transportControls?.seekTo(ms)
@@ -15,8 +15,10 @@ import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
/**
* The speech models Gearslip listens with, in two roles: a streaming one that writes the words
* while the driver talks, and an optional final one that reads the whole phrase again once they
* stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's
* GitHub releases, where its authors publish it, and kept in the app's own storage.
* stop, for a more accurate result. None ships in the APK: each is fetched once and kept in the
* app's own storage. sherpa-onnx's author publishes them twice: as .tar.bz2 archives on GitHub and
* as plain files on Hugging Face. The plain files come first where they exist. Unpacking bzip2 on
* the phone pinned a core and held a download to 150 KB/s on Wi-Fi that does 19 MB/s.
*/
object SpeechModels {
@@ -34,9 +36,11 @@ object SpeechModels {
/** Empty for Canary, which decodes with attention and has no joiner. */
val joiner: String,
val modelType: String = "",
/** Whether Hugging Face has the files unpacked under the archive's name. */
val plainFiles: Boolean = true,
) {
KROKO(
Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54,
Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 68,
"sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06",
"encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2",
),
@@ -44,20 +48,22 @@ object SpeechModels {
Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103,
"sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8",
"encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
plainFiles = false,
),
CANARY(
Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 154,
Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 198,
"sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8",
"encoder.int8.onnx", "decoder.int8.onnx", "", modelType = "canary",
),
PARAKEET_LARGE(
Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460,
Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 631,
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
"encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
),
;
val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2"
fun fileUrl(name: String) = "https://huggingface.co/csukuangfj/$archive/resolve/main/$name"
val files get() = listOf(encoder, decoder, joiner, "tokens.txt").filter { it.isNotEmpty() }
}
@@ -123,7 +129,7 @@ object SpeechModels {
if (model == _selected.value || model == _final.value) SpeechEngine.unload()
}
/** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */
/** Fetches just the files the recognizer reads, plain where it can, else out of the archive. */
fun download(context: Context, model: Model) {
val app = context.applicationContext
val state = states.getValue(model)
@@ -134,23 +140,33 @@ object SpeechModels {
val target = dir(app, model)
val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() }
try {
val conn = (URL(model.url).openConnection() as HttpURLConnection).apply {
connectTimeout = 15_000
readTimeout = 30_000
instanceFollowRedirects = true
}
check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" }
val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L)
val counting = CountingStream(conn.inputStream) { read ->
val total = model.downloadMb * 1_048_576L
var done = 0L
fun progress(read: Long) {
if (cancelled == model) throw InterruptedException("cancelled")
state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99))
state.value = State.Downloading((((done + read) * 100) / total).toInt().coerceIn(0, 99))
}
TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar ->
while (true) {
val entry = tar.nextEntry ?: break
val name = entry.name.substringAfterLast('/')
if (entry.isDirectory || name !in model.files) continue
File(partial, name).outputStream().use { tar.copyTo(it) }
if (model.plainFiles) {
for (name in model.files) {
val conn = open(model.fileUrl(name))
val length = conn.contentLengthLong
val file = File(partial, name)
CountingStream(conn.inputStream, ::progress).use { input ->
file.outputStream().use { input.copyTo(it, 256 * 1024) }
}
check(length <= 0 || file.length() == length) { "$name arrived cut short" }
done += file.length()
}
} else {
val conn = open(model.url)
val counting = CountingStream(conn.inputStream, ::progress)
TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar ->
while (true) {
val entry = tar.nextEntry ?: break
val name = entry.name.substringAfterLast('/')
if (entry.isDirectory || name !in model.files) continue
File(partial, name).outputStream().use { tar.copyTo(it) }
}
}
}
check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" }
@@ -170,6 +186,13 @@ object SpeechModels {
}
}
private fun open(url: String): HttpURLConnection =
(URL(url).openConnection() as HttpURLConnection).apply {
connectTimeout = 15_000
readTimeout = 30_000
instanceFollowRedirects = true
}.also { check(it.responseCode == 200) { "the server answered ${it.responseCode}" } }
/** Counts bytes as they pass, reporting about every megabyte. */
private class CountingStream(
private val inner: java.io.InputStream,
@@ -19,17 +19,35 @@ import app.seb3thehacker.gearslip.GearslipLog
object VoiceFocus {
private const val RELEASE_DELAY_MS = 1_500L
private const val AFTER_RELEASE_MS = 300L
private val main = Handler(Looper.getMainLooper())
private var request: AudioFocusRequest? = null
private var audio: AudioManager? = null
private var holders = 0
private val afterRelease = mutableListOf<() -> Unit>()
private val releaseNow = Runnable {
val r = request ?: return@Runnable
audio?.abandonAudioFocusRequest(r)
request = null
GearslipLog.i("voice: gave audio focus back")
request?.let { r ->
audio?.abandonAudioFocusRequest(r)
request = null
GearslipLog.i("voice: gave audio focus back")
}
val pending = afterRelease.toList()
afterRelease.clear()
// Just after the music app has had its focus back and acted on it.
if (pending.isNotEmpty()) main.postDelayed({ pending.forEach { it() } }, AFTER_RELEASE_MS)
}
/**
* Runs [action] once the music has its focus back, or now if Gearslip isn't holding it.
* Metrolist takes a pause sent while it's held as nothing and resumes when focus returns,
* so "pause" sends it again from here.
*/
fun afterRelease(action: () -> Unit) {
main.post {
if (request == null && holders == 0) action() else afterRelease += action
}
}
/** Pauses the music until a matching [release]. Safe from any thread. */
@@ -75,6 +75,17 @@ import androidx.compose.ui.graphics.BlendMode
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.CompositingStrategy
import androidx.compose.foundation.layout.wrapContentHeight
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.PlayArrow
import androidx.compose.material.icons.filled.Search
import androidx.compose.runtime.CompositionLocalProvider
import app.seb3thehacker.gearslip.car.CarSettings
import app.seb3thehacker.gearslip.car.theme.GsButton
import app.seb3thehacker.gearslip.car.theme.GsIconButton
import app.seb3thehacker.gearslip.car.theme.GsThemes
import app.seb3thehacker.gearslip.car.theme.GsTone
import app.seb3thehacker.gearslip.car.theme.LocalGsTheme
import app.seb3thehacker.gearslip.speech.SpeechModels
import app.seb3thehacker.gearslip.stats.UsageStats
import kotlin.math.PI
@@ -106,14 +117,14 @@ object Intro {
}
/** The intro's tones, taken from the phone's own Material You colours like the rest of the app. */
private class IntroColors(
internal class IntroColors(
val surface: Color, val onSurface: Color, val onVariant: Color, val outline: Color,
val primary: Color, val onPrimary: Color, val primaryC: Color, val onPrimaryC: Color,
val tertiaryC: Color, val onTertiaryC: Color, val secondaryC: Color, val onSecondaryC: Color,
)
@Composable
private fun introColors(): IntroColors {
internal fun introColors(): IntroColors {
val m = MaterialTheme.colorScheme
return IntroColors(
surface = m.surface, onSurface = m.onSurface, onVariant = m.onSurfaceVariant, outline = m.outline,
@@ -123,7 +134,7 @@ private fun introColors(): IntroColors {
)
}
private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, DRIVE }
private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, THEME, DRIVE }
private val Pop = CubicBezierEasing(0.3f, 1.6f, 0.5f, 1f)
private val Rise = CubicBezierEasing(0.2f, 0.8f, 0.2f, 1f)
@@ -183,6 +194,9 @@ fun IntroScreen(onDone: () -> Unit) {
IntroStep.APPS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Good company.",
"Free, open-source apps that work well in the car. Tap one to see it on GitHub.", step,
extra = { AppPicks(c, ::cq, ::cqSp, step) }) { StarIcon(c.onSecondaryC) }
IntroStep.THEME -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Your buttons.",
"Pick how the car's buttons look. Change it any time in the car's settings.", step,
extra = { ThemePicker(c, ::cq, ::cqSp, step) }) { ButtonsIcon(c.onTertiaryC) }
IntroStep.DRIVE -> BadgeStep(c, ::cq, ::cqSp, c.primaryC, "Let's drive.",
"That's ${Intro.release}. Plug in, or just get in.", step,
action = "Start" to finish) { CheckIcon(c.onPrimaryC) }
@@ -195,7 +209,7 @@ fun IntroScreen(onDone: () -> Unit) {
/** 0 to 1 once each time [key] changes, after [delayMs]: drives the entrance of one element. */
@Composable
private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float {
internal fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float {
val a = remember(key) { Animatable(0f) }
LaunchedEffect(key) {
delay(delayMs.toLong())
@@ -205,14 +219,14 @@ private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float {
}
/** Slides up and fades in, like the mockup's `up`. */
private fun Modifier.up(p: Float, distance: Dp) = graphicsLayer {
internal fun Modifier.up(p: Float, distance: Dp) = graphicsLayer {
val e = Rise.transform(p)
alpha = e
translationY = (1 - e) * distance.toPx()
}
/** Grows in with a bounce, like the mockup's `bounce`. */
private fun Modifier.pop(p: Float) = graphicsLayer {
internal fun Modifier.pop(p: Float) = graphicsLayer {
val e = Pop.transform(p)
alpha = p.coerceAtMost(1f)
scaleX = 0.5f + 0.5f * e
@@ -501,9 +515,79 @@ private fun AppPicks(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUni
}
}
/**
* The car's button styles side by side, each drawn in its own style, and a row of real car
* controls below that redraws in whichever is picked. Picking saves it, so the car shows it too.
*/
@Composable
private fun ThemePicker(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) {
val chosenId by CarSettings.uiTheme.collectAsState()
val chosen = GsThemes.byId(chosenId)
// Flat first: it's the default.
val themes = listOf(GsThemes.Flat) + GsThemes.builtIn.filter { it != GsThemes.Flat }
Column(
Modifier.padding(top = cq(7f)).fillMaxWidth().up(entrance(key, 500, 700), cq(7f)),
verticalArrangement = Arrangement.spacedBy(cq(6f)),
) {
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(cq(3f))) {
themes.forEach { theme ->
CompositionLocalProvider(LocalGsTheme provides theme) {
GsButton(
onClick = { CarSettings.setUiTheme(theme.id) },
modifier = Modifier.weight(1f),
tone = if (theme == chosen) GsTone.Primary else GsTone.Neutral,
contentPadding = PaddingValues(horizontal = cq(1f), vertical = cq(3f)),
) {
Text(
if (theme == chosen) "✓ ${theme.name}" else theme.name,
style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.Bold),
maxLines = 1,
)
}
}
}
}
// What the car shows: the player's controls and a full-width action.
CompositionLocalProvider(LocalGsTheme provides chosen) {
Row(
Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(cq(3f)),
verticalAlignment = Alignment.CenterVertically,
) {
GsIconButton(Icons.Filled.Search, "Search", onClick = {}, size = cq(15f), iconSize = cq(7f))
GsIconButton(Icons.Filled.PlayArrow, "Play", onClick = {}, tone = GsTone.Primary, size = cq(15f), iconSize = cq(8f))
GsButton(
onClick = {},
modifier = Modifier.weight(1f).height(cq(15f)),
tone = GsTone.Tonal,
) { Text("Navigate", style = TextStyle(fontSize = cqSp(4.6f), fontWeight = FontWeight.Bold)) }
}
}
}
}
/** Three keys of a keypad, pressing one after another. */
@Composable
private fun ButtonsIcon(color: Color) {
val t by rememberInfiniteTransition(label = "keys").animateFloat(
0f, 3f, infiniteRepeatable(tween(2400, easing = LinearEasing)), label = "keys",
)
Canvas(Modifier.fillMaxSize()) {
inIconBox {
listOf(6f, 19f, 32f).forEachIndexed { i, x ->
// Each key dips for a moment while its turn comes round.
val dip = (1f - kotlin.math.abs(t - i - 0.5f) * 2f).coerceIn(0f, 1f) * 4f
drawRoundRect(color.copy(alpha = 0.45f), Offset(x, 16f + 4f), Size(10f, 16f),
androidx.compose.ui.geometry.CornerRadius(3f))
drawRoundRect(color, Offset(x, 16f + dip), Size(10f, 16f), androidx.compose.ui.geometry.CornerRadius(3f))
}
}
}
}
/** Material's scalloped "cookie": a circle whose radius waves around its edge. */
@Composable
private fun Cookie(color: Color, modifier: Modifier) {
internal fun Cookie(color: Color, modifier: Modifier) {
Canvas(modifier) {
val unit = size.minDimension / 100f
val path = Path()
@@ -578,7 +662,7 @@ private fun DialIcon(color: Color) {
}
@Composable
private fun MicIcon(color: Color) {
internal fun MicIcon(color: Color) {
val t = rememberInfiniteTransition(label = "mic")
val waves = (0..1).map { i ->
t.animateFloat(
@@ -605,7 +689,7 @@ private fun MicIcon(color: Color) {
}
@Composable
private fun StarIcon(color: Color) {
internal fun StarIcon(color: Color) {
val turn by rememberInfiniteTransition(label = "star").animateFloat(
-8f, 8f, infiniteRepeatable(tween(1600, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "star",
)
@@ -626,7 +710,7 @@ private fun StarIcon(color: Color) {
}
@Composable
private fun ShieldIcon(color: Color, tick: Color) {
internal fun ShieldIcon(color: Color, tick: Color) {
val p = entrance(IntroStep.GRAPHENE, 800, 500)
Canvas(Modifier.fillMaxSize()) {
inIconBox {
@@ -643,7 +727,7 @@ private fun ShieldIcon(color: Color, tick: Color) {
}
@Composable
private fun HeartIcon(color: Color) {
internal fun HeartIcon(color: Color) {
val beat by rememberInfiniteTransition(label = "beat").animateFloat(
1f, 1f,
infiniteRepeatable(keyframes {
@@ -696,7 +780,7 @@ private fun DrawScope.drawPartialStroke(points: List<Offset>, progress: Float, c
}
@Composable
private fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) {
internal fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) {
Row(modifier, horizontalArrangement = Arrangement.spacedBy(cq(2f))) {
repeat(count) { i ->
val on = i == current
@@ -1,6 +1,38 @@
package app.seb3thehacker.gearslip.ui
import androidx.compose.foundation.layout.BoxWithConstraints
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.systemBarsPadding
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Build
import androidx.compose.material.icons.filled.Call
import androidx.compose.material.icons.filled.Check
import androidx.compose.material.icons.filled.Close
import androidx.compose.material.icons.filled.DateRange
import androidx.compose.material.icons.filled.Email
import androidx.compose.material.icons.filled.LocationOn
import androidx.compose.material.icons.filled.Notifications
import androidx.compose.material.icons.filled.Phone
import androidx.compose.material.icons.filled.Warning
import androidx.compose.material3.Icon
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.graphics.vector.ImageVector
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.text.TextStyle
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.TextUnit
import androidx.compose.foundation.layout.heightIn
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
@@ -27,15 +59,7 @@ import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.padding
import androidx.compose.material3.Button
import androidx.compose.material3.FilledTonalButton
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Scaffold
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
@@ -66,14 +90,23 @@ private class GuidePage(
val title: String,
/** Plain text, or an [AnnotatedString] when part of it needs to stand out. */
val explanation: CharSequence,
/** Drawn inside the page's badge, in the badge's content colour. */
val icon: @Composable (Color) -> Unit,
/** Marked "Optional" above the title: everything works without it. */
val optional: Boolean = false,
val permissions: Array<String>? = null,
val command: String? = null,
val actionLabel: String? = null,
val onAction: (() -> Unit)? = null,
/** Drawn under [explanation], inside the card. */
/** Drawn under [explanation]. */
val content: (@Composable () -> Unit)? = null,
)
/** A plain Material icon, sized to fill the badge's icon box. */
private fun glyph(icon: ImageVector): @Composable (Color) -> Unit = { tint ->
Icon(icon, contentDescription = null, tint = tint, modifier = Modifier.fillMaxSize())
}
/** True once every permission in [permissions] is already granted - nothing left for this page to do. */
private fun granted(context: Context, permissions: Array<String>): Boolean =
permissions.all { context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED }
@@ -105,9 +138,11 @@ private fun hasNotificationAccess(context: Context): Boolean {
* who set up before the notes existed see once. Leaving it without turning them on is a no.
*/
private fun usageNotesPage(context: Context) = GuidePage(
title = USAGE_STATS_TITLE,
explanation = USAGE_STATS_INTRO,
title = "Help other drivers.",
explanation = USAGE_STATS_SHORT,
icon = { HeartIcon(it) },
content = {
UsageStatsReadMore()
UsageStatsChoices()
DisposableEffect(Unit) {
onDispose { if (!UsageStats.asked(context)) UsageStats.set(context, false) }
@@ -119,11 +154,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
// --- Safety: always first, and never skipped by an already-granted check ---------
add(
GuidePage(
title = "Drive safely",
explanation = "This guide sets up the screen you will see mounted in your car. " +
"Gearslip is unofficial software: it is not made or reviewed by any carmaker, " +
"and it can fail without warning. Never interact with it while driving - set it " +
"up, mount the phone, then keep your attention on the road.",
title = "Drive safely.",
explanation = "Gearslip is unofficial. No carmaker made or checked it, and it can fail " +
"without warning. Set it up before you go, then keep your eyes on the road.",
icon = glyph(Icons.Filled.Warning),
),
)
@@ -132,10 +166,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!granted(context, arrayOf(Manifest.permission.RECORD_AUDIO))) {
add(
GuidePage(
title = "Microphone",
explanation = "Gearslip needs the microphone for two jobs: passing your music " +
"app's sound to the car, and hearing you when you reply to a message or ask " +
"the assistant. It listens only then, and your speech stays on the phone.",
title = "Microphone.",
explanation = "Gearslip uses it to send your music to the car and to hear you when you " +
"ask for something or reply to a message. What you say stays on the phone.",
icon = { MicIcon(it) },
permissions = arrayOf(Manifest.permission.RECORD_AUDIO),
),
)
@@ -143,9 +177,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!granted(context, arrayOf(Manifest.permission.ACCESS_FINE_LOCATION))) {
add(
GuidePage(
title = "Location",
explanation = "The car's map and nearby places need to know where you are. " +
"Without it, the dashboard has no map card.",
title = "Location.",
explanation = "The car's map and nearby searches need it. Without it, the dashboard " +
"shows no map.",
icon = glyph(Icons.Filled.LocationOn),
permissions = arrayOf(Manifest.permission.ACCESS_FINE_LOCATION),
),
)
@@ -153,9 +188,9 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!granted(context, arrayOf(Manifest.permission.READ_CALENDAR))) {
add(
GuidePage(
title = "Calendar",
explanation = "Gearslip reads your next event to show an agenda card on the car " +
"screen. It only reads; it never edits or deletes anything.",
title = "Calendar.",
explanation = "The car screen shows your next event. Gearslip only reads your calendar.",
icon = glyph(Icons.Filled.DateRange),
permissions = arrayOf(Manifest.permission.READ_CALENDAR),
),
)
@@ -170,11 +205,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!granted(context, callPermissions)) {
add(
GuidePage(
title = "Phone and contacts",
explanation = "When a call comes in, Gearslip matches the number against your " +
"contacts, so the car screen can show a name instead of just digits. The " +
"same permissions let the car screen's Phone app dial a number or call " +
"someone from your contacts.",
title = "Calls and contacts.",
explanation = "The car shows who's calling by name, and its Phone app can dial a " +
"number or call one of your contacts. Gearslip only reads them.",
icon = glyph(Icons.Filled.Phone),
permissions = callPermissions,
),
)
@@ -183,10 +217,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!granted(context, arrayOf(Manifest.permission.POST_NOTIFICATIONS))) {
add(
GuidePage(
title = "Notifications",
explanation = "Gearslip shows a notification while it's connected to the car or " +
"sending it audio, with a button to stop. Without this permission, Android " +
"hides it.",
title = "Notifications.",
explanation = "While Gearslip is connected, it shows a notification with a button " +
"to stop. Without this, Android hides it.",
icon = glyph(Icons.Filled.Notifications),
permissions = arrayOf(Manifest.permission.POST_NOTIFICATIONS),
),
)
@@ -194,10 +228,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!hasNotificationAccess(context)) {
add(
GuidePage(
title = "Notification access",
explanation = "The car screen shows your phone's notifications and lets you " +
"reply from there. Android only allows that once you grant Gearslip access, " +
"from the list this opens - it needs this to work at all.",
title = "Messages.",
explanation = "The car shows your messages, reads them aloud and lets you reply. " +
"Turn Gearslip on in the list this opens.",
icon = glyph(Icons.Filled.Email),
actionLabel = "Open notification access",
onAction = {
context.startActivity(Intent(Settings.ACTION_NOTIFICATION_LISTENER_SETTINGS))
@@ -212,10 +246,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (androidAutoPackage != null) {
add(
GuidePage(
title = "Remove Android Auto",
explanation = "Android Auto is still installed. It claims the same USB " +
"connection Gearslip needs, so the two fight over it. Disable or uninstall " +
"it from the app info screen this opens.",
title = "Turn off Android Auto.",
explanation = "Android Auto is still installed, and it fights Gearslip for the car's " +
"USB connection. Disable or uninstall it on the screen this opens.",
icon = glyph(Icons.Filled.Close),
actionLabel = "Open Android Auto's app info",
onAction = {
context.startActivity(
@@ -229,11 +263,10 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
// --- Default handler ---------------------------------------------------------------
add(
GuidePage(
title = "Make Gearslip the default",
explanation = "Android has no separate default-app setting for a USB accessory like " +
"the car. It just launches whichever single app matches - which is Gearslip, " +
"now that Android Auto is out of the way. If a chooser still appears when you " +
"plug in, pick Gearslip and tap Always so Android stops asking.",
title = "Choose Gearslip.",
explanation = "If your phone asks which app to open when you plug into the car, pick " +
"Gearslip and tap Always.",
icon = glyph(Icons.Filled.Check),
),
)
@@ -241,10 +274,11 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!hasProjectMediaOp(context)) {
add(
GuidePage(
title = "Optional: trust audio every drive",
explanation = "Connect the phone to a computer with USB debugging on, and run this " +
"command. It tells Android to trust Gearslip with audio from now on. Skip it " +
"and everything still works, but you tap Start once each time you set off.",
title = "No audio prompt.",
explanation = "Run this from a computer with USB debugging on, and Android trusts " +
"Gearslip with your audio for good. Without it, you tap Start once each drive.",
icon = glyph(Icons.Filled.Build),
optional = true,
command = "adb shell appops set ${BuildConfig.APPLICATION_ID} PROJECT_MEDIA allow",
),
)
@@ -252,10 +286,12 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!granted(context, arrayOf(Manifest.permission.WRITE_SECURE_SETTINGS))) {
add(
GuidePage(
title = "Optional: keep notifications readable",
explanation = "Android treats sending audio to the car like sharing your screen, and " +
"hides what your notifications say. This command switches that off only while " +
"music is going to the car, then puts it back.",
title = "Readable notifications.",
explanation = "While audio goes to the car, Android hides what notifications say, " +
"as if you were sharing your screen. This lets Gearslip show them during a " +
"drive and hide them again after.",
icon = glyph(Icons.Filled.Build),
optional = true,
command = "adb shell pm grant ${BuildConfig.APPLICATION_ID} android.permission.WRITE_SECURE_SETTINGS",
),
)
@@ -263,10 +299,11 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (!hasCallScreeningRole(context)) {
add(
GuidePage(
title = "Optional: answer and decline calls",
explanation = "This sets Gearslip as your phone's caller ID and spam-blocking app, " +
"the same kind of consent a call-blocking app asks for. Without it, calls still " +
"ring, but only the phone itself can answer or decline them.",
title = "Answer calls.",
explanation = "Make Gearslip your caller ID app, and the car can answer and decline " +
"calls. Without it, calls still ring, but only the phone can answer them.",
icon = glyph(Icons.Filled.Call),
optional = true,
actionLabel = "Set up",
onAction = onRequestCallScreening,
),
@@ -277,8 +314,9 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List<Gu
if (SHOW_RECOMMENDED_APPS) {
add(
GuidePage(
title = "Recommended apps",
title = "Good company.",
explanation = RECOMMENDED_INTRO,
icon = { StarIcon(it) },
content = { RecommendedAppsList() },
),
)
@@ -325,31 +363,23 @@ fun SetupGuideScreen(onRequestCallScreening: () -> Unit, onDone: () -> Unit, onl
context.checkSelfPermission(it) == PackageManager.PERMISSION_GRANTED
} ?: false
Scaffold(containerColor = MaterialTheme.colorScheme.background) { padding ->
Column(
Modifier
.fillMaxSize()
.padding(padding)
.padding(horizontal = 24.dp),
horizontalAlignment = Alignment.CenterHorizontally,
) {
Spacer(Modifier.height(24.dp))
// A single page has no progress to show.
if (pages.size > 1) {
LinearProgressIndicator(
progress = { (index + 1f) / pages.size },
modifier = Modifier.fillMaxWidth().height(10.dp).clip(MaterialTheme.shapes.extraLarge),
)
Spacer(Modifier.height(12.dp))
Text(
"${index + 1} of ${pages.size}",
style = MaterialTheme.typography.titleMedium,
fontWeight = FontWeight.SemiBold,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
// The page scrolls between the progress bar and the buttons, so a tall page (the usage
// note's choices, say) never pushes Next off the screen. Short pages stay centred.
val c = introColors()
// The badges take turns through the phone's three accent colours, as the release intro does.
val (badge, onBadge) = listOf(c.primaryC to c.onPrimaryC, c.secondaryC to c.onSecondaryC, c.tertiaryC to c.onTertiaryC)[index % 3]
val spin by rememberInfiniteTransition(label = "spin").animateFloat(
0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin",
)
BoxWithConstraints(Modifier.fillMaxSize().background(c.surface).systemBarsPadding()) {
val w = maxWidth
// One percent of the screen's width, as in the intro, so it scales the same way.
fun cq(n: Float): Dp = w * (n / 100f)
val density = LocalDensity.current
fun cqSp(n: Float): TextUnit = with(density) { cq(n).toSp() }
Column(Modifier.fillMaxSize().padding(horizontal = cq(8f))) {
// The page scrolls above the buttons, so a tall one (the usage notes' choices) never
// pushes them off the screen. Short pages sit in the middle.
BoxWithConstraints(Modifier.weight(1f).fillMaxWidth()) {
key(index) {
Column(
@@ -357,88 +387,114 @@ fun SetupGuideScreen(onRequestCallScreening: () -> Unit, onDone: () -> Unit, onl
.fillMaxWidth()
.verticalScroll(rememberScrollState())
.heightIn(min = maxHeight)
.padding(vertical = 24.dp),
.padding(vertical = cq(6f)),
verticalArrangement = Arrangement.Center,
) {
Surface(color = MaterialTheme.colorScheme.surfaceContainerLow, shape = MaterialTheme.shapes.extraLarge) {
Column(Modifier.fillMaxWidth().padding(24.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) {
Box(Modifier.size(cq(28f)).pop(entrance(index, 50, 800)), contentAlignment = Alignment.Center) {
Cookie(badge, Modifier.fillMaxSize().graphicsLayer { rotationZ = spin })
Box(Modifier.size(cq(12f))) { page.icon(onBadge) }
}
Spacer(Modifier.height(cq(7f)))
if (page.optional) {
Text(
"OPTIONAL", color = c.primary,
style = TextStyle(fontSize = cqSp(3.4f), fontWeight = FontWeight.Bold, letterSpacing = cqSp(0.4f)),
modifier = Modifier.up(entrance(index, 150, 700), cq(7f)),
)
Spacer(Modifier.height(cq(1.5f)))
}
Text(
page.title, color = c.onSurface,
// Long titles step down a size so they stay on two lines.
style = TextStyle(
fontSize = cqSp(if (page.title.length > 16) 8f else 10f), fontWeight = FontWeight.Black,
letterSpacing = cqSp(-0.4f), lineHeight = cqSp(if (page.title.length > 16) 8.6f else 10.5f),
),
modifier = Modifier.up(entrance(index, 200, 700), cq(7f)),
)
Spacer(Modifier.height(cq(4f)))
Column(
Modifier.up(entrance(index, 350, 700), cq(7f)),
verticalArrangement = Arrangement.spacedBy(cq(3f)),
) {
Text(
page.explanation as? AnnotatedString ?: AnnotatedString(page.explanation.toString()),
color = c.onVariant,
style = TextStyle(fontSize = cqSp(4.5f), lineHeight = cqSp(6.5f)),
)
page.command?.let { CopyableCommand(it) }
page.content?.invoke()
if (page.permissions != null && alreadyGranted) {
Text(
page.title,
style = MaterialTheme.typography.headlineSmall,
fontWeight = FontWeight.Bold,
color = MaterialTheme.colorScheme.onSurface,
"Already allowed.", color = c.primary,
style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.SemiBold),
)
Text(
page.explanation as? AnnotatedString ?: AnnotatedString(page.explanation.toString()),
style = MaterialTheme.typography.bodyLarge,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
page.command?.let { CopyableCommand(it) }
page.content?.invoke()
}
}
}
}
}
// A single page has nothing to count.
if (pages.size > 1) {
Dots(c, pages.size, index, ::cq, Modifier.align(Alignment.CenterHorizontally))
Spacer(Modifier.height(cq(5f)))
}
val back = if (index > 0) ({ index -= 1 }) else null
val next = { index += 1 }
when {
page.permissions != null && alreadyGranted -> {
Text(
"Already allowed.",
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.primary,
)
Spacer(Modifier.height(16.dp))
PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) {
Button(onClick = { index += 1 }, modifier = it) { Text("Next") }
}
}
page.permissions != null -> {
PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) {
Button(onClick = { launcher.launch(page.permissions) }, modifier = it) { Text("Allow") }
}
Spacer(Modifier.height(8.dp))
TextButton(onClick = { index += 1 }) { Text("Skip for now") }
page.permissions != null && !alreadyGranted -> {
PillRow(c, ::cq, ::cqSp, back, "Allow") { launcher.launch(page.permissions) }
Pill(c, ::cqSp, "Skip for now", PillStyle.TEXT, Modifier.align(Alignment.CenterHorizontally), next)
}
page.onAction != null -> {
FilledTonalButton(
onClick = page.onAction,
modifier = Modifier.fillMaxWidth().height(52.dp),
) { Text(page.actionLabel ?: "Open") }
Spacer(Modifier.height(8.dp))
PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) {
Button(onClick = { index += 1 }, modifier = it) { Text("Next") }
}
}
else -> {
PrimaryRow(showBack = index > 0, onBack = { index -= 1 }) {
Button(onClick = { index += 1 }, modifier = it) {
Text(if (index == pages.lastIndex) "Done" else "Next")
}
}
Pill(c, ::cqSp, page.actionLabel ?: "Open", PillStyle.OUTLINED, Modifier.fillMaxWidth().padding(bottom = cq(3f)), page.onAction)
PillRow(c, ::cq, ::cqSp, back, "Next", next)
}
else -> PillRow(c, ::cq, ::cqSp, back, if (index == pages.lastIndex) "Done" else "Next", next)
}
Spacer(Modifier.height(24.dp))
Spacer(Modifier.height(cq(6f)))
}
}
}
/**
* The bottom action row: a lower-emphasis Back on the left once there is a previous page, and the
* page's own primary button - passed [content] so the caller keeps the button's own label and
* click behavior - filling the rest of the width.
*/
private enum class PillStyle { FILLED, OUTLINED, TEXT }
/** The intro's rounded buttons: filled for the step's main action, outlined beside it, bare text to skip. */
@Composable
private fun PrimaryRow(showBack: Boolean, onBack: () -> Unit, content: @Composable (Modifier) -> Unit) {
if (showBack) {
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(12.dp)) {
OutlinedButton(
onClick = onBack,
modifier = Modifier.weight(1f).height(52.dp),
) { Text("Back") }
content(Modifier.weight(1f).height(52.dp))
}
} else {
content(Modifier.fillMaxWidth().height(52.dp))
private fun Pill(c: IntroColors, cqSp: (Float) -> TextUnit, label: String, style: PillStyle, modifier: Modifier = Modifier, onClick: () -> Unit) {
val shape = RoundedCornerShape(50)
Box(
modifier
.clip(shape)
.then(
when (style) {
PillStyle.FILLED -> Modifier.background(c.primary)
PillStyle.OUTLINED -> Modifier.border(BorderStroke(2.dp, c.outline), shape)
PillStyle.TEXT -> Modifier
},
)
.clickable(onClick = onClick)
.heightIn(min = 52.dp)
.padding(horizontal = 20.dp, vertical = 12.dp),
contentAlignment = Alignment.Center,
) {
Text(
label,
color = if (style == PillStyle.FILLED) c.onPrimary else c.primary,
style = TextStyle(fontSize = cqSp(4.4f), fontWeight = FontWeight.Bold),
)
}
}
/** Back on the left once there's a page to go back to, and the page's main action filling the rest. */
@Composable
private fun PillRow(
c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit,
onBack: (() -> Unit)?, label: String, onClick: () -> Unit,
) {
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(cq(3f))) {
if (onBack != null) Pill(c, cqSp, "Back", PillStyle.OUTLINED, Modifier.weight(1f), onBack)
Pill(c, cqSp, label, PillStyle.FILLED, Modifier.weight(if (onBack != null) 1.4f else 1f), onClick)
}
}
@@ -37,6 +37,10 @@ import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.TextButton
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.semantics.Role
import androidx.compose.ui.text.AnnotatedString
@@ -70,6 +74,31 @@ internal val USAGE_STATS_INTRO: AnnotatedString = buildAnnotatedString {
append(INTRO_END)
}
/**
* The setup guide's version of [USAGE_STATS_INTRO]: what the notes are and why, in three
* sentences. The full text is one tap away in [UsageStatsReadMore].
*/
internal val USAGE_STATS_SHORT: AnnotatedString = buildAnnotatedString {
append("Gearslip can send a short note after each drive: its version and the car details you pick below. ")
append("The notes show which cars work and where the others fail. ")
withStyle(SpanStyle(fontWeight = FontWeight.Bold)) { append("No name, account, or location goes with them.") }
}
/** "Read more", opening the whole pitch and privacy promise in a dialog. */
@Composable
internal fun UsageStatsReadMore() {
var open by remember { mutableStateOf(false) }
TextButton(onClick = { open = true }, contentPadding = PaddingValues(0.dp)) { Text("Read more") }
if (open) {
AlertDialog(
onDismissRequest = { open = false },
title = { Text("About usage notes") },
text = { Text(USAGE_STATS_INTRO, modifier = Modifier.verticalScroll(rememberScrollState())) },
confirmButton = { TextButton(onClick = { open = false }) { Text("Close") } },
)
}
}
/** The usage notes card for the setup guide: the same rows Settings shows. */
@Composable
internal fun UsageStatsChoices(modifier: Modifier = Modifier) {
@@ -27,7 +27,7 @@ import app.seb3thehacker.gearslip.speech.SpeechModels.State
/**
* Which speech model the voice assistant listens with, and downloading it. Each is fetched once
* from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening.
* from sherpa-onnx's author ([SpeechModels] says where); nothing is sent anywhere while listening.
*/
@Composable
internal fun VoiceModelSection() {
@@ -38,7 +38,7 @@ internal fun VoiceModelSection() {
SettingsSection(title = "Voice: live words") {
Text(
"The voice assistant turns speech into text on this phone, and nothing you say leaves it. " +
"It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " +
"It needs two models, about 700 MB together, downloaded once. Use Wi-Fi if you can. " +
"This one writes your words as you speak.",
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,