From af9e9a7571d549f3d6305a09652df7abfeaaa5ab Mon Sep 17 00:00:00 2001 From: Seb3thehacker Date: Mon, 12 Oct 2026 00:20:36 -0700 Subject: [PATCH] voice: play songs by name, a help list, and the phone's speech recognition as an option; Spotify connects without the wait --- .../gearslip/car/AssistantCommands.kt | 30 +++++- .../gearslip/car/CarAssistant.kt | 3 + .../seb3thehacker/gearslip/car/CarSettings.kt | 11 +++ .../gearslip/car/CarSettingsScreen.kt | 12 +++ .../app/seb3thehacker/gearslip/car/CarUi.kt | 1 + .../gearslip/car/VoiceHelpOverlay.kt | 92 +++++++++++++++++++ .../seb3thehacker/gearslip/media/CarMedia.kt | 80 ++++++++++++++-- .../gearslip/speech/SpeechEngine.kt | 13 ++- 8 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt index 12cd633..e42b43a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt @@ -17,9 +17,15 @@ import kotlinx.coroutines.withContext object AssistantCommands { suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String { - val heard = heardRaw.lowercase().trim() + val heard = heardRaw.lowercase().trim().trimEnd('.', '?', '!') + VoiceHelp.hide() + if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) { + VoiceHelp.show() + return "Here's what you can say." + } navigateCommand(context, navigator, frame, heard)?.let { return it } + playSearchCommand(heard)?.let { return it } mediaCommand(heard)?.let { return it } navCommand(navigator, heard)?.let { return it } infoCommand(heard)?.let { return it } @@ -75,6 +81,24 @@ object AssistantCommands { return "Navigating to $destination with ${chosen.label}." } + /** + * "play X" with something after it: a song, artist, album or playlist, asked of the music app + * that's open through its own search ([CarMedia.search]). A bare "play", or "play music", + * falls through to resume. + */ + private fun playSearchCommand(heard: String): String? { + if (!heard.startsWith("play ")) return null + val query = heard.removePrefix("play ").trim().removeSuffix(".").trim() + if (query.isEmpty() || query in RESUME_WORDS) return null + val media = CarServices.media + val app = media.appLabel ?: return "Open a music app first, then ask again." + if (!media.canSearchLibrary && !media.now.value.canSearch) { + return "$app ignores voice searches. Pick something on its screen instead." + } + media.playBestMatch(query) + return "Looking for $query on $app." + } + private fun mediaCommand(heard: String): String? { val media = CarServices.media return when { @@ -179,4 +203,8 @@ object AssistantCommands { } private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ") + + private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands") + + private val RESUME_WORDS = setOf("music", "the music", "something", "it", "again", "that") } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt index cf4b27a..b09a93b 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt @@ -190,6 +190,9 @@ object CarAssistant { onSpeaking: () -> Unit = {}, ) { val context = appContext ?: return onError("Speech isn't ready yet.") + if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) { + return systemListen(context, onPartial, onHeard, onResult, onError) + } SpeechEngine.listen( context, cue = cue, diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index f59ddbd..64afbda 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -39,6 +39,7 @@ object CarSettings { private const val KEY_APP_ORDER = "car_app_order" private const val KEY_EXPERIMENTAL_FEATURES = "car_experimental_features" private const val KEY_VOICE_ASSISTANT = "car_voice_assistant" + private const val KEY_SYSTEM_SPEECH = "car_system_speech" private const val KEY_CAR_SENSORS = "car_sensors_enabled" private const val KEY_MEDIA_SEARCH = "car_media_search_enabled" private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning" @@ -73,6 +74,7 @@ object CarSettings { private val appOrderFlow = MutableStateFlow>(emptyList()) private val experimentalFeaturesFlow = MutableStateFlow(false) private val voiceAssistantFlow = MutableStateFlow(false) + private val systemSpeechFlow = MutableStateFlow(false) private val carSensorsFlow = MutableStateFlow(false) private val mediaSearchFlow = MutableStateFlow(false) private val seenSafetyWarningFlow = MutableStateFlow(false) @@ -177,6 +179,9 @@ object CarSettings { */ val voiceAssistantEnabled: StateFlow = voiceAssistantFlow + /** Listen with the phone's own speech service instead of Gearslip's on-device model. */ + val systemSpeech: StateFlow = systemSpeechFlow + /** * Vehicle data is experimental and off by default: the launcher's tile for it, and the * voice assistant's "vehicle data" command, only show up once this and @@ -227,6 +232,7 @@ object CarSettings { .split(",").filter { it.isNotEmpty() } experimentalFeaturesFlow.value = AppSettings.getString(app, KEY_EXPERIMENTAL_FEATURES, "false") == "true" voiceAssistantFlow.value = AppSettings.getString(app, KEY_VOICE_ASSISTANT, "false") == "true" + systemSpeechFlow.value = AppSettings.getString(app, KEY_SYSTEM_SPEECH, "false") == "true" carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true" mediaSearchFlow.value = AppSettings.getString(app, KEY_MEDIA_SEARCH, "false") == "true" seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true" @@ -293,6 +299,11 @@ object CarSettings { AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString()) } + fun setSystemSpeech(on: Boolean) { + systemSpeechFlow.value = on + AppSettings.putString(app, KEY_SYSTEM_SPEECH, on.toString()) + } + fun setVoiceAssistantEnabled(on: Boolean) { voiceAssistantFlow.value = on AppSettings.putString(app, KEY_VOICE_ASSISTANT, on.toString()) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index 6f0f53f..05a57e3 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -220,6 +220,18 @@ private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolea checked = voiceAssistant, ) { CarSettings.setVoiceAssistantEnabled(it) } + if (voiceAssistant) { + val systemSpeech by CarSettings.systemSpeech.collectAsState() + ExperimentalFeatureRow( + title = "Use the phone's speech recognition", + description = "Understands more words, like place and brand names, but listens " + + "through the phone's microphone, not the car's. On most phones this is Google's, " + + "which may send your voice to Google. GrapheneOS has none, so it stays with " + + "Gearslip's own.", + checked = systemSpeech, + ) { CarSettings.setSystemSpeech(it) } + } + ExperimentalFeatureRow( title = "Vehicle data", description = "Adds a Vehicle data tile: a live, raw dump of the head unit's sensor " + diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt index c72f681..60ce1bd 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt @@ -256,6 +256,7 @@ fun CarUi() { modifier = Modifier.align(Alignment.TopEnd), ) // Above the popup: a call is more urgent than any notification. + VoiceHelpOverlay(Modifier.align(Alignment.Center)) CallOverlay(Modifier.align(Alignment.TopCenter)) AssistantOverlay(Modifier.align(Alignment.TopCenter)) if (app.seb3thehacker.gearslip.BuildConfig.DEBUG) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt new file mode 100644 index 0000000..fcc8368 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt @@ -0,0 +1,92 @@ +package app.seb3thehacker.gearslip.car + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.car.theme.GsIconButton +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** What the assistant understands, shown on the car screen when the driver says "help". */ +object VoiceHelp { + private val _shown = MutableStateFlow(false) + val shown: StateFlow = _shown.asStateFlow() + + fun show() { _shown.value = true } + fun hide() { _shown.value = false } + + /** Say this, and this happens. Kept in step with [AssistantCommands]. */ + val COMMANDS = listOf( + "Navigate to…" to "Directions there. Add \"with Organic Maps\" to pick the app", + "Play…" to "A song, artist, or album in the open music app", + "Pause · Resume · Next · Previous" to "Control the music", + "What's playing" to "The song and artist", + "Call…" to "A contact by name", + "Open…" to "Any app on this screen", + "Go home · Open apps · Settings" to "Move around Gearslip", + "What time is it · Weather" to "The time, or the forecast", + "Help" to "This list", + ) +} + +/** The command list over the car screen: read it at a glance, tap the cross or wait to dismiss. */ +@Composable +fun VoiceHelpOverlay(modifier: Modifier = Modifier) { + val shown by VoiceHelp.shown.collectAsState() + if (!shown) return + // A list left up would cover the map for the rest of the drive. + LaunchedEffect(Unit) { + delay(30_000) + VoiceHelp.hide() + } + Surface( + color = MaterialTheme.colorScheme.surfaceContainerHigh, + contentColor = MaterialTheme.colorScheme.onSurface, + shape = MaterialTheme.shapes.extraLarge, + tonalElevation = 6.dp, + shadowElevation = 8.dp, + modifier = modifier.padding(16.dp).widthIn(max = 640.dp), + ) { + Column(Modifier.padding(20.dp)) { + Row(Modifier.fillMaxWidth()) { + Text( + "You can say", + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.Bold, + modifier = Modifier.weight(1f).padding(top = 6.dp), + ) + GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide) + } + Column( + Modifier.verticalScroll(rememberScrollState()).padding(top = 8.dp), + verticalArrangement = Arrangement.spacedBy(10.dp), + ) { + VoiceHelp.COMMANDS.forEach { (say, does) -> + Column { + Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) + Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + } + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt index 056621d..eaa9006 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt @@ -156,6 +156,9 @@ class CarMedia(private val context: Context) { private val main = Handler(Looper.getMainLooper()) private var connecting: MediaApp? = null + /** The music app Gearslip is attached to, for the assistant to name; null when there's none. */ + val appLabel: String? get() = connecting?.label?.takeIf { controller != null } + private val _phase = MutableStateFlow(Phase.IDLE) val phase: StateFlow = _phase.asStateFlow() @@ -236,6 +239,12 @@ class CarMedia(private val context: Context) { connecting = app _phase.value = Phase.CONNECTING _browse.value = BrowseState() + // Known to refuse its library (Spotify): asking only costs the seconds it takes to say no, + // so go straight to its playback session. + if (KnownApps.playerWorks(app.component.packageName)) { + attachToSession(app) + return + } val mb = MediaBrowser(context, app.component, object : MediaBrowser.ConnectionCallback() { override fun onConnected() = this@CarMedia.onConnected(app) override fun onConnectionFailed() { @@ -273,7 +282,12 @@ class CarMedia(private val context: Context) { return } if (session == null) { - val limit = if (launched) SESSION_RETRIES_LAUNCHED else SESSION_RETRIES + // A known session-only app that isn't running won't start by waiting: say so at once. + val limit = when { + launched -> SESSION_RETRIES_LAUNCHED + KnownApps.playerWorks(app.component.packageName) -> 0 + else -> SESSION_RETRIES + } // The app only publishes its session once its service has started; give it a moment. if (attempt < limit) { main.postDelayed({ attachToSession(app, attempt + 1, launched) }, SESSION_RETRY_MS) @@ -454,7 +468,7 @@ class CarMedia(private val context: Context) { } val component = searchComponent if (component == null) { - _results.value = Results(q, emptyList(), supported = false) + noLibrarySearch(q) return } resultsQuery = q @@ -476,13 +490,13 @@ class CarMedia(private val context: Context) { override fun onConnectionFailed() { GearslipLog.w("media: no library connection for search") searchBrowser = null - resultsQuery?.let { _results.value = Results(it, emptyList(), supported = false) } + resultsQuery?.let { noLibrarySearch(it) } } }, null) searchBrowser = created runCatching { created.connect() }.onFailure { searchBrowser = null - _results.value = Results(q, emptyList(), supported = false) + noLibrarySearch(q) } } @@ -492,18 +506,42 @@ class CarMedia(private val context: Context) { override fun onSearchResult(query: String, extras: Bundle?, items: List) { if (query != resultsQuery) return GearslipLog.i("media: library search found ${items.size}") - _results.value = Results(query, items.map(::entryOfCompat)) + val entries = items.map(::entryOfCompat) + if (query == playFirstFor) { + playFirstFor = null + val first = entries.firstOrNull { it.playable } ?: entries.firstOrNull() + if (first == null) { + clearResults() + search(query) + } else { + GearslipLog.i("media: playing the library's best match") + clearResults() + searchBefore = _now.value + _search.value = Search(query) + main.removeCallbacks(searchTimeout) + controller?.transportControls?.playFromMediaId(first.id, null) + main.postDelayed(searchTimeout, SEARCH_WAIT_MS) + } + return + } + _results.value = Results(query, entries) } override fun onError(query: String, extras: Bundle?) { if (query != resultsQuery) return + if (query == playFirstFor) { + playFirstFor = null + clearResults() + search(query) + return + } GearslipLog.i("media: the app has no library search") _results.value = Results(query, emptyList(), supported = false) } }) }.onFailure { GearslipLog.w("media: library search failed: ${it.message}") - _results.value = Results(q, emptyList(), supported = false) + noLibrarySearch(q) } } @@ -520,6 +558,36 @@ class CarMedia(private val context: Context) { ) } + /** The library can't search: a voice "play X" falls back to the app's own search; a typed one says so. */ + private fun noLibrarySearch(q: String) { + if (q == playFirstFor) { + playFirstFor = null + clearResults() + search(q) + } else { + _results.value = Results(q, emptyList(), supported = false) + } + } + + /** A voice "play X" waiting on the library search, to play its first match instead of listing them. */ + private var playFirstFor: String? = null + + /** + * Plays the best match for [query], for voice. Asks the app's library first and plays its top + * result, which apps like ViVi honour where they ignore [search]; without a library search, + * falls back to [search]. + */ + fun playBestMatch(query: String) { + val q = query.trim() + if (q.isEmpty()) return + if (!canSearchLibrary) { + search(q) + return + } + playFirstFor = q + searchLibrary(q) + } + /** The connected app's service, for the search connection; null when nothing is connected. */ private var searchComponent: android.content.ComponentName? = null private var searchBrowser: MediaBrowserCompat? = null diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt index 2fc90fa..1d052fe 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt @@ -33,8 +33,11 @@ import kotlin.math.sqrt */ object SpeechEngine { - /** FUTO's ACFT Whisper tiny (Apache-2.0), re-quantized to 5 bits, inside the APK. */ - private const val MODEL = "speech/tiny_acft_q5_1.bin" + /** + * FUTO's ACFT Whisper (Apache-2.0), re-quantized to 5 bits, inside the APK: base when the build + * carries it (more accurate, about twice as slow), tiny otherwise. + */ + private val MODELS = listOf("speech/base_acft_q5_1.bin", "speech/tiny_acft_q5_1.bin") /** The error for an engine that can't run at all, which callers may answer with a fallback. */ const val NOT_LOADED = "The speech model didn't load." @@ -258,12 +261,14 @@ object SpeechEngine { libraryLoaded = true } val started = SystemClock.uptimeMillis() - handle = nativeInitAsset(context.assets, MODEL) + val model = MODELS.firstOrNull { name -> runCatching { context.assets.open(name).close() }.isSuccess } + ?: run { GearslipLog.w("speech: no model in the build"); return false } + handle = nativeInitAsset(context.assets, model) if (handle == 0L) { GearslipLog.w("speech: couldn't load the model") return false } - GearslipLog.i("speech: loaded the model in ${SystemClock.uptimeMillis() - started} ms") + GearslipLog.i("speech: loaded $model in ${SystemClock.uptimeMillis() - started} ms") return true }