voice: play songs by name, a help list, and the phone's speech recognition as an option; Spotify connects without the wait

gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt

@@ -17,9 +17,15 @@ import kotlinx.coroutines.withContext
 object AssistantCommands {
 
     suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String {
-        val heard = heardRaw.lowercase().trim()
+        val heard = heardRaw.lowercase().trim().trimEnd('.', '?', '!')
+        VoiceHelp.hide()
+        if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) {
+            VoiceHelp.show()
+            return "Here's what you can say."
+        }
 
         navigateCommand(context, navigator, frame, heard)?.let { return it }
+        playSearchCommand(heard)?.let { return it }
         mediaCommand(heard)?.let { return it }
         navCommand(navigator, heard)?.let { return it }
         infoCommand(heard)?.let { return it }
@@ -75,6 +81,24 @@ object AssistantCommands {
         return "Navigating to $destination with ${chosen.label}."
     }
 
+    /**
+     * "play X" with something after it: a song, artist, album or playlist, asked of the music app
+     * that's open through its own search ([CarMedia.search]). A bare "play", or "play music",
+     * falls through to resume.
+     */
+    private fun playSearchCommand(heard: String): String? {
+        if (!heard.startsWith("play ")) return null
+        val query = heard.removePrefix("play ").trim().removeSuffix(".").trim()
+        if (query.isEmpty() || query in RESUME_WORDS) return null
+        val media = CarServices.media
+        val app = media.appLabel ?: return "Open a music app first, then ask again."
+        if (!media.canSearchLibrary && !media.now.value.canSearch) {
+            return "$app ignores voice searches. Pick something on its screen instead."
+        }
+        media.playBestMatch(query)
+        return "Looking for $query on $app."
+    }
+
     private fun mediaCommand(heard: String): String? {
         val media = CarServices.media
         return when {
@@ -179,4 +203,8 @@ object AssistantCommands {
     }
 
     private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ")
+
+    private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands")
+
+    private val RESUME_WORDS = setOf("music", "the music", "something", "it", "again", "that")
 }

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt

@@ -190,6 +190,9 @@ object CarAssistant {
         onSpeaking: () -> Unit = {},
     ) {
         val context = appContext ?: return onError("Speech isn't ready yet.")
+        if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) {
+            return systemListen(context, onPartial, onHeard, onResult, onError)
+        }
         SpeechEngine.listen(
             context,
             cue = cue,

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt

@@ -39,6 +39,7 @@ object CarSettings {
     private const val KEY_APP_ORDER = "car_app_order"
     private const val KEY_EXPERIMENTAL_FEATURES = "car_experimental_features"
     private const val KEY_VOICE_ASSISTANT = "car_voice_assistant"
+    private const val KEY_SYSTEM_SPEECH = "car_system_speech"
     private const val KEY_CAR_SENSORS = "car_sensors_enabled"
     private const val KEY_MEDIA_SEARCH = "car_media_search_enabled"
     private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning"
@@ -73,6 +74,7 @@ object CarSettings {
     private val appOrderFlow = MutableStateFlow<List<String>>(emptyList())
     private val experimentalFeaturesFlow = MutableStateFlow(false)
     private val voiceAssistantFlow = MutableStateFlow(false)
+    private val systemSpeechFlow = MutableStateFlow(false)
     private val carSensorsFlow = MutableStateFlow(false)
     private val mediaSearchFlow = MutableStateFlow(false)
     private val seenSafetyWarningFlow = MutableStateFlow(false)
@@ -177,6 +179,9 @@ object CarSettings {
      */
     val voiceAssistantEnabled: StateFlow<Boolean> = voiceAssistantFlow
 
+    /** Listen with the phone's own speech service instead of Gearslip's on-device model. */
+    val systemSpeech: StateFlow<Boolean> = systemSpeechFlow
+
     /**
      * Vehicle data is experimental and off by default: the launcher's tile for it, and the
      * voice assistant's "vehicle data" command, only show up once this and
@@ -227,6 +232,7 @@ object CarSettings {
             .split(",").filter { it.isNotEmpty() }
         experimentalFeaturesFlow.value = AppSettings.getString(app, KEY_EXPERIMENTAL_FEATURES, "false") == "true"
         voiceAssistantFlow.value = AppSettings.getString(app, KEY_VOICE_ASSISTANT, "false") == "true"
+        systemSpeechFlow.value = AppSettings.getString(app, KEY_SYSTEM_SPEECH, "false") == "true"
         carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true"
         mediaSearchFlow.value = AppSettings.getString(app, KEY_MEDIA_SEARCH, "false") == "true"
         seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true"
@@ -293,6 +299,11 @@ object CarSettings {
         AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
     }
 
+    fun setSystemSpeech(on: Boolean) {
+        systemSpeechFlow.value = on
+        AppSettings.putString(app, KEY_SYSTEM_SPEECH, on.toString())
+    }
+
     fun setVoiceAssistantEnabled(on: Boolean) {
         voiceAssistantFlow.value = on
         AppSettings.putString(app, KEY_VOICE_ASSISTANT, on.toString())

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt

@@ -220,6 +220,18 @@ private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolea
             checked = voiceAssistant,
         ) { CarSettings.setVoiceAssistantEnabled(it) }
 
+        if (voiceAssistant) {
+            val systemSpeech by CarSettings.systemSpeech.collectAsState()
+            ExperimentalFeatureRow(
+                title = "Use the phone's speech recognition",
+                description = "Understands more words, like place and brand names, but listens " +
+                    "through the phone's microphone, not the car's. On most phones this is Google's, " +
+                    "which may send your voice to Google. GrapheneOS has none, so it stays with " +
+                    "Gearslip's own.",
+                checked = systemSpeech,
+            ) { CarSettings.setSystemSpeech(it) }
+        }
+
         ExperimentalFeatureRow(
             title = "Vehicle data",
             description = "Adds a Vehicle data tile: a live, raw dump of the head unit's sensor " +

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt

@@ -256,6 +256,7 @@ fun CarUi() {
                             modifier = Modifier.align(Alignment.TopEnd),
                         )
                         // Above the popup: a call is more urgent than any notification.
+                        VoiceHelpOverlay(Modifier.align(Alignment.Center))
                         CallOverlay(Modifier.align(Alignment.TopCenter))
                         AssistantOverlay(Modifier.align(Alignment.TopCenter))
                         if (app.seb3thehacker.gearslip.BuildConfig.DEBUG) {

gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt

@@ -0,0 +1,92 @@
+package app.seb3thehacker.gearslip.car
+
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.layout.widthIn
+import androidx.compose.foundation.rememberScrollState
+import androidx.compose.foundation.verticalScroll
+import androidx.compose.material.icons.Icons
+import androidx.compose.material.icons.filled.Close
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.Surface
+import androidx.compose.material3.Text
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.LaunchedEffect
+import androidx.compose.runtime.collectAsState
+import androidx.compose.runtime.getValue
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.text.font.FontWeight
+import androidx.compose.ui.unit.dp
+import app.seb3thehacker.gearslip.car.theme.GsIconButton
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+
+/** What the assistant understands, shown on the car screen when the driver says "help". */
+object VoiceHelp {
+    private val _shown = MutableStateFlow(false)
+    val shown: StateFlow<Boolean> = _shown.asStateFlow()
+
+    fun show() { _shown.value = true }
+    fun hide() { _shown.value = false }
+
+    /** Say this, and this happens. Kept in step with [AssistantCommands]. */
+    val COMMANDS = listOf(
+        "Navigate to…" to "Directions there. Add \"with Organic Maps\" to pick the app",
+        "Play…" to "A song, artist, or album in the open music app",
+        "Pause · Resume · Next · Previous" to "Control the music",
+        "What's playing" to "The song and artist",
+        "Call…" to "A contact by name",
+        "Open…" to "Any app on this screen",
+        "Go home · Open apps · Settings" to "Move around Gearslip",
+        "What time is it · Weather" to "The time, or the forecast",
+        "Help" to "This list",
+    )
+}
+
+/** The command list over the car screen: read it at a glance, tap the cross or wait to dismiss. */
+@Composable
+fun VoiceHelpOverlay(modifier: Modifier = Modifier) {
+    val shown by VoiceHelp.shown.collectAsState()
+    if (!shown) return
+    // A list left up would cover the map for the rest of the drive.
+    LaunchedEffect(Unit) {
+        delay(30_000)
+        VoiceHelp.hide()
+    }
+    Surface(
+        color = MaterialTheme.colorScheme.surfaceContainerHigh,
+        contentColor = MaterialTheme.colorScheme.onSurface,
+        shape = MaterialTheme.shapes.extraLarge,
+        tonalElevation = 6.dp,
+        shadowElevation = 8.dp,
+        modifier = modifier.padding(16.dp).widthIn(max = 640.dp),
+    ) {
+        Column(Modifier.padding(20.dp)) {
+            Row(Modifier.fillMaxWidth()) {
+                Text(
+                    "You can say",
+                    style = MaterialTheme.typography.titleLarge,
+                    fontWeight = FontWeight.Bold,
+                    modifier = Modifier.weight(1f).padding(top = 6.dp),
+                )
+                GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide)
+            }
+            Column(
+                Modifier.verticalScroll(rememberScrollState()).padding(top = 8.dp),
+                verticalArrangement = Arrangement.spacedBy(10.dp),
+            ) {
+                VoiceHelp.COMMANDS.forEach { (say, does) ->
+                    Column {
+                        Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold)
+                        Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
+                    }
+                }
+            }
+        }
+    }
+}

gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt

@@ -156,6 +156,9 @@ class CarMedia(private val context: Context) {
     private val main = Handler(Looper.getMainLooper())
     private var connecting: MediaApp? = null
 
+    /** The music app Gearslip is attached to, for the assistant to name; null when there's none. */
+    val appLabel: String? get() = connecting?.label?.takeIf { controller != null }
+
     private val _phase = MutableStateFlow(Phase.IDLE)
     val phase: StateFlow<Phase> = _phase.asStateFlow()
 
@@ -236,6 +239,12 @@ class CarMedia(private val context: Context) {
         connecting = app
         _phase.value = Phase.CONNECTING
         _browse.value = BrowseState()
+        // Known to refuse its library (Spotify): asking only costs the seconds it takes to say no,
+        // so go straight to its playback session.
+        if (KnownApps.playerWorks(app.component.packageName)) {
+            attachToSession(app)
+            return
+        }
         val mb = MediaBrowser(context, app.component, object : MediaBrowser.ConnectionCallback() {
             override fun onConnected() = this@CarMedia.onConnected(app)
             override fun onConnectionFailed() {
@@ -273,7 +282,12 @@ class CarMedia(private val context: Context) {
             return
         }
         if (session == null) {
-            val limit = if (launched) SESSION_RETRIES_LAUNCHED else SESSION_RETRIES
+            // A known session-only app that isn't running won't start by waiting: say so at once.
+            val limit = when {
+                launched -> SESSION_RETRIES_LAUNCHED
+                KnownApps.playerWorks(app.component.packageName) -> 0
+                else -> SESSION_RETRIES
+            }
             // The app only publishes its session once its service has started; give it a moment.
             if (attempt < limit) {
                 main.postDelayed({ attachToSession(app, attempt + 1, launched) }, SESSION_RETRY_MS)
@@ -454,7 +468,7 @@ class CarMedia(private val context: Context) {
         }
         val component = searchComponent
         if (component == null) {
-            _results.value = Results(q, emptyList(), supported = false)
+            noLibrarySearch(q)
             return
         }
         resultsQuery = q
@@ -476,13 +490,13 @@ class CarMedia(private val context: Context) {
             override fun onConnectionFailed() {
                 GearslipLog.w("media: no library connection for search")
                 searchBrowser = null
-                resultsQuery?.let { _results.value = Results(it, emptyList(), supported = false) }
+                resultsQuery?.let { noLibrarySearch(it) }
             }
         }, null)
         searchBrowser = created
         runCatching { created.connect() }.onFailure {
             searchBrowser = null
-            _results.value = Results(q, emptyList(), supported = false)
+            noLibrarySearch(q)
         }
     }
 
@@ -492,18 +506,42 @@ class CarMedia(private val context: Context) {
                 override fun onSearchResult(query: String, extras: Bundle?, items: List<MediaBrowserCompat.MediaItem>) {
                     if (query != resultsQuery) return
                     GearslipLog.i("media: library search found ${items.size}")
-                    _results.value = Results(query, items.map(::entryOfCompat))
+                    val entries = items.map(::entryOfCompat)
+                    if (query == playFirstFor) {
+                        playFirstFor = null
+                        val first = entries.firstOrNull { it.playable } ?: entries.firstOrNull()
+                        if (first == null) {
+                            clearResults()
+                            search(query)
+                        } else {
+                            GearslipLog.i("media: playing the library's best match")
+                            clearResults()
+                            searchBefore = _now.value
+                            _search.value = Search(query)
+                            main.removeCallbacks(searchTimeout)
+                            controller?.transportControls?.playFromMediaId(first.id, null)
+                            main.postDelayed(searchTimeout, SEARCH_WAIT_MS)
+                        }
+                        return
+                    }
+                    _results.value = Results(query, entries)
                 }
 
                 override fun onError(query: String, extras: Bundle?) {
                     if (query != resultsQuery) return
+                    if (query == playFirstFor) {
+                        playFirstFor = null
+                        clearResults()
+                        search(query)
+                        return
+                    }
                     GearslipLog.i("media: the app has no library search")
                     _results.value = Results(query, emptyList(), supported = false)
                 }
             })
         }.onFailure {
             GearslipLog.w("media: library search failed: ${it.message}")
-            _results.value = Results(q, emptyList(), supported = false)
+            noLibrarySearch(q)
         }
     }
 
@@ -520,6 +558,36 @@ class CarMedia(private val context: Context) {
         )
     }
 
+    /** The library can't search: a voice "play X" falls back to the app's own search; a typed one says so. */
+    private fun noLibrarySearch(q: String) {
+        if (q == playFirstFor) {
+            playFirstFor = null
+            clearResults()
+            search(q)
+        } else {
+            _results.value = Results(q, emptyList(), supported = false)
+        }
+    }
+
+    /** A voice "play X" waiting on the library search, to play its first match instead of listing them. */
+    private var playFirstFor: String? = null
+
+    /**
+     * Plays the best match for [query], for voice. Asks the app's library first and plays its top
+     * result, which apps like ViVi honour where they ignore [search]; without a library search,
+     * falls back to [search].
+     */
+    fun playBestMatch(query: String) {
+        val q = query.trim()
+        if (q.isEmpty()) return
+        if (!canSearchLibrary) {
+            search(q)
+            return
+        }
+        playFirstFor = q
+        searchLibrary(q)
+    }
+
     /** The connected app's service, for the search connection; null when nothing is connected. */
     private var searchComponent: android.content.ComponentName? = null
     private var searchBrowser: MediaBrowserCompat? = null

gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt

@@ -33,8 +33,11 @@ import kotlin.math.sqrt
  */
 object SpeechEngine {
 
-    /** FUTO's ACFT Whisper tiny (Apache-2.0), re-quantized to 5 bits, inside the APK. */
+    /**
-    private const val MODEL = "speech/tiny_acft_q5_1.bin"
+     * FUTO's ACFT Whisper (Apache-2.0), re-quantized to 5 bits, inside the APK: base when the build
+     * carries it (more accurate, about twice as slow), tiny otherwise.
+     */
+    private val MODELS = listOf("speech/base_acft_q5_1.bin", "speech/tiny_acft_q5_1.bin")
 
     /** The error for an engine that can't run at all, which callers may answer with a fallback. */
     const val NOT_LOADED = "The speech model didn't load."
@@ -258,12 +261,14 @@ object SpeechEngine {
             libraryLoaded = true
         }
         val started = SystemClock.uptimeMillis()
-        handle = nativeInitAsset(context.assets, MODEL)
+        val model = MODELS.firstOrNull { name -> runCatching { context.assets.open(name).close() }.isSuccess }
+            ?: run { GearslipLog.w("speech: no model in the build"); return false }
+        handle = nativeInitAsset(context.assets, model)
         if (handle == 0L) {
             GearslipLog.w("speech: couldn't load the model")
             return false
         }
-        GearslipLog.i("speech: loaded the model in ${SystemClock.uptimeMillis() - started} ms")
+        GearslipLog.i("speech: loaded $model in ${SystemClock.uptimeMillis() - started} ms")
         return true
     }