voice: understand natural phrasing for every command, open Gearslip's own apps by name, and say "exit" to hand the screen back

gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt

@@ -4,7 +4,9 @@ import android.content.Context
 import java.text.DateFormat
 import java.util.Date
 import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
 import kotlinx.coroutines.withContext
+import kotlinx.coroutines.withTimeoutOrNull
 
 /**
  * Turns what [CarAssistant] heard into an action plus something to say back.
@@ -17,7 +19,7 @@ import kotlinx.coroutines.withContext
 object AssistantCommands {
 
     suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String {
-        val heard = heardRaw.lowercase().trim().trimEnd('.', '?', '!')
+        val heard = natural(heardRaw)
         VoiceHelp.hide()
         if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) {
             VoiceHelp.show()
@@ -25,7 +27,7 @@ object AssistantCommands {
         }
 
         navigateCommand(context, navigator, frame, heard)?.let { return it }
-        playSearchCommand(heard)?.let { return it }
+        playSearchCommand(context, navigator, heard)?.let { return it }
         mediaCommand(heard)?.let { return it }
         navCommand(navigator, heard)?.let { return it }
         infoCommand(heard)?.let { return it }
@@ -35,6 +37,42 @@ object AssistantCommands {
         return "I don't know how to do that yet."
     }
 
+    /**
+     * What was said, minus the padding people wrap a command in: "hey", "can you", "I want to",
+     * "please", "thanks". Each command below then only has to know its own words.
+     */
+    private fun natural(raw: String): String {
+        var t = raw.lowercase().replace(Regex("[.,!?]"), " ").replace(Regex("\\s+"), " ").trim()
+        var changed = true
+        while (changed) {
+            changed = false
+            for (lead in LEAD_FILLERS) if (t.startsWith("$lead ")) { t = t.removePrefix("$lead ").trim(); changed = true }
+            for (tail in TAIL_FILLERS) if (t.endsWith(" $tail")) { t = t.removeSuffix(" $tail").trim(); changed = true }
+        }
+        return t
+    }
+
+    /** The part after the first of [triggers] that [text] starts with, or null. */
+    private fun after(text: String, triggers: List<String>): String? =
+        triggers.firstOrNull { text.startsWith(it) }?.let { text.removePrefix(it).trim() }
+
+    private suspend fun allEntries(context: Context): List<Entry> {
+        val vehicleData = CarSettings.experimentalFeaturesEnabled.value && CarSettings.carSensorsEnabled.value
+        val builtIn = BuiltInApps.entries.filter { it.builtIn !== BuiltInApps.VehicleData || vehicleData }
+        return builtIn + (LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) })
+    }
+
+    /** Best label match: exact, then a label starting with [name], then one containing it. */
+    private fun <T> bestMatch(items: List<T>, name: String, label: (T) -> String): T? {
+        val n = name.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
+        if (n.isEmpty()) return null
+        items.firstOrNull { label(it).lowercase() == n }?.let { return it }
+        // Loose matches need a real word: "pizza in a" mustn't pick "Organic Maps" for its "a".
+        if (n.length < 3) return null
+        return items.firstOrNull { label(it).lowercase().startsWith(n) }
+            ?: items.firstOrNull { label(it).lowercase().contains(n) }
+    }
+
     /**
      * "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it,
      * whichever nav app was used last, or the only one installed, or the first one found. Starts
@@ -48,29 +86,45 @@ object AssistantCommands {
         frame: CarEnvironment.Frame,
         heard: String,
     ): String? {
-        val trigger = NAVIGATE_TRIGGERS.firstOrNull { heard.startsWith(it) } ?: return null
+        // "use Organic Maps to get to X", "open OsmAnd and navigate to X": the app comes first.
-        var destination = heard.removePrefix(trigger).trim()
+        var text = heard
+        var leadingApp: String? = null
+        Regex("^(?:use|using|with|open|in|on) (.+?) (?:to |and )(.+)$").find(text)?.let { m ->
+            if (after(m.groupValues[2], NAVIGATE_TRIGGERS) != null) {
+                leadingApp = m.groupValues[1]
+                text = m.groupValues[2]
+            }
+        }
+        var destination = after(text, NAVIGATE_TRIGGERS) ?: return null
         if (destination.isBlank()) return "Where to?"
 
-        var appName: String? = null
+        val entries = allEntries(context)
-        for (sep in listOf(" with ", " using ", " on ", " via ")) {
+        // "go to Spotify" means the app, and "go to settings" the screen; "go to Target" is a place.
-            val at = destination.lastIndexOf(sep)
+        val asApp = destination.removePrefix("the ").removeSuffix(" app")
-            if (at > 0) {
+        if (text.startsWith("go to ") &&
-                appName = destination.substring(at + sep.length).trim()
+            (asApp in SCREEN_WORDS || asApp in APP_NICKNAMES || entries.any { it.label.lowercase() == asApp })
+        ) {
+            return null
+        }
+        val navApps = entries.filter { it.template?.isNavigation == true }
+        if (navApps.isEmpty()) return "You don't have any navigation apps installed."
+
+        // "X with Organic Maps", "X on OsmAnd": split only when the end names a nav app, so a
+        // place like "Starbucks on Main Street" stays whole.
+        var chosenApp = leadingApp?.let { name -> bestMatch(navApps, name) { it.label } ?: return "I couldn't find a navigation app called $name." }
+        if (chosenApp == null) {
+            for (sep in listOf(" with ", " using ", " on ", " via ", " in ", " through ")) {
+                val at = destination.lastIndexOf(sep)
+                if (at <= 0) continue
+                val app = bestMatch(navApps, destination.substring(at + sep.length)) { it.label } ?: continue
+                chosenApp = app
                 destination = destination.substring(0, at).trim()
                 break
             }
         }
         if (destination.isBlank()) return "Where to?"
 
-        val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
+        val chosen = chosenApp ?: when {
-        val navApps = entries.filter { it.template?.isNavigation == true }
-        if (navApps.isEmpty()) return "You don't have any navigation apps installed."
-
-        val requestedApp = appName
-        val chosen = when {
-            requestedApp != null -> navApps.firstOrNull { it.label.lowercase().contains(requestedApp) }
-                ?: return "I couldn't find a navigation app called $requestedApp."
             navApps.size == 1 -> navApps.first()
             else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first()
         }
@@ -86,11 +140,35 @@ object AssistantCommands {
      * that's open through its own search ([CarMedia.search]). A bare "play", or "play music",
      * falls through to resume.
      */
-    private fun playSearchCommand(heard: String): String? {
+    private suspend fun playSearchCommand(context: Context, navigator: CarNavigator, heard: String): String? {
-        if (!heard.startsWith("play ")) return null
+        var query = after(heard, PLAY_TRIGGERS) ?: return null
-        val query = heard.removePrefix("play ").trim().removeSuffix(".").trim()
+        query = query.removePrefix("some ").trim()
         if (query.isEmpty() || query in RESUME_WORDS) return null
         val media = CarServices.media
+
+        // "play Daft Punk on Metrolist": switch to that app first, if it's one we know.
+        for (sep in listOf(" on ", " in ", " from ", " with ", " using ")) {
+            val at = query.lastIndexOf(sep)
+            if (at <= 0) continue
+            val apps = withContext(Dispatchers.IO) { app.seb3thehacker.gearslip.media.MediaCatalog.installed(context) }
+            val wanted = bestMatch(apps, query.substring(at + sep.length)) { it.label } ?: continue
+            query = query.substring(0, at).trim()
+            if (query.isEmpty() || query in RESUME_WORDS) {
+                CarServices.openMedia(wanted, autoplay = true)
+                navigator.media()
+                return "Opening ${wanted.label}."
+            }
+            if (CarServices.mediaApp.value?.component != wanted.component) {
+                CarServices.openMedia(wanted)
+                navigator.media()
+                // Give it a moment to connect before asking it to search.
+                withTimeoutOrNull(APP_CONNECT_MS) {
+                    while (media.appLabel == null || (!media.canSearchLibrary && !media.now.value.canSearch)) delay(200)
+                }
+            }
+            break
+        }
+
         val app = media.appLabel ?: return "Open a music app first, then ask again."
         if (!media.canSearchLibrary && !media.now.value.canSearch) {
             return "$app ignores voice searches. Pick something on its screen instead."
@@ -102,23 +180,27 @@ object AssistantCommands {
     private fun mediaCommand(heard: String): String? {
         val media = CarServices.media
         return when {
-            "pause" in heard || heard.startsWith("stop") -> {
+            "pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> {
                 if (media.now.value.playing) media.togglePlay()
                 "Paused."
             }
-            "resume" in heard || (heard.startsWith("play") && "playing" !in heard) -> {
+            "resume" in heard || "unpause" in heard || heard in RESUME_PHRASES ||
+                (heard.startsWith("play") && "playing" !in heard) -> {
                 if (!media.now.value.playing) media.togglePlay()
                 "Playing."
             }
-            "next" in heard || "skip" in heard -> {
+            "next" in heard || "skip" in heard || heard.startsWith("change the song") -> {
                 media.next()
                 "Skipping."
             }
-            "previous" in heard || "last song" in heard || "back song" in heard -> {
+            "previous" in heard || "last song" in heard || "back song" in heard || "go back a song" in heard ||
+                "play that again" in heard || "replay" in heard -> {
                 media.previous()
                 "Playing the previous track."
             }
-            "what" in heard && "playing" in heard -> {
+            ("what" in heard && "playing" in heard) || "what song" in heard || "who sings" in heard ||
+                "who's singing" in heard || "who is singing" in heard || "what's this song" in heard ||
+                "name of this song" in heard -> {
                 val now = media.now.value
                 if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ")
                 else "Nothing's playing."
@@ -128,11 +210,19 @@ object AssistantCommands {
     }
 
     private fun navCommand(navigator: CarNavigator, heard: String): String? = when {
-        "go home" in heard || heard == "home" -> {
+        heard == "go back" || heard == "back" -> {
+            navigator.back()
+            "Going back."
+        }
+        heard in EXIT_WORDS -> {
+            BuiltInApps.Exit.open(navigator)
+            "Back to the car's screen."
+        }
+        "go home" in heard || heard == "home" || heard == "home screen" || "go to the home screen" in heard -> {
             navigator.home()
             "Heading home."
         }
-        "open apps" in heard || heard == "apps" || "app list" in heard -> {
+        "open apps" in heard || heard == "apps" || "app list" in heard || "all apps" in heard || "my apps" in heard || heard == "go to apps" -> {
             navigator.apps()
             "Here are your apps."
         }
@@ -152,9 +242,10 @@ object AssistantCommands {
     }
 
     private fun infoCommand(heard: String): String? = when {
-        "what time" in heard || "what's the time" in heard ->
+        "what time" in heard || "what's the time" in heard || heard == "time" || "the time" in heard ->
             "It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}."
-        "weather" in heard -> {
+        "weather" in heard || "going to rain" in heard || "gonna rain" in heard || "how hot" in heard ||
+            "how cold" in heard || "temperature outside" in heard -> {
             val data = (Weather.state.value as? WeatherState.Ready)?.data
             if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}."
             else "I don't have the weather yet."
@@ -163,12 +254,11 @@ object AssistantCommands {
     }
 
     private suspend fun callCommand(context: Context, heard: String): String? {
-        val name = when {
+        val name = (
-            heard.startsWith("call ") -> heard.removePrefix("call ")
+            after(heard, CALL_TRIGGERS)
-            heard.startsWith("phone ") -> heard.removePrefix("phone ")
+                ?: Regex("^give (.+) a (?:call|ring)$").find(heard)?.groupValues?.get(1)
-            heard.startsWith("dial ") -> heard.removePrefix("dial ")
+                ?: return null
-            else -> return null
+            ).replace(Regex(" (?:on|at) (?:their |his |her )?(?:mobile|cell|phone|home|work)$"), "").trim()
-        }.trim()
         if (name.isBlank()) return "Who should I call?"
         if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen."
 
@@ -186,25 +276,59 @@ object AssistantCommands {
         frame: CarEnvironment.Frame,
         heard: String,
     ): String? {
-        val name = when {
+        val name = (after(heard, OPEN_TRIGGERS) ?: return null)
-            heard.startsWith("open ") -> heard.removePrefix("open ")
+            .removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
-            heard.startsWith("launch ") -> heard.removePrefix("launch ")
-            heard.startsWith("start ") -> heard.removePrefix("start ")
-            else -> return null
-        }.trim()
         if (name.isBlank()) return null
+        val wanted = APP_NICKNAMES[name] ?: name
 
-        val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
+        // Gearslip's own apps come first in the list, so "phone" opens its dialer, not the phone's.
-        val match = entries.firstOrNull { it.label.lowercase().contains(name) }
+        val match = bestMatch(allEntries(context), wanted) { it.label }
             ?: return "I couldn't find an app called $name."
 
         launchEntry(match, navigator, frame)
         return "Opening ${match.label}."
     }
 
-    private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ")
+    private val EXIT_WORDS = setOf("exit", "exit gearslip", "close gearslip", "quit", "quit gearslip", "open exit")
+
+    /** Other names people use for Gearslip's own apps. */
+    private val APP_NICKNAMES = mapOf(
+        "web" to "browser", "internet" to "browser", "the web" to "browser", "web browser" to "browser",
+        "dialer" to "phone", "the dialer" to "phone", "keypad" to "phone",
+        "screen share" to "screen sharing", "screen mirroring" to "screen sharing", "mirroring" to "screen sharing",
+    )
+
+    private val NAVIGATE_TRIGGERS = listOf(
+        "navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ",
+        "take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ",
+        "how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ",
+    )
+
+    private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ")
+
+    private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ")
+
+    private val OPEN_TRIGGERS = listOf("open ", "launch ", "start ", "go to ", "switch to ", "show me ", "bring up ", "pull up ")
+
+    /** Screens "go to X" means rather than a place. */
+    private val SCREEN_WORDS = setOf("settings", "home", "the home screen", "home screen", "apps", "my apps", "all apps")
+
+    /** Padding stripped from the front of a command, longest first where they overlap. */
+    private val LEAD_FILLERS = listOf(
+        "hey gearslip", "ok gearslip", "okay gearslip", "gearslip", "hey", "ok", "okay", "um", "uh", "so",
+        "please", "can you please", "can you", "could you", "would you", "will you", "can i", "could i",
+        "i want you to", "i want to", "i'd like to", "i would like to", "i wanna", "i need to", "i need you to",
+        "let's", "lets", "go ahead and", "just", "now", "quickly",
+    )
+
+    /** Longest first, so "right now" goes whole rather than leaving "right". */
+    private val TAIL_FILLERS = listOf("thank you very much", "thank you", "right now", "for me", "please", "thanks", "now")
+
+    private const val APP_CONNECT_MS = 5_000L
 
     private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands")
 
-    private val RESUME_WORDS = setOf("music", "the music", "something", "it", "again", "that")
+    private val RESUME_WORDS = setOf("music", "the music", "some music", "something", "it", "again", "that", "my music")
+
+    private val RESUME_PHRASES = setOf("keep playing", "continue", "keep going", "start the music", "music on", "turn the music back on")
 }

gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt

@@ -41,7 +41,8 @@ object VoiceHelp {
         "Pause · Resume · Next · Previous" to "Control the music",
         "What's playing" to "The song and artist",
         "Call…" to "A contact by name",
-        "Open…" to "Any app on this screen",
+        "Open…" to "Any app, like Browser, Phone, or Screen sharing",
+        "Exit" to "Back to the car's own screen",
         "Go home · Open apps · Settings" to "Move around Gearslip",
         "What time is it · Weather" to "The time, or the forecast",
         "Help" to "This list",