diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt index e42b43a..a846fd8 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt @@ -4,7 +4,9 @@ import android.content.Context import java.text.DateFormat import java.util.Date import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull /** * Turns what [CarAssistant] heard into an action plus something to say back. @@ -17,7 +19,7 @@ import kotlinx.coroutines.withContext object AssistantCommands { suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String { - val heard = heardRaw.lowercase().trim().trimEnd('.', '?', '!') + val heard = natural(heardRaw) VoiceHelp.hide() if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) { VoiceHelp.show() @@ -25,7 +27,7 @@ object AssistantCommands { } navigateCommand(context, navigator, frame, heard)?.let { return it } - playSearchCommand(heard)?.let { return it } + playSearchCommand(context, navigator, heard)?.let { return it } mediaCommand(heard)?.let { return it } navCommand(navigator, heard)?.let { return it } infoCommand(heard)?.let { return it } @@ -35,6 +37,42 @@ object AssistantCommands { return "I don't know how to do that yet." } + /** + * What was said, minus the padding people wrap a command in: "hey", "can you", "I want to", + * "please", "thanks". Each command below then only has to know its own words. + */ + private fun natural(raw: String): String { + var t = raw.lowercase().replace(Regex("[.,!?]"), " ").replace(Regex("\\s+"), " ").trim() + var changed = true + while (changed) { + changed = false + for (lead in LEAD_FILLERS) if (t.startsWith("$lead ")) { t = t.removePrefix("$lead ").trim(); changed = true } + for (tail in TAIL_FILLERS) if (t.endsWith(" $tail")) { t = t.removeSuffix(" $tail").trim(); changed = true } + } + return t + } + + /** The part after the first of [triggers] that [text] starts with, or null. */ + private fun after(text: String, triggers: List): String? = + triggers.firstOrNull { text.startsWith(it) }?.let { text.removePrefix(it).trim() } + + private suspend fun allEntries(context: Context): List { + val vehicleData = CarSettings.experimentalFeaturesEnabled.value && CarSettings.carSensorsEnabled.value + val builtIn = BuiltInApps.entries.filter { it.builtIn !== BuiltInApps.VehicleData || vehicleData } + return builtIn + (LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }) + } + + /** Best label match: exact, then a label starting with [name], then one containing it. */ + private fun bestMatch(items: List, name: String, label: (T) -> String): T? { + val n = name.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim() + if (n.isEmpty()) return null + items.firstOrNull { label(it).lowercase() == n }?.let { return it } + // Loose matches need a real word: "pizza in a" mustn't pick "Organic Maps" for its "a". + if (n.length < 3) return null + return items.firstOrNull { label(it).lowercase().startsWith(n) } + ?: items.firstOrNull { label(it).lowercase().contains(n) } + } + /** * "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it, * whichever nav app was used last, or the only one installed, or the first one found. Starts @@ -48,29 +86,45 @@ object AssistantCommands { frame: CarEnvironment.Frame, heard: String, ): String? { - val trigger = NAVIGATE_TRIGGERS.firstOrNull { heard.startsWith(it) } ?: return null - var destination = heard.removePrefix(trigger).trim() + // "use Organic Maps to get to X", "open OsmAnd and navigate to X": the app comes first. + var text = heard + var leadingApp: String? = null + Regex("^(?:use|using|with|open|in|on) (.+?) (?:to |and )(.+)$").find(text)?.let { m -> + if (after(m.groupValues[2], NAVIGATE_TRIGGERS) != null) { + leadingApp = m.groupValues[1] + text = m.groupValues[2] + } + } + var destination = after(text, NAVIGATE_TRIGGERS) ?: return null if (destination.isBlank()) return "Where to?" - var appName: String? = null - for (sep in listOf(" with ", " using ", " on ", " via ")) { - val at = destination.lastIndexOf(sep) - if (at > 0) { - appName = destination.substring(at + sep.length).trim() + val entries = allEntries(context) + // "go to Spotify" means the app, and "go to settings" the screen; "go to Target" is a place. + val asApp = destination.removePrefix("the ").removeSuffix(" app") + if (text.startsWith("go to ") && + (asApp in SCREEN_WORDS || asApp in APP_NICKNAMES || entries.any { it.label.lowercase() == asApp }) + ) { + return null + } + val navApps = entries.filter { it.template?.isNavigation == true } + if (navApps.isEmpty()) return "You don't have any navigation apps installed." + + // "X with Organic Maps", "X on OsmAnd": split only when the end names a nav app, so a + // place like "Starbucks on Main Street" stays whole. + var chosenApp = leadingApp?.let { name -> bestMatch(navApps, name) { it.label } ?: return "I couldn't find a navigation app called $name." } + if (chosenApp == null) { + for (sep in listOf(" with ", " using ", " on ", " via ", " in ", " through ")) { + val at = destination.lastIndexOf(sep) + if (at <= 0) continue + val app = bestMatch(navApps, destination.substring(at + sep.length)) { it.label } ?: continue + chosenApp = app destination = destination.substring(0, at).trim() break } } if (destination.isBlank()) return "Where to?" - val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) } - val navApps = entries.filter { it.template?.isNavigation == true } - if (navApps.isEmpty()) return "You don't have any navigation apps installed." - - val requestedApp = appName - val chosen = when { - requestedApp != null -> navApps.firstOrNull { it.label.lowercase().contains(requestedApp) } - ?: return "I couldn't find a navigation app called $requestedApp." + val chosen = chosenApp ?: when { navApps.size == 1 -> navApps.first() else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first() } @@ -86,11 +140,35 @@ object AssistantCommands { * that's open through its own search ([CarMedia.search]). A bare "play", or "play music", * falls through to resume. */ - private fun playSearchCommand(heard: String): String? { - if (!heard.startsWith("play ")) return null - val query = heard.removePrefix("play ").trim().removeSuffix(".").trim() + private suspend fun playSearchCommand(context: Context, navigator: CarNavigator, heard: String): String? { + var query = after(heard, PLAY_TRIGGERS) ?: return null + query = query.removePrefix("some ").trim() if (query.isEmpty() || query in RESUME_WORDS) return null val media = CarServices.media + + // "play Daft Punk on Metrolist": switch to that app first, if it's one we know. + for (sep in listOf(" on ", " in ", " from ", " with ", " using ")) { + val at = query.lastIndexOf(sep) + if (at <= 0) continue + val apps = withContext(Dispatchers.IO) { app.seb3thehacker.gearslip.media.MediaCatalog.installed(context) } + val wanted = bestMatch(apps, query.substring(at + sep.length)) { it.label } ?: continue + query = query.substring(0, at).trim() + if (query.isEmpty() || query in RESUME_WORDS) { + CarServices.openMedia(wanted, autoplay = true) + navigator.media() + return "Opening ${wanted.label}." + } + if (CarServices.mediaApp.value?.component != wanted.component) { + CarServices.openMedia(wanted) + navigator.media() + // Give it a moment to connect before asking it to search. + withTimeoutOrNull(APP_CONNECT_MS) { + while (media.appLabel == null || (!media.canSearchLibrary && !media.now.value.canSearch)) delay(200) + } + } + break + } + val app = media.appLabel ?: return "Open a music app first, then ask again." if (!media.canSearchLibrary && !media.now.value.canSearch) { return "$app ignores voice searches. Pick something on its screen instead." @@ -102,23 +180,27 @@ object AssistantCommands { private fun mediaCommand(heard: String): String? { val media = CarServices.media return when { - "pause" in heard || heard.startsWith("stop") -> { + "pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> { if (media.now.value.playing) media.togglePlay() "Paused." } - "resume" in heard || (heard.startsWith("play") && "playing" !in heard) -> { + "resume" in heard || "unpause" in heard || heard in RESUME_PHRASES || + (heard.startsWith("play") && "playing" !in heard) -> { if (!media.now.value.playing) media.togglePlay() "Playing." } - "next" in heard || "skip" in heard -> { + "next" in heard || "skip" in heard || heard.startsWith("change the song") -> { media.next() "Skipping." } - "previous" in heard || "last song" in heard || "back song" in heard -> { + "previous" in heard || "last song" in heard || "back song" in heard || "go back a song" in heard || + "play that again" in heard || "replay" in heard -> { media.previous() "Playing the previous track." } - "what" in heard && "playing" in heard -> { + ("what" in heard && "playing" in heard) || "what song" in heard || "who sings" in heard || + "who's singing" in heard || "who is singing" in heard || "what's this song" in heard || + "name of this song" in heard -> { val now = media.now.value if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ") else "Nothing's playing." @@ -128,11 +210,19 @@ object AssistantCommands { } private fun navCommand(navigator: CarNavigator, heard: String): String? = when { - "go home" in heard || heard == "home" -> { + heard == "go back" || heard == "back" -> { + navigator.back() + "Going back." + } + heard in EXIT_WORDS -> { + BuiltInApps.Exit.open(navigator) + "Back to the car's screen." + } + "go home" in heard || heard == "home" || heard == "home screen" || "go to the home screen" in heard -> { navigator.home() "Heading home." } - "open apps" in heard || heard == "apps" || "app list" in heard -> { + "open apps" in heard || heard == "apps" || "app list" in heard || "all apps" in heard || "my apps" in heard || heard == "go to apps" -> { navigator.apps() "Here are your apps." } @@ -152,9 +242,10 @@ object AssistantCommands { } private fun infoCommand(heard: String): String? = when { - "what time" in heard || "what's the time" in heard -> + "what time" in heard || "what's the time" in heard || heard == "time" || "the time" in heard -> "It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}." - "weather" in heard -> { + "weather" in heard || "going to rain" in heard || "gonna rain" in heard || "how hot" in heard || + "how cold" in heard || "temperature outside" in heard -> { val data = (Weather.state.value as? WeatherState.Ready)?.data if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}." else "I don't have the weather yet." @@ -163,12 +254,11 @@ object AssistantCommands { } private suspend fun callCommand(context: Context, heard: String): String? { - val name = when { - heard.startsWith("call ") -> heard.removePrefix("call ") - heard.startsWith("phone ") -> heard.removePrefix("phone ") - heard.startsWith("dial ") -> heard.removePrefix("dial ") - else -> return null - }.trim() + val name = ( + after(heard, CALL_TRIGGERS) + ?: Regex("^give (.+) a (?:call|ring)$").find(heard)?.groupValues?.get(1) + ?: return null + ).replace(Regex(" (?:on|at) (?:their |his |her )?(?:mobile|cell|phone|home|work)$"), "").trim() if (name.isBlank()) return "Who should I call?" if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen." @@ -186,25 +276,59 @@ object AssistantCommands { frame: CarEnvironment.Frame, heard: String, ): String? { - val name = when { - heard.startsWith("open ") -> heard.removePrefix("open ") - heard.startsWith("launch ") -> heard.removePrefix("launch ") - heard.startsWith("start ") -> heard.removePrefix("start ") - else -> return null - }.trim() + val name = (after(heard, OPEN_TRIGGERS) ?: return null) + .removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim() if (name.isBlank()) return null + val wanted = APP_NICKNAMES[name] ?: name - val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) } - val match = entries.firstOrNull { it.label.lowercase().contains(name) } + // Gearslip's own apps come first in the list, so "phone" opens its dialer, not the phone's. + val match = bestMatch(allEntries(context), wanted) { it.label } ?: return "I couldn't find an app called $name." launchEntry(match, navigator, frame) return "Opening ${match.label}." } - private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ") + private val EXIT_WORDS = setOf("exit", "exit gearslip", "close gearslip", "quit", "quit gearslip", "open exit") + + /** Other names people use for Gearslip's own apps. */ + private val APP_NICKNAMES = mapOf( + "web" to "browser", "internet" to "browser", "the web" to "browser", "web browser" to "browser", + "dialer" to "phone", "the dialer" to "phone", "keypad" to "phone", + "screen share" to "screen sharing", "screen mirroring" to "screen sharing", "mirroring" to "screen sharing", + ) + + private val NAVIGATE_TRIGGERS = listOf( + "navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ", + "take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ", + "how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ", + ) + + private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ") + + private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ") + + private val OPEN_TRIGGERS = listOf("open ", "launch ", "start ", "go to ", "switch to ", "show me ", "bring up ", "pull up ") + + /** Screens "go to X" means rather than a place. */ + private val SCREEN_WORDS = setOf("settings", "home", "the home screen", "home screen", "apps", "my apps", "all apps") + + /** Padding stripped from the front of a command, longest first where they overlap. */ + private val LEAD_FILLERS = listOf( + "hey gearslip", "ok gearslip", "okay gearslip", "gearslip", "hey", "ok", "okay", "um", "uh", "so", + "please", "can you please", "can you", "could you", "would you", "will you", "can i", "could i", + "i want you to", "i want to", "i'd like to", "i would like to", "i wanna", "i need to", "i need you to", + "let's", "lets", "go ahead and", "just", "now", "quickly", + ) + + /** Longest first, so "right now" goes whole rather than leaving "right". */ + private val TAIL_FILLERS = listOf("thank you very much", "thank you", "right now", "for me", "please", "thanks", "now") + + private const val APP_CONNECT_MS = 5_000L private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands") - private val RESUME_WORDS = setOf("music", "the music", "something", "it", "again", "that") + private val RESUME_WORDS = setOf("music", "the music", "some music", "something", "it", "again", "that", "my music") + + private val RESUME_PHRASES = setOf("keep playing", "continue", "keep going", "start the music", "music on", "turn the music back on") } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt index 2e1c2af..5257a31 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt @@ -41,7 +41,8 @@ object VoiceHelp { "Pause · Resume · Next · Previous" to "Control the music", "What's playing" to "The song and artist", "Call…" to "A contact by name", - "Open…" to "Any app on this screen", + "Open…" to "Any app, like Browser, Phone, or Screen sharing", + "Exit" to "Back to the car's own screen", "Go home · Open apps · Settings" to "Move around Gearslip", "What time is it · Weather" to "The time, or the forecast", "Help" to "This list",