Clone/Gearslip
1
0 Fork 0
mirror of https://github.com/twinksoftware/Gearslip.git synced 2026-11-10 16:02:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

voice: understand natural phrasing for every command, open Gearslip's own apps by name, and say "exit" to hand the screen back

Browse Source
This commit is contained in:
Seb3thehacker committed 2026-10-12 16:10:23 -07:00
1 parent d099193ead
commit 4ac3d172d3
2 files changed
+171 -46
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt
+169 -45
View File
@@ -4,7 +4,9 @@ import android.content.Context
import java.text.DateFormat
import java.util.Date
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
/**
* Turns what [CarAssistant] heard into an action plus something to say back.
@@ -17,7 +19,7 @@ import kotlinx.coroutines.withContext
object AssistantCommands {
suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String {
val heard = heardRaw.lowercase().trim().trimEnd('.', '?', '!')
val heard = natural(heardRaw)
VoiceHelp.hide()
if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) {
VoiceHelp.show()
@@ -25,7 +27,7 @@ object AssistantCommands {
}
navigateCommand(context, navigator, frame, heard)?.let { return it }
playSearchCommand(heard)?.let { return it }
playSearchCommand(context, navigator, heard)?.let { return it }
mediaCommand(heard)?.let { return it }
navCommand(navigator, heard)?.let { return it }
infoCommand(heard)?.let { return it }
@@ -35,6 +37,42 @@ object AssistantCommands {
return "I don't know how to do that yet."
}
/**
* What was said, minus the padding people wrap a command in: "hey", "can you", "I want to",
* "please", "thanks". Each command below then only has to know its own words.
*/
private fun natural(raw: String): String {
var t = raw.lowercase().replace(Regex("[.,!?]"), " ").replace(Regex("\\s+"), " ").trim()
var changed = true
while (changed) {
changed = false
for (lead in LEAD_FILLERS) if (t.startsWith("$lead ")) { t = t.removePrefix("$lead ").trim(); changed = true }
for (tail in TAIL_FILLERS) if (t.endsWith(" $tail")) { t = t.removeSuffix(" $tail").trim(); changed = true }
}
return t
}
/** The part after the first of [triggers] that [text] starts with, or null. */
private fun after(text: String, triggers: List<String>): String? =
triggers.firstOrNull { text.startsWith(it) }?.let { text.removePrefix(it).trim() }
private suspend fun allEntries(context: Context): List<Entry> {
val vehicleData = CarSettings.experimentalFeaturesEnabled.value && CarSettings.carSensorsEnabled.value
val builtIn = BuiltInApps.entries.filter { it.builtIn !== BuiltInApps.VehicleData || vehicleData }
return builtIn + (LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) })
}
/** Best label match: exact, then a label starting with [name], then one containing it. */
private fun <T> bestMatch(items: List<T>, name: String, label: (T) -> String): T? {
val n = name.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
if (n.isEmpty()) return null
items.firstOrNull { label(it).lowercase() == n }?.let { return it }
// Loose matches need a real word: "pizza in a" mustn't pick "Organic Maps" for its "a".
if (n.length < 3) return null
return items.firstOrNull { label(it).lowercase().startsWith(n) }
?: items.firstOrNull { label(it).lowercase().contains(n) }
}
/**
* "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it,
* whichever nav app was used last, or the only one installed, or the first one found. Starts
@@ -48,29 +86,45 @@ object AssistantCommands {
frame: CarEnvironment.Frame,
heard: String,
): String? {
val trigger = NAVIGATE_TRIGGERS.firstOrNull { heard.startsWith(it) } ?: return null
var destination = heard.removePrefix(trigger).trim()
// "use Organic Maps to get to X", "open OsmAnd and navigate to X": the app comes first.
var text = heard
var leadingApp: String? = null
Regex("^(?:use|using|with|open|in|on) (.+?) (?:to |and )(.+)$").find(text)?.let { m ->
if (after(m.groupValues[2], NAVIGATE_TRIGGERS) != null) {
leadingApp = m.groupValues[1]
text = m.groupValues[2]
}
}
var destination = after(text, NAVIGATE_TRIGGERS) ?: return null
if (destination.isBlank()) return "Where to?"
var appName: String? = null
for (sep in listOf(" with ", " using ", " on ", " via ")) {
val at = destination.lastIndexOf(sep)
if (at > 0) {
appName = destination.substring(at + sep.length).trim()
val entries = allEntries(context)
// "go to Spotify" means the app, and "go to settings" the screen; "go to Target" is a place.
val asApp = destination.removePrefix("the ").removeSuffix(" app")
if (text.startsWith("go to ") &&
(asApp in SCREEN_WORDS || asApp in APP_NICKNAMES || entries.any { it.label.lowercase() == asApp })
) {
return null
}
val navApps = entries.filter { it.template?.isNavigation == true }
if (navApps.isEmpty()) return "You don't have any navigation apps installed."
// "X with Organic Maps", "X on OsmAnd": split only when the end names a nav app, so a
// place like "Starbucks on Main Street" stays whole.
var chosenApp = leadingApp?.let { name -> bestMatch(navApps, name) { it.label } ?: return "I couldn't find a navigation app called $name." }
if (chosenApp == null) {
for (sep in listOf(" with ", " using ", " on ", " via ", " in ", " through ")) {
val at = destination.lastIndexOf(sep)
if (at <= 0) continue
val app = bestMatch(navApps, destination.substring(at + sep.length)) { it.label } ?: continue
chosenApp = app
destination = destination.substring(0, at).trim()
break
}
}
if (destination.isBlank()) return "Where to?"
val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
val navApps = entries.filter { it.template?.isNavigation == true }
if (navApps.isEmpty()) return "You don't have any navigation apps installed."
val requestedApp = appName
val chosen = when {
requestedApp != null -> navApps.firstOrNull { it.label.lowercase().contains(requestedApp) }
?: return "I couldn't find a navigation app called $requestedApp."
val chosen = chosenApp ?: when {
navApps.size == 1 -> navApps.first()
else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first()
}
@@ -86,11 +140,35 @@ object AssistantCommands {
* that's open through its own search ([CarMedia.search]). A bare "play", or "play music",
* falls through to resume.
*/
private fun playSearchCommand(heard: String): String? {
if (!heard.startsWith("play ")) return null
val query = heard.removePrefix("play ").trim().removeSuffix(".").trim()
private suspend fun playSearchCommand(context: Context, navigator: CarNavigator, heard: String): String? {
var query = after(heard, PLAY_TRIGGERS) ?: return null
query = query.removePrefix("some ").trim()
if (query.isEmpty() || query in RESUME_WORDS) return null
val media = CarServices.media
// "play Daft Punk on Metrolist": switch to that app first, if it's one we know.
for (sep in listOf(" on ", " in ", " from ", " with ", " using ")) {
val at = query.lastIndexOf(sep)
if (at <= 0) continue
val apps = withContext(Dispatchers.IO) { app.seb3thehacker.gearslip.media.MediaCatalog.installed(context) }
val wanted = bestMatch(apps, query.substring(at + sep.length)) { it.label } ?: continue
query = query.substring(0, at).trim()
if (query.isEmpty() || query in RESUME_WORDS) {
CarServices.openMedia(wanted, autoplay = true)
navigator.media()
return "Opening ${wanted.label}."
}
if (CarServices.mediaApp.value?.component != wanted.component) {
CarServices.openMedia(wanted)
navigator.media()
// Give it a moment to connect before asking it to search.
withTimeoutOrNull(APP_CONNECT_MS) {
while (media.appLabel == null || (!media.canSearchLibrary && !media.now.value.canSearch)) delay(200)
}
}
break
}
val app = media.appLabel ?: return "Open a music app first, then ask again."
if (!media.canSearchLibrary && !media.now.value.canSearch) {
return "$app ignores voice searches. Pick something on its screen instead."
@@ -102,23 +180,27 @@ object AssistantCommands {
private fun mediaCommand(heard: String): String? {
val media = CarServices.media
return when {
"pause" in heard || heard.startsWith("stop") -> {
"pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> {
if (media.now.value.playing) media.togglePlay()
"Paused."
}
"resume" in heard || (heard.startsWith("play") && "playing" !in heard) -> {
"resume" in heard || "unpause" in heard || heard in RESUME_PHRASES ||
(heard.startsWith("play") && "playing" !in heard) -> {
if (!media.now.value.playing) media.togglePlay()
"Playing."
}
"next" in heard || "skip" in heard -> {
"next" in heard || "skip" in heard || heard.startsWith("change the song") -> {
media.next()
"Skipping."
}
"previous" in heard || "last song" in heard || "back song" in heard -> {
"previous" in heard || "last song" in heard || "back song" in heard || "go back a song" in heard ||
"play that again" in heard || "replay" in heard -> {
media.previous()
"Playing the previous track."
}
"what" in heard && "playing" in heard -> {
("what" in heard && "playing" in heard) || "what song" in heard || "who sings" in heard ||
"who's singing" in heard || "who is singing" in heard || "what's this song" in heard ||
"name of this song" in heard -> {
val now = media.now.value
if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ")
else "Nothing's playing."
@@ -128,11 +210,19 @@ object AssistantCommands {
}
private fun navCommand(navigator: CarNavigator, heard: String): String? = when {
"go home" in heard || heard == "home" -> {
heard == "go back" || heard == "back" -> {
navigator.back()
"Going back."
}
heard in EXIT_WORDS -> {
BuiltInApps.Exit.open(navigator)
"Back to the car's screen."
}
"go home" in heard || heard == "home" || heard == "home screen" || "go to the home screen" in heard -> {
navigator.home()
"Heading home."
}
"open apps" in heard || heard == "apps" || "app list" in heard -> {
"open apps" in heard || heard == "apps" || "app list" in heard || "all apps" in heard || "my apps" in heard || heard == "go to apps" -> {
navigator.apps()
"Here are your apps."
}
@@ -152,9 +242,10 @@ object AssistantCommands {
}
private fun infoCommand(heard: String): String? = when {
"what time" in heard || "what's the time" in heard ->
"what time" in heard || "what's the time" in heard || heard == "time" || "the time" in heard ->
"It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}."
"weather" in heard -> {
"weather" in heard || "going to rain" in heard || "gonna rain" in heard || "how hot" in heard ||
"how cold" in heard || "temperature outside" in heard -> {
val data = (Weather.state.value as? WeatherState.Ready)?.data
if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}."
else "I don't have the weather yet."
@@ -163,12 +254,11 @@ object AssistantCommands {
}
private suspend fun callCommand(context: Context, heard: String): String? {
val name = when {
heard.startsWith("call ") -> heard.removePrefix("call ")
heard.startsWith("phone ") -> heard.removePrefix("phone ")
heard.startsWith("dial ") -> heard.removePrefix("dial ")
else -> return null
}.trim()
val name = (
after(heard, CALL_TRIGGERS)
?: Regex("^give (.+) a (?:call|ring)$").find(heard)?.groupValues?.get(1)
?: return null
).replace(Regex(" (?:on|at) (?:their |his |her )?(?:mobile|cell|phone|home|work)$"), "").trim()
if (name.isBlank()) return "Who should I call?"
if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen."
@@ -186,25 +276,59 @@ object AssistantCommands {
frame: CarEnvironment.Frame,
heard: String,
): String? {
val name = when {
heard.startsWith("open ") -> heard.removePrefix("open ")
heard.startsWith("launch ") -> heard.removePrefix("launch ")
heard.startsWith("start ") -> heard.removePrefix("start ")
else -> return null
}.trim()
val name = (after(heard, OPEN_TRIGGERS) ?: return null)
.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
if (name.isBlank()) return null
val wanted = APP_NICKNAMES[name] ?: name
val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
val match = entries.firstOrNull { it.label.lowercase().contains(name) }
// Gearslip's own apps come first in the list, so "phone" opens its dialer, not the phone's.
val match = bestMatch(allEntries(context), wanted) { it.label }
?: return "I couldn't find an app called $name."
launchEntry(match, navigator, frame)
return "Opening ${match.label}."
}
private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ")
private val EXIT_WORDS = setOf("exit", "exit gearslip", "close gearslip", "quit", "quit gearslip", "open exit")
/** Other names people use for Gearslip's own apps. */
private val APP_NICKNAMES = mapOf(
"web" to "browser", "internet" to "browser", "the web" to "browser", "web browser" to "browser",
"dialer" to "phone", "the dialer" to "phone", "keypad" to "phone",
"screen share" to "screen sharing", "screen mirroring" to "screen sharing", "mirroring" to "screen sharing",
)
private val NAVIGATE_TRIGGERS = listOf(
"navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ",
"take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ",
"how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ",
)
private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ")
private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ")
private val OPEN_TRIGGERS = listOf("open ", "launch ", "start ", "go to ", "switch to ", "show me ", "bring up ", "pull up ")
/** Screens "go to X" means rather than a place. */
private val SCREEN_WORDS = setOf("settings", "home", "the home screen", "home screen", "apps", "my apps", "all apps")
/** Padding stripped from the front of a command, longest first where they overlap. */
private val LEAD_FILLERS = listOf(
"hey gearslip", "ok gearslip", "okay gearslip", "gearslip", "hey", "ok", "okay", "um", "uh", "so",
"please", "can you please", "can you", "could you", "would you", "will you", "can i", "could i",
"i want you to", "i want to", "i'd like to", "i would like to", "i wanna", "i need to", "i need you to",
"let's", "lets", "go ahead and", "just", "now", "quickly",
)
/** Longest first, so "right now" goes whole rather than leaving "right". */
private val TAIL_FILLERS = listOf("thank you very much", "thank you", "right now", "for me", "please", "thanks", "now")
private const val APP_CONNECT_MS = 5_000L
private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands")
private val RESUME_WORDS = setOf("music", "the music", "something", "it", "again", "that")
private val RESUME_WORDS = setOf("music", "the music", "some music", "something", "it", "again", "that", "my music")
private val RESUME_PHRASES = setOf("keep playing", "continue", "keep going", "start the music", "music on", "turn the music back on")
}
gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt
+2 -1
View File
@@ -41,7 +41,8 @@ object VoiceHelp {
"Pause · Resume · Next · Previous" to "Control the music",
"What's playing" to "The song and artist",
"Call…" to "A contact by name",
"Open…" to "Any app on this screen",
"Open…" to "Any app, like Browser, Phone, or Screen sharing",
"Exit" to "Back to the car's own screen",
"Go home · Open apps · Settings" to "Move around Gearslip",
"What time is it · Weather" to "The time, or the forecast",
"Help" to "This list",