voice: start listening sooner, say so while it gets ready, and scroll the help list from anywhere on it

gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt

@@ -53,14 +53,14 @@ fun AssistantOverlay(modifier: Modifier = Modifier) {
     val state by CarAssistant.state.collectAsState()
     if (state.phase == CarAssistant.Phase.IDLE) return
     val context = LocalContext.current
-    val listening = state.phase == CarAssistant.Phase.LISTENING
+    val listening = state.phase == CarAssistant.Phase.LISTENING && state.ready
     val glow = MaterialTheme.colorScheme.primary
     val glow2 = MaterialTheme.colorScheme.tertiary
     val breathe by rememberInfiniteTransition(label = "glow").animateFloat(
         0.55f, 1f, infiniteRepeatable(tween(900, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "glow",
     )
     val strength = when (state.phase) {
-        CarAssistant.Phase.LISTENING -> breathe
+        CarAssistant.Phase.LISTENING -> if (state.ready) breathe else 0.3f
         CarAssistant.Phase.ERROR -> 0.3f
         else -> 0.6f
     }
@@ -87,7 +87,7 @@ fun AssistantOverlay(modifier: Modifier = Modifier) {
             PulsingMic(listening)
             Column(Modifier.weight(1f), horizontalAlignment = Alignment.CenterHorizontally) {
                 val (small, big) = when (state.phase) {
-                    CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { "Listening…" }
+                    CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { if (state.ready) "Listening…" else "One moment…" }
                     CarAssistant.Phase.THINKING -> "" to state.heard.ifBlank { "…" }
                     else -> state.heard to state.reply
                 }
@@ -96,7 +96,7 @@ fun AssistantOverlay(modifier: Modifier = Modifier) {
                 }
                 Text(
                     big,
-                    color = if (big == "Listening…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White,
+                    color = if (big == "Listening…" || big == "One moment…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White,
                     style = MaterialTheme.typography.headlineSmall,
                     fontWeight = FontWeight.SemiBold,
                     textAlign = TextAlign.Center,

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt

@@ -38,6 +38,8 @@ object CarAssistant {
         val heard: String = "",
         val reply: String = "",
         val missingPermission: Boolean = false,
+        /** False from the press until the microphone is actually open, so the overlay doesn't claim to listen early. */
+        val ready: Boolean = false,
     )
 
     private val _state = MutableStateFlow(State())
@@ -89,6 +91,7 @@ object CarAssistant {
         tts?.stop()
         _state.value = State(Phase.LISTENING)
         listenOnce(
+            onListening = { _state.value = _state.value.copy(ready = true) },
             onPartial = { text -> _state.value = _state.value.copy(heard = text) },
             onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) },
             onResult = { text ->
@@ -174,7 +177,7 @@ object CarAssistant {
         onSpeaking: () -> Unit = {},
     ) {
         cancel()
-        listenOnce(onPartial, onHeard = {}, onResult, onError, cue, onSpeaking)
+        listenOnce(onPartial, onHeard = {}, onResult = onResult, onError = onError, cue = cue, onSpeaking = onSpeaking)
     }
 
     /**
@@ -183,6 +186,7 @@ object CarAssistant {
      */
     private fun listenOnce(
         onPartial: (String) -> Unit,
+        onListening: () -> Unit = {},
         onHeard: () -> Unit,
         onResult: (String) -> Unit,
         onError: (String) -> Unit,
@@ -191,13 +195,14 @@ object CarAssistant {
     ) {
         val context = appContext ?: return onError("Speech isn't ready yet.")
         if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) {
-            return systemListen(context, onPartial, onHeard, onResult, onError)
+            return systemListen(context, onListening, onPartial, onHeard, onResult, onError)
         }
         SpeechEngine.listen(
             context,
             cue = cue,
             onSpeaking = onSpeaking,
             onPartial = onPartial,
+            onListening = onListening,
             onHeard = onHeard,
             onResult = onResult,
             onError = onError,
@@ -206,6 +211,7 @@ object CarAssistant {
 
     private fun systemListen(
         context: Context,
+        onListening: () -> Unit,
         onPartial: (String) -> Unit,
         onHeard: () -> Unit,
         onResult: (String) -> Unit,
@@ -214,7 +220,7 @@ object CarAssistant {
         val r = SpeechRecognizer.createSpeechRecognizer(context)
         recognizer = r
         r.setRecognitionListener(object : RecognitionListener {
-            override fun onReadyForSpeech(params: Bundle?) {}
+            override fun onReadyForSpeech(params: Bundle?) = onListening()
             override fun onBeginningOfSpeech() {}
             override fun onRmsChanged(rmsdB: Float) {}
             override fun onBufferReceived(buffer: ByteArray?) {}

gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt

@@ -76,12 +76,13 @@ fun VoiceHelpOverlay(modifier: Modifier = Modifier) {
                 )
                 GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide)
             }
+            // Full width, so a drag anywhere on the card scrolls, not just one on the words.
             Column(
-                Modifier.verticalScroll(rememberScrollState()).padding(top = 8.dp),
+                Modifier.fillMaxWidth().verticalScroll(rememberScrollState()).padding(top = 8.dp),
                 verticalArrangement = Arrangement.spacedBy(10.dp),
             ) {
                 VoiceHelp.COMMANDS.forEach { (say, does) ->
-                    Column {
+                    Column(Modifier.fillMaxWidth()) {
                         Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold)
                         Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
                     }

gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt

@@ -64,6 +64,12 @@ object SpeechEngine {
     private const val CUE_TAIL_MS = 250L
 
     private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") }
+    /**
+     * Parakeet large takes seconds to load. On [worker], the first press after connecting
+     * waited behind it before listening at all; here it loads while the driver talks.
+     */
+    private val finalWorker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-final") }
+    private val finalLock = Object()
     private val main = Handler(Looper.getMainLooper())
 
     private var recognizer: OnlineRecognizer? = null
@@ -76,9 +82,10 @@ object SpeechEngine {
     fun warm(context: Context) {
         val app = context.applicationContext
         SpeechModels.init(app)
+        // Kroko first, alone: loading both at once slowed it from a fraction of a second to six.
         worker.execute {
             ensureLoaded(app)
-            ensureFinalLoaded(app)
+            finalWorker.execute { ensureFinalLoaded(app) }
         }
     }
 
@@ -89,9 +96,13 @@ object SpeechEngine {
             recognizer?.release()
             recognizer = null
             loadedModel = null
-            finalRecognizer?.release()
+        }
-            finalRecognizer = null
+        finalWorker.execute {
-            loadedFinal = null
+            synchronized(finalLock) {
+                finalRecognizer?.release()
+                finalRecognizer = null
+                loadedFinal = null
+            }
         }
     }
 
@@ -106,6 +117,7 @@ object SpeechEngine {
         cue: Boolean = true,
         onSpeaking: () -> Unit = {},
         onPartial: (String) -> Unit = {},
+        onListening: () -> Unit = {},
         onHeard: () -> Unit,
         onResult: (String) -> Unit,
         onError: (String) -> Unit,
@@ -125,6 +137,7 @@ object SpeechEngine {
             val text = try {
                 if (cue) cue()
                 if (id != run) return@execute
+                post(onListening)
                 stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) })
             } catch (t: Throwable) {
                 GearslipLog.e("speech: listening failed", t)
@@ -201,17 +214,20 @@ object SpeechEngine {
             "speech: $audioMs ms of audio, decoding took $decodeMs ms " +
                 "(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)",
         )
-        if (last.isEmpty() || id != run) return last
+        if (last.isEmpty() || id != run) { phrase.fill(0f); return last }
-        val final = ensureFinalLoaded(context) ?: return last
         onStopped()
         val started = SystemClock.uptimeMillis()
-        val reread = final.createStream().run {
+        // Held for the whole read, so a model swap in settings can't free it mid-decode.
-            try {
+        val reread = synchronized(finalLock) {
-                acceptWaveform(phrase.copyOf(held), RATE)
+            val final = ensureFinalLoaded(context) ?: run { phrase.fill(0f); return last }
-                final.decode(this)
+            final.createStream().run {
-                clean(final.getResult(this).text)
+                try {
-            } finally {
+                    acceptWaveform(phrase.copyOf(held), RATE)
-                release()
+                    final.decode(this)
+                    clean(final.getResult(this).text)
+                } finally {
+                    release()
+                }
             }
         }
         phrase.fill(0f)
@@ -222,7 +238,7 @@ object SpeechEngine {
     }
 
     /** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */
-    private fun ensureFinalLoaded(context: Context): OfflineRecognizer? {
+    private fun ensureFinalLoaded(context: Context): OfflineRecognizer? = synchronized(finalLock) {
         val model = SpeechModels.final.value ?: return null
         if (finalRecognizer != null && loadedFinal == model) return finalRecognizer
         finalRecognizer?.release()