diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt index 961d3ec..35f5814 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt @@ -53,14 +53,14 @@ fun AssistantOverlay(modifier: Modifier = Modifier) { val state by CarAssistant.state.collectAsState() if (state.phase == CarAssistant.Phase.IDLE) return val context = LocalContext.current - val listening = state.phase == CarAssistant.Phase.LISTENING + val listening = state.phase == CarAssistant.Phase.LISTENING && state.ready val glow = MaterialTheme.colorScheme.primary val glow2 = MaterialTheme.colorScheme.tertiary val breathe by rememberInfiniteTransition(label = "glow").animateFloat( 0.55f, 1f, infiniteRepeatable(tween(900, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "glow", ) val strength = when (state.phase) { - CarAssistant.Phase.LISTENING -> breathe + CarAssistant.Phase.LISTENING -> if (state.ready) breathe else 0.3f CarAssistant.Phase.ERROR -> 0.3f else -> 0.6f } @@ -87,7 +87,7 @@ fun AssistantOverlay(modifier: Modifier = Modifier) { PulsingMic(listening) Column(Modifier.weight(1f), horizontalAlignment = Alignment.CenterHorizontally) { val (small, big) = when (state.phase) { - CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { "Listening…" } + CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { if (state.ready) "Listening…" else "One moment…" } CarAssistant.Phase.THINKING -> "" to state.heard.ifBlank { "…" } else -> state.heard to state.reply } @@ -96,7 +96,7 @@ fun AssistantOverlay(modifier: Modifier = Modifier) { } Text( big, - color = if (big == "Listening…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White, + color = if (big == "Listening…" || big == "One moment…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White, style = MaterialTheme.typography.headlineSmall, fontWeight = FontWeight.SemiBold, textAlign = TextAlign.Center, diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt index 888fa16..96e564d 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt @@ -38,6 +38,8 @@ object CarAssistant { val heard: String = "", val reply: String = "", val missingPermission: Boolean = false, + /** False from the press until the microphone is actually open, so the overlay doesn't claim to listen early. */ + val ready: Boolean = false, ) private val _state = MutableStateFlow(State()) @@ -89,6 +91,7 @@ object CarAssistant { tts?.stop() _state.value = State(Phase.LISTENING) listenOnce( + onListening = { _state.value = _state.value.copy(ready = true) }, onPartial = { text -> _state.value = _state.value.copy(heard = text) }, onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) }, onResult = { text -> @@ -174,7 +177,7 @@ object CarAssistant { onSpeaking: () -> Unit = {}, ) { cancel() - listenOnce(onPartial, onHeard = {}, onResult, onError, cue, onSpeaking) + listenOnce(onPartial, onHeard = {}, onResult = onResult, onError = onError, cue = cue, onSpeaking = onSpeaking) } /** @@ -183,6 +186,7 @@ object CarAssistant { */ private fun listenOnce( onPartial: (String) -> Unit, + onListening: () -> Unit = {}, onHeard: () -> Unit, onResult: (String) -> Unit, onError: (String) -> Unit, @@ -191,13 +195,14 @@ object CarAssistant { ) { val context = appContext ?: return onError("Speech isn't ready yet.") if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) { - return systemListen(context, onPartial, onHeard, onResult, onError) + return systemListen(context, onListening, onPartial, onHeard, onResult, onError) } SpeechEngine.listen( context, cue = cue, onSpeaking = onSpeaking, onPartial = onPartial, + onListening = onListening, onHeard = onHeard, onResult = onResult, onError = onError, @@ -206,6 +211,7 @@ object CarAssistant { private fun systemListen( context: Context, + onListening: () -> Unit, onPartial: (String) -> Unit, onHeard: () -> Unit, onResult: (String) -> Unit, @@ -214,7 +220,7 @@ object CarAssistant { val r = SpeechRecognizer.createSpeechRecognizer(context) recognizer = r r.setRecognitionListener(object : RecognitionListener { - override fun onReadyForSpeech(params: Bundle?) {} + override fun onReadyForSpeech(params: Bundle?) = onListening() override fun onBeginningOfSpeech() {} override fun onRmsChanged(rmsdB: Float) {} override fun onBufferReceived(buffer: ByteArray?) {} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt index fcc8368..2e1c2af 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt @@ -76,12 +76,13 @@ fun VoiceHelpOverlay(modifier: Modifier = Modifier) { ) GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide) } + // Full width, so a drag anywhere on the card scrolls, not just one on the words. Column( - Modifier.verticalScroll(rememberScrollState()).padding(top = 8.dp), + Modifier.fillMaxWidth().verticalScroll(rememberScrollState()).padding(top = 8.dp), verticalArrangement = Arrangement.spacedBy(10.dp), ) { VoiceHelp.COMMANDS.forEach { (say, does) -> - Column { + Column(Modifier.fillMaxWidth()) { Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt index 10032ae..1ea248e 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt @@ -64,6 +64,12 @@ object SpeechEngine { private const val CUE_TAIL_MS = 250L private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") } + /** + * Parakeet large takes seconds to load. On [worker], the first press after connecting + * waited behind it before listening at all; here it loads while the driver talks. + */ + private val finalWorker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-final") } + private val finalLock = Object() private val main = Handler(Looper.getMainLooper()) private var recognizer: OnlineRecognizer? = null @@ -76,9 +82,10 @@ object SpeechEngine { fun warm(context: Context) { val app = context.applicationContext SpeechModels.init(app) + // Kroko first, alone: loading both at once slowed it from a fraction of a second to six. worker.execute { ensureLoaded(app) - ensureFinalLoaded(app) + finalWorker.execute { ensureFinalLoaded(app) } } } @@ -89,9 +96,13 @@ object SpeechEngine { recognizer?.release() recognizer = null loadedModel = null - finalRecognizer?.release() - finalRecognizer = null - loadedFinal = null + } + finalWorker.execute { + synchronized(finalLock) { + finalRecognizer?.release() + finalRecognizer = null + loadedFinal = null + } } } @@ -106,6 +117,7 @@ object SpeechEngine { cue: Boolean = true, onSpeaking: () -> Unit = {}, onPartial: (String) -> Unit = {}, + onListening: () -> Unit = {}, onHeard: () -> Unit, onResult: (String) -> Unit, onError: (String) -> Unit, @@ -125,6 +137,7 @@ object SpeechEngine { val text = try { if (cue) cue() if (id != run) return@execute + post(onListening) stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) }) } catch (t: Throwable) { GearslipLog.e("speech: listening failed", t) @@ -201,17 +214,20 @@ object SpeechEngine { "speech: $audioMs ms of audio, decoding took $decodeMs ms " + "(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)", ) - if (last.isEmpty() || id != run) return last - val final = ensureFinalLoaded(context) ?: return last + if (last.isEmpty() || id != run) { phrase.fill(0f); return last } onStopped() val started = SystemClock.uptimeMillis() - val reread = final.createStream().run { - try { - acceptWaveform(phrase.copyOf(held), RATE) - final.decode(this) - clean(final.getResult(this).text) - } finally { - release() + // Held for the whole read, so a model swap in settings can't free it mid-decode. + val reread = synchronized(finalLock) { + val final = ensureFinalLoaded(context) ?: run { phrase.fill(0f); return last } + final.createStream().run { + try { + acceptWaveform(phrase.copyOf(held), RATE) + final.decode(this) + clean(final.getResult(this).text) + } finally { + release() + } } } phrase.fill(0f) @@ -222,7 +238,7 @@ object SpeechEngine { } /** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */ - private fun ensureFinalLoaded(context: Context): OfflineRecognizer? { + private fun ensureFinalLoaded(context: Context): OfflineRecognizer? = synchronized(finalLock) { val model = SpeechModels.final.value ?: return null if (finalRecognizer != null && loadedFinal == model) return finalRecognizer finalRecognizer?.release()