voice: replace Whisper with sherpa-onnx, live words from Kroko and the final words from Parakeet

Models download in Settings instead of shipping in the APK. Credits for the engine and models are in About.

gearslip/build.gradle.kts

@@ -83,21 +83,10 @@ android {
     }
 
     defaultConfig {
-        // Speech-to-text ships for 64-bit ARM only: every phone Gearslip runs on, and one copy
-        // of the native library instead of four.
+        // Speech-to-text (sherpa-onnx, in jniLibs) ships for 64-bit ARM only: every phone
+        // Gearslip runs on, and one copy of the native library instead of four.
         ndk { abiFilters += "arm64-v8a" }
     }
-    ndkVersion = "27.1.12297006"
-    externalNativeBuild {
-        cmake {
-            path = file("src/main/cpp/CMakeLists.txt")
-            version = "3.22.1"
-        }
-    }
-    androidResources {
-        // Model weights barely compress, and stored plainly they stream straight out of the APK.
-        noCompress += "bin"
-    }
 
     testOptions {
         // GearslipLog calls android.util.Log, which is a stub on the local JVM.
@@ -130,6 +119,7 @@ dependencies {
     implementation(platform(libs.androidx.compose.bom))
     implementation(libs.androidx.compose.ui)
     implementation(libs.androidx.compose.material3)
+    implementation(libs.commons.compress)
     implementation(libs.androidx.compose.material.icons.core)
     implementation(libs.androidx.activity.compose)
     implementation(libs.androidx.lifecycle.runtime.compose)

gearslip/src/main/cpp/CMakeLists.txt

@@ -1,36 +0,0 @@
-# Speech-to-text for voice replies and the assistant: whisper.cpp, pinned to one commit and
-# fetched at build time, so nothing outside this folder (cloned-repos included) is needed.
-cmake_minimum_required(VERSION 3.22)
-project(gearslip_speech CXX C)
-
-set(CMAKE_CXX_STANDARD 17)
-
-# Optimised in debug builds too. Unoptimised, Whisper runs several times slower, and a dev build
-# would make voice look far worse than what ships.
-foreach(lang C CXX)
-    set(CMAKE_${lang}_FLAGS_DEBUG "-O3 -DNDEBUG")
-endforeach()
-
-include(FetchContent)
-FetchContent_Declare(
-    whisper
-    URL https://github.com/ggml-org/whisper.cpp/archive/6e4ab854f67f743900934a703d5603419384c961.tar.gz
-    URL_HASH SHA256=55b585aec5e1bc03e2a5cea61da97c6406fa833996cf7daeed704972273cc039
-)
-
-set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
-set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
-set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE)
-set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE)
-set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
-set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
-# ARMv8.2 with half floats and dot products: every phone that runs Android 12 or later in
-# practice, and what makes Whisper fast enough to answer in under a second.
-set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE)
-FetchContent_MakeAvailable(whisper)
-
-add_library(gearslip_speech SHARED speech_jni.cpp)
-target_compile_options(gearslip_speech PRIVATE -O3)
-find_library(LOG_LIB log)
-target_link_libraries(gearslip_speech whisper android ${LOG_LIB})
-target_link_options(gearslip_speech PRIVATE -Wl,--gc-sections -Wl,-z,max-page-size=16384)

gearslip/src/main/cpp/speech_jni.cpp

@@ -1,86 +0,0 @@
-// The bridge between Kotlin's SpeechEngine and whisper.cpp: load a model, turn one recorded
-// phrase into text, free the model. Recording and deciding when a phrase ends stay in Kotlin.
-
-#include <jni.h>
-#include <android/asset_manager.h>
-#include <android/asset_manager_jni.h>
-#include <android/log.h>
-#include <algorithm>
-#include <cmath>
-#include <string>
-#include "whisper.h"
-
-#define TAG "GearslipSpeech"
-#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
-
-static whisper_context_params context_params() {
-    whisper_context_params p = whisper_context_default_params();
-    p.use_gpu = false;
-    return p;
-}
-
-// Streams a model straight out of the APK, so the bundled one is never copied to storage.
-static size_t asset_read(void *ctx, void *output, size_t size) {
-    int n = AAsset_read(static_cast<AAsset *>(ctx), output, size);
-    return n > 0 ? static_cast<size_t>(n) : 0;
-}
-static bool asset_eof(void *ctx) { return AAsset_getRemainingLength64(static_cast<AAsset *>(ctx)) <= 0; }
-static void asset_close(void *ctx) { AAsset_close(static_cast<AAsset *>(ctx)); }
-
-extern "C" JNIEXPORT jlong JNICALL
-Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeInitAsset(JNIEnv *env, jobject, jobject assets, jstring jpath) {
-    AAssetManager *manager = AAssetManager_fromJava(env, assets);
-    const char *path = env->GetStringUTFChars(jpath, nullptr);
-    AAsset *asset = AAssetManager_open(manager, path, AASSET_MODE_STREAMING);
-    env->ReleaseStringUTFChars(jpath, path);
-    if (!asset) return 0;
-    whisper_model_loader loader = {asset, asset_read, asset_eof, asset_close};
-    return reinterpret_cast<jlong>(whisper_init_with_params(&loader, context_params()));
-}
-
-// Bytes, not a jstring: NewStringUTF wants modified UTF-8 and aborts on some of what Whisper can
-// write (four-byte characters), so Kotlin decodes the plain UTF-8 instead.
-extern "C" JNIEXPORT jbyteArray JNICALL
-Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeTranscribe(
-    JNIEnv *env, jobject, jlong handle, jfloatArray jsamples, jstring jlang, jint threads) {
-    auto *ctx = reinterpret_cast<whisper_context *>(handle);
-    if (!ctx) return env->NewByteArray(0);
-    jsize n = env->GetArrayLength(jsamples);
-    jfloat *samples = env->GetFloatArrayElements(jsamples, nullptr);
-    const char *lang = env->GetStringUTFChars(jlang, nullptr);
-
-    whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
-    p.n_threads = threads;
-    p.language = lang;
-    p.translate = false;
-    p.no_context = true;
-    p.no_timestamps = true;
-    p.single_segment = true;
-    p.print_progress = false;
-    p.print_realtime = false;
-    p.print_special = false;
-    p.print_timestamps = false;
-    p.suppress_blank = true;
-    // The ACFT models are trained for a context cut to the audio's own length (1500 frames is
-    // 30 s, so 50 per second), which is what makes a three-second phrase fast. A little extra
-    // keeps the last word from being clipped.
-    p.audio_ctx = std::min(1500, static_cast<int>(std::ceil(n / 320.0)) + 32);
-
-    int rc = whisper_full(ctx, p, samples, n);
-    env->ReleaseStringUTFChars(jlang, lang);
-    env->ReleaseFloatArrayElements(jsamples, samples, JNI_ABORT);
-    if (rc != 0) {
-        LOGI("whisper_full failed: %d", rc);
-        return env->NewByteArray(0);
-    }
-    std::string text;
-    for (int i = 0; i < whisper_full_n_segments(ctx); i++) text += whisper_full_get_segment_text(ctx, i);
-    jbyteArray out = env->NewByteArray(static_cast<jsize>(text.size()));
-    env->SetByteArrayRegion(out, 0, static_cast<jsize>(text.size()), reinterpret_cast<const jbyte *>(text.data()));
-    return out;
-}
-
-extern "C" JNIEXPORT void JNICALL
-Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeFree(JNIEnv *, jobject, jlong handle) {
-    if (handle) whisper_free(reinterpret_cast<whisper_context *>(handle));
-}

gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt

@@ -178,8 +178,8 @@ object CarAssistant {
     }
 
     /**
-     * One phrase, through Gearslip's own [SpeechEngine]. The phone's recognizer is only a fallback
-     * for when the engine itself can't run, since plenty of phones (GrapheneOS among them) have none.
+     * One phrase, through Gearslip's own [SpeechEngine], or the phone's recognizer when the driver
+     * chose it in settings. Never the phone's on its own: on most phones that's Google's.
      */
     private fun listenOnce(
         onPartial: (String) -> Unit,
@@ -197,15 +197,10 @@ object CarAssistant {
             context,
             cue = cue,
             onSpeaking = onSpeaking,
+            onPartial = onPartial,
             onHeard = onHeard,
             onResult = onResult,
-            onError = { message ->
-                if (message == SpeechEngine.NOT_LOADED && SpeechRecognizer.isRecognitionAvailable(context)) {
-                    systemListen(context, onPartial, onHeard, onResult, onError)
-                } else {
-                    onError(message)
-                }
-            },
+            onError = onError,
         )
     }
 

gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt

@@ -4,7 +4,6 @@ import android.Manifest
 import android.annotation.SuppressLint
 import android.content.Context
 import android.content.pm.PackageManager
-import android.content.res.AssetManager
 import android.media.AudioFormat
 import android.media.AudioManager
 import android.media.AudioRecord
@@ -15,40 +14,47 @@ import android.os.Looper
 import android.os.SystemClock
 import app.seb3thehacker.gearslip.GearslipLog
 import app.seb3thehacker.gearslip.audio.CarMic
-import java.util.Locale
+import com.k2fsa.sherpa.onnx.EndpointConfig
+import com.k2fsa.sherpa.onnx.EndpointRule
+import com.k2fsa.sherpa.onnx.FeatureConfig
+import com.k2fsa.sherpa.onnx.OfflineModelConfig
+import com.k2fsa.sherpa.onnx.OfflineRecognizer
+import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
+import com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig
+import com.k2fsa.sherpa.onnx.OnlineModelConfig
+import com.k2fsa.sherpa.onnx.OnlineRecognizer
+import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig
+import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig
+import java.io.File
 import java.util.concurrent.Executors
-import kotlin.math.log10
 import kotlin.math.max
-import kotlin.math.sqrt
 
 /**
  * Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships
- * none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply
- * of a few seconds comes back in well under a second on a recent phone.
+ * none). sherpa-onnx runs a streaming model ([SpeechModels]) on the phone: words come out while
+ * the driver is still talking, and the model itself decides when they've finished. A larger model
+ * then reads the whole phrase once more for the final words, since a model that waits for the end
+ * hears names and places far better than one that has to keep up. The phrase is held in memory
+ * for that and nowhere else, and dropped as soon as it's read.
  *
- * Whisper needs the whole phrase before it starts, so this records from the car's microphone (the
- * phone's when the car has none or sends nothing),
- * decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes
- * once. Everything runs on one worker thread; callbacks arrive on the main thread.
+ * Audio comes from the car's microphone, or the phone's when the car has none or sends nothing.
+ * Everything runs on one worker thread; callbacks arrive on the main thread.
  */
 object SpeechEngine {
 
-    /**
-     * FUTO's ACFT Whisper (Apache-2.0), re-quantized to 5 bits, inside the APK: base when the build
-     * carries it (more accurate, about twice as slow), tiny otherwise.
-     */
-    private val MODELS = listOf("speech/base_acft_q5_1.bin", "speech/tiny_acft_q5_1.bin")
-
     /** The error for an engine that can't run at all, which callers may answer with a fallback. */
     const val NOT_LOADED = "The speech model didn't load."
 
+    /** Said when the chosen model hasn't been downloaded yet. */
+    const val NOT_DOWNLOADED = "Download a voice model in Gearslip's settings on the phone first."
+
     private const val RATE = 16_000
-    private const val FRAME = 480 // 30 ms
-    private const val MAX_MS = 15_000L
-    private const val NO_SPEECH_MS = 6_000L
-    private const val END_SILENCE_MS = 1_100L
-    private const val CALIBRATE_MS = 300L
-    private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word
+    private const val CHUNK = 1_600 // 100 ms
+    private const val MAX_S = 15f
+    /** Quiet before the first word that counts as "nobody spoke". */
+    private const val NO_SPEECH_S = 6f
+    /** Quiet after the last word that ends the phrase. */
+    private const val END_SILENCE_S = 1.0f
     private const val CUE_VOLUME = 80
     /** How long the car gets to start sending once asked, before the phone's microphone takes over. */
     private const val CAR_FIRST_AUDIO_MS = 1_500L
@@ -60,25 +66,46 @@ object SpeechEngine {
     private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") }
     private val main = Handler(Looper.getMainLooper())
 
-    private var libraryLoaded = false
-    private var handle = 0L
+    private var recognizer: OnlineRecognizer? = null
+    private var loadedModel: SpeechModels.Model? = null
+    private var finalRecognizer: OfflineRecognizer? = null
+    private var loadedFinal: SpeechModels.Model? = null
     @Volatile private var run = 0
 
-    /** Loads the model ahead of time, so the first reply doesn't wait for it. */
+    /** Loads the model ahead of time, so the first command doesn't wait for it. */
     fun warm(context: Context) {
         val app = context.applicationContext
-        worker.execute { ensureLoaded(app) }
+        SpeechModels.init(app)
+        worker.execute {
+            ensureLoaded(app)
+            ensureFinalLoaded(app)
+        }
+    }
+
+    /** Drops the loaded model, after the driver picks another or deletes it. */
+    fun unload() {
+        run++
+        worker.execute {
+            recognizer?.release()
+            recognizer = null
+            loadedModel = null
+            finalRecognizer?.release()
+            finalRecognizer = null
+            loadedFinal = null
+        }
     }
 
     /**
      * Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk.
-     * [onSpeaking] fires at the first word, [onHeard] when the driver stops and the words are being
-     * worked out; [onResult] gets the text, [onError] a sentence fit to show on the car screen.
+     * [onPartial] gets the words so far as they're recognized, [onSpeaking] fires at the first
+     * word, [onHeard] when the driver stops; [onResult] gets the text, [onError] a sentence fit to
+     * show on the car screen.
      */
     fun listen(
         context: Context,
         cue: Boolean = true,
         onSpeaking: () -> Unit = {},
+        onPartial: (String) -> Unit = {},
         onHeard: () -> Unit,
         onResult: (String) -> Unit,
         onError: (String) -> Unit,
@@ -92,26 +119,21 @@ object SpeechEngine {
         fun post(block: () -> Unit) { main.post { if (id == run) block() } }
         worker.execute {
             if (id != run) return@execute
-            if (!ensureLoaded(app)) return@execute post { onError(NOT_LOADED) }
+            if (!SpeechModels.isReady(app, SpeechModels.selected.value)) return@execute post { onError(NOT_DOWNLOADED) }
+            val r = ensureLoaded(app) ?: return@execute post { onError(NOT_LOADED) }
             VoiceFocus.acquire(app)
-            val audio = try {
+            val text = try {
                 if (cue) cue()
                 if (id != run) return@execute
-                record(id) { post(onSpeaking) }
+                stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) })
             } catch (t: Throwable) {
-                GearslipLog.e("speech: recording failed", t)
+                GearslipLog.e("speech: listening failed", t)
                 return@execute post { onError("The microphone isn't available right now.") }
             } finally {
                 VoiceFocus.release()
             }
             if (id != run) return@execute
-            if (audio == null) return@execute post { onError("Didn't catch that.") }
             post(onHeard)
-            val started = SystemClock.uptimeMillis()
-            val text = clean(transcribe(audio))
-            GearslipLog.i(
-                "speech: ${audio.size * 1000L / RATE} ms of audio took ${SystemClock.uptimeMillis() - started} ms",
-            )
             post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) }
         }
     }
@@ -119,6 +141,165 @@ object SpeechEngine {
     /** Stops listening; a phrase already being worked out is dropped. */
     fun cancel() { run++ }
 
+    /**
+     * Feeds the microphone to the recognizer in 100 ms pieces, reporting the words as they firm
+     * up, until the model hears the end of the phrase (or nobody speaks, or it runs too long).
+     */
+    private fun stream(
+        context: Context,
+        r: OnlineRecognizer,
+        id: Int,
+        onSpeaking: () -> Unit,
+        onPartial: (String) -> Unit,
+        onStopped: () -> Unit,
+    ): String {
+        val mic = carMic() ?: phoneMic()
+        val s = r.createStream()
+        val frame = ShortArray(CHUNK)
+        val samples = FloatArray(CHUNK)
+        // The whole phrase, for the final read; [MAX_S] and a little over is the most it can hold.
+        val phrase = FloatArray((RATE * (MAX_S + 1)).toInt())
+        var held = 0
+        var last = ""
+        var spoke = false
+        var audioMs = 0L
+        var decodeMs = 0L
+        try {
+            while (id == run) {
+                val n = mic.read(frame)
+                if (n < 0) break
+                if (n == 0) continue
+                for (i in 0 until n) samples[i] = frame[i] / 32768f
+                val keep = minOf(n, phrase.size - held)
+                System.arraycopy(samples, 0, phrase, held, keep)
+                held += keep
+                s.acceptWaveform(if (n == CHUNK) samples else samples.copyOf(n), RATE)
+                audioMs += n * 1000L / RATE
+                val started = SystemClock.uptimeMillis()
+                while (r.isReady(s)) r.decode(s)
+                decodeMs += SystemClock.uptimeMillis() - started
+                val text = clean(r.getResult(s).text)
+                if (text != last) {
+                    last = text
+                    if (text.isNotEmpty() && !spoke) {
+                        spoke = true
+                        onSpeaking()
+                    }
+                    onPartial(text)
+                }
+                if (r.isEndpoint(s)) break
+            }
+            s.inputFinished()
+            while (r.isReady(s)) r.decode(s)
+            last = clean(r.getResult(s).text)
+        } finally {
+            mic.close()
+            s.release()
+        }
+        // Under 1.0 keeps up with the driver; over it, the words fall behind the voice.
+        GearslipLog.i(
+            "speech: $audioMs ms of audio, decoding took $decodeMs ms " +
+                "(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)",
+        )
+        if (last.isEmpty() || id != run) return last
+        val final = ensureFinalLoaded(context) ?: return last
+        onStopped()
+        val started = SystemClock.uptimeMillis()
+        val reread = final.createStream().run {
+            try {
+                acceptWaveform(phrase.copyOf(held), RATE)
+                final.decode(this)
+                clean(final.getResult(this).text)
+            } finally {
+                release()
+            }
+        }
+        phrase.fill(0f)
+        GearslipLog.i("speech: the final read took ${SystemClock.uptimeMillis() - started} ms")
+        if (reread.isEmpty()) return last
+        if (reread != last) onPartial(reread)
+        return reread
+    }
+
+    /** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */
+    private fun ensureFinalLoaded(context: Context): OfflineRecognizer? {
+        val model = SpeechModels.final.value ?: return null
+        if (finalRecognizer != null && loadedFinal == model) return finalRecognizer
+        finalRecognizer?.release()
+        finalRecognizer = null
+        if (!SpeechModels.isReady(context, model)) return null
+        val dir = SpeechModels.dir(context, model)
+        fun path(name: String) = File(dir, name).absolutePath
+        val started = SystemClock.uptimeMillis()
+        return runCatching {
+            OfflineRecognizer(
+                config = OfflineRecognizerConfig(
+                    featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80),
+                    modelConfig = OfflineModelConfig(
+                        transducer = OfflineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)),
+                        tokens = path("tokens.txt"),
+                        numThreads = threads(),
+                        modelType = model.modelType,
+                    ),
+                    decodingMethod = "greedy_search",
+                ),
+            )
+        }.onSuccess {
+            finalRecognizer = it
+            loadedFinal = model
+            GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms")
+        }.onFailure {
+            GearslipLog.e("speech: couldn't load ${model.label}", it)
+        }.getOrNull()
+    }
+
+    private fun ensureLoaded(context: Context): OnlineRecognizer? {
+        val model = SpeechModels.selected.value
+        if (recognizer != null && loadedModel == model) return recognizer
+        recognizer?.release()
+        recognizer = null
+        if (!SpeechModels.isReady(context, model)) return null
+        val dir = SpeechModels.dir(context, model)
+        fun path(name: String) = File(dir, name).absolutePath
+        val started = SystemClock.uptimeMillis()
+        return runCatching {
+            OnlineRecognizer(
+                config = OnlineRecognizerConfig(
+                    featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80),
+                    modelConfig = OnlineModelConfig(
+                        transducer = OnlineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)),
+                        tokens = path("tokens.txt"),
+                        numThreads = threads(),
+                        modelType = model.modelType,
+                    ),
+                    endpointConfig = EndpointConfig(
+                        rule1 = EndpointRule(false, NO_SPEECH_S, 0f),
+                        rule2 = EndpointRule(true, END_SILENCE_S, 0f),
+                        rule3 = EndpointRule(false, 0f, MAX_S),
+                    ),
+                    enableEndpoint = true,
+                    decodingMethod = "greedy_search",
+                ),
+            )
+        }.onSuccess {
+            recognizer = it
+            loadedModel = model
+            GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms")
+        }.onFailure {
+            GearslipLog.e("speech: couldn't load ${model.label}", it)
+        }.getOrNull()
+    }
+
+    /** The big cores only: more threads than that slow the model down on a phone. */
+    private fun threads() = Runtime.getRuntime().availableProcessors().let { if (it >= 8) 4 else max(1, it / 2) }
+
+    /**
+     * Models write in upper case, or with punctuation and capitals of their own; commands are
+     * matched in plain lower case, so only words and apostrophes are kept.
+     */
+    private fun clean(text: String): String =
+        text.lowercase().replace(Regex("[^\\p{L}\\p{N}' ]"), " ").replace(Regex("\\s+"), " ").trim()
+
     /**
      * A short beep on the media stream, so it plays through the car like everything else, then a
      * pause long enough for it to finish there before the microphone opens and hears it.
@@ -133,14 +314,9 @@ object SpeechEngine {
         }
     }
 
-    // --- recording --------------------------------------------------------------------------
+    // --- microphones -----------------------------------------------------------------------
 
-    /**
-     * Records until the driver stops talking. Returns the phrase from just before the first word,
-     * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
-     * loud cabin needs a louder voice than a quiet one to count as talking.
-     */
-    /** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */
+    /** Where a phrase is heard from: [read] fills [frame] and returns how many samples it got, -1 at the end. */
     private interface Mic {
         fun read(frame: ShortArray): Int
         fun close()
@@ -151,10 +327,11 @@ object SpeechEngine {
         val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
         val recorder = AudioRecord(
             MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE,
-            AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8),
+            AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, CHUNK * 4),
         )
         check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" }
         recorder.startRecording()
+        GearslipLog.i("speech: listening through the phone's microphone")
         return object : Mic {
             override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size)
             override fun close() {
@@ -166,7 +343,7 @@ object SpeechEngine {
 
     /**
      * The car's microphone, which hears the driver better than a phone in a cup holder. The car
-     * sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car
+     * sends audio in its own sized pieces; they're cut to [CHUNK]s here. Null when there's no car
      * microphone, or it sends nothing at first, so the phone's is used instead.
      */
     private fun carMic(): Mic? {
@@ -193,107 +370,10 @@ object SpeechEngine {
                     at += n
                     if (at == chunk.size) { pending = null; at = 0 }
                 }
-                // The car stopped sending mid-phrase: end the recording rather than wait forever.
+                // The car stopped sending mid-phrase: end the phrase rather than wait forever.
                 return if (filled == 0) -1 else filled
             }
             override fun close() = car.stop()
         }
     }
-
-    /**
-     * Records until the driver stops talking. Returns the phrase from just before the first word,
-     * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
-     * loud cabin needs a louder voice than a quiet one to count as talking.
-     */
-    private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? {
-        val mic = carMic() ?: phoneMic()
-        val samples = FloatArray((RATE * MAX_MS / 1000).toInt())
-        var count = 0
-        val frame = ShortArray(FRAME)
-        var noise = 0.0
-        var noiseFrames = 0
-        var speechStart = -1
-        var lastSpeech = 0
-        try {
-            while (id == run && count + FRAME <= samples.size) {
-                val n = mic.read(frame)
-                if (n < 0) break
-                if (n == 0) continue
-                var sum = 0.0
-                for (i in 0 until n) {
-                    val v = frame[i] / 32768f
-                    samples[count + i] = v
-                    sum += v * v
-                }
-                count += n
-                val db = 20 * log10(sqrt(sum / n) + 1e-9)
-                val ms = count * 1000L / RATE
-                if (ms <= CALIBRATE_MS) {
-                    noise += db; noiseFrames++
-                    continue
-                }
-                val floor = if (noiseFrames > 0) noise / noiseFrames else -60.0
-                val talking = db > max(floor + 12, -45.0)
-                if (talking) {
-                    if (speechStart < 0) {
-                        speechStart = max(0, count - n - PRE_ROLL)
-                        onSpeaking()
-                    }
-                    lastSpeech = count
-                }
-                if (speechStart < 0 && ms > NO_SPEECH_MS) return null
-                if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break
-            }
-        } finally {
-            mic.close()
-        }
-        if (speechStart < 0) return null
-        return samples.copyOfRange(speechStart, count)
-    }
-
-    // --- the model --------------------------------------------------------------------------
-
-    private fun ensureLoaded(context: Context): Boolean {
-        if (handle != 0L) return true
-        if (!libraryLoaded) {
-            runCatching { System.loadLibrary("gearslip_speech") }
-                .onFailure { GearslipLog.e("speech: native library missing", it); return false }
-            libraryLoaded = true
-        }
-        val started = SystemClock.uptimeMillis()
-        val model = MODELS.firstOrNull { name -> runCatching { context.assets.open(name).close() }.isSuccess }
-            ?: run { GearslipLog.w("speech: no model in the build"); return false }
-        handle = nativeInitAsset(context.assets, model)
-        if (handle == 0L) {
-            GearslipLog.w("speech: couldn't load the model")
-            return false
-        }
-        GearslipLog.i("speech: loaded $model in ${SystemClock.uptimeMillis() - started} ms")
-        return true
-    }
-
-    private fun transcribe(audio: FloatArray): String {
-        val bytes = nativeTranscribe(handle, audio, language(), threads())
-        return String(bytes, Charsets.UTF_8)
-    }
-
-    /** The phone's language, as Whisper names it. Older Java codes differ for a few. */
-    private fun language(): String = when (val lang = Locale.getDefault().language) {
-        "iw" -> "he"
-        "in" -> "id"
-        "ji" -> "yi"
-        "" -> "en"
-        else -> lang
-    }
-
-    /** The big cores only: more threads than that slow Whisper down on a phone. */
-    private fun threads() = Runtime.getRuntime().availableProcessors().coerceIn(1, 8).let { if (it >= 8) 4 else max(1, it / 2) }
-
-    /** Whisper marks silence and noise as "[BLANK_AUDIO]" or "(wind blowing)"; none of it is words. */
-    private fun clean(text: String): String =
-        text.replace(Regex("""\[[^\]]*\]|\([^)]*\)|\*[^*]*\*"""), " ").replace(Regex("\\s+"), " ").trim()
-
-    private external fun nativeInitAsset(assets: AssetManager, path: String): Long
-    private external fun nativeTranscribe(handle: Long, samples: FloatArray, language: String, threads: Int): ByteArray
-    private external fun nativeFree(handle: Long)
 }

gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt

@@ -0,0 +1,187 @@
+package app.seb3thehacker.gearslip.speech
+
+import android.content.Context
+import app.seb3thehacker.gearslip.GearslipLog
+import java.io.File
+import java.net.HttpURLConnection
+import java.net.URL
+import java.util.concurrent.Executors
+import kotlinx.coroutines.flow.MutableStateFlow
+import kotlinx.coroutines.flow.StateFlow
+import kotlinx.coroutines.flow.asStateFlow
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
+import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
+
+/**
+ * The speech models Gearslip listens with, in two roles: a streaming one that writes the words
+ * while the driver talks, and an optional final one that reads the whole phrase again once they
+ * stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's
+ * GitHub releases, where its authors publish it, and kept in the app's own storage.
+ */
+object SpeechModels {
+
+    enum class Role { LIVE, FINAL }
+
+    enum class Model(
+        val role: Role,
+        val id: String,
+        val label: String,
+        val detail: String,
+        val downloadMb: Int,
+        private val archive: String,
+        val encoder: String,
+        val decoder: String,
+        val joiner: String,
+        val modelType: String = "",
+    ) {
+        KROKO(
+            Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54,
+            "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06",
+            "encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2",
+        ),
+        PARAKEET_SMALL(
+            Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103,
+            "sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8",
+            "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
+        ),
+        PARAKEET_LARGE(
+            Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460,
+            "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
+            "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
+        ),
+        ;
+
+        val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2"
+        val files get() = listOf(encoder, decoder, joiner, "tokens.txt")
+    }
+
+    sealed interface State {
+        data object Missing : State
+        data class Downloading(val percent: Int) : State
+        data object Ready : State
+        data class Failed(val reason: String) : State
+    }
+
+    private const val PREFS = "speech"
+    private const val KEY_MODEL = "model"
+    private const val KEY_FINAL = "final_model"
+
+    private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-download") }
+    private val states = Model.entries.associateWith { MutableStateFlow<State>(State.Missing) }
+    private val _selected = MutableStateFlow(Model.KROKO)
+    val selected: StateFlow<Model> = _selected.asStateFlow()
+    private val _final = MutableStateFlow<Model?>(Model.PARAKEET_LARGE)
+
+    /** The model that rereads the whole phrase at the end; null to keep the live model's words. */
+    val final: StateFlow<Model?> = _final.asStateFlow()
+    @Volatile private var cancelled: Model? = null
+
+    fun init(context: Context) {
+        val id = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_MODEL, null)
+        _selected.value = Model.entries.firstOrNull { it.id == id && it.role == Role.LIVE } ?: Model.KROKO
+        val finalId = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_FINAL, Model.PARAKEET_LARGE.id)
+        _final.value = Model.entries.firstOrNull { it.id == finalId && it.role == Role.FINAL }
+        Model.entries.forEach { m -> if (isReady(context, m)) states.getValue(m).value = State.Ready }
+        // Models an older build offered and this one doesn't: hundreds of MB nobody can use.
+        val known = Model.entries.map { it.id }.toSet()
+        File(context.filesDir, "speech").listFiles()?.filter { it.isDirectory && it.name !in known }?.forEach {
+            GearslipLog.i("speech: removing ${it.name}, no longer offered")
+            it.deleteRecursively()
+        }
+    }
+
+    fun state(model: Model): StateFlow<State> = states.getValue(model).asStateFlow()
+
+    fun dir(context: Context, model: Model) = File(File(context.filesDir, "speech"), model.id)
+
+    fun isReady(context: Context, model: Model) = model.files.all { File(dir(context, model), it).isFile }
+
+    fun select(context: Context, model: Model) {
+        if (model.role == Role.FINAL) return selectFinal(context, model)
+        _selected.value = model
+        context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_MODEL, model.id).apply()
+        SpeechEngine.unload()
+    }
+
+    /** [model] null keeps the live words as they are. */
+    fun selectFinal(context: Context, model: Model?) {
+        _final.value = model
+        context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_FINAL, model?.id ?: "none").apply()
+        SpeechEngine.unload()
+    }
+
+    fun delete(context: Context, model: Model) {
+        cancelled = model
+        dir(context, model).deleteRecursively()
+        states.getValue(model).value = State.Missing
+        if (model == _selected.value || model == _final.value) SpeechEngine.unload()
+    }
+
+    /** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */
+    fun download(context: Context, model: Model) {
+        val app = context.applicationContext
+        val state = states.getValue(model)
+        if (state.value is State.Downloading || state.value == State.Ready) return
+        state.value = State.Downloading(0)
+        cancelled = null
+        worker.execute {
+            val target = dir(app, model)
+            val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() }
+            try {
+                val conn = (URL(model.url).openConnection() as HttpURLConnection).apply {
+                    connectTimeout = 15_000
+                    readTimeout = 30_000
+                    instanceFollowRedirects = true
+                }
+                check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" }
+                val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L)
+                val counting = CountingStream(conn.inputStream) { read ->
+                    if (cancelled == model) throw InterruptedException("cancelled")
+                    state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99))
+                }
+                TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar ->
+                    while (true) {
+                        val entry = tar.nextEntry ?: break
+                        val name = entry.name.substringAfterLast('/')
+                        if (entry.isDirectory || name !in model.files) continue
+                        File(partial, name).outputStream().use { tar.copyTo(it) }
+                    }
+                }
+                check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" }
+                target.deleteRecursively()
+                check(partial.renameTo(target)) { "couldn't store the model" }
+                GearslipLog.i("speech: downloaded ${model.label}")
+                state.value = State.Ready
+            } catch (e: Exception) {
+                partial.deleteRecursively()
+                if (cancelled == model) {
+                    state.value = State.Missing
+                } else {
+                    GearslipLog.w("speech: download of ${model.label} failed: ${e.message}")
+                    state.value = State.Failed(e.message ?: e.javaClass.simpleName)
+                }
+            }
+        }
+    }
+
+    /** Counts bytes as they pass, reporting about every megabyte. */
+    private class CountingStream(
+        private val inner: java.io.InputStream,
+        private val onProgress: (Long) -> Unit,
+    ) : java.io.InputStream() {
+        private var read = 0L
+        private var reported = 0L
+
+        override fun read(): Int = inner.read().also { if (it >= 0) count(1) }
+        override fun read(b: ByteArray, off: Int, len: Int): Int = inner.read(b, off, len).also { if (it > 0) count(it.toLong()) }
+        override fun close() = inner.close()
+
+        private fun count(n: Long) {
+            read += n
+            if (read - reported >= 1_048_576) {
+                reported = read
+                onProgress(read)
+            }
+        }
+    }
+}

gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt

@@ -109,6 +109,7 @@ fun SettingsScreen(
             AboutSection(onOpenWhatsNew, onReplayTutorial, onReplayIntro)
             ConnectionSection()
             CarAudioSection()
+            VoiceModelSection()
             if (SHOW_RECOMMENDED_APPS) AppsSection(onOpenRecommendedApps)
             CertificateSection()
             SettingsSection("Usage notes", header = USAGE_STATS_INTRO) { UsageStatsRows() }

gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt

@@ -151,4 +151,11 @@ internal fun AboutLinkRows() {
         icon = Icons.Filled.Build,
         trailing = arrow,
     )
+    SettingsDivider()
+    SettingsRow(
+        "Voice credits",
+        subtitle = "Speech recognition by sherpa-onnx (Apache 2.0), with Kroko ASR by Banafo " +
+            "(CC BY-SA 4.0) and Parakeet by NVIDIA (CC BY 4.0)",
+        icon = Icons.Filled.Favorite,
+    )
 }

gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt

@@ -0,0 +1,125 @@
+package app.seb3thehacker.gearslip.ui
+
+import androidx.compose.foundation.layout.Arrangement
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.fillMaxWidth
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.selection.selectable
+import androidx.compose.material3.LinearProgressIndicator
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.OutlinedButton
+import androidx.compose.material3.RadioButton
+import androidx.compose.material3.Text
+import androidx.compose.material3.TextButton
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.collectAsState
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.remember
+import androidx.compose.ui.Alignment
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.semantics.Role
+import androidx.compose.ui.unit.dp
+import app.seb3thehacker.gearslip.speech.SpeechModels
+import app.seb3thehacker.gearslip.speech.SpeechModels.Model
+import app.seb3thehacker.gearslip.speech.SpeechModels.State
+
+/**
+ * Which speech model the voice assistant listens with, and downloading it. Each is fetched once
+ * from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening.
+ */
+@Composable
+internal fun VoiceModelSection() {
+    val context = LocalContext.current
+    remember { SpeechModels.init(context) }
+    val selected by SpeechModels.selected.collectAsState()
+    val final by SpeechModels.final.collectAsState()
+    SettingsSection(title = "Voice: live words") {
+        Text(
+            "The voice assistant turns speech into text on this phone, and nothing you say leaves it. " +
+                "It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " +
+                "This one writes your words as you speak.",
+            style = MaterialTheme.typography.bodyMedium,
+            color = MaterialTheme.colorScheme.onSurfaceVariant,
+            modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
+        )
+        Model.entries.filter { it.role == SpeechModels.Role.LIVE }.forEachIndexed { i, model ->
+            if (i > 0) SettingsDivider()
+            ModelRow(model, model == selected)
+        }
+    }
+    SettingsSection(title = "Voice: final words") {
+        Text(
+            "When you stop talking, this model reads the whole phrase again. It catches names and " +
+                "places the live model misses.",
+            style = MaterialTheme.typography.bodyMedium,
+            color = MaterialTheme.colorScheme.onSurfaceVariant,
+            modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
+        )
+        val context = LocalContext.current
+        Row(
+            Modifier
+                .fillMaxWidth()
+                .selectable(final == null, role = Role.RadioButton) { SpeechModels.selectFinal(context, null) }
+                .padding(horizontal = 8.dp, vertical = 8.dp),
+            verticalAlignment = Alignment.CenterVertically,
+        ) {
+            RadioButton(final == null, onClick = null, modifier = Modifier.padding(8.dp))
+            Column {
+                Text("None", style = MaterialTheme.typography.titleMedium)
+                Text(
+                    "Keep the live words. Fastest, least accurate",
+                    style = MaterialTheme.typography.bodySmall,
+                    color = MaterialTheme.colorScheme.onSurfaceVariant,
+                )
+            }
+        }
+        Model.entries.filter { it.role == SpeechModels.Role.FINAL }.forEach { model ->
+            SettingsDivider()
+            ModelRow(model, model == final)
+        }
+    }
+}
+
+@Composable
+private fun ModelRow(model: Model, chosen: Boolean) {
+    val context = LocalContext.current
+    val state by SpeechModels.state(model).collectAsState()
+    Column(
+        Modifier
+            .fillMaxWidth()
+            .selectable(chosen, role = Role.RadioButton) { SpeechModels.select(context, model) }
+            .padding(horizontal = 8.dp, vertical = 8.dp),
+    ) {
+        Row(verticalAlignment = Alignment.CenterVertically) {
+            RadioButton(chosen, onClick = null, modifier = Modifier.padding(8.dp))
+            Column(Modifier.weight(1f)) {
+                Text("${model.label} · ${model.downloadMb} MB", style = MaterialTheme.typography.titleMedium)
+                Text(
+                    when (val s = state) {
+                        State.Ready -> "Downloaded. ${model.detail}"
+                        is State.Downloading -> "Downloading, ${s.percent}%"
+                        is State.Failed -> "Download failed: ${s.reason}"
+                        State.Missing -> model.detail
+                    },
+                    style = MaterialTheme.typography.bodySmall,
+                    color = MaterialTheme.colorScheme.onSurfaceVariant,
+                )
+            }
+            Row(horizontalArrangement = Arrangement.spacedBy(4.dp)) {
+                when (state) {
+                    State.Ready -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Delete") }
+                    is State.Downloading -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Cancel") }
+                    else -> OutlinedButton(onClick = { SpeechModels.download(context, model) }) { Text("Download") }
+                }
+            }
+        }
+        (state as? State.Downloading)?.let {
+            LinearProgressIndicator(
+                progress = { it.percent / 100f },
+                modifier = Modifier.fillMaxWidth().padding(start = 48.dp, end = 8.dp, top = 4.dp),
+            )
+        }
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt

@@ -0,0 +1,12 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+data class FeatureConfig(
+    var sampleRate: Int = 16000,
+    var featureDim: Int = 80,
+    var dither: Float = 0.0f
+)
+
+fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig {
+    return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim)
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt

@@ -0,0 +1,8 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+data class HomophoneReplacerConfig(
+    var dictDir: String = "", // unused
+    var lexicon: String = "",
+    var ruleFsts: String = "",
+)

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt

@@ -0,0 +1,255 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
+package com.k2fsa.sherpa.onnx
+
+import android.content.res.AssetManager
+
+data class OfflineRecognizerResult(
+    val text: String,
+    val tokens: Array<String>,
+    val timestamps: FloatArray,
+    val lang: String,
+    val emotion: String,
+    val event: String,
+
+    // valid only for TDT models
+    val durations: FloatArray,
+)
+
+data class OfflineTransducerModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var joiner: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineParaformerModelConfig(
+    var model: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineNemoEncDecCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineDolphinModelConfig(
+    var model: String = "",
+)
+
+data class OfflineZipformerCtcModelConfig(
+    var model: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineWenetCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineOmnilingualAsrCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineMedAsrCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineFireRedAsrCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineFunAsrNanoModelConfig(
+    var encoderAdaptor: String = "",
+    var llm: String = "",
+    var embedding: String = "",
+    var tokenizer: String = "",
+    var systemPrompt: String = "You are a helpful assistant.",
+    var userPrompt: String = "语音转写：",
+    var maxNewTokens: Int = 512,
+    var temperature: Float = 1e-6f,
+    var topP: Float = 0.8f,
+    var seed: Int = 42,
+    var language: String = "",
+    var itn: Boolean = true,
+    var hotwords: String = "",
+)
+
+data class OfflineQwen3AsrModelConfig(
+    var convFrontend: String = "",
+    var encoder: String = "",
+    var decoder: String = "",
+    var tokenizer: String = "",
+    var maxTotalLen: Int = 512,
+    var maxNewTokens: Int = 128,
+    var temperature: Float = 1e-6f,
+    var topP: Float = 0.8f,
+    var seed: Int = 42,
+    var hotwords: String = "",
+)
+
+data class OfflineWhisperModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var language: String = "en", // Used with multilingual model
+    var task: String = "transcribe", // transcribe or translate
+    var tailPaddings: Int = 1000, // Padding added at the end of the samples
+    var enableTokenTimestamps: Boolean = false,
+    var enableSegmentTimestamps: Boolean = false,
+)
+
+data class OfflineCanaryModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var srcLang: String = "en",
+    var tgtLang: String = "en",
+    var usePnc: Boolean = true,
+)
+
+data class OfflineCohereTranscribeModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var language: String = "",
+    var usePunct: Boolean = true,
+    var useItn: Boolean = true,
+)
+
+data class OfflineFireRedAsrModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+)
+
+// For moonshine v1, you need four models.
+// For moonshine v2, you need two models.
+// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder
+// - v2: encoder, mergedDecoder
+data class OfflineMoonshineModelConfig(
+    var preprocessor: String = "",
+    var encoder: String = "",
+    var uncachedDecoder: String = "",
+    var cachedDecoder: String = "",
+    var mergedDecoder: String = "",
+)
+
+data class OfflineSenseVoiceModelConfig(
+    var model: String = "",
+    var language: String = "",
+    var useInverseTextNormalization: Boolean = true,
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineModelConfig(
+    var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
+    var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
+    var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
+    var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(),
+    var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(),
+    var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(),
+    var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
+    var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
+    var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
+    var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(),
+    var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(),
+    var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(),
+    var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(),
+    var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(),
+    var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(),
+    var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
+    var cohereTranscribe: OfflineCohereTranscribeModelConfig =
+        OfflineCohereTranscribeModelConfig(),
+    var teleSpeech: String = "",
+    var numThreads: Int = 1,
+    var debug: Boolean = false,
+    var provider: String = "cpu",
+    var modelType: String = "",
+    var tokens: String = "",
+    var modelingUnit: String = "",
+    var bpeVocab: String = "",
+)
+
+data class OfflineRecognizerConfig(
+    var featConfig: FeatureConfig = FeatureConfig(),
+    var modelConfig: OfflineModelConfig = OfflineModelConfig(),
+    // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it
+    var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
+    var decodingMethod: String = "greedy_search",
+    var maxActivePaths: Int = 4,
+    var hotwordsFile: String = "",
+    var hotwordsScore: Float = 1.5f,
+    var ruleFsts: String = "",
+    var ruleFars: String = "",
+    var blankPenalty: Float = 0.0f,
+)
+
+class OfflineRecognizer(
+    assetManager: AssetManager? = null,
+    val config: OfflineRecognizerConfig,
+) {
+    private var ptr: Long
+
+    init {
+        ptr = if (assetManager != null) {
+            newFromAsset(assetManager, config)
+        } else {
+            newFromFile(config)
+        }
+        require(ptr != 0L) {
+            "Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer"
+        }
+    }
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun createStream(): OfflineStream {
+        val p = createStream(ptr)
+        return OfflineStream(p)
+    }
+
+    fun createStream(hotwords: String): OfflineStream {
+        val p = createStreamWithHotwords(ptr, hotwords)
+        return OfflineStream(p)
+    }
+
+    fun getResult(stream: OfflineStream): OfflineRecognizerResult {
+        return getResult(stream.ptr)
+    }
+
+    fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
+
+    fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config)
+
+    private external fun delete(ptr: Long)
+
+    private external fun createStream(ptr: Long): Long
+
+    private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long
+
+    private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig)
+
+    private external fun newFromAsset(
+        assetManager: AssetManager,
+        config: OfflineRecognizerConfig,
+    ): Long
+
+    private external fun newFromFile(
+        config: OfflineRecognizerConfig,
+    ): Long
+
+    private external fun decode(ptr: Long, streamPtr: Long)
+
+    private external fun getResult(streamPtr: Long): OfflineRecognizerResult
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+
+        @JvmStatic
+        external fun prependAdspLibraryPath(newPath: String) // for qnn
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt

@@ -0,0 +1,43 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+class OfflineStream(var ptr: Long) {
+    init {
+        require(ptr != 0L) { "Failed to create native OfflineStream" }
+    }
+
+    fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
+        acceptWaveform(ptr, samples, sampleRate)
+
+    fun setOption(key: String, value: String) = setOption(ptr, key, value)
+
+    fun getOption(key: String): String = getOption(ptr, key)
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun use(block: (OfflineStream) -> Unit) {
+        try {
+            block(this)
+        } finally {
+            release()
+        }
+    }
+
+    private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
+    private external fun setOption(ptr: Long, key: String, value: String)
+    private external fun getOption(ptr: Long, key: String): String
+    private external fun delete(ptr: Long)
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt

@@ -0,0 +1,158 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
+package com.k2fsa.sherpa.onnx
+
+import android.content.res.AssetManager
+
+data class EndpointRule(
+    var mustContainNonSilence: Boolean,
+    var minTrailingSilence: Float,
+    var minUtteranceLength: Float,
+)
+
+data class EndpointConfig(
+    var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f),
+    var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f),
+    var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f)
+)
+
+data class OnlineTransducerModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var joiner: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OnlineParaformerModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+)
+
+data class OnlineZipformer2CtcModelConfig(
+    var model: String = "",
+)
+
+data class OnlineNeMoCtcModelConfig(
+    var model: String = "",
+)
+
+data class OnlineToneCtcModelConfig(
+    var model: String = "",
+)
+
+data class OnlineModelConfig(
+    var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
+    var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
+    var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
+    var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(),
+    var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(),
+    var tokens: String = "",
+    var numThreads: Int = 1,
+    var debug: Boolean = false,
+    var provider: String = "cpu",
+    var modelType: String = "",
+    var modelingUnit: String = "",
+    var bpeVocab: String = "",
+)
+
+data class OnlineLMConfig(
+    var model: String = "",
+    var scale: Float = 0.5f,
+)
+
+data class OnlineCtcFstDecoderConfig(
+    var graph: String = "",
+    var maxActive: Int = 3000,
+)
+
+data class OnlineRecognizerConfig(
+    var featConfig: FeatureConfig = FeatureConfig(),
+    var modelConfig: OnlineModelConfig = OnlineModelConfig(),
+    var lmConfig: OnlineLMConfig = OnlineLMConfig(),
+    var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(),
+    var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
+    var endpointConfig: EndpointConfig = EndpointConfig(),
+    var enableEndpoint: Boolean = true,
+    var decodingMethod: String = "greedy_search",
+    var maxActivePaths: Int = 4,
+    var hotwordsFile: String = "",
+    var hotwordsScore: Float = 1.5f,
+    var ruleFsts: String = "",
+    var ruleFars: String = "",
+    var blankPenalty: Float = 0.0f,
+)
+
+data class OnlineRecognizerResult(
+    val text: String,
+    val tokens: Array<String>,
+    val timestamps: FloatArray,
+    val ysProbs: FloatArray,
+    // TODO(fangjun): Add more fields
+)
+
+class OnlineRecognizer(
+    assetManager: AssetManager? = null,
+    val config: OnlineRecognizerConfig,
+) {
+    private var ptr: Long
+
+    init {
+        ptr = if (assetManager != null) {
+            newFromAsset(assetManager, config)
+        } else {
+            newFromFile(config)
+        }
+        require(ptr != 0L) {
+            "Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer"
+        }
+    }
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun createStream(hotwords: String = ""): OnlineStream {
+        val p = createStream(ptr, hotwords)
+        return OnlineStream(p)
+    }
+
+    fun reset(stream: OnlineStream) = reset(ptr, stream.ptr)
+    fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
+    fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr)
+    fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
+    fun getResult(stream: OnlineStream): OnlineRecognizerResult {
+        return getResult(ptr, stream.ptr)
+    }
+
+    private external fun delete(ptr: Long)
+
+    private external fun newFromAsset(
+        assetManager: AssetManager,
+        config: OnlineRecognizerConfig,
+    ): Long
+
+    private external fun newFromFile(
+        config: OnlineRecognizerConfig,
+    ): Long
+
+    private external fun createStream(ptr: Long, hotwords: String): Long
+    private external fun reset(ptr: Long, streamPtr: Long)
+    private external fun decode(ptr: Long, streamPtr: Long)
+    private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean
+    private external fun isReady(ptr: Long, streamPtr: Long): Boolean
+    private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+
+        @JvmStatic
+        external fun prependAdspLibraryPath(newPath: String) // for qnn
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt

@@ -0,0 +1,47 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+class OnlineStream(var ptr: Long = 0) {
+    init {
+        require(ptr != 0L) { "Failed to create native OnlineStream" }
+    }
+
+    fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
+        acceptWaveform(ptr, samples, sampleRate)
+
+    fun inputFinished() = inputFinished(ptr)
+
+    fun setOption(key: String, value: String) = setOption(ptr, key, value)
+
+    fun getOption(key: String): String = getOption(ptr, key)
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun use(block: (OnlineStream) -> Unit) {
+        try {
+            block(this)
+        } finally {
+            release()
+        }
+    }
+
+    private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
+    private external fun inputFinished(ptr: Long)
+    private external fun setOption(ptr: Long, key: String, value: String)
+    private external fun getOption(ptr: Long, key: String): String
+    private external fun delete(ptr: Long)
+
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt

@@ -0,0 +1,8 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+data class QnnConfig(
+    var backendLib: String = "",
+    var contextBinary: String = "",
+    var systemLib: String = "",
+)

gradle/libs.versions.toml

@@ -14,6 +14,7 @@ material = "1.14.0"
 constraintlayout = "2.2.1"
 zxing = "3.5.4"
 okhttp = "5.4.0"
+commonsCompress = "1.28.0"
 kotlinxSerializationJson = "1.11.0"
 ossLicenses = "17.5.1"
 webkit = "1.16.0"
@@ -49,6 +50,7 @@ androidx-constraintlayout = { group = "androidx.constraintlayout", name = "const
 google-material = { group = "com.google.android.material", name = "material", version.ref = "material" }
 zxing-core = { group = "com.google.zxing", name = "core", version.ref = "zxing" }
 okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
+commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commonsCompress" }
 kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
 google-oss-licenses = { group = "com.google.android.gms", name = "play-services-oss-licenses", version.ref = "ossLicenses" }
 androidx-webkit = { group = "androidx.webkit", name = "webkit", version.ref = "webkit" }