voice: replace Whisper with sherpa-onnx, live words from Kroko and the final words from Parakeet

Models download in Settings instead of shipping in the APK. Credits for the engine and models are in About.

gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt

@@ -0,0 +1,12 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+data class FeatureConfig(
+    var sampleRate: Int = 16000,
+    var featureDim: Int = 80,
+    var dither: Float = 0.0f
+)
+
+fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig {
+    return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim)
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt

@@ -0,0 +1,8 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+data class HomophoneReplacerConfig(
+    var dictDir: String = "", // unused
+    var lexicon: String = "",
+    var ruleFsts: String = "",
+)

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt

@@ -0,0 +1,255 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
+package com.k2fsa.sherpa.onnx
+
+import android.content.res.AssetManager
+
+data class OfflineRecognizerResult(
+    val text: String,
+    val tokens: Array<String>,
+    val timestamps: FloatArray,
+    val lang: String,
+    val emotion: String,
+    val event: String,
+
+    // valid only for TDT models
+    val durations: FloatArray,
+)
+
+data class OfflineTransducerModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var joiner: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineParaformerModelConfig(
+    var model: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineNemoEncDecCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineDolphinModelConfig(
+    var model: String = "",
+)
+
+data class OfflineZipformerCtcModelConfig(
+    var model: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineWenetCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineOmnilingualAsrCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineMedAsrCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineFireRedAsrCtcModelConfig(
+    var model: String = "",
+)
+
+data class OfflineFunAsrNanoModelConfig(
+    var encoderAdaptor: String = "",
+    var llm: String = "",
+    var embedding: String = "",
+    var tokenizer: String = "",
+    var systemPrompt: String = "You are a helpful assistant.",
+    var userPrompt: String = "语音转写：",
+    var maxNewTokens: Int = 512,
+    var temperature: Float = 1e-6f,
+    var topP: Float = 0.8f,
+    var seed: Int = 42,
+    var language: String = "",
+    var itn: Boolean = true,
+    var hotwords: String = "",
+)
+
+data class OfflineQwen3AsrModelConfig(
+    var convFrontend: String = "",
+    var encoder: String = "",
+    var decoder: String = "",
+    var tokenizer: String = "",
+    var maxTotalLen: Int = 512,
+    var maxNewTokens: Int = 128,
+    var temperature: Float = 1e-6f,
+    var topP: Float = 0.8f,
+    var seed: Int = 42,
+    var hotwords: String = "",
+)
+
+data class OfflineWhisperModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var language: String = "en", // Used with multilingual model
+    var task: String = "transcribe", // transcribe or translate
+    var tailPaddings: Int = 1000, // Padding added at the end of the samples
+    var enableTokenTimestamps: Boolean = false,
+    var enableSegmentTimestamps: Boolean = false,
+)
+
+data class OfflineCanaryModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var srcLang: String = "en",
+    var tgtLang: String = "en",
+    var usePnc: Boolean = true,
+)
+
+data class OfflineCohereTranscribeModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var language: String = "",
+    var usePunct: Boolean = true,
+    var useItn: Boolean = true,
+)
+
+data class OfflineFireRedAsrModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+)
+
+// For moonshine v1, you need four models.
+// For moonshine v2, you need two models.
+// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder
+// - v2: encoder, mergedDecoder
+data class OfflineMoonshineModelConfig(
+    var preprocessor: String = "",
+    var encoder: String = "",
+    var uncachedDecoder: String = "",
+    var cachedDecoder: String = "",
+    var mergedDecoder: String = "",
+)
+
+data class OfflineSenseVoiceModelConfig(
+    var model: String = "",
+    var language: String = "",
+    var useInverseTextNormalization: Boolean = true,
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OfflineModelConfig(
+    var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
+    var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
+    var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
+    var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(),
+    var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(),
+    var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(),
+    var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
+    var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
+    var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
+    var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(),
+    var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(),
+    var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(),
+    var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(),
+    var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(),
+    var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(),
+    var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
+    var cohereTranscribe: OfflineCohereTranscribeModelConfig =
+        OfflineCohereTranscribeModelConfig(),
+    var teleSpeech: String = "",
+    var numThreads: Int = 1,
+    var debug: Boolean = false,
+    var provider: String = "cpu",
+    var modelType: String = "",
+    var tokens: String = "",
+    var modelingUnit: String = "",
+    var bpeVocab: String = "",
+)
+
+data class OfflineRecognizerConfig(
+    var featConfig: FeatureConfig = FeatureConfig(),
+    var modelConfig: OfflineModelConfig = OfflineModelConfig(),
+    // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it
+    var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
+    var decodingMethod: String = "greedy_search",
+    var maxActivePaths: Int = 4,
+    var hotwordsFile: String = "",
+    var hotwordsScore: Float = 1.5f,
+    var ruleFsts: String = "",
+    var ruleFars: String = "",
+    var blankPenalty: Float = 0.0f,
+)
+
+class OfflineRecognizer(
+    assetManager: AssetManager? = null,
+    val config: OfflineRecognizerConfig,
+) {
+    private var ptr: Long
+
+    init {
+        ptr = if (assetManager != null) {
+            newFromAsset(assetManager, config)
+        } else {
+            newFromFile(config)
+        }
+        require(ptr != 0L) {
+            "Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer"
+        }
+    }
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun createStream(): OfflineStream {
+        val p = createStream(ptr)
+        return OfflineStream(p)
+    }
+
+    fun createStream(hotwords: String): OfflineStream {
+        val p = createStreamWithHotwords(ptr, hotwords)
+        return OfflineStream(p)
+    }
+
+    fun getResult(stream: OfflineStream): OfflineRecognizerResult {
+        return getResult(stream.ptr)
+    }
+
+    fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
+
+    fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config)
+
+    private external fun delete(ptr: Long)
+
+    private external fun createStream(ptr: Long): Long
+
+    private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long
+
+    private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig)
+
+    private external fun newFromAsset(
+        assetManager: AssetManager,
+        config: OfflineRecognizerConfig,
+    ): Long
+
+    private external fun newFromFile(
+        config: OfflineRecognizerConfig,
+    ): Long
+
+    private external fun decode(ptr: Long, streamPtr: Long)
+
+    private external fun getResult(streamPtr: Long): OfflineRecognizerResult
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+
+        @JvmStatic
+        external fun prependAdspLibraryPath(newPath: String) // for qnn
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt

@@ -0,0 +1,43 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+class OfflineStream(var ptr: Long) {
+    init {
+        require(ptr != 0L) { "Failed to create native OfflineStream" }
+    }
+
+    fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
+        acceptWaveform(ptr, samples, sampleRate)
+
+    fun setOption(key: String, value: String) = setOption(ptr, key, value)
+
+    fun getOption(key: String): String = getOption(ptr, key)
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun use(block: (OfflineStream) -> Unit) {
+        try {
+            block(this)
+        } finally {
+            release()
+        }
+    }
+
+    private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
+    private external fun setOption(ptr: Long, key: String, value: String)
+    private external fun getOption(ptr: Long, key: String): String
+    private external fun delete(ptr: Long)
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt

@@ -0,0 +1,158 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
+package com.k2fsa.sherpa.onnx
+
+import android.content.res.AssetManager
+
+data class EndpointRule(
+    var mustContainNonSilence: Boolean,
+    var minTrailingSilence: Float,
+    var minUtteranceLength: Float,
+)
+
+data class EndpointConfig(
+    var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f),
+    var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f),
+    var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f)
+)
+
+data class OnlineTransducerModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+    var joiner: String = "",
+    var qnnConfig: QnnConfig = QnnConfig(),
+)
+
+data class OnlineParaformerModelConfig(
+    var encoder: String = "",
+    var decoder: String = "",
+)
+
+data class OnlineZipformer2CtcModelConfig(
+    var model: String = "",
+)
+
+data class OnlineNeMoCtcModelConfig(
+    var model: String = "",
+)
+
+data class OnlineToneCtcModelConfig(
+    var model: String = "",
+)
+
+data class OnlineModelConfig(
+    var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
+    var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
+    var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
+    var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(),
+    var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(),
+    var tokens: String = "",
+    var numThreads: Int = 1,
+    var debug: Boolean = false,
+    var provider: String = "cpu",
+    var modelType: String = "",
+    var modelingUnit: String = "",
+    var bpeVocab: String = "",
+)
+
+data class OnlineLMConfig(
+    var model: String = "",
+    var scale: Float = 0.5f,
+)
+
+data class OnlineCtcFstDecoderConfig(
+    var graph: String = "",
+    var maxActive: Int = 3000,
+)
+
+data class OnlineRecognizerConfig(
+    var featConfig: FeatureConfig = FeatureConfig(),
+    var modelConfig: OnlineModelConfig = OnlineModelConfig(),
+    var lmConfig: OnlineLMConfig = OnlineLMConfig(),
+    var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(),
+    var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
+    var endpointConfig: EndpointConfig = EndpointConfig(),
+    var enableEndpoint: Boolean = true,
+    var decodingMethod: String = "greedy_search",
+    var maxActivePaths: Int = 4,
+    var hotwordsFile: String = "",
+    var hotwordsScore: Float = 1.5f,
+    var ruleFsts: String = "",
+    var ruleFars: String = "",
+    var blankPenalty: Float = 0.0f,
+)
+
+data class OnlineRecognizerResult(
+    val text: String,
+    val tokens: Array<String>,
+    val timestamps: FloatArray,
+    val ysProbs: FloatArray,
+    // TODO(fangjun): Add more fields
+)
+
+class OnlineRecognizer(
+    assetManager: AssetManager? = null,
+    val config: OnlineRecognizerConfig,
+) {
+    private var ptr: Long
+
+    init {
+        ptr = if (assetManager != null) {
+            newFromAsset(assetManager, config)
+        } else {
+            newFromFile(config)
+        }
+        require(ptr != 0L) {
+            "Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer"
+        }
+    }
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun createStream(hotwords: String = ""): OnlineStream {
+        val p = createStream(ptr, hotwords)
+        return OnlineStream(p)
+    }
+
+    fun reset(stream: OnlineStream) = reset(ptr, stream.ptr)
+    fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
+    fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr)
+    fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
+    fun getResult(stream: OnlineStream): OnlineRecognizerResult {
+        return getResult(ptr, stream.ptr)
+    }
+
+    private external fun delete(ptr: Long)
+
+    private external fun newFromAsset(
+        assetManager: AssetManager,
+        config: OnlineRecognizerConfig,
+    ): Long
+
+    private external fun newFromFile(
+        config: OnlineRecognizerConfig,
+    ): Long
+
+    private external fun createStream(ptr: Long, hotwords: String): Long
+    private external fun reset(ptr: Long, streamPtr: Long)
+    private external fun decode(ptr: Long, streamPtr: Long)
+    private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean
+    private external fun isReady(ptr: Long, streamPtr: Long): Boolean
+    private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+
+        @JvmStatic
+        external fun prependAdspLibraryPath(newPath: String) // for qnn
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt

@@ -0,0 +1,47 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+class OnlineStream(var ptr: Long = 0) {
+    init {
+        require(ptr != 0L) { "Failed to create native OnlineStream" }
+    }
+
+    fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
+        acceptWaveform(ptr, samples, sampleRate)
+
+    fun inputFinished() = inputFinished(ptr)
+
+    fun setOption(key: String, value: String) = setOption(ptr, key, value)
+
+    fun getOption(key: String): String = getOption(ptr, key)
+
+    protected fun finalize() {
+        if (ptr != 0L) {
+            delete(ptr)
+            ptr = 0
+        }
+    }
+
+    fun release() = finalize()
+
+    fun use(block: (OnlineStream) -> Unit) {
+        try {
+            block(this)
+        } finally {
+            release()
+        }
+    }
+
+    private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
+    private external fun inputFinished(ptr: Long)
+    private external fun setOption(ptr: Long, key: String, value: String)
+    private external fun getOption(ptr: Long, key: String): String
+    private external fun delete(ptr: Long)
+
+
+    companion object {
+        init {
+            System.loadLibrary("sherpa-onnx-jni")
+        }
+    }
+}

gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt

@@ -0,0 +1,8 @@
+// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
+package com.k2fsa.sherpa.onnx
+
+data class QnnConfig(
+    var backendLib: String = "",
+    var contextBinary: String = "",
+    var systemLib: String = "",
+)