diff --git a/gearslip/build.gradle.kts b/gearslip/build.gradle.kts index 6eed1ad..e6e946a 100644 --- a/gearslip/build.gradle.kts +++ b/gearslip/build.gradle.kts @@ -83,21 +83,10 @@ android { } defaultConfig { - // Speech-to-text ships for 64-bit ARM only: every phone Gearslip runs on, and one copy - // of the native library instead of four. + // Speech-to-text (sherpa-onnx, in jniLibs) ships for 64-bit ARM only: every phone + // Gearslip runs on, and one copy of the native library instead of four. ndk { abiFilters += "arm64-v8a" } } - ndkVersion = "27.1.12297006" - externalNativeBuild { - cmake { - path = file("src/main/cpp/CMakeLists.txt") - version = "3.22.1" - } - } - androidResources { - // Model weights barely compress, and stored plainly they stream straight out of the APK. - noCompress += "bin" - } testOptions { // GearslipLog calls android.util.Log, which is a stub on the local JVM. @@ -130,6 +119,7 @@ dependencies { implementation(platform(libs.androidx.compose.bom)) implementation(libs.androidx.compose.ui) implementation(libs.androidx.compose.material3) + implementation(libs.commons.compress) implementation(libs.androidx.compose.material.icons.core) implementation(libs.androidx.activity.compose) implementation(libs.androidx.lifecycle.runtime.compose) diff --git a/gearslip/src/main/cpp/CMakeLists.txt b/gearslip/src/main/cpp/CMakeLists.txt deleted file mode 100644 index f4ab8b5..0000000 --- a/gearslip/src/main/cpp/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -# Speech-to-text for voice replies and the assistant: whisper.cpp, pinned to one commit and -# fetched at build time, so nothing outside this folder (cloned-repos included) is needed. -cmake_minimum_required(VERSION 3.22) -project(gearslip_speech CXX C) - -set(CMAKE_CXX_STANDARD 17) - -# Optimised in debug builds too. Unoptimised, Whisper runs several times slower, and a dev build -# would make voice look far worse than what ships. -foreach(lang C CXX) - set(CMAKE_${lang}_FLAGS_DEBUG "-O3 -DNDEBUG") -endforeach() - -include(FetchContent) -FetchContent_Declare( - whisper - URL https://github.com/ggml-org/whisper.cpp/archive/6e4ab854f67f743900934a703d5603419384c961.tar.gz - URL_HASH SHA256=55b585aec5e1bc03e2a5cea61da97c6406fa833996cf7daeed704972273cc039 -) - -set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) -set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) -set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE) -set(GGML_OPENMP OFF CACHE BOOL "" FORCE) -set(GGML_NATIVE OFF CACHE BOOL "" FORCE) -# ARMv8.2 with half floats and dot products: every phone that runs Android 12 or later in -# practice, and what makes Whisper fast enough to answer in under a second. -set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE) -FetchContent_MakeAvailable(whisper) - -add_library(gearslip_speech SHARED speech_jni.cpp) -target_compile_options(gearslip_speech PRIVATE -O3) -find_library(LOG_LIB log) -target_link_libraries(gearslip_speech whisper android ${LOG_LIB}) -target_link_options(gearslip_speech PRIVATE -Wl,--gc-sections -Wl,-z,max-page-size=16384) diff --git a/gearslip/src/main/cpp/speech_jni.cpp b/gearslip/src/main/cpp/speech_jni.cpp deleted file mode 100644 index bbd79c3..0000000 --- a/gearslip/src/main/cpp/speech_jni.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// The bridge between Kotlin's SpeechEngine and whisper.cpp: load a model, turn one recorded -// phrase into text, free the model. Recording and deciding when a phrase ends stay in Kotlin. - -#include -#include -#include -#include -#include -#include -#include -#include "whisper.h" - -#define TAG "GearslipSpeech" -#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) - -static whisper_context_params context_params() { - whisper_context_params p = whisper_context_default_params(); - p.use_gpu = false; - return p; -} - -// Streams a model straight out of the APK, so the bundled one is never copied to storage. -static size_t asset_read(void *ctx, void *output, size_t size) { - int n = AAsset_read(static_cast(ctx), output, size); - return n > 0 ? static_cast(n) : 0; -} -static bool asset_eof(void *ctx) { return AAsset_getRemainingLength64(static_cast(ctx)) <= 0; } -static void asset_close(void *ctx) { AAsset_close(static_cast(ctx)); } - -extern "C" JNIEXPORT jlong JNICALL -Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeInitAsset(JNIEnv *env, jobject, jobject assets, jstring jpath) { - AAssetManager *manager = AAssetManager_fromJava(env, assets); - const char *path = env->GetStringUTFChars(jpath, nullptr); - AAsset *asset = AAssetManager_open(manager, path, AASSET_MODE_STREAMING); - env->ReleaseStringUTFChars(jpath, path); - if (!asset) return 0; - whisper_model_loader loader = {asset, asset_read, asset_eof, asset_close}; - return reinterpret_cast(whisper_init_with_params(&loader, context_params())); -} - -// Bytes, not a jstring: NewStringUTF wants modified UTF-8 and aborts on some of what Whisper can -// write (four-byte characters), so Kotlin decodes the plain UTF-8 instead. -extern "C" JNIEXPORT jbyteArray JNICALL -Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeTranscribe( - JNIEnv *env, jobject, jlong handle, jfloatArray jsamples, jstring jlang, jint threads) { - auto *ctx = reinterpret_cast(handle); - if (!ctx) return env->NewByteArray(0); - jsize n = env->GetArrayLength(jsamples); - jfloat *samples = env->GetFloatArrayElements(jsamples, nullptr); - const char *lang = env->GetStringUTFChars(jlang, nullptr); - - whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); - p.n_threads = threads; - p.language = lang; - p.translate = false; - p.no_context = true; - p.no_timestamps = true; - p.single_segment = true; - p.print_progress = false; - p.print_realtime = false; - p.print_special = false; - p.print_timestamps = false; - p.suppress_blank = true; - // The ACFT models are trained for a context cut to the audio's own length (1500 frames is - // 30 s, so 50 per second), which is what makes a three-second phrase fast. A little extra - // keeps the last word from being clipped. - p.audio_ctx = std::min(1500, static_cast(std::ceil(n / 320.0)) + 32); - - int rc = whisper_full(ctx, p, samples, n); - env->ReleaseStringUTFChars(jlang, lang); - env->ReleaseFloatArrayElements(jsamples, samples, JNI_ABORT); - if (rc != 0) { - LOGI("whisper_full failed: %d", rc); - return env->NewByteArray(0); - } - std::string text; - for (int i = 0; i < whisper_full_n_segments(ctx); i++) text += whisper_full_get_segment_text(ctx, i); - jbyteArray out = env->NewByteArray(static_cast(text.size())); - env->SetByteArrayRegion(out, 0, static_cast(text.size()), reinterpret_cast(text.data())); - return out; -} - -extern "C" JNIEXPORT void JNICALL -Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeFree(JNIEnv *, jobject, jlong handle) { - if (handle) whisper_free(reinterpret_cast(handle)); -} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt index b09a93b..888fa16 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt @@ -178,8 +178,8 @@ object CarAssistant { } /** - * One phrase, through Gearslip's own [SpeechEngine]. The phone's recognizer is only a fallback - * for when the engine itself can't run, since plenty of phones (GrapheneOS among them) have none. + * One phrase, through Gearslip's own [SpeechEngine], or the phone's recognizer when the driver + * chose it in settings. Never the phone's on its own: on most phones that's Google's. */ private fun listenOnce( onPartial: (String) -> Unit, @@ -197,15 +197,10 @@ object CarAssistant { context, cue = cue, onSpeaking = onSpeaking, + onPartial = onPartial, onHeard = onHeard, onResult = onResult, - onError = { message -> - if (message == SpeechEngine.NOT_LOADED && SpeechRecognizer.isRecognitionAvailable(context)) { - systemListen(context, onPartial, onHeard, onResult, onError) - } else { - onError(message) - } - }, + onError = onError, ) } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt index 1d052fe..10032ae 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt @@ -4,7 +4,6 @@ import android.Manifest import android.annotation.SuppressLint import android.content.Context import android.content.pm.PackageManager -import android.content.res.AssetManager import android.media.AudioFormat import android.media.AudioManager import android.media.AudioRecord @@ -15,40 +14,47 @@ import android.os.Looper import android.os.SystemClock import app.seb3thehacker.gearslip.GearslipLog import app.seb3thehacker.gearslip.audio.CarMic -import java.util.Locale +import com.k2fsa.sherpa.onnx.EndpointConfig +import com.k2fsa.sherpa.onnx.EndpointRule +import com.k2fsa.sherpa.onnx.FeatureConfig +import com.k2fsa.sherpa.onnx.OfflineModelConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizer +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig +import com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig +import com.k2fsa.sherpa.onnx.OnlineModelConfig +import com.k2fsa.sherpa.onnx.OnlineRecognizer +import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig +import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig +import java.io.File import java.util.concurrent.Executors -import kotlin.math.log10 import kotlin.math.max -import kotlin.math.sqrt /** * Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships - * none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply - * of a few seconds comes back in well under a second on a recent phone. + * none). sherpa-onnx runs a streaming model ([SpeechModels]) on the phone: words come out while + * the driver is still talking, and the model itself decides when they've finished. A larger model + * then reads the whole phrase once more for the final words, since a model that waits for the end + * hears names and places far better than one that has to keep up. The phrase is held in memory + * for that and nowhere else, and dropped as soon as it's read. * - * Whisper needs the whole phrase before it starts, so this records from the car's microphone (the - * phone's when the car has none or sends nothing), - * decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes - * once. Everything runs on one worker thread; callbacks arrive on the main thread. + * Audio comes from the car's microphone, or the phone's when the car has none or sends nothing. + * Everything runs on one worker thread; callbacks arrive on the main thread. */ object SpeechEngine { - /** - * FUTO's ACFT Whisper (Apache-2.0), re-quantized to 5 bits, inside the APK: base when the build - * carries it (more accurate, about twice as slow), tiny otherwise. - */ - private val MODELS = listOf("speech/base_acft_q5_1.bin", "speech/tiny_acft_q5_1.bin") - /** The error for an engine that can't run at all, which callers may answer with a fallback. */ const val NOT_LOADED = "The speech model didn't load." + /** Said when the chosen model hasn't been downloaded yet. */ + const val NOT_DOWNLOADED = "Download a voice model in Gearslip's settings on the phone first." + private const val RATE = 16_000 - private const val FRAME = 480 // 30 ms - private const val MAX_MS = 15_000L - private const val NO_SPEECH_MS = 6_000L - private const val END_SILENCE_MS = 1_100L - private const val CALIBRATE_MS = 300L - private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word + private const val CHUNK = 1_600 // 100 ms + private const val MAX_S = 15f + /** Quiet before the first word that counts as "nobody spoke". */ + private const val NO_SPEECH_S = 6f + /** Quiet after the last word that ends the phrase. */ + private const val END_SILENCE_S = 1.0f private const val CUE_VOLUME = 80 /** How long the car gets to start sending once asked, before the phone's microphone takes over. */ private const val CAR_FIRST_AUDIO_MS = 1_500L @@ -60,25 +66,46 @@ object SpeechEngine { private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") } private val main = Handler(Looper.getMainLooper()) - private var libraryLoaded = false - private var handle = 0L + private var recognizer: OnlineRecognizer? = null + private var loadedModel: SpeechModels.Model? = null + private var finalRecognizer: OfflineRecognizer? = null + private var loadedFinal: SpeechModels.Model? = null @Volatile private var run = 0 - /** Loads the model ahead of time, so the first reply doesn't wait for it. */ + /** Loads the model ahead of time, so the first command doesn't wait for it. */ fun warm(context: Context) { val app = context.applicationContext - worker.execute { ensureLoaded(app) } + SpeechModels.init(app) + worker.execute { + ensureLoaded(app) + ensureFinalLoaded(app) + } + } + + /** Drops the loaded model, after the driver picks another or deletes it. */ + fun unload() { + run++ + worker.execute { + recognizer?.release() + recognizer = null + loadedModel = null + finalRecognizer?.release() + finalRecognizer = null + loadedFinal = null + } } /** * Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk. - * [onSpeaking] fires at the first word, [onHeard] when the driver stops and the words are being - * worked out; [onResult] gets the text, [onError] a sentence fit to show on the car screen. + * [onPartial] gets the words so far as they're recognized, [onSpeaking] fires at the first + * word, [onHeard] when the driver stops; [onResult] gets the text, [onError] a sentence fit to + * show on the car screen. */ fun listen( context: Context, cue: Boolean = true, onSpeaking: () -> Unit = {}, + onPartial: (String) -> Unit = {}, onHeard: () -> Unit, onResult: (String) -> Unit, onError: (String) -> Unit, @@ -92,26 +119,21 @@ object SpeechEngine { fun post(block: () -> Unit) { main.post { if (id == run) block() } } worker.execute { if (id != run) return@execute - if (!ensureLoaded(app)) return@execute post { onError(NOT_LOADED) } + if (!SpeechModels.isReady(app, SpeechModels.selected.value)) return@execute post { onError(NOT_DOWNLOADED) } + val r = ensureLoaded(app) ?: return@execute post { onError(NOT_LOADED) } VoiceFocus.acquire(app) - val audio = try { + val text = try { if (cue) cue() if (id != run) return@execute - record(id) { post(onSpeaking) } + stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) }) } catch (t: Throwable) { - GearslipLog.e("speech: recording failed", t) + GearslipLog.e("speech: listening failed", t) return@execute post { onError("The microphone isn't available right now.") } } finally { VoiceFocus.release() } if (id != run) return@execute - if (audio == null) return@execute post { onError("Didn't catch that.") } post(onHeard) - val started = SystemClock.uptimeMillis() - val text = clean(transcribe(audio)) - GearslipLog.i( - "speech: ${audio.size * 1000L / RATE} ms of audio took ${SystemClock.uptimeMillis() - started} ms", - ) post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) } } } @@ -119,6 +141,165 @@ object SpeechEngine { /** Stops listening; a phrase already being worked out is dropped. */ fun cancel() { run++ } + /** + * Feeds the microphone to the recognizer in 100 ms pieces, reporting the words as they firm + * up, until the model hears the end of the phrase (or nobody speaks, or it runs too long). + */ + private fun stream( + context: Context, + r: OnlineRecognizer, + id: Int, + onSpeaking: () -> Unit, + onPartial: (String) -> Unit, + onStopped: () -> Unit, + ): String { + val mic = carMic() ?: phoneMic() + val s = r.createStream() + val frame = ShortArray(CHUNK) + val samples = FloatArray(CHUNK) + // The whole phrase, for the final read; [MAX_S] and a little over is the most it can hold. + val phrase = FloatArray((RATE * (MAX_S + 1)).toInt()) + var held = 0 + var last = "" + var spoke = false + var audioMs = 0L + var decodeMs = 0L + try { + while (id == run) { + val n = mic.read(frame) + if (n < 0) break + if (n == 0) continue + for (i in 0 until n) samples[i] = frame[i] / 32768f + val keep = minOf(n, phrase.size - held) + System.arraycopy(samples, 0, phrase, held, keep) + held += keep + s.acceptWaveform(if (n == CHUNK) samples else samples.copyOf(n), RATE) + audioMs += n * 1000L / RATE + val started = SystemClock.uptimeMillis() + while (r.isReady(s)) r.decode(s) + decodeMs += SystemClock.uptimeMillis() - started + val text = clean(r.getResult(s).text) + if (text != last) { + last = text + if (text.isNotEmpty() && !spoke) { + spoke = true + onSpeaking() + } + onPartial(text) + } + if (r.isEndpoint(s)) break + } + s.inputFinished() + while (r.isReady(s)) r.decode(s) + last = clean(r.getResult(s).text) + } finally { + mic.close() + s.release() + } + // Under 1.0 keeps up with the driver; over it, the words fall behind the voice. + GearslipLog.i( + "speech: $audioMs ms of audio, decoding took $decodeMs ms " + + "(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)", + ) + if (last.isEmpty() || id != run) return last + val final = ensureFinalLoaded(context) ?: return last + onStopped() + val started = SystemClock.uptimeMillis() + val reread = final.createStream().run { + try { + acceptWaveform(phrase.copyOf(held), RATE) + final.decode(this) + clean(final.getResult(this).text) + } finally { + release() + } + } + phrase.fill(0f) + GearslipLog.i("speech: the final read took ${SystemClock.uptimeMillis() - started} ms") + if (reread.isEmpty()) return last + if (reread != last) onPartial(reread) + return reread + } + + /** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */ + private fun ensureFinalLoaded(context: Context): OfflineRecognizer? { + val model = SpeechModels.final.value ?: return null + if (finalRecognizer != null && loadedFinal == model) return finalRecognizer + finalRecognizer?.release() + finalRecognizer = null + if (!SpeechModels.isReady(context, model)) return null + val dir = SpeechModels.dir(context, model) + fun path(name: String) = File(dir, name).absolutePath + val started = SystemClock.uptimeMillis() + return runCatching { + OfflineRecognizer( + config = OfflineRecognizerConfig( + featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80), + modelConfig = OfflineModelConfig( + transducer = OfflineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)), + tokens = path("tokens.txt"), + numThreads = threads(), + modelType = model.modelType, + ), + decodingMethod = "greedy_search", + ), + ) + }.onSuccess { + finalRecognizer = it + loadedFinal = model + GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms") + }.onFailure { + GearslipLog.e("speech: couldn't load ${model.label}", it) + }.getOrNull() + } + + private fun ensureLoaded(context: Context): OnlineRecognizer? { + val model = SpeechModels.selected.value + if (recognizer != null && loadedModel == model) return recognizer + recognizer?.release() + recognizer = null + if (!SpeechModels.isReady(context, model)) return null + val dir = SpeechModels.dir(context, model) + fun path(name: String) = File(dir, name).absolutePath + val started = SystemClock.uptimeMillis() + return runCatching { + OnlineRecognizer( + config = OnlineRecognizerConfig( + featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80), + modelConfig = OnlineModelConfig( + transducer = OnlineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)), + tokens = path("tokens.txt"), + numThreads = threads(), + modelType = model.modelType, + ), + endpointConfig = EndpointConfig( + rule1 = EndpointRule(false, NO_SPEECH_S, 0f), + rule2 = EndpointRule(true, END_SILENCE_S, 0f), + rule3 = EndpointRule(false, 0f, MAX_S), + ), + enableEndpoint = true, + decodingMethod = "greedy_search", + ), + ) + }.onSuccess { + recognizer = it + loadedModel = model + GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms") + }.onFailure { + GearslipLog.e("speech: couldn't load ${model.label}", it) + }.getOrNull() + } + + /** The big cores only: more threads than that slow the model down on a phone. */ + private fun threads() = Runtime.getRuntime().availableProcessors().let { if (it >= 8) 4 else max(1, it / 2) } + + /** + * Models write in upper case, or with punctuation and capitals of their own; commands are + * matched in plain lower case, so only words and apostrophes are kept. + */ + private fun clean(text: String): String = + text.lowercase().replace(Regex("[^\\p{L}\\p{N}' ]"), " ").replace(Regex("\\s+"), " ").trim() + /** * A short beep on the media stream, so it plays through the car like everything else, then a * pause long enough for it to finish there before the microphone opens and hears it. @@ -133,14 +314,9 @@ object SpeechEngine { } } - // --- recording -------------------------------------------------------------------------- + // --- microphones ----------------------------------------------------------------------- - /** - * Records until the driver stops talking. Returns the phrase from just before the first word, - * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a - * loud cabin needs a louder voice than a quiet one to count as talking. - */ - /** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */ + /** Where a phrase is heard from: [read] fills [frame] and returns how many samples it got, -1 at the end. */ private interface Mic { fun read(frame: ShortArray): Int fun close() @@ -151,10 +327,11 @@ object SpeechEngine { val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) val recorder = AudioRecord( MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE, - AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8), + AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, CHUNK * 4), ) check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" } recorder.startRecording() + GearslipLog.i("speech: listening through the phone's microphone") return object : Mic { override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size) override fun close() { @@ -166,7 +343,7 @@ object SpeechEngine { /** * The car's microphone, which hears the driver better than a phone in a cup holder. The car - * sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car + * sends audio in its own sized pieces; they're cut to [CHUNK]s here. Null when there's no car * microphone, or it sends nothing at first, so the phone's is used instead. */ private fun carMic(): Mic? { @@ -193,107 +370,10 @@ object SpeechEngine { at += n if (at == chunk.size) { pending = null; at = 0 } } - // The car stopped sending mid-phrase: end the recording rather than wait forever. + // The car stopped sending mid-phrase: end the phrase rather than wait forever. return if (filled == 0) -1 else filled } override fun close() = car.stop() } } - - /** - * Records until the driver stops talking. Returns the phrase from just before the first word, - * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a - * loud cabin needs a louder voice than a quiet one to count as talking. - */ - private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? { - val mic = carMic() ?: phoneMic() - val samples = FloatArray((RATE * MAX_MS / 1000).toInt()) - var count = 0 - val frame = ShortArray(FRAME) - var noise = 0.0 - var noiseFrames = 0 - var speechStart = -1 - var lastSpeech = 0 - try { - while (id == run && count + FRAME <= samples.size) { - val n = mic.read(frame) - if (n < 0) break - if (n == 0) continue - var sum = 0.0 - for (i in 0 until n) { - val v = frame[i] / 32768f - samples[count + i] = v - sum += v * v - } - count += n - val db = 20 * log10(sqrt(sum / n) + 1e-9) - val ms = count * 1000L / RATE - if (ms <= CALIBRATE_MS) { - noise += db; noiseFrames++ - continue - } - val floor = if (noiseFrames > 0) noise / noiseFrames else -60.0 - val talking = db > max(floor + 12, -45.0) - if (talking) { - if (speechStart < 0) { - speechStart = max(0, count - n - PRE_ROLL) - onSpeaking() - } - lastSpeech = count - } - if (speechStart < 0 && ms > NO_SPEECH_MS) return null - if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break - } - } finally { - mic.close() - } - if (speechStart < 0) return null - return samples.copyOfRange(speechStart, count) - } - - // --- the model -------------------------------------------------------------------------- - - private fun ensureLoaded(context: Context): Boolean { - if (handle != 0L) return true - if (!libraryLoaded) { - runCatching { System.loadLibrary("gearslip_speech") } - .onFailure { GearslipLog.e("speech: native library missing", it); return false } - libraryLoaded = true - } - val started = SystemClock.uptimeMillis() - val model = MODELS.firstOrNull { name -> runCatching { context.assets.open(name).close() }.isSuccess } - ?: run { GearslipLog.w("speech: no model in the build"); return false } - handle = nativeInitAsset(context.assets, model) - if (handle == 0L) { - GearslipLog.w("speech: couldn't load the model") - return false - } - GearslipLog.i("speech: loaded $model in ${SystemClock.uptimeMillis() - started} ms") - return true - } - - private fun transcribe(audio: FloatArray): String { - val bytes = nativeTranscribe(handle, audio, language(), threads()) - return String(bytes, Charsets.UTF_8) - } - - /** The phone's language, as Whisper names it. Older Java codes differ for a few. */ - private fun language(): String = when (val lang = Locale.getDefault().language) { - "iw" -> "he" - "in" -> "id" - "ji" -> "yi" - "" -> "en" - else -> lang - } - - /** The big cores only: more threads than that slow Whisper down on a phone. */ - private fun threads() = Runtime.getRuntime().availableProcessors().coerceIn(1, 8).let { if (it >= 8) 4 else max(1, it / 2) } - - /** Whisper marks silence and noise as "[BLANK_AUDIO]" or "(wind blowing)"; none of it is words. */ - private fun clean(text: String): String = - text.replace(Regex("""\[[^\]]*\]|\([^)]*\)|\*[^*]*\*"""), " ").replace(Regex("\\s+"), " ").trim() - - private external fun nativeInitAsset(assets: AssetManager, path: String): Long - private external fun nativeTranscribe(handle: Long, samples: FloatArray, language: String, threads: Int): ByteArray - private external fun nativeFree(handle: Long) } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt new file mode 100644 index 0000000..06484d4 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt @@ -0,0 +1,187 @@ +package app.seb3thehacker.gearslip.speech + +import android.content.Context +import app.seb3thehacker.gearslip.GearslipLog +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.Executors +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream + +/** + * The speech models Gearslip listens with, in two roles: a streaming one that writes the words + * while the driver talks, and an optional final one that reads the whole phrase again once they + * stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's + * GitHub releases, where its authors publish it, and kept in the app's own storage. + */ +object SpeechModels { + + enum class Role { LIVE, FINAL } + + enum class Model( + val role: Role, + val id: String, + val label: String, + val detail: String, + val downloadMb: Int, + private val archive: String, + val encoder: String, + val decoder: String, + val joiner: String, + val modelType: String = "", + ) { + KROKO( + Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54, + "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06", + "encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2", + ), + PARAKEET_SMALL( + Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103, + "sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8", + "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer", + ), + PARAKEET_LARGE( + Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460, + "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", + "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer", + ), + ; + + val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2" + val files get() = listOf(encoder, decoder, joiner, "tokens.txt") + } + + sealed interface State { + data object Missing : State + data class Downloading(val percent: Int) : State + data object Ready : State + data class Failed(val reason: String) : State + } + + private const val PREFS = "speech" + private const val KEY_MODEL = "model" + private const val KEY_FINAL = "final_model" + + private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-download") } + private val states = Model.entries.associateWith { MutableStateFlow(State.Missing) } + private val _selected = MutableStateFlow(Model.KROKO) + val selected: StateFlow = _selected.asStateFlow() + private val _final = MutableStateFlow(Model.PARAKEET_LARGE) + + /** The model that rereads the whole phrase at the end; null to keep the live model's words. */ + val final: StateFlow = _final.asStateFlow() + @Volatile private var cancelled: Model? = null + + fun init(context: Context) { + val id = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_MODEL, null) + _selected.value = Model.entries.firstOrNull { it.id == id && it.role == Role.LIVE } ?: Model.KROKO + val finalId = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_FINAL, Model.PARAKEET_LARGE.id) + _final.value = Model.entries.firstOrNull { it.id == finalId && it.role == Role.FINAL } + Model.entries.forEach { m -> if (isReady(context, m)) states.getValue(m).value = State.Ready } + // Models an older build offered and this one doesn't: hundreds of MB nobody can use. + val known = Model.entries.map { it.id }.toSet() + File(context.filesDir, "speech").listFiles()?.filter { it.isDirectory && it.name !in known }?.forEach { + GearslipLog.i("speech: removing ${it.name}, no longer offered") + it.deleteRecursively() + } + } + + fun state(model: Model): StateFlow = states.getValue(model).asStateFlow() + + fun dir(context: Context, model: Model) = File(File(context.filesDir, "speech"), model.id) + + fun isReady(context: Context, model: Model) = model.files.all { File(dir(context, model), it).isFile } + + fun select(context: Context, model: Model) { + if (model.role == Role.FINAL) return selectFinal(context, model) + _selected.value = model + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_MODEL, model.id).apply() + SpeechEngine.unload() + } + + /** [model] null keeps the live words as they are. */ + fun selectFinal(context: Context, model: Model?) { + _final.value = model + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_FINAL, model?.id ?: "none").apply() + SpeechEngine.unload() + } + + fun delete(context: Context, model: Model) { + cancelled = model + dir(context, model).deleteRecursively() + states.getValue(model).value = State.Missing + if (model == _selected.value || model == _final.value) SpeechEngine.unload() + } + + /** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */ + fun download(context: Context, model: Model) { + val app = context.applicationContext + val state = states.getValue(model) + if (state.value is State.Downloading || state.value == State.Ready) return + state.value = State.Downloading(0) + cancelled = null + worker.execute { + val target = dir(app, model) + val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() } + try { + val conn = (URL(model.url).openConnection() as HttpURLConnection).apply { + connectTimeout = 15_000 + readTimeout = 30_000 + instanceFollowRedirects = true + } + check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" } + val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L) + val counting = CountingStream(conn.inputStream) { read -> + if (cancelled == model) throw InterruptedException("cancelled") + state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99)) + } + TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar -> + while (true) { + val entry = tar.nextEntry ?: break + val name = entry.name.substringAfterLast('/') + if (entry.isDirectory || name !in model.files) continue + File(partial, name).outputStream().use { tar.copyTo(it) } + } + } + check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" } + target.deleteRecursively() + check(partial.renameTo(target)) { "couldn't store the model" } + GearslipLog.i("speech: downloaded ${model.label}") + state.value = State.Ready + } catch (e: Exception) { + partial.deleteRecursively() + if (cancelled == model) { + state.value = State.Missing + } else { + GearslipLog.w("speech: download of ${model.label} failed: ${e.message}") + state.value = State.Failed(e.message ?: e.javaClass.simpleName) + } + } + } + } + + /** Counts bytes as they pass, reporting about every megabyte. */ + private class CountingStream( + private val inner: java.io.InputStream, + private val onProgress: (Long) -> Unit, + ) : java.io.InputStream() { + private var read = 0L + private var reported = 0L + + override fun read(): Int = inner.read().also { if (it >= 0) count(1) } + override fun read(b: ByteArray, off: Int, len: Int): Int = inner.read(b, off, len).also { if (it > 0) count(it.toLong()) } + override fun close() = inner.close() + + private fun count(n: Long) { + read += n + if (read - reported >= 1_048_576) { + reported = read + onProgress(read) + } + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt index a33cf85..53f6146 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt @@ -109,6 +109,7 @@ fun SettingsScreen( AboutSection(onOpenWhatsNew, onReplayTutorial, onReplayIntro) ConnectionSection() CarAudioSection() + VoiceModelSection() if (SHOW_RECOMMENDED_APPS) AppsSection(onOpenRecommendedApps) CertificateSection() SettingsSection("Usage notes", header = USAGE_STATS_INTRO) { UsageStatsRows() } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt index 924e0ef..7599961 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt @@ -151,4 +151,11 @@ internal fun AboutLinkRows() { icon = Icons.Filled.Build, trailing = arrow, ) + SettingsDivider() + SettingsRow( + "Voice credits", + subtitle = "Speech recognition by sherpa-onnx (Apache 2.0), with Kroko ASR by Banafo " + + "(CC BY-SA 4.0) and Parakeet by NVIDIA (CC BY 4.0)", + icon = Icons.Filled.Favorite, + ) } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt new file mode 100644 index 0000000..c11665d --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt @@ -0,0 +1,125 @@ +package app.seb3thehacker.gearslip.ui + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.selection.selectable +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.RadioButton +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.speech.SpeechModels +import app.seb3thehacker.gearslip.speech.SpeechModels.Model +import app.seb3thehacker.gearslip.speech.SpeechModels.State + +/** + * Which speech model the voice assistant listens with, and downloading it. Each is fetched once + * from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening. + */ +@Composable +internal fun VoiceModelSection() { + val context = LocalContext.current + remember { SpeechModels.init(context) } + val selected by SpeechModels.selected.collectAsState() + val final by SpeechModels.final.collectAsState() + SettingsSection(title = "Voice: live words") { + Text( + "The voice assistant turns speech into text on this phone, and nothing you say leaves it. " + + "It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " + + "This one writes your words as you speak.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + ) + Model.entries.filter { it.role == SpeechModels.Role.LIVE }.forEachIndexed { i, model -> + if (i > 0) SettingsDivider() + ModelRow(model, model == selected) + } + } + SettingsSection(title = "Voice: final words") { + Text( + "When you stop talking, this model reads the whole phrase again. It catches names and " + + "places the live model misses.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + ) + val context = LocalContext.current + Row( + Modifier + .fillMaxWidth() + .selectable(final == null, role = Role.RadioButton) { SpeechModels.selectFinal(context, null) } + .padding(horizontal = 8.dp, vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + RadioButton(final == null, onClick = null, modifier = Modifier.padding(8.dp)) + Column { + Text("None", style = MaterialTheme.typography.titleMedium) + Text( + "Keep the live words. Fastest, least accurate", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + Model.entries.filter { it.role == SpeechModels.Role.FINAL }.forEach { model -> + SettingsDivider() + ModelRow(model, model == final) + } + } +} + +@Composable +private fun ModelRow(model: Model, chosen: Boolean) { + val context = LocalContext.current + val state by SpeechModels.state(model).collectAsState() + Column( + Modifier + .fillMaxWidth() + .selectable(chosen, role = Role.RadioButton) { SpeechModels.select(context, model) } + .padding(horizontal = 8.dp, vertical = 8.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + RadioButton(chosen, onClick = null, modifier = Modifier.padding(8.dp)) + Column(Modifier.weight(1f)) { + Text("${model.label} · ${model.downloadMb} MB", style = MaterialTheme.typography.titleMedium) + Text( + when (val s = state) { + State.Ready -> "Downloaded. ${model.detail}" + is State.Downloading -> "Downloading, ${s.percent}%" + is State.Failed -> "Download failed: ${s.reason}" + State.Missing -> model.detail + }, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + Row(horizontalArrangement = Arrangement.spacedBy(4.dp)) { + when (state) { + State.Ready -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Delete") } + is State.Downloading -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Cancel") } + else -> OutlinedButton(onClick = { SpeechModels.download(context, model) }) { Text("Download") } + } + } + } + (state as? State.Downloading)?.let { + LinearProgressIndicator( + progress = { it.percent / 100f }, + modifier = Modifier.fillMaxWidth().padding(start = 48.dp, end = 8.dp, top = 4.dp), + ) + } + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt new file mode 100644 index 0000000..44491c7 --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt @@ -0,0 +1,12 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +data class FeatureConfig( + var sampleRate: Int = 16000, + var featureDim: Int = 80, + var dither: Float = 0.0f +) + +fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig { + return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim) +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt new file mode 100644 index 0000000..d282f0d --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt @@ -0,0 +1,8 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +data class HomophoneReplacerConfig( + var dictDir: String = "", // unused + var lexicon: String = "", + var ruleFsts: String = "", +) diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt new file mode 100644 index 0000000..ac229ac --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt @@ -0,0 +1,255 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out. +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class OfflineRecognizerResult( + val text: String, + val tokens: Array, + val timestamps: FloatArray, + val lang: String, + val emotion: String, + val event: String, + + // valid only for TDT models + val durations: FloatArray, +) + +data class OfflineTransducerModelConfig( + var encoder: String = "", + var decoder: String = "", + var joiner: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineParaformerModelConfig( + var model: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineNemoEncDecCtcModelConfig( + var model: String = "", +) + +data class OfflineDolphinModelConfig( + var model: String = "", +) + +data class OfflineZipformerCtcModelConfig( + var model: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineWenetCtcModelConfig( + var model: String = "", +) + +data class OfflineOmnilingualAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineMedAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineFireRedAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineFunAsrNanoModelConfig( + var encoderAdaptor: String = "", + var llm: String = "", + var embedding: String = "", + var tokenizer: String = "", + var systemPrompt: String = "You are a helpful assistant.", + var userPrompt: String = "语音转写：", + var maxNewTokens: Int = 512, + var temperature: Float = 1e-6f, + var topP: Float = 0.8f, + var seed: Int = 42, + var language: String = "", + var itn: Boolean = true, + var hotwords: String = "", +) + +data class OfflineQwen3AsrModelConfig( + var convFrontend: String = "", + var encoder: String = "", + var decoder: String = "", + var tokenizer: String = "", + var maxTotalLen: Int = 512, + var maxNewTokens: Int = 128, + var temperature: Float = 1e-6f, + var topP: Float = 0.8f, + var seed: Int = 42, + var hotwords: String = "", +) + +data class OfflineWhisperModelConfig( + var encoder: String = "", + var decoder: String = "", + var language: String = "en", // Used with multilingual model + var task: String = "transcribe", // transcribe or translate + var tailPaddings: Int = 1000, // Padding added at the end of the samples + var enableTokenTimestamps: Boolean = false, + var enableSegmentTimestamps: Boolean = false, +) + +data class OfflineCanaryModelConfig( + var encoder: String = "", + var decoder: String = "", + var srcLang: String = "en", + var tgtLang: String = "en", + var usePnc: Boolean = true, +) + +data class OfflineCohereTranscribeModelConfig( + var encoder: String = "", + var decoder: String = "", + var language: String = "", + var usePunct: Boolean = true, + var useItn: Boolean = true, +) + +data class OfflineFireRedAsrModelConfig( + var encoder: String = "", + var decoder: String = "", +) + +// For moonshine v1, you need four models. +// For moonshine v2, you need two models. +// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder +// - v2: encoder, mergedDecoder +data class OfflineMoonshineModelConfig( + var preprocessor: String = "", + var encoder: String = "", + var uncachedDecoder: String = "", + var cachedDecoder: String = "", + var mergedDecoder: String = "", +) + +data class OfflineSenseVoiceModelConfig( + var model: String = "", + var language: String = "", + var useInverseTextNormalization: Boolean = true, + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineModelConfig( + var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(), + var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(), + var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(), + var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(), + var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(), + var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(), + var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(), + var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(), + var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(), + var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(), + var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(), + var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(), + var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(), + var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(), + var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(), + var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(), + var cohereTranscribe: OfflineCohereTranscribeModelConfig = + OfflineCohereTranscribeModelConfig(), + var teleSpeech: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", + var modelType: String = "", + var tokens: String = "", + var modelingUnit: String = "", + var bpeVocab: String = "", +) + +data class OfflineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OfflineModelConfig = OfflineModelConfig(), + // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it + var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(), + var decodingMethod: String = "greedy_search", + var maxActivePaths: Int = 4, + var hotwordsFile: String = "", + var hotwordsScore: Float = 1.5f, + var ruleFsts: String = "", + var ruleFars: String = "", + var blankPenalty: Float = 0.0f, +) + +class OfflineRecognizer( + assetManager: AssetManager? = null, + val config: OfflineRecognizerConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(): OfflineStream { + val p = createStream(ptr) + return OfflineStream(p) + } + + fun createStream(hotwords: String): OfflineStream { + val p = createStreamWithHotwords(ptr, hotwords) + return OfflineStream(p) + } + + fun getResult(stream: OfflineStream): OfflineRecognizerResult { + return getResult(stream.ptr) + } + + fun decode(stream: OfflineStream) = decode(ptr, stream.ptr) + + fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config) + + private external fun delete(ptr: Long) + + private external fun createStream(ptr: Long): Long + + private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long + + private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OfflineRecognizerConfig, + ): Long + + private external fun newFromFile( + config: OfflineRecognizerConfig, + ): Long + + private external fun decode(ptr: Long, streamPtr: Long) + + private external fun getResult(streamPtr: Long): OfflineRecognizerResult + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + + @JvmStatic + external fun prependAdspLibraryPath(newPath: String) // for qnn + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt new file mode 100644 index 0000000..12ac836 --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt @@ -0,0 +1,43 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +class OfflineStream(var ptr: Long) { + init { + require(ptr != 0L) { "Failed to create native OfflineStream" } + } + + fun acceptWaveform(samples: FloatArray, sampleRate: Int) = + acceptWaveform(ptr, samples, sampleRate) + + fun setOption(key: String, value: String) = setOption(ptr, key, value) + + fun getOption(key: String): String = getOption(ptr, key) + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun use(block: (OfflineStream) -> Unit) { + try { + block(this) + } finally { + release() + } + } + + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) + private external fun setOption(ptr: Long, key: String, value: String) + private external fun getOption(ptr: Long, key: String): String + private external fun delete(ptr: Long) + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt new file mode 100644 index 0000000..3cd4a16 --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt @@ -0,0 +1,158 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out. +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class EndpointRule( + var mustContainNonSilence: Boolean, + var minTrailingSilence: Float, + var minUtteranceLength: Float, +) + +data class EndpointConfig( + var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f), + var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f), + var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f) +) + +data class OnlineTransducerModelConfig( + var encoder: String = "", + var decoder: String = "", + var joiner: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OnlineParaformerModelConfig( + var encoder: String = "", + var decoder: String = "", +) + +data class OnlineZipformer2CtcModelConfig( + var model: String = "", +) + +data class OnlineNeMoCtcModelConfig( + var model: String = "", +) + +data class OnlineToneCtcModelConfig( + var model: String = "", +) + +data class OnlineModelConfig( + var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(), + var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(), + var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(), + var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(), + var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(), + var tokens: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", + var modelType: String = "", + var modelingUnit: String = "", + var bpeVocab: String = "", +) + +data class OnlineLMConfig( + var model: String = "", + var scale: Float = 0.5f, +) + +data class OnlineCtcFstDecoderConfig( + var graph: String = "", + var maxActive: Int = 3000, +) + +data class OnlineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OnlineModelConfig = OnlineModelConfig(), + var lmConfig: OnlineLMConfig = OnlineLMConfig(), + var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(), + var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(), + var endpointConfig: EndpointConfig = EndpointConfig(), + var enableEndpoint: Boolean = true, + var decodingMethod: String = "greedy_search", + var maxActivePaths: Int = 4, + var hotwordsFile: String = "", + var hotwordsScore: Float = 1.5f, + var ruleFsts: String = "", + var ruleFars: String = "", + var blankPenalty: Float = 0.0f, +) + +data class OnlineRecognizerResult( + val text: String, + val tokens: Array, + val timestamps: FloatArray, + val ysProbs: FloatArray, + // TODO(fangjun): Add more fields +) + +class OnlineRecognizer( + assetManager: AssetManager? = null, + val config: OnlineRecognizerConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(hotwords: String = ""): OnlineStream { + val p = createStream(ptr, hotwords) + return OnlineStream(p) + } + + fun reset(stream: OnlineStream) = reset(ptr, stream.ptr) + fun decode(stream: OnlineStream) = decode(ptr, stream.ptr) + fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr) + fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr) + fun getResult(stream: OnlineStream): OnlineRecognizerResult { + return getResult(ptr, stream.ptr) + } + + private external fun delete(ptr: Long) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OnlineRecognizerConfig, + ): Long + + private external fun newFromFile( + config: OnlineRecognizerConfig, + ): Long + + private external fun createStream(ptr: Long, hotwords: String): Long + private external fun reset(ptr: Long, streamPtr: Long) + private external fun decode(ptr: Long, streamPtr: Long) + private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean + private external fun isReady(ptr: Long, streamPtr: Long): Boolean + private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + + @JvmStatic + external fun prependAdspLibraryPath(newPath: String) // for qnn + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt new file mode 100644 index 0000000..6eebf5e --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt @@ -0,0 +1,47 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +class OnlineStream(var ptr: Long = 0) { + init { + require(ptr != 0L) { "Failed to create native OnlineStream" } + } + + fun acceptWaveform(samples: FloatArray, sampleRate: Int) = + acceptWaveform(ptr, samples, sampleRate) + + fun inputFinished() = inputFinished(ptr) + + fun setOption(key: String, value: String) = setOption(ptr, key, value) + + fun getOption(key: String): String = getOption(ptr, key) + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun use(block: (OnlineStream) -> Unit) { + try { + block(this) + } finally { + release() + } + } + + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) + private external fun inputFinished(ptr: Long) + private external fun setOption(ptr: Long, key: String, value: String) + private external fun getOption(ptr: Long, key: String): String + private external fun delete(ptr: Long) + + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt new file mode 100644 index 0000000..95be41c --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt @@ -0,0 +1,8 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +data class QnnConfig( + var backendLib: String = "", + var contextBinary: String = "", + var systemLib: String = "", +) diff --git a/gearslip/src/main/assets/speech/tiny_acft_q5_1.bin b/gearslip/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so similarity index 59% rename from gearslip/src/main/assets/speech/tiny_acft_q5_1.bin rename to gearslip/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so index 541f1df..b2f18d1 100644 Binary files a/gearslip/src/main/assets/speech/tiny_acft_q5_1.bin and b/gearslip/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so differ diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f967a0d..0075398 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -14,6 +14,7 @@ material = "1.14.0" constraintlayout = "2.2.1" zxing = "3.5.4" okhttp = "5.4.0" +commonsCompress = "1.28.0" kotlinxSerializationJson = "1.11.0" ossLicenses = "17.5.1" webkit = "1.16.0" @@ -49,6 +50,7 @@ androidx-constraintlayout = { group = "androidx.constraintlayout", name = "const google-material = { group = "com.google.android.material", name = "material", version.ref = "material" } zxing-core = { group = "com.google.zxing", name = "core", version.ref = "zxing" } okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } +commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commonsCompress" } kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } google-oss-licenses = { group = "com.google.android.gms", name = "play-services-oss-licenses", version.ref = "ossLicenses" } androidx-webkit = { group = "androidx.webkit", name = "webkit", version.ref = "webkit" }