diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..b3d82ba --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# Weekly pull requests for newer Gradle libraries and GitHub Actions, aimed at experimental, +# where unreleased code lands. Security fixes come separately, as soon as an alert opens. +version: 2 +updates: + - package-ecosystem: gradle + directory: / + target-branch: experimental + schedule: + interval: weekly + open-pull-requests-limit: 5 + - package-ecosystem: github-actions + directory: / + target-branch: experimental + schedule: + interval: weekly diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..dd5b338 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,50 @@ +name: CodeQL + +# Kotlin can only be scanned while it compiles, so this builds the debug app, the same way +# release.yml does but without a signing key. Workflow files need no build. +on: + push: + branches: [main, experimental] + pull_request: + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + +jobs: + analyze: + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + actions: read + strategy: + fail-fast: false + matrix: + include: + - language: actions + build-mode: none + - language: java-kotlin + build-mode: manual + steps: + - uses: actions/checkout@v4 + + - uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + + - if: matrix.build-mode == 'manual' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "21" + + - if: matrix.build-mode == 'manual' + uses: gradle/actions/setup-gradle@v4 + + - if: matrix.build-mode == 'manual' + run: ./gradlew :gearslip:assembleDebug --no-daemon + + - uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml new file mode 100644 index 0000000..c310dc0 --- /dev/null +++ b/.github/workflows/trivy.yml @@ -0,0 +1,34 @@ +name: Security scan + +# Trivy looks through the repository for committed secrets and risky workflow or build +# settings. Findings show under the Security tab, next to CodeQL's and Dependabot's. +on: + push: + branches: [main, experimental] + pull_request: + schedule: + - cron: "0 9 * * 1" + workflow_dispatch: + +jobs: + trivy: + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - uses: actions/checkout@v4 + + - uses: aquasecurity/trivy-action@v0.36.0 + with: + scan-type: fs + scanners: vuln,secret,misconfig + format: sarif + output: trivy.sarif + severity: MEDIUM,HIGH,CRITICAL + + - uses: github/codeql-action/upload-sarif@v4 + if: always() + with: + sarif_file: trivy.sarif + category: trivy diff --git a/gearslip/build.gradle.kts b/gearslip/build.gradle.kts index 6eed1ad..e6e946a 100644 --- a/gearslip/build.gradle.kts +++ b/gearslip/build.gradle.kts @@ -83,21 +83,10 @@ android { } defaultConfig { - // Speech-to-text ships for 64-bit ARM only: every phone Gearslip runs on, and one copy - // of the native library instead of four. + // Speech-to-text (sherpa-onnx, in jniLibs) ships for 64-bit ARM only: every phone + // Gearslip runs on, and one copy of the native library instead of four. ndk { abiFilters += "arm64-v8a" } } - ndkVersion = "27.1.12297006" - externalNativeBuild { - cmake { - path = file("src/main/cpp/CMakeLists.txt") - version = "3.22.1" - } - } - androidResources { - // Model weights barely compress, and stored plainly they stream straight out of the APK. - noCompress += "bin" - } testOptions { // GearslipLog calls android.util.Log, which is a stub on the local JVM. @@ -130,6 +119,7 @@ dependencies { implementation(platform(libs.androidx.compose.bom)) implementation(libs.androidx.compose.ui) implementation(libs.androidx.compose.material3) + implementation(libs.commons.compress) implementation(libs.androidx.compose.material.icons.core) implementation(libs.androidx.activity.compose) implementation(libs.androidx.lifecycle.runtime.compose) diff --git a/gearslip/src/main/cpp/CMakeLists.txt b/gearslip/src/main/cpp/CMakeLists.txt deleted file mode 100644 index f4ab8b5..0000000 --- a/gearslip/src/main/cpp/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -# Speech-to-text for voice replies and the assistant: whisper.cpp, pinned to one commit and -# fetched at build time, so nothing outside this folder (cloned-repos included) is needed. -cmake_minimum_required(VERSION 3.22) -project(gearslip_speech CXX C) - -set(CMAKE_CXX_STANDARD 17) - -# Optimised in debug builds too. Unoptimised, Whisper runs several times slower, and a dev build -# would make voice look far worse than what ships. -foreach(lang C CXX) - set(CMAKE_${lang}_FLAGS_DEBUG "-O3 -DNDEBUG") -endforeach() - -include(FetchContent) -FetchContent_Declare( - whisper - URL https://github.com/ggml-org/whisper.cpp/archive/6e4ab854f67f743900934a703d5603419384c961.tar.gz - URL_HASH SHA256=55b585aec5e1bc03e2a5cea61da97c6406fa833996cf7daeed704972273cc039 -) - -set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) -set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) -set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE) -set(GGML_OPENMP OFF CACHE BOOL "" FORCE) -set(GGML_NATIVE OFF CACHE BOOL "" FORCE) -# ARMv8.2 with half floats and dot products: every phone that runs Android 12 or later in -# practice, and what makes Whisper fast enough to answer in under a second. -set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE) -FetchContent_MakeAvailable(whisper) - -add_library(gearslip_speech SHARED speech_jni.cpp) -target_compile_options(gearslip_speech PRIVATE -O3) -find_library(LOG_LIB log) -target_link_libraries(gearslip_speech whisper android ${LOG_LIB}) -target_link_options(gearslip_speech PRIVATE -Wl,--gc-sections -Wl,-z,max-page-size=16384) diff --git a/gearslip/src/main/cpp/speech_jni.cpp b/gearslip/src/main/cpp/speech_jni.cpp deleted file mode 100644 index bbd79c3..0000000 --- a/gearslip/src/main/cpp/speech_jni.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// The bridge between Kotlin's SpeechEngine and whisper.cpp: load a model, turn one recorded -// phrase into text, free the model. Recording and deciding when a phrase ends stay in Kotlin. - -#include -#include -#include -#include -#include -#include -#include -#include "whisper.h" - -#define TAG "GearslipSpeech" -#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) - -static whisper_context_params context_params() { - whisper_context_params p = whisper_context_default_params(); - p.use_gpu = false; - return p; -} - -// Streams a model straight out of the APK, so the bundled one is never copied to storage. -static size_t asset_read(void *ctx, void *output, size_t size) { - int n = AAsset_read(static_cast(ctx), output, size); - return n > 0 ? static_cast(n) : 0; -} -static bool asset_eof(void *ctx) { return AAsset_getRemainingLength64(static_cast(ctx)) <= 0; } -static void asset_close(void *ctx) { AAsset_close(static_cast(ctx)); } - -extern "C" JNIEXPORT jlong JNICALL -Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeInitAsset(JNIEnv *env, jobject, jobject assets, jstring jpath) { - AAssetManager *manager = AAssetManager_fromJava(env, assets); - const char *path = env->GetStringUTFChars(jpath, nullptr); - AAsset *asset = AAssetManager_open(manager, path, AASSET_MODE_STREAMING); - env->ReleaseStringUTFChars(jpath, path); - if (!asset) return 0; - whisper_model_loader loader = {asset, asset_read, asset_eof, asset_close}; - return reinterpret_cast(whisper_init_with_params(&loader, context_params())); -} - -// Bytes, not a jstring: NewStringUTF wants modified UTF-8 and aborts on some of what Whisper can -// write (four-byte characters), so Kotlin decodes the plain UTF-8 instead. -extern "C" JNIEXPORT jbyteArray JNICALL -Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeTranscribe( - JNIEnv *env, jobject, jlong handle, jfloatArray jsamples, jstring jlang, jint threads) { - auto *ctx = reinterpret_cast(handle); - if (!ctx) return env->NewByteArray(0); - jsize n = env->GetArrayLength(jsamples); - jfloat *samples = env->GetFloatArrayElements(jsamples, nullptr); - const char *lang = env->GetStringUTFChars(jlang, nullptr); - - whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); - p.n_threads = threads; - p.language = lang; - p.translate = false; - p.no_context = true; - p.no_timestamps = true; - p.single_segment = true; - p.print_progress = false; - p.print_realtime = false; - p.print_special = false; - p.print_timestamps = false; - p.suppress_blank = true; - // The ACFT models are trained for a context cut to the audio's own length (1500 frames is - // 30 s, so 50 per second), which is what makes a three-second phrase fast. A little extra - // keeps the last word from being clipped. - p.audio_ctx = std::min(1500, static_cast(std::ceil(n / 320.0)) + 32); - - int rc = whisper_full(ctx, p, samples, n); - env->ReleaseStringUTFChars(jlang, lang); - env->ReleaseFloatArrayElements(jsamples, samples, JNI_ABORT); - if (rc != 0) { - LOGI("whisper_full failed: %d", rc); - return env->NewByteArray(0); - } - std::string text; - for (int i = 0; i < whisper_full_n_segments(ctx); i++) text += whisper_full_get_segment_text(ctx, i); - jbyteArray out = env->NewByteArray(static_cast(text.size())); - env->SetByteArrayRegion(out, 0, static_cast(text.size()), reinterpret_cast(text.data())); - return out; -} - -extern "C" JNIEXPORT void JNICALL -Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeFree(JNIEnv *, jobject, jlong handle) { - if (handle) whisper_free(reinterpret_cast(handle)); -} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt index 7c9cf73..bf4608d 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt @@ -9,7 +9,6 @@ object AppSettings { private const val PREFS = "gearslip_settings" private const val KEY_STARTUP_URL = "startup_url" - private const val KEY_SEEN_COMPAT_WARNING = "seen_compat_warning" private const val KEY_SKIPPED_CERT_SETUP = "skipped_cert_setup" private const val KEY_SEEN_PERMISSIONS_SETUP = "seen_permissions_setup" private const val KEY_GEARSLIP_ENABLED = "gearslip_enabled" @@ -32,14 +31,6 @@ object AppSettings { prefs(context).edit().putString(KEY_STARTUP_URL, value.trim()).apply() } - /** Whether the newer-car compatibility warning has already been shown and dismissed once. */ - fun hasSeenCompatWarning(context: Context): Boolean = - prefs(context).getBoolean(KEY_SEEN_COMPAT_WARNING, false) - - fun setSeenCompatWarning(context: Context) { - prefs(context).edit().putBoolean(KEY_SEEN_COMPAT_WARNING, true).apply() - } - /** Whether the certificate setup screen has been deliberately skipped. */ fun hasSkippedCertSetup(context: Context): Boolean = prefs(context).getBoolean(KEY_SKIPPED_CERT_SETUP, false) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt index c35b303..0cb49cb 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt @@ -8,8 +8,10 @@ import android.hardware.usb.UsbAccessory import android.hardware.usb.UsbManager import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager +import android.net.Uri import android.os.Bundle import android.os.ParcelFileDescriptor +import android.provider.Settings import android.view.MotionEvent import android.view.Surface import android.view.WindowManager @@ -76,7 +78,13 @@ class GearslipActivity : ComponentActivity(), Projection { private val audioPermission = registerForActivityResult( androidx.activity.result.contract.ActivityResultContracts.RequestPermission(), ) { granted -> - if (granted) askAudioCapture() else { + if (granted) askAudioCapture() + else if (!shouldShowRequestPermissionRationale(android.Manifest.permission.RECORD_AUDIO)) { + // Refused twice: Android now says no without showing anything, so the phone must. + GearslipLog.w("audio: record permission blocked - Android no longer asks") + CarAudio.micBlocked() + micBlockedNotice.value = true + } else { GearslipLog.w("audio: record permission declined") CarAudio.denied() } @@ -93,6 +101,9 @@ class GearslipActivity : ComponentActivity(), Projection { } } + /** Raised when the microphone permission is blocked; see [MicBlockedDialog]. */ + private val micBlockedNotice = kotlinx.coroutines.flow.MutableStateFlow(false) + /** Raised once, after the first "no" to the capture dialog; see [AudioChoiceDialog]. */ private val offerBluetooth = kotlinx.coroutines.flow.MutableStateFlow(false) @@ -154,6 +165,18 @@ class GearslipActivity : ComponentActivity(), Projection { onKeepAsking = ::close, ) } + val micNotice by micBlockedNotice.collectAsState() + if (micNotice) { + app.seb3thehacker.gearslip.ui.MicBlockedDialog( + onOpenSettings = { + micBlockedNotice.value = false + startActivity( + Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", packageName, null)), + ) + }, + onDismiss = { micBlockedNotice.value = false }, + ) + } } } @@ -179,6 +202,10 @@ class GearslipActivity : ComponentActivity(), Projection { Prefetch.warm(this) UsageStats.onAppOpened(this) app.seb3thehacker.gearslip.update.UpdateChecker.onAppOpened(this) + // Back from the app's settings with the microphone turned on: try car audio again. + if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) == android.content.pm.PackageManager.PERMISSION_GRANTED) { + CarAudio.micAllowed() + } } override fun onNewIntent(intent: Intent) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index f80a860..d39b1bc 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -45,7 +45,10 @@ class GearslipRunner( private lateinit var identityTask: FutureTask /** Sends every message, in order, so no other thread ever blocks on the USB link. See [send]. */ - private val writer = java.util.concurrent.Executors.newSingleThreadExecutor { Thread(it, "gearslip-writer") } + private val writer = java.util.concurrent.Executors.newSingleThreadExecutor { + // Sound goes out through here too, so it gets audio's priority: late writes are skips. + Thread({ android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); it.run() }, "gearslip-writer") + } /** Waits for the certificate only when TLS is first needed; see [run]. */ private val tls: PhoneTls by lazy { identityTask.get() } @@ -88,6 +91,8 @@ class GearslipRunner( @Volatile private var lastKeyframeSentAt = 0L fun run() { + // This thread reads the car's acks; audio waits on them, so it can't be left at the back. + android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO) SessionReport.begin() log.i("--- session starting ---") SessionStatus.connecting("Connected to the head unit") @@ -260,11 +265,10 @@ class GearslipRunner( log.i("<- VersionRequest: head unit speaks $major.$minor") SessionReport.protocolVersion(major, minor) - // PROTOCOL-DOWNGRADE EXPERIMENT. The 2018 Uconnect speaks 1.3 and ACCEPTS the JVC cert; - // the 2025 unit speaks 4.3 and rejects it at AuthComplete (-3). If the 4.3 identity check - // is tied to the *negotiated* protocol version rather than baked into the firmware, - // claiming an older version makes the newer unit skip the check entirely. CLAIM_VERSION - // = null restores the previous echo-the-head-unit behaviour. + // Always 1.3, on purpose. Every car that accepts Gearslip was tested this way, 1.5 units + // included. Echoing the car's own version was tried too: a 4.3 unit rejected the + // certificate either way, so the version doesn't decide acceptance. The log says so, since + // a car offering 1.5 and getting 1.3 back looks like a bug to anyone reading it. val claim = CLAIM_VERSION ?: (major to minor) val response = byteArrayOf( ((claim.first shr 8) and 0xFF).toByte(), (claim.first and 0xFF).toByte(), @@ -274,7 +278,10 @@ class GearslipRunner( send(MSG_VERSION_RESPONSE, response, encrypted = false) log.i( "-> VersionResponse: ${claim.first}.${claim.second} status=0 (STATUS_SUCCESS)" + - if (CLAIM_VERSION != null) " [FORCED; head unit offered $major.$minor]" else "", + if (CLAIM_VERSION != null && claim != (major to minor)) { + " - intended: Gearslip always answers ${claim.first}.${claim.second}, which cars " + + "offering $major.$minor accept too" + } else "", ) state = State.TLS_HANDSHAKE SessionStatus.connecting("Securing the link") @@ -1209,9 +1216,9 @@ class GearslipRunner( private companion object { /** - * Force the protocol version Gearslip claims in VersionResponse, as (major, minor), or - * null to echo the head unit. 1.3 is what the 2018 Uconnect (which accepts the JVC cert) - * speaks. This is the 2025 identity-check downgrade experiment: see onVersionRequest. + * The protocol version Gearslip answers every car with, as (major, minor), or null to echo + * the head unit's. Kept at 1.3 because every working car was tested with it; see + * onVersionRequest. */ val CLAIM_VERSION: Pair? = 1 to 3 diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt index cbdb536..6948015 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt @@ -16,6 +16,7 @@ import android.media.AudioRecord import android.media.projection.MediaProjection import android.media.projection.MediaProjectionManager import android.os.IBinder +import android.os.Process import app.seb3thehacker.gearslip.GearslipLog import kotlin.math.abs @@ -147,6 +148,9 @@ class AudioCaptureService : Service() { reading = true thread = Thread({ + // A busy phone (a map app drawing, the video encoder) would otherwise starve this + // loop, and the car hears every late read as a skip. + Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO) val buffer = ByteArray(CHUNK_BYTES) var peak = 0 var heardAt = System.currentTimeMillis() diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index d904be6..f250b1c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -147,7 +147,8 @@ class AudioLink( fun onFocus(state: Int) { val name = FOCUS_STATES[state] ?: "unknown($state)" GearslipLog.i("audio: <- AudioFocusResponse state=$name") - focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT + focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT || + state == FOCUS_STATE_GAIN_MEDIA_ONLY if (focusGranted) startStream() else stop() } @@ -257,13 +258,16 @@ class AudioLink( private const val FOCUS_RELEASE = 4 private const val FOCUS_STATE_GAIN = 1 private const val FOCUS_STATE_GAIN_TRANSIENT = 2 + private const val FOCUS_STATE_GAIN_MEDIA_ONLY = 6 private const val SESSION_ID = 2 private const val STALL_MS = 2_000L private const val CHUNK_LOG_EVERY = 200 // about every four seconds private const val CONFIG_WAIT_MS = 1_000L + // AudioFocusStateNotification.AudioFocusState in aasdk. private val FOCUS_STATES = mapOf( - 1 to "GAIN", 2 to "GAIN_TRANSIENT", 3 to "LOSS", 4 to "LOSS_TRANSIENT", + 1 to "GAIN", 2 to "GAIN_TRANSIENT", 3 to "LOSS", 4 to "LOSS_TRANSIENT_CAN_DUCK", + 5 to "LOSS_TRANSIENT", 6 to "GAIN_MEDIA_ONLY", 7 to "GAIN_TRANSIENT_GUIDANCE_ONLY", ) } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt index 13a0073..21c6cf2 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/CarAudio.kt @@ -100,6 +100,26 @@ object CarAudio { _capture.value = Capture.DENIED } + private val _micBlocked = MutableStateFlow(false) + + /** The microphone permission is off for good: Android refuses without asking, so say why. */ + val micBlocked: StateFlow = _micBlocked.asStateFlow() + + fun micBlocked() { + _micBlocked.value = true + denied() + } + + /** The driver turned the permission back on mid-drive: ask for capture again. */ + fun micAllowed() { + if (!_micBlocked.value) return + _micBlocked.value = false + if (_capture.value == Capture.DENIED) { + _capture.value = Capture.OFF + if (wanted) request() + } + } + internal fun captureStarted() { _capture.value = Capture.ON link?.start() diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt index 12cd633..aaa448c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantCommands.kt @@ -1,10 +1,13 @@ package app.seb3thehacker.gearslip.car import android.content.Context +import app.seb3thehacker.gearslip.media.CarMedia import java.text.DateFormat import java.util.Date import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay import kotlinx.coroutines.withContext +import kotlinx.coroutines.withTimeoutOrNull /** * Turns what [CarAssistant] heard into an action plus something to say back. @@ -17,9 +20,15 @@ import kotlinx.coroutines.withContext object AssistantCommands { suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String { - val heard = heardRaw.lowercase().trim() + val heard = natural(heardRaw) + VoiceHelp.hide() + if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) { + VoiceHelp.show() + return "Here's what you can say." + } navigateCommand(context, navigator, frame, heard)?.let { return it } + playSearchCommand(context, navigator, heard)?.let { return it } mediaCommand(heard)?.let { return it } navCommand(navigator, heard)?.let { return it } infoCommand(heard)?.let { return it } @@ -29,6 +38,42 @@ object AssistantCommands { return "I don't know how to do that yet." } + /** + * What was said, minus the padding people wrap a command in: "hey", "can you", "I want to", + * "please", "thanks". Each command below then only has to know its own words. + */ + private fun natural(raw: String): String { + var t = raw.lowercase().replace(Regex("[.,!?]"), " ").replace(Regex("\\s+"), " ").trim() + var changed = true + while (changed) { + changed = false + for (lead in LEAD_FILLERS) if (t.startsWith("$lead ")) { t = t.removePrefix("$lead ").trim(); changed = true } + for (tail in TAIL_FILLERS) if (t.endsWith(" $tail")) { t = t.removeSuffix(" $tail").trim(); changed = true } + } + return t + } + + /** The part after the first of [triggers] that [text] starts with, or null. */ + private fun after(text: String, triggers: List): String? = + triggers.firstOrNull { text.startsWith(it) }?.let { text.removePrefix(it).trim() } + + private suspend fun allEntries(context: Context): List { + val vehicleData = CarSettings.experimentalFeaturesEnabled.value && CarSettings.carSensorsEnabled.value + val builtIn = BuiltInApps.entries.filter { it.builtIn !== BuiltInApps.VehicleData || vehicleData } + return builtIn + (LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }) + } + + /** Best label match: exact, then a label starting with [name], then one containing it. */ + private fun bestMatch(items: List, name: String, label: (T) -> String): T? { + val n = name.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim() + if (n.isEmpty()) return null + items.firstOrNull { label(it).lowercase() == n }?.let { return it } + // Loose matches need a real word: "pizza in a" mustn't pick "Organic Maps" for its "a". + if (n.length < 3) return null + return items.firstOrNull { label(it).lowercase().startsWith(n) } + ?: items.firstOrNull { label(it).lowercase().contains(n) } + } + /** * "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it, * whichever nav app was used last, or the only one installed, or the first one found. Starts @@ -42,29 +87,45 @@ object AssistantCommands { frame: CarEnvironment.Frame, heard: String, ): String? { - val trigger = NAVIGATE_TRIGGERS.firstOrNull { heard.startsWith(it) } ?: return null - var destination = heard.removePrefix(trigger).trim() + // "use Organic Maps to get to X", "open OsmAnd and navigate to X": the app comes first. + var text = heard + var leadingApp: String? = null + Regex("^(?:use|using|with|open|in|on) (.+?) (?:to |and )(.+)$").find(text)?.let { m -> + if (after(m.groupValues[2], NAVIGATE_TRIGGERS) != null) { + leadingApp = m.groupValues[1] + text = m.groupValues[2] + } + } + var destination = after(text, NAVIGATE_TRIGGERS) ?: return null if (destination.isBlank()) return "Where to?" - var appName: String? = null - for (sep in listOf(" with ", " using ", " on ", " via ")) { - val at = destination.lastIndexOf(sep) - if (at > 0) { - appName = destination.substring(at + sep.length).trim() + val entries = allEntries(context) + // "go to Spotify" means the app, and "go to settings" the screen; "go to Target" is a place. + val asApp = destination.removePrefix("the ").removeSuffix(" app") + if (text.startsWith("go to ") && + (asApp in SCREEN_WORDS || asApp in APP_NICKNAMES || entries.any { it.label.lowercase() == asApp }) + ) { + return null + } + val navApps = entries.filter { it.template?.isNavigation == true } + if (navApps.isEmpty()) return "You don't have any navigation apps installed." + + // "X with Organic Maps", "X on OsmAnd": split only when the end names a nav app, so a + // place like "Starbucks on Main Street" stays whole. + var chosenApp = leadingApp?.let { name -> bestMatch(navApps, name) { it.label } ?: return "I couldn't find a navigation app called $name." } + if (chosenApp == null) { + for (sep in listOf(" with ", " using ", " on ", " via ", " in ", " through ")) { + val at = destination.lastIndexOf(sep) + if (at <= 0) continue + val app = bestMatch(navApps, destination.substring(at + sep.length)) { it.label } ?: continue + chosenApp = app destination = destination.substring(0, at).trim() break } } if (destination.isBlank()) return "Where to?" - val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) } - val navApps = entries.filter { it.template?.isNavigation == true } - if (navApps.isEmpty()) return "You don't have any navigation apps installed." - - val requestedApp = appName - val chosen = when { - requestedApp != null -> navApps.firstOrNull { it.label.lowercase().contains(requestedApp) } - ?: return "I couldn't find a navigation app called $requestedApp." + val chosen = chosenApp ?: when { navApps.size == 1 -> navApps.first() else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first() } @@ -75,26 +136,84 @@ object AssistantCommands { return "Navigating to $destination with ${chosen.label}." } + /** + * "play X" with something after it: a song, artist, album or playlist, asked of the music app + * that's open through its own search ([CarMedia.search]). A bare "play", or "play music", + * falls through to resume. + */ + private suspend fun playSearchCommand(context: Context, navigator: CarNavigator, heard: String): String? { + var query = after(heard, PLAY_TRIGGERS) ?: return null + query = query.removePrefix("some ").trim() + if (query.isEmpty() || query in RESUME_WORDS) return null + val media = CarServices.media + + // "play Daft Punk on Metrolist": switch to that app first, if it's one we know. + for (sep in listOf(" on ", " in ", " from ", " with ", " using ")) { + val at = query.lastIndexOf(sep) + if (at <= 0) continue + val apps = withContext(Dispatchers.IO) { app.seb3thehacker.gearslip.media.MediaCatalog.installed(context) } + val wanted = bestMatch(apps, query.substring(at + sep.length)) { it.label } ?: continue + query = query.substring(0, at).trim() + if (query.isEmpty() || query in RESUME_WORDS) { + CarServices.openMedia(wanted, autoplay = true) + navigator.media() + return "Opening ${wanted.label}." + } + if (CarServices.mediaApp.value?.component != wanted.component) { + CarServices.openMedia(wanted) + navigator.media() + // Give it a moment to connect before asking it to search. + withTimeoutOrNull(APP_CONNECT_MS) { + while (media.appLabel == null || (!media.canSearchLibrary && !media.now.value.canSearch)) delay(200) + } + } + break + } + + // No music app open, or its connection is gone (the app was closed): use the one played + // last, the way "play X" works on Android Auto. + val phase = media.phase.value + if (media.appLabel == null || phase == CarMedia.Phase.IDLE || phase == CarMedia.Phase.REJECTED) { + if (!CarServices.reopenLastMedia()) return "Open a music app first, then ask again." + navigator.media() + } + withTimeoutOrNull(APP_CONNECT_MS) { + while (media.phase.value != CarMedia.Phase.READY || media.appLabel == null || + (!media.canSearchLibrary && !media.now.value.canSearch) + ) delay(200) + } + val app = media.appLabel ?: return "Your music app didn't open in time. Try again." + if (!media.canSearchLibrary && !media.now.value.canSearch) { + return "$app ignores voice searches. Pick something on its screen instead." + } + media.playBestMatch(query) + return "Looking for $query on $app." + } + private fun mediaCommand(heard: String): String? { val media = CarServices.media return when { - "pause" in heard || heard.startsWith("stop") -> { + "pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> { if (media.now.value.playing) media.togglePlay() "Paused." } - "resume" in heard || (heard.startsWith("play") && "playing" !in heard) -> { + "resume" in heard || "unpause" in heard || heard in RESUME_PHRASES || + (heard.startsWith("play") && "playing" !in heard) -> { if (!media.now.value.playing) media.togglePlay() "Playing." } - "next" in heard || "skip" in heard -> { + "next" in heard || "skip" in heard || heard.startsWith("change the song") -> { media.next() "Skipping." } - "previous" in heard || "last song" in heard || "back song" in heard -> { + "previous" in heard || "last song" in heard || "back song" in heard || "go back a song" in heard || + "play that again" in heard || "replay" in heard -> { media.previous() "Playing the previous track." } - "what" in heard && "playing" in heard -> { + ("what" in heard && "playing" in heard) || "what song" in heard || "who sings" in heard || + "who's singing" in heard || "who is singing" in heard || "what's this song" in heard || + "name of this song" in heard -> { val now = media.now.value if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ") else "Nothing's playing." @@ -104,11 +223,19 @@ object AssistantCommands { } private fun navCommand(navigator: CarNavigator, heard: String): String? = when { - "go home" in heard || heard == "home" -> { + heard == "go back" || heard == "back" -> { + navigator.back() + "Going back." + } + heard in EXIT_WORDS -> { + BuiltInApps.Exit.open(navigator) + "Back to the car's screen." + } + "go home" in heard || heard == "home" || heard == "home screen" || "go to the home screen" in heard -> { navigator.home() "Heading home." } - "open apps" in heard || heard == "apps" || "app list" in heard -> { + "open apps" in heard || heard == "apps" || "app list" in heard || "all apps" in heard || "my apps" in heard || heard == "go to apps" -> { navigator.apps() "Here are your apps." } @@ -128,9 +255,10 @@ object AssistantCommands { } private fun infoCommand(heard: String): String? = when { - "what time" in heard || "what's the time" in heard -> + "what time" in heard || "what's the time" in heard || heard == "time" || "the time" in heard -> "It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}." - "weather" in heard -> { + "weather" in heard || "going to rain" in heard || "gonna rain" in heard || "how hot" in heard || + "how cold" in heard || "temperature outside" in heard -> { val data = (Weather.state.value as? WeatherState.Ready)?.data if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}." else "I don't have the weather yet." @@ -139,12 +267,11 @@ object AssistantCommands { } private suspend fun callCommand(context: Context, heard: String): String? { - val name = when { - heard.startsWith("call ") -> heard.removePrefix("call ") - heard.startsWith("phone ") -> heard.removePrefix("phone ") - heard.startsWith("dial ") -> heard.removePrefix("dial ") - else -> return null - }.trim() + val name = ( + after(heard, CALL_TRIGGERS) + ?: Regex("^give (.+) a (?:call|ring)$").find(heard)?.groupValues?.get(1) + ?: return null + ).replace(Regex(" (?:on|at) (?:their |his |her )?(?:mobile|cell|phone|home|work)$"), "").trim() if (name.isBlank()) return "Who should I call?" if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen." @@ -162,21 +289,59 @@ object AssistantCommands { frame: CarEnvironment.Frame, heard: String, ): String? { - val name = when { - heard.startsWith("open ") -> heard.removePrefix("open ") - heard.startsWith("launch ") -> heard.removePrefix("launch ") - heard.startsWith("start ") -> heard.removePrefix("start ") - else -> return null - }.trim() + val name = (after(heard, OPEN_TRIGGERS) ?: return null) + .removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim() if (name.isBlank()) return null + val wanted = APP_NICKNAMES[name] ?: name - val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) } - val match = entries.firstOrNull { it.label.lowercase().contains(name) } + // Gearslip's own apps come first in the list, so "phone" opens its dialer, not the phone's. + val match = bestMatch(allEntries(context), wanted) { it.label } ?: return "I couldn't find an app called $name." launchEntry(match, navigator, frame) return "Opening ${match.label}." } - private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ") + private val EXIT_WORDS = setOf("exit", "exit gearslip", "close gearslip", "quit", "quit gearslip", "open exit") + + /** Other names people use for Gearslip's own apps. */ + private val APP_NICKNAMES = mapOf( + "web" to "browser", "internet" to "browser", "the web" to "browser", "web browser" to "browser", + "dialer" to "phone", "the dialer" to "phone", "keypad" to "phone", + "screen share" to "screen sharing", "screen mirroring" to "screen sharing", "mirroring" to "screen sharing", + ) + + private val NAVIGATE_TRIGGERS = listOf( + "navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ", + "take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ", + "how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ", + ) + + private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ") + + private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ") + + private val OPEN_TRIGGERS = listOf("open ", "launch ", "start ", "go to ", "switch to ", "show me ", "bring up ", "pull up ") + + /** Screens "go to X" means rather than a place. */ + private val SCREEN_WORDS = setOf("settings", "home", "the home screen", "home screen", "apps", "my apps", "all apps") + + /** Padding stripped from the front of a command, longest first where they overlap. */ + private val LEAD_FILLERS = listOf( + "hey gearslip", "ok gearslip", "okay gearslip", "gearslip", "hey", "ok", "okay", "um", "uh", "so", + "please", "can you please", "can you", "could you", "would you", "will you", "can i", "could i", + "i want you to", "i want to", "i'd like to", "i would like to", "i wanna", "i need to", "i need you to", + "let's", "lets", "go ahead and", "just", "now", "quickly", + ) + + /** Longest first, so "right now" goes whole rather than leaving "right". */ + private val TAIL_FILLERS = listOf("thank you very much", "thank you", "right now", "for me", "please", "thanks", "now") + + private const val APP_CONNECT_MS = 5_000L + + private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands") + + private val RESUME_WORDS = setOf("music", "the music", "some music", "something", "it", "again", "that", "my music") + + private val RESUME_PHRASES = setOf("keep playing", "continue", "keep going", "start the music", "music on", "turn the music back on") } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt index 8c7a161..35f5814 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/AssistantOverlay.kt @@ -4,7 +4,21 @@ import app.seb3thehacker.gearslip.car.theme.* import android.content.Intent import android.net.Uri import android.provider.Settings +import androidx.compose.animation.core.FastOutSlowInEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.ui.draw.drawBehind +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.scale +import androidx.compose.ui.text.style.TextAlign +import androidx.compose.ui.text.style.TextOverflow import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.Row import androidx.compose.foundation.layout.fillMaxWidth @@ -30,47 +44,66 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp /** - * The band that appears while [CarAssistant] is listening, thinking or speaking - a thin strip - * across the top, same idea as [CallOverlay]'s active-call banner, so it never blocks the screen - * the driver was already looking at. + * What the assistant hears and says, laid over the bottom of whatever screen is up: a dark shade + * so white words read over a bright map, and a glow along the bottom edge in the phone's own + * colours that breathes while listening. The shade passes touches through; only the buttons take them. */ @Composable fun AssistantOverlay(modifier: Modifier = Modifier) { val state by CarAssistant.state.collectAsState() if (state.phase == CarAssistant.Phase.IDLE) return val context = LocalContext.current + val listening = state.phase == CarAssistant.Phase.LISTENING && state.ready + val glow = MaterialTheme.colorScheme.primary + val glow2 = MaterialTheme.colorScheme.tertiary + val breathe by rememberInfiniteTransition(label = "glow").animateFloat( + 0.55f, 1f, infiniteRepeatable(tween(900, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "glow", + ) + val strength = when (state.phase) { + CarAssistant.Phase.LISTENING -> if (state.ready) breathe else 0.3f + CarAssistant.Phase.ERROR -> 0.3f + else -> 0.6f + } - Surface( - color = MaterialTheme.colorScheme.primaryContainer, - contentColor = MaterialTheme.colorScheme.onPrimaryContainer, - modifier = modifier.fillMaxWidth(), - ) { - Row( - Modifier.padding(horizontal = 16.dp, vertical = 10.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(12.dp), - ) { - PulsingMic(state.phase == CarAssistant.Phase.LISTENING) - Column(Modifier.weight(1f)) { - Text( - when (state.phase) { - CarAssistant.Phase.LISTENING -> "Listening…" - CarAssistant.Phase.THINKING -> "…" - CarAssistant.Phase.SPEAKING -> state.reply - CarAssistant.Phase.ERROR -> state.reply - CarAssistant.Phase.IDLE -> "" - }, - style = MaterialTheme.typography.titleSmall, - fontWeight = FontWeight.SemiBold, - maxLines = 2, - ) - if (state.heard.isNotBlank() && state.phase != CarAssistant.Phase.SPEAKING) { - Text( - "\"${state.heard}\"", - style = MaterialTheme.typography.bodySmall, + Box( + modifier + .fillMaxWidth() + .drawBehind { + drawRect(Brush.verticalGradient(0f to Color.Transparent, 0.45f to Color.Black.copy(alpha = 0.6f), 1f to Color.Black.copy(alpha = 0.85f))) + // A wide, flat oval of light rising from the bottom edge. + scale(1f, 0.22f, Offset(size.width / 2, size.height)) { + drawCircle( + Brush.radialGradient( + 0f to glow.copy(alpha = 0.9f * strength), 0.5f to glow2.copy(alpha = 0.35f * strength), 1f to Color.Transparent, + center = Offset(size.width / 2, size.height), radius = size.width * 0.55f, + ), + radius = size.width * 0.55f, center = Offset(size.width / 2, size.height), ) } } + .padding(start = 16.dp, end = 16.dp, top = 56.dp, bottom = 20.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(16.dp)) { + PulsingMic(listening) + Column(Modifier.weight(1f), horizontalAlignment = Alignment.CenterHorizontally) { + val (small, big) = when (state.phase) { + CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { if (state.ready) "Listening…" else "One moment…" } + CarAssistant.Phase.THINKING -> "" to state.heard.ifBlank { "…" } + else -> state.heard to state.reply + } + if (small.isNotBlank()) { + Text(small, color = Color.White.copy(alpha = 0.7f), style = MaterialTheme.typography.bodyLarge, maxLines = 1, overflow = TextOverflow.Ellipsis) + } + Text( + big, + color = if (big == "Listening…" || big == "One moment…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White, + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.SemiBold, + textAlign = TextAlign.Center, + maxLines = 2, + overflow = TextOverflow.Ellipsis, + ) + } if (state.missingPermission) { GsButton(tone = GsTone.Tonal, onClick = { context.startActivity( diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarApps.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarApps.kt index 51e41f8..bea237a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarApps.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarApps.kt @@ -22,7 +22,7 @@ class CarApp( object CarApps { val all: List = listOf( - CarApp("web", "Web", Icons.Filled.Search) { + CarApp("web", "Browser", Icons.Filled.Search) { WebApp(content = ProjectedContent.resolve(LocalContext.current)) }, CarApp("phone", "Screen sharing", Icons.Filled.Share) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt index cf4b27a..96e564d 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt @@ -38,6 +38,8 @@ object CarAssistant { val heard: String = "", val reply: String = "", val missingPermission: Boolean = false, + /** False from the press until the microphone is actually open, so the overlay doesn't claim to listen early. */ + val ready: Boolean = false, ) private val _state = MutableStateFlow(State()) @@ -89,6 +91,7 @@ object CarAssistant { tts?.stop() _state.value = State(Phase.LISTENING) listenOnce( + onListening = { _state.value = _state.value.copy(ready = true) }, onPartial = { text -> _state.value = _state.value.copy(heard = text) }, onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) }, onResult = { text -> @@ -174,15 +177,16 @@ object CarAssistant { onSpeaking: () -> Unit = {}, ) { cancel() - listenOnce(onPartial, onHeard = {}, onResult, onError, cue, onSpeaking) + listenOnce(onPartial, onHeard = {}, onResult = onResult, onError = onError, cue = cue, onSpeaking = onSpeaking) } /** - * One phrase, through Gearslip's own [SpeechEngine]. The phone's recognizer is only a fallback - * for when the engine itself can't run, since plenty of phones (GrapheneOS among them) have none. + * One phrase, through Gearslip's own [SpeechEngine], or the phone's recognizer when the driver + * chose it in settings. Never the phone's on its own: on most phones that's Google's. */ private fun listenOnce( onPartial: (String) -> Unit, + onListening: () -> Unit = {}, onHeard: () -> Unit, onResult: (String) -> Unit, onError: (String) -> Unit, @@ -190,24 +194,24 @@ object CarAssistant { onSpeaking: () -> Unit = {}, ) { val context = appContext ?: return onError("Speech isn't ready yet.") + if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) { + return systemListen(context, onListening, onPartial, onHeard, onResult, onError) + } SpeechEngine.listen( context, cue = cue, onSpeaking = onSpeaking, + onPartial = onPartial, + onListening = onListening, onHeard = onHeard, onResult = onResult, - onError = { message -> - if (message == SpeechEngine.NOT_LOADED && SpeechRecognizer.isRecognitionAvailable(context)) { - systemListen(context, onPartial, onHeard, onResult, onError) - } else { - onError(message) - } - }, + onError = onError, ) } private fun systemListen( context: Context, + onListening: () -> Unit, onPartial: (String) -> Unit, onHeard: () -> Unit, onResult: (String) -> Unit, @@ -216,7 +220,7 @@ object CarAssistant { val r = SpeechRecognizer.createSpeechRecognizer(context) recognizer = r r.setRecognitionListener(object : RecognitionListener { - override fun onReadyForSpeech(params: Bundle?) {} + override fun onReadyForSpeech(params: Bundle?) = onListening() override fun onBeginningOfSpeech() {} override fun onRmsChanged(rmsdB: Float) {} override fun onBufferReceived(buffer: ByteArray?) {} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeyboard.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeyboard.kt index 0dfa2fc..6139770 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeyboard.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeyboard.kt @@ -138,18 +138,18 @@ fun CarKeyboard( } // Control row: no backspace here - it stays with the letters above, not next to the - // key most likely to be hit right after it by mistake. Dismiss (when offered) is the - // last key, bottom-right corner of the keyboard. + // key most likely to be hit right after it by mistake. Search/enter takes the + // bottom-right corner, where a thumb expects it; dismiss (when offered) sits left of it. Row( Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(4.dp), ) { Key(if (symbols) "abc" else "?123", Modifier.weight(1.4f)) { symbols = !symbols } Key("space", Modifier.weight(if (onDismiss != null) 3.2f else 4f)) { onTextChange("$text ") } - SubmitKey(Modifier.weight(1.6f), submitIcon, submitImage, onSubmit) if (onDismiss != null) { DismissKey(onDismiss, Modifier.weight(1.4f)) } + SubmitKey(Modifier.weight(1.6f), submitIcon, submitImage, onSubmit) } } } @@ -222,9 +222,9 @@ private fun SubmitKey(modifier: Modifier, search: Boolean, image: ImageVector?, * Big enough to hit on a bumpy road, small enough that four rows plus a field plus a few results * still fit. A 480-tall screen sets the floor; a taller one (LIVI's 1280x720) gets bigger keys. */ -private fun keyHeight(screenHeightDp: Int): Dp = (screenHeightDp * 0.075f).dp.coerceIn(36.dp, 52.dp) +private fun keyHeight(screenHeightDp: Int): Dp = (screenHeightDp * 0.09f).dp.coerceIn(44.dp, 64.dp) -private val LocalKeyHeight = compositionLocalOf { 36.dp } +private val LocalKeyHeight = compositionLocalOf { 44.dp } private val LETTER_ROWS = listOf( "qwertyuiop".map { it.toString() }, diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt index ed4b4f0..3eed2a1 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt @@ -102,15 +102,10 @@ object CarKeys { /** * The voice key: answers the message on screen if there is one, otherwise starts the - * assistant when it's turned on. + * assistant. */ private fun voice() = main.post { - if (VoiceReply.onVoiceKey()) return@post - if (CarSettings.experimentalFeaturesEnabled.value && CarSettings.voiceAssistantEnabled.value) { - CarAssistant.start() - } else { - GearslipLog.i("keys: voice pressed, but the assistant is off in settings") - } + if (!VoiceReply.onVoiceKey()) CarAssistant.start() } /** Answers a ringing call; with nothing ringing, opens the phone screen. */ diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt index 6c6d920..a4992dc 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt @@ -247,6 +247,8 @@ object CarServices { runCatching { media.disconnect() } runCatching { browse.disconnect() } CarAssistant.cancel() + // About 900 MB that nothing needs once the car's gone. + app.seb3thehacker.gearslip.speech.SpeechEngine.unload() CarAudio.release() _mediaApp.value = null navStopped = false diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index f59ddbd..a0b3024 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -38,9 +38,8 @@ object CarSettings { private const val KEY_PINNED = "car_pinned_apps" private const val KEY_APP_ORDER = "car_app_order" private const val KEY_EXPERIMENTAL_FEATURES = "car_experimental_features" - private const val KEY_VOICE_ASSISTANT = "car_voice_assistant" + private const val KEY_SYSTEM_SPEECH = "car_system_speech" private const val KEY_CAR_SENSORS = "car_sensors_enabled" - private const val KEY_MEDIA_SEARCH = "car_media_search_enabled" private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning" private const val KEY_UNITS = "car_units" private const val KEY_WHATS_NEW = "car_whats_new_after_updates" @@ -72,9 +71,8 @@ object CarSettings { private val pinnedFlow = MutableStateFlow>(emptySet()) private val appOrderFlow = MutableStateFlow>(emptyList()) private val experimentalFeaturesFlow = MutableStateFlow(false) - private val voiceAssistantFlow = MutableStateFlow(false) + private val systemSpeechFlow = MutableStateFlow(false) private val carSensorsFlow = MutableStateFlow(false) - private val mediaSearchFlow = MutableStateFlow(false) private val seenSafetyWarningFlow = MutableStateFlow(false) private val unitsFlow = MutableStateFlow(Units.AUTO) @@ -170,12 +168,8 @@ object CarSettings { */ val experimentalFeaturesEnabled: StateFlow = experimentalFeaturesFlow - /** - * The voice assistant is experimental and off by default; a driver has to turn it on - * knowingly, from the settings screen, after reading the warning shown there. This flag - * alone does not run the assistant - [experimentalFeaturesEnabled] must also be on. - */ - val voiceAssistantEnabled: StateFlow = voiceAssistantFlow + /** Listen with the phone's own speech service instead of Gearslip's on-device model. */ + val systemSpeech: StateFlow = systemSpeechFlow /** * Vehicle data is experimental and off by default: the launcher's tile for it, and the @@ -185,13 +179,6 @@ object CarSettings { */ val carSensorsEnabled: StateFlow = carSensorsFlow - /** - * Searching inside a media app is experimental and off by default: many apps claim to take - * searches and then ignore them or play something else. The media screen's search button - * shows only once this and [experimentalFeaturesEnabled] are both on. - */ - val mediaSearchEnabled: StateFlow = mediaSearchFlow - /** * Whether the car screen's own "drive safely" warning has already been shown and dismissed * once, from this same install. Separate from the phone side's own compat/safety dialogs - @@ -226,9 +213,8 @@ object CarSettings { appOrderFlow.value = AppSettings.getString(app, KEY_APP_ORDER, "") .split(",").filter { it.isNotEmpty() } experimentalFeaturesFlow.value = AppSettings.getString(app, KEY_EXPERIMENTAL_FEATURES, "false") == "true" - voiceAssistantFlow.value = AppSettings.getString(app, KEY_VOICE_ASSISTANT, "false") == "true" + systemSpeechFlow.value = AppSettings.getString(app, KEY_SYSTEM_SPEECH, "false") == "true" carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true" - mediaSearchFlow.value = AppSettings.getString(app, KEY_MEDIA_SEARCH, "false") == "true" seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true" unitsFlow.value = runCatching { Units.valueOf(AppSettings.getString(app, KEY_UNITS, Units.AUTO.name)) @@ -293,9 +279,9 @@ object CarSettings { AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString()) } - fun setVoiceAssistantEnabled(on: Boolean) { - voiceAssistantFlow.value = on - AppSettings.putString(app, KEY_VOICE_ASSISTANT, on.toString()) + fun setSystemSpeech(on: Boolean) { + systemSpeechFlow.value = on + AppSettings.putString(app, KEY_SYSTEM_SPEECH, on.toString()) } fun setCarSensorsEnabled(on: Boolean) { @@ -303,11 +289,6 @@ object CarSettings { AppSettings.putString(app, KEY_CAR_SENSORS, on.toString()) } - fun setMediaSearchEnabled(on: Boolean) { - mediaSearchFlow.value = on - AppSettings.putString(app, KEY_MEDIA_SEARCH, on.toString()) - } - fun setNightMode(mode: NightMode) { nightFlow.value = mode AppSettings.putString(app, KEY_NIGHT, mode.name) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index 6f0f53f..2fec3e0 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -58,9 +58,8 @@ fun CarSettingsScreen() { val theme by CarSettings.appTheme.collectAsState() val uiTheme by CarSettings.uiTheme.collectAsState() val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState() - val voiceAssistant by CarSettings.voiceAssistantEnabled.collectAsState() val carSensors by CarSettings.carSensorsEnabled.collectAsState() - val mediaSearch by CarSettings.mediaSearchEnabled.collectAsState() + val systemSpeech by CarSettings.systemSpeech.collectAsState() val display by CarEnvironment.display.collectAsState() val vehicle by CarEnvironment.vehicle.collectAsState() val context = LocalContext.current @@ -138,6 +137,14 @@ fun CarSettingsScreen() { color = MaterialTheme.colorScheme.onSurfaceVariant, ) + ExperimentalFeatureRow( + title = "Use the phone's speech recognition", + description = "Listens through the phone's own speech service instead of Gearslip's. On " + + "most phones that's Google's, which may send your voice to Google. GrapheneOS has " + + "none, so Gearslip keeps using its own.", + checked = systemSpeech, + ) { CarSettings.setSystemSpeech(it) } + WebPageSetting() ScreenFit(vehicle) @@ -148,7 +155,7 @@ fun CarSettingsScreen() { experimentalFeatures, ) { CarSettings.setExperimentalFeaturesEnabled(it) } if (experimentalFeatures) { - ExperimentalFeaturesList(voiceAssistant, carSensors, mediaSearch) + ExperimentalFeaturesList(carSensors) } Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { @@ -205,21 +212,11 @@ private fun ChoiceRow(title: String, options: List>, selecte * so the list reads fine whether there's one entry or several. */ @Composable -private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolean, mediaSearch: Boolean) { +private fun ExperimentalFeaturesList(carSensors: Boolean) { Column( Modifier.fillMaxWidth(), verticalArrangement = Arrangement.spacedBy(10.dp), ) { - ExperimentalFeatureRow( - title = "Voice assistant", - description = "Push-to-talk and the car's voice button start listening: navigate, " + - "play media, call, or open an app by saying so. Can mishear you or act on the " + - "wrong command - keep your eyes on the road, and confirm anything it does " + - "before you rely on it.", - warning = true, - checked = voiceAssistant, - ) { CarSettings.setVoiceAssistantEnabled(it) } - ExperimentalFeatureRow( title = "Vehicle data", description = "Adds a Vehicle data tile: a live, raw dump of the head unit's sensor " + @@ -228,13 +225,6 @@ private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolea "guess, not a fact, until you've watched one change for yourself.", checked = carSensors, ) { CarSettings.setCarSensorsEnabled(it) } - - ExperimentalFeatureRow( - title = "Media search", - description = "Adds a search button to the media screen. Some apps ignore the search " + - "or play something else.", - checked = mediaSearch, - ) { CarSettings.setMediaSearchEnabled(it) } } } @@ -272,7 +262,7 @@ private fun ExperimentalFeatureRow( } } -/** The URL (or raw HTML) the Web app shows. Edited with [CarKeyboard] - no phone in hand needed. */ +/** The URL (or raw HTML) the Browser app shows. Edited with [CarKeyboard] - no phone in hand needed. */ @Composable private fun WebPageSetting() { val context = LocalContext.current @@ -297,7 +287,7 @@ private fun WebPageSetting() { ) } Text( - "Shown by the Web app on the launcher. Applies on the next connection.", + "Shown by the Browser app on the launcher. Applies on the next connection.", style = MaterialTheme.typography.bodySmall, color = MaterialTheme.colorScheme.onSurfaceVariant, ) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt index c72f681..99649b7 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt @@ -256,8 +256,9 @@ fun CarUi() { modifier = Modifier.align(Alignment.TopEnd), ) // Above the popup: a call is more urgent than any notification. + VoiceHelpOverlay(Modifier.align(Alignment.Center)) CallOverlay(Modifier.align(Alignment.TopCenter)) - AssistantOverlay(Modifier.align(Alignment.TopCenter)) + AssistantOverlay(Modifier.align(Alignment.BottomCenter)) if (app.seb3thehacker.gearslip.BuildConfig.DEBUG) { // Bottom corner, not top: a screen's own header almost always puts // something in the top-right (settings, search, a Liked Songs @@ -469,12 +470,8 @@ private fun CarNavBar(navigator: CarNavigator) { NavHomeButton(navigator, entries, running) Spacer(Modifier.width(6.dp)) NavItem(MediaIcons.Apps, navigator.current == CarScreen.Apps) { navigator.apps() } - val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState() - val voiceEnabled by CarSettings.voiceAssistantEnabled.collectAsState() - if (experimentalFeatures && voiceEnabled) { - Spacer(Modifier.width(6.dp)) - NavAssistantIcon() - } + Spacer(Modifier.width(6.dp)) + NavAssistantIcon() // Pinned shortcuts, then whatever else is open right now. A pinned app that's also // open stays in its pinned spot, marked with a dot, instead of showing up twice. diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MediaScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MediaScreen.kt index 40fc17e..13cf66e 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MediaScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MediaScreen.kt @@ -61,7 +61,6 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.style.TextOverflow import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle -import app.seb3thehacker.gearslip.audio.CarAudio import app.seb3thehacker.gearslip.media.CarMedia import app.seb3thehacker.gearslip.media.MediaApp import app.seb3thehacker.gearslip.media.MediaArt @@ -88,11 +87,9 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) { val queue by media.queue.collectAsStateWithLifecycle() val browse by media.browse.collectAsStateWithLifecycle() val rejection by media.rejection.collectAsStateWithLifecycle() - val capture by CarAudio.capture.collectAsStateWithLifecycle() val search by media.search.collectAsStateWithLifecycle() var tab by remember { mutableStateOf(Tab.NOW_PLAYING) } var searching by remember { mutableStateOf(false) } - val searchAllowed = CarSettings.experimentalFeaturesEnabled.collectAsState().value && CarSettings.mediaSearchEnabled.collectAsState().value // The connection outlives this screen: the home screen's player is the same one. LaunchedEffect(app) { CarServices.openMedia(app) } @@ -174,7 +171,6 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) { } else { CompactPlayer(now, media, Modifier.weight(1f).fillMaxWidth()) } - AudioBadge(capture) } Column(Modifier.weight(1.2f).fillMaxHeight()) { Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) { @@ -182,7 +178,7 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) { MediaTab("Browse", tab == Tab.BROWSE, Modifier.weight(1f)) { tab = Tab.BROWSE } MediaTab("Up next", tab == Tab.QUEUE, Modifier.weight(1f)) { tab = Tab.QUEUE } MediaTab("Lyrics", tab == Tab.LYRICS, Modifier.weight(1f)) { tab = Tab.LYRICS } - if (searchAllowed && (now.canSearch || media.canSearchLibrary)) { + if (now.canSearch || media.canSearchLibrary) { GsIconButton( Icons.Filled.Search, "Search ${app.label}", @@ -447,17 +443,6 @@ private sealed interface LyricsState { class Found(val result: LyricsResult) : LyricsState } -@Composable -private fun AudioBadge(capture: CarAudio.Capture) { - val text = when (capture) { - CarAudio.Capture.ON -> "Audio to car" - CarAudio.Capture.ASKING -> "Allow audio capture on the phone" - CarAudio.Capture.DENIED -> "Audio stays on the phone" - CarAudio.Capture.OFF -> return - } - Text(text, style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurfaceVariant) -} - @Composable private fun Notice(title: String, detail: String?, action: Pair Unit>? = null) { Column( diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NavApps.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NavApps.kt index ce706fe..0a50ca5 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NavApps.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NavApps.kt @@ -72,7 +72,7 @@ internal class BuiltInApp( ) internal object BuiltInApps { - val Web = BuiltInApp("web", "Web", Icons.Filled.Search, CarScreen.App("web")) { it.open("web") } + val Web = BuiltInApp("web", "Browser", Icons.Filled.Search, CarScreen.App("web")) { it.open("web") } val ScreenSharing = BuiltInApp("phone", "Screen sharing", Icons.Filled.Share, CarScreen.App("phone")) { it.open("phone") } val Phone = BuiltInApp("dialer", "Phone", Icons.Filled.Call, CarScreen.App("dialer")) { it.open("dialer") } val VehicleData = BuiltInApp("vehicle", "Vehicle data", Icons.Filled.Info, CarScreen.VehicleData) { it.vehicleData() } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/TemplateContent.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/TemplateContent.kt index cd3c04e..4b29ac9 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/TemplateContent.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/TemplateContent.kt @@ -45,6 +45,7 @@ import androidx.car.app.model.MessageTemplate import androidx.car.app.model.PaneTemplate import androidx.car.app.model.Row as CarRow import androidx.car.app.model.SearchTemplate +import androidx.compose.foundation.clickable import androidx.car.app.model.Section import androidx.car.app.model.SectionedItemTemplate import androidx.car.app.model.TabTemplate @@ -238,6 +239,9 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) { // initialSearchText would fight whatever is being typed. var query by remember { mutableStateOf(template.initialSearchText.orEmpty()) } val delegate = template.searchCallbackDelegate + // An app that opens on results ("navigate to Starbucks") asks for the keyboard to stay down, + // so the results aren't hidden under it. Tapping the field brings it up. + var typing by remember { mutableStateOf(template.isShowKeyboardByDefault) } ContentSurface(modifier) { // Back button and field share a row: on a 480px screen the keyboard and the results @@ -251,7 +255,7 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) { SearchField( query, template.searchHint.orEmpty(), - Modifier.weight(1f), + Modifier.weight(1f).clickable { typing = true }, onClear = { query = "" delegate.textChanged("") @@ -265,14 +269,17 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) { else ItemListColumn(template.itemList, modifier = Modifier.fillMaxSize()) } - CarKeyboard( - text = query, - onTextChange = { - query = it - delegate.textChanged(it) - }, - onSubmit = { delegate.submitted(query) }, - ) + if (typing) { + CarKeyboard( + text = query, + onTextChange = { + query = it + delegate.textChanged(it) + }, + onSubmit = { delegate.submitted(query) }, + onDismiss = { typing = false }, + ) + } } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt new file mode 100644 index 0000000..5257a31 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceHelpOverlay.kt @@ -0,0 +1,94 @@ +package app.seb3thehacker.gearslip.car + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.verticalScroll +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.car.theme.GsIconButton +import kotlinx.coroutines.delay +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** What the assistant understands, shown on the car screen when the driver says "help". */ +object VoiceHelp { + private val _shown = MutableStateFlow(false) + val shown: StateFlow = _shown.asStateFlow() + + fun show() { _shown.value = true } + fun hide() { _shown.value = false } + + /** Say this, and this happens. Kept in step with [AssistantCommands]. */ + val COMMANDS = listOf( + "Navigate to…" to "Directions there. Add \"with Organic Maps\" to pick the app", + "Play…" to "A song, artist, or album in the open music app", + "Pause · Resume · Next · Previous" to "Control the music", + "What's playing" to "The song and artist", + "Call…" to "A contact by name", + "Open…" to "Any app, like Browser, Phone, or Screen sharing", + "Exit" to "Back to the car's own screen", + "Go home · Open apps · Settings" to "Move around Gearslip", + "What time is it · Weather" to "The time, or the forecast", + "Help" to "This list", + ) +} + +/** The command list over the car screen: read it at a glance, tap the cross or wait to dismiss. */ +@Composable +fun VoiceHelpOverlay(modifier: Modifier = Modifier) { + val shown by VoiceHelp.shown.collectAsState() + if (!shown) return + // A list left up would cover the map for the rest of the drive. + LaunchedEffect(Unit) { + delay(30_000) + VoiceHelp.hide() + } + Surface( + color = MaterialTheme.colorScheme.surfaceContainerHigh, + contentColor = MaterialTheme.colorScheme.onSurface, + shape = MaterialTheme.shapes.extraLarge, + tonalElevation = 6.dp, + shadowElevation = 8.dp, + modifier = modifier.padding(16.dp).widthIn(max = 640.dp), + ) { + Column(Modifier.padding(20.dp)) { + Row(Modifier.fillMaxWidth()) { + Text( + "You can say", + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.Bold, + modifier = Modifier.weight(1f).padding(top = 6.dp), + ) + GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide) + } + // Full width, so a drag anywhere on the card scrolls, not just one on the words. + Column( + Modifier.fillMaxWidth().verticalScroll(rememberScrollState()).padding(top = 8.dp), + verticalArrangement = Arrangement.spacedBy(10.dp), + ) { + VoiceHelp.COMMANDS.forEach { (say, does) -> + Column(Modifier.fillMaxWidth()) { + Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) + Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant) + } + } + } + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/WebApp.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/WebApp.kt index 7883fc1..54cbd47 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/WebApp.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/WebApp.kt @@ -59,7 +59,7 @@ fun WebApp(content: String) { Column(Modifier.fillMaxSize()) { // Back steps through the page's history first (the interceptor above), then leaves. - ScreenHeader("Web", Modifier.padding(horizontal = 16.dp, vertical = 8.dp)) + ScreenHeader("Browser", Modifier.padding(horizontal = 16.dp, vertical = 8.dp)) AndroidView( modifier = Modifier.weight(1f).fillMaxSize(), factory = { context -> diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt index 14d8e0a..f363f3f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt @@ -301,11 +301,14 @@ class CarAppConnection(private val context: Context) { /** * "navigate to X" deep link, the same one Android Auto's own Assistant integration sends - * [ACTION_NAVIGATE] with the destination as a `geo:` search query, rather than anything - * app-specific. + * app-specific. Centred on the phone when there's a fix: `geo:0,0` made Organic Maps pick the + * first Del Taco it found anywhere, not the nearest. The location goes only to the map app. */ - private fun navigateIntent(destination: String) = - Intent(CarContext.ACTION_NAVIGATE, Uri.parse("geo:0,0?q=" + Uri.encode(destination))) + private fun navigateIntent(destination: String): Intent { + val near = app.seb3thehacker.gearslip.notify.CarMotion.nearHere(context) ?: "0,0" + return Intent(CarContext.ACTION_NAVIGATE, Uri.parse("geo:$near?q=" + Uri.encode(destination))) .setComponent(component) + } /** * Sends a fresh destination to an app that's already running, instead of tearing it down diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt index 056621d..eaa9006 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/media/CarMedia.kt @@ -156,6 +156,9 @@ class CarMedia(private val context: Context) { private val main = Handler(Looper.getMainLooper()) private var connecting: MediaApp? = null + /** The music app Gearslip is attached to, for the assistant to name; null when there's none. */ + val appLabel: String? get() = connecting?.label?.takeIf { controller != null } + private val _phase = MutableStateFlow(Phase.IDLE) val phase: StateFlow = _phase.asStateFlow() @@ -236,6 +239,12 @@ class CarMedia(private val context: Context) { connecting = app _phase.value = Phase.CONNECTING _browse.value = BrowseState() + // Known to refuse its library (Spotify): asking only costs the seconds it takes to say no, + // so go straight to its playback session. + if (KnownApps.playerWorks(app.component.packageName)) { + attachToSession(app) + return + } val mb = MediaBrowser(context, app.component, object : MediaBrowser.ConnectionCallback() { override fun onConnected() = this@CarMedia.onConnected(app) override fun onConnectionFailed() { @@ -273,7 +282,12 @@ class CarMedia(private val context: Context) { return } if (session == null) { - val limit = if (launched) SESSION_RETRIES_LAUNCHED else SESSION_RETRIES + // A known session-only app that isn't running won't start by waiting: say so at once. + val limit = when { + launched -> SESSION_RETRIES_LAUNCHED + KnownApps.playerWorks(app.component.packageName) -> 0 + else -> SESSION_RETRIES + } // The app only publishes its session once its service has started; give it a moment. if (attempt < limit) { main.postDelayed({ attachToSession(app, attempt + 1, launched) }, SESSION_RETRY_MS) @@ -454,7 +468,7 @@ class CarMedia(private val context: Context) { } val component = searchComponent if (component == null) { - _results.value = Results(q, emptyList(), supported = false) + noLibrarySearch(q) return } resultsQuery = q @@ -476,13 +490,13 @@ class CarMedia(private val context: Context) { override fun onConnectionFailed() { GearslipLog.w("media: no library connection for search") searchBrowser = null - resultsQuery?.let { _results.value = Results(it, emptyList(), supported = false) } + resultsQuery?.let { noLibrarySearch(it) } } }, null) searchBrowser = created runCatching { created.connect() }.onFailure { searchBrowser = null - _results.value = Results(q, emptyList(), supported = false) + noLibrarySearch(q) } } @@ -492,18 +506,42 @@ class CarMedia(private val context: Context) { override fun onSearchResult(query: String, extras: Bundle?, items: List) { if (query != resultsQuery) return GearslipLog.i("media: library search found ${items.size}") - _results.value = Results(query, items.map(::entryOfCompat)) + val entries = items.map(::entryOfCompat) + if (query == playFirstFor) { + playFirstFor = null + val first = entries.firstOrNull { it.playable } ?: entries.firstOrNull() + if (first == null) { + clearResults() + search(query) + } else { + GearslipLog.i("media: playing the library's best match") + clearResults() + searchBefore = _now.value + _search.value = Search(query) + main.removeCallbacks(searchTimeout) + controller?.transportControls?.playFromMediaId(first.id, null) + main.postDelayed(searchTimeout, SEARCH_WAIT_MS) + } + return + } + _results.value = Results(query, entries) } override fun onError(query: String, extras: Bundle?) { if (query != resultsQuery) return + if (query == playFirstFor) { + playFirstFor = null + clearResults() + search(query) + return + } GearslipLog.i("media: the app has no library search") _results.value = Results(query, emptyList(), supported = false) } }) }.onFailure { GearslipLog.w("media: library search failed: ${it.message}") - _results.value = Results(q, emptyList(), supported = false) + noLibrarySearch(q) } } @@ -520,6 +558,36 @@ class CarMedia(private val context: Context) { ) } + /** The library can't search: a voice "play X" falls back to the app's own search; a typed one says so. */ + private fun noLibrarySearch(q: String) { + if (q == playFirstFor) { + playFirstFor = null + clearResults() + search(q) + } else { + _results.value = Results(q, emptyList(), supported = false) + } + } + + /** A voice "play X" waiting on the library search, to play its first match instead of listing them. */ + private var playFirstFor: String? = null + + /** + * Plays the best match for [query], for voice. Asks the app's library first and plays its top + * result, which apps like ViVi honour where they ignore [search]; without a library search, + * falls back to [search]. + */ + fun playBestMatch(query: String) { + val q = query.trim() + if (q.isEmpty()) return + if (!canSearchLibrary) { + search(q) + return + } + playFirstFor = q + searchLibrary(q) + } + /** The connected app's service, for the search connection; null when nothing is connected. */ private var searchComponent: android.content.ComponentName? = null private var searchBrowser: MediaBrowserCompat? = null diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarMotion.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarMotion.kt index fc87fbc..3d95990 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarMotion.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarMotion.kt @@ -22,6 +22,33 @@ object CarMotion { private var listener: LocationListener? = null private var speed = 0f private var fixAt = 0L + @Volatile private var last: Location? = null + + /** A fix older than this is somewhere the car has left. */ + private const val NEAR_STALE_MS = 10 * 60_000L + + /** + * Where the phone is, as "lat,lng", for "navigate to Del Taco" to find the nearest one. From + * this drive's GPS, or Android's last known fix; null with no permission or nothing recent. + */ + fun nearHere(context: Context): String? { + val fix = last?.takeIf { SystemClock.elapsedRealtime() - fixAt < NEAR_STALE_MS } + ?: lastKnown(context) + ?: return null + return String.format(java.util.Locale.US, "%.5f,%.5f", fix.latitude, fix.longitude) + } + + private fun lastKnown(context: Context): Location? { + if (context.checkSelfPermission(Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED && + context.checkSelfPermission(Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED + ) return null + val manager = context.getSystemService(LocationManager::class.java) ?: return null + val now = System.currentTimeMillis() + return manager.getProviders(true) + .mapNotNull { runCatching { manager.getLastKnownLocation(it) }.getOrNull() } + .filter { now - it.time < NEAR_STALE_MS } + .maxByOrNull { it.time } + } val moving: Boolean get() = SystemClock.elapsedRealtime() - fixAt < STALE_MS && speed > MOVING_MS @@ -34,6 +61,7 @@ object CarMotion { override fun onLocationChanged(location: Location) { speed = if (location.hasSpeed()) location.speed else 0f fixAt = SystemClock.elapsedRealtime() + last = location } } listener = l diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt index 2fc90fa..525c55c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt @@ -4,7 +4,6 @@ import android.Manifest import android.annotation.SuppressLint import android.content.Context import android.content.pm.PackageManager -import android.content.res.AssetManager import android.media.AudioFormat import android.media.AudioManager import android.media.AudioRecord @@ -15,38 +14,50 @@ import android.os.Looper import android.os.SystemClock import app.seb3thehacker.gearslip.GearslipLog import app.seb3thehacker.gearslip.audio.CarMic -import java.util.Locale +import com.k2fsa.sherpa.onnx.EndpointConfig +import com.k2fsa.sherpa.onnx.EndpointRule +import com.k2fsa.sherpa.onnx.FeatureConfig +import com.k2fsa.sherpa.onnx.OfflineCanaryModelConfig +import com.k2fsa.sherpa.onnx.OfflineModelConfig +import com.k2fsa.sherpa.onnx.OfflineRecognizer +import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig +import com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig +import com.k2fsa.sherpa.onnx.OnlineModelConfig +import com.k2fsa.sherpa.onnx.OnlineRecognizer +import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig +import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig +import java.io.File import java.util.concurrent.Executors -import kotlin.math.log10 import kotlin.math.max -import kotlin.math.sqrt /** * Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships - * none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply - * of a few seconds comes back in well under a second on a recent phone. + * none). sherpa-onnx runs a streaming model ([SpeechModels]) on the phone: words come out while + * the driver is still talking, and the model itself decides when they've finished. A larger model + * then reads the whole phrase once more for the final words, since a model that waits for the end + * hears names and places far better than one that has to keep up. The phrase is held in memory + * for that and nowhere else, and dropped as soon as it's read. * - * Whisper needs the whole phrase before it starts, so this records from the car's microphone (the - * phone's when the car has none or sends nothing), - * decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes - * once. Everything runs on one worker thread; callbacks arrive on the main thread. + * Audio comes from the car's microphone, or the phone's when the car has none or sends nothing. + * Everything runs on one worker thread; callbacks arrive on the main thread. */ object SpeechEngine { - /** FUTO's ACFT Whisper tiny (Apache-2.0), re-quantized to 5 bits, inside the APK. */ - private const val MODEL = "speech/tiny_acft_q5_1.bin" - /** The error for an engine that can't run at all, which callers may answer with a fallback. */ const val NOT_LOADED = "The speech model didn't load." + /** Said when the chosen model hasn't been downloaded yet. */ + const val NOT_DOWNLOADED = "Download a voice model in Gearslip's settings on the phone first." + private const val RATE = 16_000 - private const val FRAME = 480 // 30 ms - private const val MAX_MS = 15_000L - private const val NO_SPEECH_MS = 6_000L - private const val END_SILENCE_MS = 1_100L - private const val CALIBRATE_MS = 300L - private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word + private const val CHUNK = 1_600 // 100 ms + private const val MAX_S = 15f + /** Quiet before the first word that counts as "nobody spoke". */ + private const val NO_SPEECH_S = 6f + /** Quiet after the last word that ends the phrase. */ + private const val END_SILENCE_S = 1.0f private const val CUE_VOLUME = 80 + private const val IDLE_UNLOAD_MS = 5 * 60_000L /** How long the car gets to start sending once asked, before the phone's microphone takes over. */ private const val CAR_FIRST_AUDIO_MS = 1_500L private const val CAR_GAP_MS = 1_000L @@ -55,27 +66,79 @@ object SpeechEngine { private const val CUE_TAIL_MS = 250L private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") } + /** + * Parakeet large takes seconds to load. On [worker], the first press after connecting + * waited behind it before listening at all; here it loads while the driver talks. + */ + private val finalWorker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-final") } + private val finalLock = Object() private val main = Handler(Looper.getMainLooper()) - private var libraryLoaded = false - private var handle = 0L + private var recognizer: OnlineRecognizer? = null + private var loadedModel: SpeechModels.Model? = null + @Volatile private var finalRecognizer: OfflineRecognizer? = null + @Volatile private var loadedFinal: SpeechModels.Model? = null + + /** + * Parakeet large holds most of the ~900 MB the two models take, so it goes after a few + * minutes without a command. Kroko stays: it's small, and keeps listening instant. + */ + private val idleUnload = Runnable { unloadFinal("after ${IDLE_UNLOAD_MS / 60_000} minutes without use") } @Volatile private var run = 0 - /** Loads the model ahead of time, so the first reply doesn't wait for it. */ + /** Loads the model ahead of time, so the first command doesn't wait for it. */ fun warm(context: Context) { val app = context.applicationContext - worker.execute { ensureLoaded(app) } + SpeechModels.init(app) + // Kroko first, alone: loading both at once slowed it from a fraction of a second to six. + worker.execute { + ensureLoaded(app) + finalWorker.execute { ensureFinalLoaded(app) } + } + } + + private fun unloadFinal(why: String) { + finalWorker.execute { + synchronized(finalLock) { + val model = loadedFinal ?: return@execute + finalRecognizer?.release() + finalRecognizer = null + loadedFinal = null + GearslipLog.i("speech: unloaded ${model.label} $why") + } + } + } + + /** Drops the loaded models: the driver picked another or deleted one, or the car went away. */ + fun unload() { + main.removeCallbacks(idleUnload) + run++ + worker.execute { + recognizer?.release() + recognizer = null + loadedModel = null + } + finalWorker.execute { + synchronized(finalLock) { + finalRecognizer?.release() + finalRecognizer = null + loadedFinal = null + } + } } /** * Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk. - * [onSpeaking] fires at the first word, [onHeard] when the driver stops and the words are being - * worked out; [onResult] gets the text, [onError] a sentence fit to show on the car screen. + * [onPartial] gets the words so far as they're recognized, [onSpeaking] fires at the first + * word, [onHeard] when the driver stops; [onResult] gets the text, [onError] a sentence fit to + * show on the car screen. */ fun listen( context: Context, cue: Boolean = true, onSpeaking: () -> Unit = {}, + onPartial: (String) -> Unit = {}, + onListening: () -> Unit = {}, onHeard: () -> Unit, onResult: (String) -> Unit, onError: (String) -> Unit, @@ -87,28 +150,29 @@ object SpeechEngine { } val id = ++run fun post(block: () -> Unit) { main.post { if (id == run) block() } } + // Brings Parakeet back if it was unloaded, while the driver talks. + main.removeCallbacks(idleUnload) + finalWorker.execute { ensureFinalLoaded(app) } worker.execute { if (id != run) return@execute - if (!ensureLoaded(app)) return@execute post { onError(NOT_LOADED) } + if (!SpeechModels.isReady(app, SpeechModels.selected.value)) return@execute post { onError(NOT_DOWNLOADED) } + val r = ensureLoaded(app) ?: return@execute post { onError(NOT_LOADED) } VoiceFocus.acquire(app) - val audio = try { + val text = try { if (cue) cue() if (id != run) return@execute - record(id) { post(onSpeaking) } + post(onListening) + stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) }) } catch (t: Throwable) { - GearslipLog.e("speech: recording failed", t) + GearslipLog.e("speech: listening failed", t) return@execute post { onError("The microphone isn't available right now.") } } finally { VoiceFocus.release() } + main.removeCallbacks(idleUnload) + main.postDelayed(idleUnload, IDLE_UNLOAD_MS) if (id != run) return@execute - if (audio == null) return@execute post { onError("Didn't catch that.") } post(onHeard) - val started = SystemClock.uptimeMillis() - val text = clean(transcribe(audio)) - GearslipLog.i( - "speech: ${audio.size * 1000L / RATE} ms of audio took ${SystemClock.uptimeMillis() - started} ms", - ) post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) } } } @@ -116,6 +180,185 @@ object SpeechEngine { /** Stops listening; a phrase already being worked out is dropped. */ fun cancel() { run++ } + /** + * Feeds the microphone to the recognizer in 100 ms pieces, reporting the words as they firm + * up, until the model hears the end of the phrase (or nobody speaks, or it runs too long). + */ + private fun stream( + context: Context, + r: OnlineRecognizer, + id: Int, + onSpeaking: () -> Unit, + onPartial: (String) -> Unit, + onStopped: () -> Unit, + ): String { + val mic = carMic() ?: phoneMic() + val s = r.createStream() + val frame = ShortArray(CHUNK) + val samples = FloatArray(CHUNK) + // The whole phrase, for the final read; [MAX_S] and a little over is the most it can hold. + val phrase = FloatArray((RATE * (MAX_S + 1)).toInt()) + var held = 0 + var last = "" + var spoke = false + var audioMs = 0L + var decodeMs = 0L + try { + while (id == run) { + val n = mic.read(frame) + if (n < 0) break + if (n == 0) continue + for (i in 0 until n) samples[i] = frame[i] / 32768f + val keep = minOf(n, phrase.size - held) + System.arraycopy(samples, 0, phrase, held, keep) + held += keep + s.acceptWaveform(if (n == CHUNK) samples else samples.copyOf(n), RATE) + audioMs += n * 1000L / RATE + val started = SystemClock.uptimeMillis() + while (r.isReady(s)) r.decode(s) + decodeMs += SystemClock.uptimeMillis() - started + val text = clean(r.getResult(s).text) + if (text != last) { + last = text + if (text.isNotEmpty() && !spoke) { + spoke = true + onSpeaking() + } + onPartial(text) + } + if (r.isEndpoint(s)) break + } + s.inputFinished() + while (r.isReady(s)) r.decode(s) + last = clean(r.getResult(s).text) + } finally { + mic.close() + s.release() + } + // Under 1.0 keeps up with the driver; over it, the words fall behind the voice. + GearslipLog.i( + "speech: $audioMs ms of audio, decoding took $decodeMs ms " + + "(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)", + ) + if (last.isEmpty() || id != run) { phrase.fill(0f); return last } + // Still loading (just connected, or back after going idle): the live words now beat + // Parakeet's in several seconds. + val wanted = SpeechModels.final.value + if (wanted == null || finalRecognizer == null || loadedFinal != wanted) { + if (wanted != null) GearslipLog.i("speech: ${wanted.label} isn't loaded yet, keeping the live words") + phrase.fill(0f) + return last + } + onStopped() + val started = SystemClock.uptimeMillis() + // Held for the whole read, so a model swap or the idle unload can't free it mid-decode. + val reread = synchronized(finalLock) { + val final = finalRecognizer?.takeIf { loadedFinal == wanted } ?: run { phrase.fill(0f); return last } + final.createStream().run { + try { + acceptWaveform(phrase.copyOf(held), RATE) + final.decode(this) + clean(final.getResult(this).text) + } finally { + release() + } + } + } + phrase.fill(0f) + GearslipLog.i("speech: the final read took ${SystemClock.uptimeMillis() - started} ms") + if (reread.isEmpty()) return last + if (reread != last) onPartial(reread) + return reread + } + + /** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */ + private fun ensureFinalLoaded(context: Context): OfflineRecognizer? = synchronized(finalLock) { + val model = SpeechModels.final.value ?: return null + if (finalRecognizer != null && loadedFinal == model) return finalRecognizer + finalRecognizer?.release() + finalRecognizer = null + if (!SpeechModels.isReady(context, model)) return null + val dir = SpeechModels.dir(context, model) + fun path(name: String) = File(dir, name).absolutePath + val started = SystemClock.uptimeMillis() + return runCatching { + OfflineRecognizer( + config = OfflineRecognizerConfig( + featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80), + modelConfig = if (model.modelType == "canary") { + // English in, English out, and no punctuation: clean() strips it anyway. + OfflineModelConfig( + canary = OfflineCanaryModelConfig(path(model.encoder), path(model.decoder), srcLang = "en", tgtLang = "en", usePnc = false), + tokens = path("tokens.txt"), + numThreads = threads(), + ) + } else { + OfflineModelConfig( + transducer = OfflineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)), + tokens = path("tokens.txt"), + numThreads = threads(), + modelType = model.modelType, + ) + }, + decodingMethod = "greedy_search", + ), + ) + }.onSuccess { + finalRecognizer = it + loadedFinal = model + GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms") + }.onFailure { + GearslipLog.e("speech: couldn't load ${model.label}", it) + }.getOrNull() + } + + private fun ensureLoaded(context: Context): OnlineRecognizer? { + val model = SpeechModels.selected.value + if (recognizer != null && loadedModel == model) return recognizer + recognizer?.release() + recognizer = null + if (!SpeechModels.isReady(context, model)) return null + val dir = SpeechModels.dir(context, model) + fun path(name: String) = File(dir, name).absolutePath + val started = SystemClock.uptimeMillis() + return runCatching { + OnlineRecognizer( + config = OnlineRecognizerConfig( + featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80), + modelConfig = OnlineModelConfig( + transducer = OnlineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)), + tokens = path("tokens.txt"), + numThreads = threads(), + modelType = model.modelType, + ), + endpointConfig = EndpointConfig( + rule1 = EndpointRule(false, NO_SPEECH_S, 0f), + rule2 = EndpointRule(true, END_SILENCE_S, 0f), + rule3 = EndpointRule(false, 0f, MAX_S), + ), + enableEndpoint = true, + decodingMethod = "greedy_search", + ), + ) + }.onSuccess { + recognizer = it + loadedModel = model + GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms") + }.onFailure { + GearslipLog.e("speech: couldn't load ${model.label}", it) + }.getOrNull() + } + + /** The big cores only: more threads than that slow the model down on a phone. */ + private fun threads() = Runtime.getRuntime().availableProcessors().let { if (it >= 8) 4 else max(1, it / 2) } + + /** + * Models write in upper case, or with punctuation and capitals of their own; commands are + * matched in plain lower case, so only words and apostrophes are kept. + */ + private fun clean(text: String): String = + text.lowercase().replace(Regex("[^\\p{L}\\p{N}' ]"), " ").replace(Regex("\\s+"), " ").trim() + /** * A short beep on the media stream, so it plays through the car like everything else, then a * pause long enough for it to finish there before the microphone opens and hears it. @@ -130,14 +373,9 @@ object SpeechEngine { } } - // --- recording -------------------------------------------------------------------------- + // --- microphones ----------------------------------------------------------------------- - /** - * Records until the driver stops talking. Returns the phrase from just before the first word, - * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a - * loud cabin needs a louder voice than a quiet one to count as talking. - */ - /** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */ + /** Where a phrase is heard from: [read] fills [frame] and returns how many samples it got, -1 at the end. */ private interface Mic { fun read(frame: ShortArray): Int fun close() @@ -148,10 +386,11 @@ object SpeechEngine { val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) val recorder = AudioRecord( MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE, - AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8), + AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, CHUNK * 4), ) check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" } recorder.startRecording() + GearslipLog.i("speech: listening through the phone's microphone") return object : Mic { override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size) override fun close() { @@ -163,7 +402,7 @@ object SpeechEngine { /** * The car's microphone, which hears the driver better than a phone in a cup holder. The car - * sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car + * sends audio in its own sized pieces; they're cut to [CHUNK]s here. Null when there's no car * microphone, or it sends nothing at first, so the phone's is used instead. */ private fun carMic(): Mic? { @@ -190,105 +429,10 @@ object SpeechEngine { at += n if (at == chunk.size) { pending = null; at = 0 } } - // The car stopped sending mid-phrase: end the recording rather than wait forever. + // The car stopped sending mid-phrase: end the phrase rather than wait forever. return if (filled == 0) -1 else filled } override fun close() = car.stop() } } - - /** - * Records until the driver stops talking. Returns the phrase from just before the first word, - * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a - * loud cabin needs a louder voice than a quiet one to count as talking. - */ - private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? { - val mic = carMic() ?: phoneMic() - val samples = FloatArray((RATE * MAX_MS / 1000).toInt()) - var count = 0 - val frame = ShortArray(FRAME) - var noise = 0.0 - var noiseFrames = 0 - var speechStart = -1 - var lastSpeech = 0 - try { - while (id == run && count + FRAME <= samples.size) { - val n = mic.read(frame) - if (n < 0) break - if (n == 0) continue - var sum = 0.0 - for (i in 0 until n) { - val v = frame[i] / 32768f - samples[count + i] = v - sum += v * v - } - count += n - val db = 20 * log10(sqrt(sum / n) + 1e-9) - val ms = count * 1000L / RATE - if (ms <= CALIBRATE_MS) { - noise += db; noiseFrames++ - continue - } - val floor = if (noiseFrames > 0) noise / noiseFrames else -60.0 - val talking = db > max(floor + 12, -45.0) - if (talking) { - if (speechStart < 0) { - speechStart = max(0, count - n - PRE_ROLL) - onSpeaking() - } - lastSpeech = count - } - if (speechStart < 0 && ms > NO_SPEECH_MS) return null - if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break - } - } finally { - mic.close() - } - if (speechStart < 0) return null - return samples.copyOfRange(speechStart, count) - } - - // --- the model -------------------------------------------------------------------------- - - private fun ensureLoaded(context: Context): Boolean { - if (handle != 0L) return true - if (!libraryLoaded) { - runCatching { System.loadLibrary("gearslip_speech") } - .onFailure { GearslipLog.e("speech: native library missing", it); return false } - libraryLoaded = true - } - val started = SystemClock.uptimeMillis() - handle = nativeInitAsset(context.assets, MODEL) - if (handle == 0L) { - GearslipLog.w("speech: couldn't load the model") - return false - } - GearslipLog.i("speech: loaded the model in ${SystemClock.uptimeMillis() - started} ms") - return true - } - - private fun transcribe(audio: FloatArray): String { - val bytes = nativeTranscribe(handle, audio, language(), threads()) - return String(bytes, Charsets.UTF_8) - } - - /** The phone's language, as Whisper names it. Older Java codes differ for a few. */ - private fun language(): String = when (val lang = Locale.getDefault().language) { - "iw" -> "he" - "in" -> "id" - "ji" -> "yi" - "" -> "en" - else -> lang - } - - /** The big cores only: more threads than that slow Whisper down on a phone. */ - private fun threads() = Runtime.getRuntime().availableProcessors().coerceIn(1, 8).let { if (it >= 8) 4 else max(1, it / 2) } - - /** Whisper marks silence and noise as "[BLANK_AUDIO]" or "(wind blowing)"; none of it is words. */ - private fun clean(text: String): String = - text.replace(Regex("""\[[^\]]*\]|\([^)]*\)|\*[^*]*\*"""), " ").replace(Regex("\\s+"), " ").trim() - - private external fun nativeInitAsset(assets: AssetManager, path: String): Long - private external fun nativeTranscribe(handle: Long, samples: FloatArray, language: String, threads: Int): ByteArray - private external fun nativeFree(handle: Long) } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt new file mode 100644 index 0000000..74437fb --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechModels.kt @@ -0,0 +1,193 @@ +package app.seb3thehacker.gearslip.speech + +import android.content.Context +import app.seb3thehacker.gearslip.GearslipLog +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.concurrent.Executors +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream + +/** + * The speech models Gearslip listens with, in two roles: a streaming one that writes the words + * while the driver talks, and an optional final one that reads the whole phrase again once they + * stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's + * GitHub releases, where its authors publish it, and kept in the app's own storage. + */ +object SpeechModels { + + enum class Role { LIVE, FINAL } + + enum class Model( + val role: Role, + val id: String, + val label: String, + val detail: String, + val downloadMb: Int, + private val archive: String, + val encoder: String, + val decoder: String, + /** Empty for Canary, which decodes with attention and has no joiner. */ + val joiner: String, + val modelType: String = "", + ) { + KROKO( + Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54, + "sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06", + "encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2", + ), + PARAKEET_SMALL( + Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103, + "sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8", + "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer", + ), + CANARY( + Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 154, + "sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8", + "encoder.int8.onnx", "decoder.int8.onnx", "", modelType = "canary", + ), + PARAKEET_LARGE( + Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460, + "sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8", + "encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer", + ), + ; + + val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2" + val files get() = listOf(encoder, decoder, joiner, "tokens.txt").filter { it.isNotEmpty() } + } + + sealed interface State { + data object Missing : State + data class Downloading(val percent: Int) : State + data object Ready : State + data class Failed(val reason: String) : State + } + + private const val PREFS = "speech" + private const val KEY_MODEL = "model" + private const val KEY_FINAL = "final_model" + + private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-download") } + private val states = Model.entries.associateWith { MutableStateFlow(State.Missing) } + private val _selected = MutableStateFlow(Model.KROKO) + val selected: StateFlow = _selected.asStateFlow() + private val _final = MutableStateFlow(Model.PARAKEET_LARGE) + + /** The model that rereads the whole phrase at the end; null to keep the live model's words. */ + val final: StateFlow = _final.asStateFlow() + @Volatile private var cancelled: Model? = null + + fun init(context: Context) { + val id = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_MODEL, null) + _selected.value = Model.entries.firstOrNull { it.id == id && it.role == Role.LIVE } ?: Model.KROKO + val finalId = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_FINAL, Model.PARAKEET_LARGE.id) + _final.value = Model.entries.firstOrNull { it.id == finalId && it.role == Role.FINAL } + Model.entries.forEach { m -> if (isReady(context, m)) states.getValue(m).value = State.Ready } + // Models an older build offered and this one doesn't: hundreds of MB nobody can use. + val known = Model.entries.map { it.id }.toSet() + File(context.filesDir, "speech").listFiles()?.filter { it.isDirectory && it.name !in known }?.forEach { + GearslipLog.i("speech: removing ${it.name}, no longer offered") + it.deleteRecursively() + } + } + + fun state(model: Model): StateFlow = states.getValue(model).asStateFlow() + + fun dir(context: Context, model: Model) = File(File(context.filesDir, "speech"), model.id) + + fun isReady(context: Context, model: Model) = model.files.all { File(dir(context, model), it).isFile } + + fun select(context: Context, model: Model) { + if (model.role == Role.FINAL) return selectFinal(context, model) + _selected.value = model + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_MODEL, model.id).apply() + SpeechEngine.unload() + } + + /** [model] null keeps the live words as they are. */ + fun selectFinal(context: Context, model: Model?) { + _final.value = model + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_FINAL, model?.id ?: "none").apply() + SpeechEngine.unload() + } + + fun delete(context: Context, model: Model) { + cancelled = model + dir(context, model).deleteRecursively() + states.getValue(model).value = State.Missing + if (model == _selected.value || model == _final.value) SpeechEngine.unload() + } + + /** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */ + fun download(context: Context, model: Model) { + val app = context.applicationContext + val state = states.getValue(model) + if (state.value is State.Downloading || state.value == State.Ready) return + state.value = State.Downloading(0) + cancelled = null + worker.execute { + val target = dir(app, model) + val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() } + try { + val conn = (URL(model.url).openConnection() as HttpURLConnection).apply { + connectTimeout = 15_000 + readTimeout = 30_000 + instanceFollowRedirects = true + } + check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" } + val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L) + val counting = CountingStream(conn.inputStream) { read -> + if (cancelled == model) throw InterruptedException("cancelled") + state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99)) + } + TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar -> + while (true) { + val entry = tar.nextEntry ?: break + val name = entry.name.substringAfterLast('/') + if (entry.isDirectory || name !in model.files) continue + File(partial, name).outputStream().use { tar.copyTo(it) } + } + } + check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" } + target.deleteRecursively() + check(partial.renameTo(target)) { "couldn't store the model" } + GearslipLog.i("speech: downloaded ${model.label}") + state.value = State.Ready + } catch (e: Exception) { + partial.deleteRecursively() + if (cancelled == model) { + state.value = State.Missing + } else { + GearslipLog.w("speech: download of ${model.label} failed: ${e.message}") + state.value = State.Failed(e.message ?: e.javaClass.simpleName) + } + } + } + } + + /** Counts bytes as they pass, reporting about every megabyte. */ + private class CountingStream( + private val inner: java.io.InputStream, + private val onProgress: (Long) -> Unit, + ) : java.io.InputStream() { + private var read = 0L + private var reported = 0L + + override fun read(): Int = inner.read().also { if (it >= 0) count(1) } + override fun read(b: ByteArray, off: Int, len: Int): Int = inner.read(b, off, len).also { if (it > 0) count(it.toLong()) } + override fun close() = inner.close() + + private fun count(n: Long) { + read += n + if (read - reported >= 1_048_576) { + reported = read + onProgress(read) + } + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/AudioChoiceDialog.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/AudioChoiceDialog.kt index f53ecde..0314586 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/AudioChoiceDialog.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/AudioChoiceDialog.kt @@ -47,3 +47,25 @@ fun AudioChoiceDialog(onUseBluetooth: () -> Unit, onKeepAsking: () -> Unit) { dismissButton = { TextButton(onClick = onKeepAsking) { Text("Keep asking") } }, ) } + +/** + * Android lets an app capture other apps' audio only while it holds the microphone permission. + * Once the driver has refused that twice, Android stops asking, and music silently stays on the + * phone; this says why and where to turn it back on. + */ +@Composable +fun MicBlockedDialog(onOpenSettings: () -> Unit, onDismiss: () -> Unit) { + AlertDialog( + onDismissRequest = onDismiss, + title = { Text("Car audio needs the microphone permission") }, + text = { + Text( + "Android lets an app send other apps' sound to the car only if it has the " + + "microphone permission. It's turned off for Gearslip, so music plays on the " + + "phone. Turn it on under Permissions, then Microphone.", + ) + }, + confirmButton = { TextButton(onClick = onOpenSettings) { Text("Open app settings") } }, + dismissButton = { TextButton(onClick = onDismiss) { Text("Not now") } }, + ) +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt index 9c5a3db..f3dbd09 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt @@ -16,11 +16,9 @@ import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height import androidx.compose.foundation.layout.statusBars import androidx.compose.foundation.layout.windowInsetsPadding -import androidx.compose.material3.AlertDialog import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text -import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue @@ -39,7 +37,7 @@ import app.seb3thehacker.gearslip.BuildConfig import app.seb3thehacker.gearslip.CertProvider import app.seb3thehacker.gearslip.stats.UsageStats -private enum class Screen { HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, USAGE_NOTES, HELP, WHATS_NEW, RECOMMENDED_APPS } +private enum class Screen { INTRO, HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, USAGE_NOTES, HELP, WHATS_NEW, RECOMMENDED_APPS } /** Three screens and a back stack of depth one: no navigation library needed. */ @Composable @@ -49,20 +47,22 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) { !AppSettings.hasSkippedCertSetup(context) && !CertProvider.hasAnyCert(context) } val needsSetupGuide = remember { !AppSettings.hasSeenPermissionsSetup(context) } - var screen by rememberSaveable { - mutableStateOf( - when { - needsCertSetup -> Screen.CERT_SETUP - needsSetupGuide -> Screen.SETUP_GUIDE - // Set up before usage notes existed: ask once, on the setup guide's own page. - !UsageStats.asked(context) -> Screen.USAGE_NOTES - else -> Screen.HOME - }, - ) + // Where launch goes once the intro (if any) is done. + val afterIntro = remember { + when { + needsCertSetup -> Screen.CERT_SETUP + needsSetupGuide -> Screen.SETUP_GUIDE + // Set up before usage notes existed: ask once, on the setup guide's own page. + !UsageStats.asked(context) -> Screen.USAGE_NOTES + else -> Screen.HOME + } } + var screen by rememberSaveable { mutableStateOf(if (Intro.due(context)) Screen.INTRO else afterIntro) } + // A replay from About goes back to Settings; the launch one carries on into setup. + var introReturn by rememberSaveable { mutableStateOf(afterIntro) } // Home's usage notes row opens Settings at that section rather than at the top. var settingsAtUsageNotes by rememberSaveable { mutableStateOf(false) } - BackHandler(enabled = screen != Screen.HOME) { + BackHandler(enabled = screen != Screen.HOME && screen != Screen.INTRO) { screen = if (screen == Screen.RECOMMENDED_APPS) Screen.SETTINGS else Screen.HOME } @@ -95,10 +95,11 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) { .windowInsetsPadding(WindowInsets.statusBars) .consumeWindowInsets(WindowInsets.statusBars), ) { - if (BuildConfig.DEBUG && screen != Screen.CAR_PREVIEW) DevBuildBanner() + if (BuildConfig.DEBUG && screen != Screen.CAR_PREVIEW && screen != Screen.INTRO) DevBuildBanner() Column(Modifier.weight(1f)) { when (screen) { + Screen.INTRO -> IntroScreen(onDone = { screen = introReturn }) Screen.HOME -> HomeScreen( onOpenLogs = { screen = Screen.LOGS }, onOpenSettings = { settingsAtUsageNotes = false; screen = Screen.SETTINGS }, @@ -114,6 +115,7 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) { onOpenRecommendedApps = { settingsAtUsageNotes = false; screen = Screen.RECOMMENDED_APPS }, onOpenWhatsNew = { settingsAtUsageNotes = false; screen = Screen.WHATS_NEW }, onReplayTutorial = { screen = Screen.SETUP_GUIDE }, + onReplayIntro = { introReturn = Screen.SETTINGS; screen = Screen.INTRO }, atUsageNotes = settingsAtUsageNotes, ) Screen.RECOMMENDED_APPS -> RecommendedAppsScreen(onBack = { screen = Screen.SETTINGS }) @@ -141,8 +143,6 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) { // A car keyboard is open: offer the phone's own, for a passenger. if (screen != Screen.CAR_PREVIEW) PhoneTypingBar() } - - CompatWarning() } /** @@ -167,29 +167,3 @@ private fun DevBuildBanner() { } } } - -/** Shown once, on first run: the certificate is known to fail on head units newer than ~2020. */ -@Composable -private fun CompatWarning() { - val context = LocalContext.current - var visible by rememberSaveable { mutableStateOf(!AppSettings.hasSeenCompatWarning(context)) } - - if (visible) { - AlertDialog( - onDismissRequest = {}, - title = { Text("Before you plug in") }, - text = { - Text( - "Gearslip works with cars built before 2020. " + - "Newer firmware often blocks the connection.", - ) - }, - confirmButton = { - TextButton(onClick = { - AppSettings.setSeenCompatWarning(context) - visible = false - }) { Text("Got it") } - }, - ) - } -} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt index cc58673..b293639 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt @@ -112,6 +112,10 @@ fun HomeScreen( Spacer(Modifier.height(16.dp)) WirelessCard() } + if (!live) { + Spacer(Modifier.height(16.dp)) + SupportCard() + } } // Sits at the bottom, where a thumb reaches it. diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt new file mode 100644 index 0000000..84b94be --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/IntroScreen.kt @@ -0,0 +1,726 @@ +package app.seb3thehacker.gearslip.ui + +import android.content.Context +import android.content.Intent +import android.net.Uri +import android.provider.Settings +import androidx.activity.compose.BackHandler +import androidx.compose.animation.core.Animatable +import androidx.compose.animation.core.CubicBezierEasing +import androidx.compose.animation.core.FastOutSlowInEasing +import androidx.compose.animation.core.LinearEasing +import androidx.compose.animation.core.RepeatMode +import androidx.compose.animation.core.animateDpAsState +import androidx.compose.animation.core.animateFloat +import androidx.compose.animation.core.infiniteRepeatable +import androidx.compose.animation.core.keyframes +import androidx.compose.animation.core.rememberInfiniteTransition +import androidx.compose.animation.core.tween +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.BorderStroke +import androidx.compose.foundation.background +import androidx.compose.foundation.border +import androidx.compose.foundation.clickable +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.offset +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.width +import androidx.compose.foundation.layout.wrapContentWidth +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableIntStateOf +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.Path +import androidx.compose.ui.graphics.StrokeCap +import androidx.compose.ui.graphics.StrokeJoin +import androidx.compose.ui.graphics.drawscope.DrawScope +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.graphics.drawscope.rotate +import androidx.compose.ui.graphics.drawscope.scale +import androidx.compose.ui.graphics.graphicsLayer +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalDensity +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.Dp +import androidx.compose.ui.unit.TextUnit +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.BuildConfig +import androidx.compose.runtime.collectAsState +import androidx.compose.ui.draw.clipToBounds +import androidx.compose.ui.draw.drawWithContent +import androidx.compose.ui.graphics.BlendMode +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.CompositingStrategy +import androidx.compose.foundation.layout.wrapContentHeight +import app.seb3thehacker.gearslip.speech.SpeechModels +import app.seb3thehacker.gearslip.stats.UsageStats +import kotlin.math.PI +import kotlin.math.cos +import kotlin.math.atan2 +import kotlin.math.sqrt +import kotlin.math.sin +import kotlin.random.Random +import kotlinx.coroutines.delay + +/** + * The release intro: a few tap-through screens shown once after each new release (and on a new + * install), before setup. Its look follows the 0.3 mockup: Material shapes in the phone's own colours, + * with every size taken from the screen's width so it scales from small phones to tablets. + */ +object Intro { + private const val PREFS = "intro" + private const val KEY_SEEN = "seen_release" + + /** "0.3" from "0.3.00-alpha": the intro belongs to a release, not to each patch. */ + val release: String = BuildConfig.VERSION_NAME.split('.').take(2).joinToString(".") + + fun due(context: Context): Boolean = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_SEEN, null) != release + + fun markSeen(context: Context) { + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_SEEN, release).apply() + } +} + +/** The intro's tones, taken from the phone's own Material You colours like the rest of the app. */ +private class IntroColors( + val surface: Color, val onSurface: Color, val onVariant: Color, val outline: Color, + val primary: Color, val onPrimary: Color, val primaryC: Color, val onPrimaryC: Color, + val tertiaryC: Color, val onTertiaryC: Color, val secondaryC: Color, val onSecondaryC: Color, +) + +@Composable +private fun introColors(): IntroColors { + val m = MaterialTheme.colorScheme + return IntroColors( + surface = m.surface, onSurface = m.onSurface, onVariant = m.onSurfaceVariant, outline = m.outline, + primary = m.primary, onPrimary = m.onPrimary, primaryC = m.primaryContainer, onPrimaryC = m.onPrimaryContainer, + tertiaryC = m.tertiaryContainer, onTertiaryC = m.onTertiaryContainer, + secondaryC = m.secondaryContainer, onSecondaryC = m.onSecondaryContainer, + ) +} + +private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, DRIVE } + +private val Pop = CubicBezierEasing(0.3f, 1.6f, 0.5f, 1f) +private val Rise = CubicBezierEasing(0.2f, 0.8f, 0.2f, 1f) + +@Composable +fun IntroScreen(onDone: () -> Unit) { + val context = LocalContext.current + val c = introColors() + // The GrapheneOS screen is only news to people running it. + val steps = remember { + IntroStep.entries.filter { it != IntroStep.GRAPHENE || UsageStats.osName(context) == "GrapheneOS" } + } + var index by remember { mutableIntStateOf(0) } + val step = steps[index] + val finish = { + Intro.markSeen(context) + onDone() + } + BackHandler(enabled = index > 0) { index-- } + + BoxWithConstraints( + Modifier + .fillMaxSize() + .background(c.surface) + .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) { + if (index < steps.lastIndex) index++ + }, + ) { + val w = maxWidth + // The mockup's cqw: one percent of the screen's width. + fun cq(n: Float): Dp = w * (n / 100f) + val density = LocalDensity.current + fun cqSp(n: Float): TextUnit = with(density) { cq(n).toSp() } + + when (step) { + IntroStep.WELCOME -> Welcome(c, ::cq, ::cqSp, maxHeight) + IntroStep.WIRELESS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Wireless.", + "If your car does wireless Android Auto, pair once and leave the cable at home.", step) { WifiIcon(c.onSecondaryC) } + IntroStep.KNOBS -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Knobs work.", + "Turn, push, done. No touchscreen? Gearslip doesn't need one anymore.", step) { DialIcon(c.onTertiaryC) } + IntroStep.VOICE -> VoiceStep(c, ::cq, ::cqSp, step) + IntroStep.GRAPHENE -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Protections back on.", + "On GrapheneOS, Gearslip no longer needs exploit protection compatibility mode. If you " + + "turned it on for Gearslip, turn it off to get full protection back.", step, + action = "Open app settings" to { + runCatching { + context.startActivity( + Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", context.packageName, null)), + ) + } + }, + outlined = true, + ) { ShieldIcon(c.onSecondaryC, c.secondaryC) } + IntroStep.THANKS -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Thank you.", + "To everyone sending usage notes: they show me which cars Gearslip works in and which " + + "ones still fail, so I know what to fix next. Every note helps.", step) { HeartIcon(c.onTertiaryC) } + IntroStep.APPS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Good company.", + "Free, open-source apps that work well in the car. Tap one to see it on GitHub.", step, + extra = { AppPicks(c, ::cq, ::cqSp, step) }) { StarIcon(c.onSecondaryC) } + IntroStep.DRIVE -> BadgeStep(c, ::cq, ::cqSp, c.primaryC, "Let's drive.", + "That's ${Intro.release}. Plug in, or just get in.", step, + action = "Start" to finish) { CheckIcon(c.onPrimaryC) } + } + + Dots(c, steps.size, index, ::cq, Modifier.align(Alignment.BottomCenter).padding(bottom = maxHeight * 0.05f)) + if (step != IntroStep.DRIVE) Hint(c, ::cqSp, step, Modifier.align(Alignment.BottomCenter).padding(bottom = maxHeight * 0.09f)) + } +} + +/** 0 to 1 once each time [key] changes, after [delayMs]: drives the entrance of one element. */ +@Composable +private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float { + val a = remember(key) { Animatable(0f) } + LaunchedEffect(key) { + delay(delayMs.toLong()) + a.animateTo(1f, tween(durationMs, easing = LinearEasing)) + } + return a.value +} + +/** Slides up and fades in, like the mockup's `up`. */ +private fun Modifier.up(p: Float, distance: Dp) = graphicsLayer { + val e = Rise.transform(p) + alpha = e + translationY = (1 - e) * distance.toPx() +} + +/** Grows in with a bounce, like the mockup's `bounce`. */ +private fun Modifier.pop(p: Float) = graphicsLayer { + val e = Pop.transform(p) + alpha = p.coerceAtMost(1f) + scaleX = 0.5f + 0.5f * e + scaleY = 0.5f + 0.5f * e +} + +@Composable +private fun Welcome(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, height: Dp) { + val key = IntroStep.WELCOME + val spin by rememberInfiniteTransition(label = "spin").animateFloat( + 0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin", + ) + Box(Modifier.fillMaxSize()) { + Cookie(c.primaryC, Modifier.size(cq(120f)).offset(x = cq(28f), y = -cq(22f)).align(Alignment.TopEnd).graphicsLayer { rotationZ = spin; alpha = 0.9f }) + Ticker(c.secondaryC, c.onSecondaryC, "OPEN SOURCE ✦ NO ACCOUNT ✦ YOUR CAR, YOUR PHONE ✦ ", cq, cqSp, reverse = true, tilt = 3f, + key = key, modifier = Modifier.align(Alignment.TopCenter).padding(top = height * 0.11f)) + + Column(Modifier.align(Alignment.CenterStart).padding(horizontal = cq(8f))) { + Text("Welcome to", color = c.onVariant, style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.SemiBold), + modifier = Modifier.up(entrance(key, 100), cq(7f))) + Text( + Intro.release, color = c.primary, + style = TextStyle(fontSize = cqSp(44f), fontWeight = FontWeight.Black, letterSpacing = cqSp(-2.4f), lineHeight = cqSp(38f)), + modifier = Modifier.offset(x = -cq(1.5f)).pop(entrance(key, 250, 900)), + ) + Text( + "alpha · the big one", color = c.onTertiaryC, + style = TextStyle(fontSize = cqSp(4f), fontWeight = FontWeight.Bold), + modifier = Modifier.padding(top = cq(4f)).up(entrance(key, 700), cq(7f)) + .clip(RoundedCornerShape(50)).background(c.tertiaryC).padding(horizontal = cq(4.4f), vertical = cq(2.2f)), + ) + } + + Ticker(c.primary, c.onPrimary, "WIRELESS ✦ KNOB SUPPORT ✦ OFFLINE VOICE ✦ FASTER VIDEO ✦ AUTO UPDATES ✦ ", cq, cqSp, reverse = false, tilt = -4f, + key = key, modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = height * 0.15f)) + Confetti(c, key, Modifier.fillMaxSize()) + } +} + +/** A tilted band of words scrolling past, slid in from the side when the step opens. */ +@Composable +private fun Ticker( + bg: Color, fg: Color, words: String, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, + reverse: Boolean, tilt: Float, key: Any, modifier: Modifier, +) { + val enter = entrance(key, 400, 800) + val scroll by rememberInfiniteTransition(label = "ticker").animateFloat( + 0f, 1f, infiniteRepeatable(tween(if (reverse) 20_000 else 16_000, easing = LinearEasing)), label = "ticker", + ) + Box( + modifier + .fillMaxWidth() + .graphicsLayer { + rotationZ = tilt + scaleX = 1.2f + val e = Rise.transform(enter) + alpha = e + translationX = (1 - e) * -cq(30f).toPx() + } + .background(bg) + .padding(vertical = cq(2.6f)), + ) { + Text( + words.repeat(4), + color = fg, maxLines = 1, softWrap = false, + style = TextStyle(fontSize = cqSp(5f), fontWeight = FontWeight.ExtraBold), + modifier = Modifier + .wrapContentWidth(Alignment.Start, unbounded = true) + .graphicsLayer { + // The text holds the words four times; sliding by half loops cleanly. + val shift = size.width / 2f * (if (reverse) 1 - scroll else scroll) + translationX = -shift + }, + ) + } +} + +/** Material shapes thrown once from behind the version number. */ +@Composable +private fun Confetti(c: IntroColors, key: Any, modifier: Modifier) { + val p = entrance(key, 450, 1300) + val bits = remember { + val r = Random(7) + List(22) { i -> + val a = i / 22f * 2 * PI.toFloat() + r.nextFloat() * 0.4f + val d = 30f + r.nextFloat() * 45f + Bit(cos(a) * d, sin(a) * d * 1.2f, r.nextInt(-270, 270).toFloat(), i % 4, i % 3) + } + } + if (p <= 0f || p >= 1f) return + val tones = listOf(c.primary, c.primaryC, c.tertiaryC, c.secondaryC) + Canvas(modifier) { + val unit = size.width / 100f + val e = CubicBezierEasing(0.15f, 0.7f, 0.3f, 1f).transform(p) + val alpha = if (p < 0.7f) 1f else 1f - (p - 0.7f) / 0.3f + val origin = Offset(size.width * 0.3f, size.height * 0.42f) + bits.forEach { b -> + val at = origin + Offset(b.dx * unit * e, b.dy * unit * e) + val s = 3.6f * unit * (0.3f + 0.7f * e) + rotate(b.spin * e, at) { + val color = tones[b.tone].copy(alpha = alpha) + when (b.shape) { + 0 -> drawCircle(color, s / 2, at) + 1 -> drawRoundRect(color, at - Offset(s / 2, s / 2), Size(s, s), androidx.compose.ui.geometry.CornerRadius(unit)) + else -> drawRoundRect(color, at - Offset(s / 2, s / 4), Size(s, s / 2), androidx.compose.ui.geometry.CornerRadius(s / 4)) + } + } + } + } +} + +private class Bit(val dx: Float, val dy: Float, val spin: Float, val tone: Int, val shape: Int) + +/** One idea per screen: a spinning cookie with an icon, a headline, a line of text, maybe a button. */ +@Composable +private fun BadgeStep( + c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, badge: Color, + title: String, body: String, key: Any, + action: Pair Unit>? = null, outlined: Boolean = false, + extra: (@Composable () -> Unit)? = null, + icon: @Composable () -> Unit, +) { + val spin by rememberInfiniteTransition(label = "spin").animateFloat( + 0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin", + ) + Column( + Modifier.fillMaxSize().padding(horizontal = cq(8f)), + verticalArrangement = Arrangement.Center, + ) { + Box(Modifier.size(cq(34f)).pop(entrance(key, 50, 800)), contentAlignment = Alignment.Center) { + Cookie(badge, Modifier.fillMaxSize().graphicsLayer { rotationZ = spin }) + Box(Modifier.size(cq(15f))) { icon() } + } + Spacer(Modifier.height(cq(8f))) + Text( + title, color = c.onSurface, + style = TextStyle(fontSize = cqSp(13.5f), fontWeight = FontWeight.Black, letterSpacing = cqSp(-0.6f), lineHeight = cqSp(12.5f)), + modifier = Modifier.up(entrance(key, 200, 700), cq(7f)), + ) + Spacer(Modifier.height(cq(4f))) + Text( + body, color = c.onVariant, + style = TextStyle(fontSize = cqSp(4.5f), lineHeight = cqSp(6.5f)), + modifier = Modifier.up(entrance(key, 350, 700), cq(7f)), + ) + extra?.invoke() + if (action != null) { + val (label, onClick) = action + Spacer(Modifier.height(if (outlined) cq(7f) else cq(9f))) + Text( + label, + color = if (outlined) c.primary else c.onPrimary, + style = TextStyle(fontSize = cqSp(if (outlined) 4.4f else 4.6f), fontWeight = FontWeight.Bold), + modifier = Modifier + .up(entrance(key, 500, 700), cq(7f)) + .clip(RoundedCornerShape(50)) + .then( + if (outlined) Modifier.border(BorderStroke(2.dp, c.outline), RoundedCornerShape(50)) + else Modifier.background(c.primary), + ) + .clickable(onClick = onClick) + .padding(horizontal = if (outlined) cq(7f) else cq(9f), vertical = if (outlined) cq(3.6f) else cq(4.2f)), + ) + } + } +} + +/** + * Voice: a few things to say, drifting past, and a button that fetches both speech models so the + * assistant works the first time someone tries it in the car. + */ +@Composable +private fun VoiceStep(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) { + val context = LocalContext.current + remember { SpeechModels.init(context) } + val needed = listOf(SpeechModels.Model.KROKO, SpeechModels.Model.PARAKEET_LARGE) + val states = needed.map { SpeechModels.state(it).collectAsState().value } + val ready = states.all { it == SpeechModels.State.Ready } + val label = when { + states.any { it is SpeechModels.State.Downloading } -> + "Downloading, ${needed.zip(states).sumOf { (m, st) -> m.downloadMb * percentOf(st) } / needed.sumOf { it.downloadMb }}%" + states.any { it is SpeechModels.State.Failed } -> "Download failed. Try again" + else -> "Download voice, ${needed.sumOf { it.downloadMb }} MB" + } + BadgeStep( + c, cq, cqSp, c.primaryC, "Just say it.", + "Tap the mic, or press the talk button on your wheel, and speak. Your phone turns speech " + + "into text by itself, and nothing you say leaves it. Say \"help\" in the car for the full list.", + key, + action = if (ready) null else label to { needed.forEach { SpeechModels.download(context, it) } }, + outlined = true, + extra = { CommandDrift(c, cq, cqSp, key) }, + ) { MicIcon(c.onPrimaryC) } +} + +private fun percentOf(state: SpeechModels.State) = when (state) { + SpeechModels.State.Ready -> 100 + is SpeechModels.State.Downloading -> state.percent + else -> 0 +} + +private val SAMPLE_COMMANDS = listOf( + "Play Daft Punk", "What's playing?", "Next song", + "Call Mom", "Open Organic Maps", "Pause", "Help", +) + +/** Example commands rising slowly through a window that fades at both edges. */ +@Composable +private fun CommandDrift(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: Any) { + val scroll by rememberInfiniteTransition(label = "drift").animateFloat( + 0f, 1f, infiniteRepeatable(tween(SAMPLE_COMMANDS.size * 2_200, easing = LinearEasing)), label = "drift", + ) + Box( + Modifier + .padding(top = cq(6f)) + .fillMaxWidth() + .height(cq(34f)) + .up(entrance(key, 500, 700), cq(7f)) + .graphicsLayer { compositingStrategy = CompositingStrategy.Offscreen } + .drawWithContent { + drawContent() + drawRect( + Brush.verticalGradient(0f to Color.Transparent, 0.25f to Color.Black, 0.75f to Color.Black, 1f to Color.Transparent), + blendMode = BlendMode.DstIn, + ) + } + .clipToBounds(), + ) { + Column( + Modifier + .wrapContentHeight(Alignment.Top, unbounded = true) + // The list is there twice; rising by half loops cleanly. + .graphicsLayer { translationY = -size.height / 2f * scroll }, + verticalArrangement = Arrangement.spacedBy(cq(2.4f)), + ) { + repeat(2) { + SAMPLE_COMMANDS.forEach { say -> + Text( + "\u201c$say\u201d", color = c.onSecondaryC, + style = TextStyle(fontSize = cqSp(4.4f), fontWeight = FontWeight.SemiBold), + modifier = Modifier + .clip(RoundedCornerShape(50)) + .background(c.secondaryC) + .padding(horizontal = cq(4.4f), vertical = cq(2f)), + ) + } + } + } + } +} + +private val APP_PICKS = listOf( + Triple("Music", "Metrolist", "https://github.com/MetrolistGroup/Metrolist"), + Triple("Maps", "Vela", "https://github.com/PimpinPumpkin/Vela"), + Triple("Podcasts", "AntennaPod", "https://github.com/AntennaPod/AntennaPod"), +) + +/** One row per pick, each opening the app's GitHub page; they arrive one after another. */ +@Composable +private fun AppPicks(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) { + val context = LocalContext.current + Column(Modifier.padding(top = cq(6f)), verticalArrangement = Arrangement.spacedBy(cq(2.6f))) { + APP_PICKS.forEachIndexed { i, (kind, name, url) -> + Row( + Modifier + .fillMaxWidth() + .up(entrance(key, 500 + i * 120, 700), cq(7f)) + .clip(RoundedCornerShape(cq(5f))) + .background(c.secondaryC) + .clickable { + runCatching { + context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(url)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + } + } + .padding(horizontal = cq(5f), vertical = cq(3.4f)), + verticalAlignment = Alignment.CenterVertically, + ) { + Column(Modifier.weight(1f)) { + Text(kind.uppercase(), color = c.onSecondaryC.copy(alpha = 0.7f), + style = TextStyle(fontSize = cqSp(3.2f), fontWeight = FontWeight.Bold, letterSpacing = cqSp(0.3f))) + Text(name, color = c.onSecondaryC, style = TextStyle(fontSize = cqSp(5.2f), fontWeight = FontWeight.ExtraBold)) + } + Text("GitHub ↗", color = c.onSecondaryC, style = TextStyle(fontSize = cqSp(3.8f), fontWeight = FontWeight.SemiBold)) + } + } + } +} + +/** Material's scalloped "cookie": a circle whose radius waves around its edge. */ +@Composable +private fun Cookie(color: Color, modifier: Modifier) { + Canvas(modifier) { + val unit = size.minDimension / 100f + val path = Path() + var deg = 0 + while (deg <= 360) { + val a = deg * PI / 180 + val r = (44 + 4.5 * cos(9 * a)) * unit + val x = center.x + (r * cos(a)).toFloat() + val y = center.y + (r * sin(a)).toFloat() + if (deg == 0) path.moveTo(x, y) else path.lineTo(x, y) + deg += 3 + } + path.close() + drawPath(path, color) + } +} + +/** Draws in the mockup's 48-unit icon box. */ +private inline fun DrawScope.inIconBox(block: DrawScope.() -> Unit) { + val s = size.minDimension / 48f + scale(s, s, Offset.Zero) { block() } +} + +@Composable +private fun WifiIcon(color: Color) { + val t = rememberInfiniteTransition(label = "wifi") + val waves = (0..2).map { i -> + t.animateFloat( + 0.35f, 1f, + infiniteRepeatable(tween(800, delayMillis = 0, easing = FastOutSlowInEasing), RepeatMode.Reverse, + initialStartOffset = androidx.compose.animation.core.StartOffset(i * 200)), + label = "wave$i", + ) + } + Canvas(Modifier.fillMaxSize()) { + inIconBox { + val stroke = Stroke(5f, cap = StrokeCap.Round) + // The mockup's three arcs, inner to outer: radius r through (24 ± half, y). + listOf(Triple(8f, 6f, 34f), Triple(18f, 13f, 26f), Triple(28f, 20f, 18f)).forEachIndexed { i, (r, half, y) -> + val cy = y + sqrt(r * r - half * half) + val start = Math.toDegrees(atan2((y - cy).toDouble(), (-half).toDouble())).toFloat() + val end = Math.toDegrees(atan2((y - cy).toDouble(), half.toDouble())).toFloat() + drawArc(color.copy(alpha = waves[i].value), start, (end - start + 360f) % 360f, false, + Offset(24f - r, cy - r), Size(2 * r, 2 * r), style = stroke) + } + drawCircle(color, 3.6f, Offset(24f, 41f)) + } + } +} + +@Composable +private fun DialIcon(color: Color) { + val angle by rememberInfiniteTransition(label = "dial").animateFloat( + -70f, -70f, + infiniteRepeatable(keyframes { + durationMillis = 2600 + -70f at 0 using CubicBezierEasing(0.5f, 0f, 0.3f, 1f) + 70f at 1040 + 70f at 1430 using CubicBezierEasing(0.5f, 0f, 0.3f, 1f) + -70f at 2470 + }), + label = "dial", + ) + Canvas(Modifier.fillMaxSize()) { + inIconBox { + drawCircle(color, 19f, Offset(24f, 24f), style = Stroke(5f)) + rotate(angle, Offset(24f, 24f)) { + drawRoundRect(color, Offset(21.5f, 7f), Size(5f, 13f), androidx.compose.ui.geometry.CornerRadius(2.5f)) + } + } + } +} + +@Composable +private fun MicIcon(color: Color) { + val t = rememberInfiniteTransition(label = "mic") + val waves = (0..1).map { i -> + t.animateFloat( + 0.25f, 1f, + infiniteRepeatable(tween(700, easing = FastOutSlowInEasing), RepeatMode.Reverse, + initialStartOffset = androidx.compose.animation.core.StartOffset(i * 250)), + label = "mic$i", + ) + } + Canvas(Modifier.fillMaxSize()) { + inIconBox { + drawRoundRect(color, Offset(18f, 5f), Size(12f, 22f), androidx.compose.ui.geometry.CornerRadius(6f)) + val stroke = Stroke(4f, cap = StrokeCap.Round) + drawArc(color, 0f, 180f, false, Offset(12f, 14f), Size(24f, 22f), style = stroke) + drawLine(color, Offset(24f, 36f), Offset(24f, 42f), 4f, StrokeCap.Round) + // Sound waves either side, pulsing as if someone's talking. + listOf(1f to 0, -1f to 0, 1f to 1, -1f to 1).forEach { (side, i) -> + val r = if (i == 0) 15f else 21f + drawArc(color.copy(alpha = waves[i].value), if (side > 0) -35f else 145f, 70f, false, + Offset(24f - r, 16f - r), Size(2 * r, 2 * r), style = stroke) + } + } + } +} + +@Composable +private fun StarIcon(color: Color) { + val turn by rememberInfiniteTransition(label = "star").animateFloat( + -8f, 8f, infiniteRepeatable(tween(1600, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "star", + ) + Canvas(Modifier.fillMaxSize()) { + inIconBox { + val star = Path() + for (i in 0 until 10) { + val r = if (i % 2 == 0) 20f else 8.5f + val a = (-90 + i * 36) * PI / 180 + val x = 24f + (r * cos(a)).toFloat() + val y = 25f + (r * sin(a)).toFloat() + if (i == 0) star.moveTo(x, y) else star.lineTo(x, y) + } + star.close() + rotate(turn, Offset(24f, 25f)) { drawPath(star, color) } + } + } +} + +@Composable +private fun ShieldIcon(color: Color, tick: Color) { + val p = entrance(IntroStep.GRAPHENE, 800, 500) + Canvas(Modifier.fillMaxSize()) { + inIconBox { + val shield = Path().apply { + moveTo(24f, 4f); lineTo(40f, 10f); lineTo(40f, 22f) + cubicTo(40f, 32f, 33f, 40f, 24f, 44f) + cubicTo(15f, 40f, 8f, 32f, 8f, 22f) + lineTo(8f, 10f); close() + } + drawPath(shield, color) + drawPartialStroke(listOf(Offset(16f, 24f), Offset(22f, 30f), Offset(33f, 18f)), p, tick, 4.5f) + } + } +} + +@Composable +private fun HeartIcon(color: Color) { + val beat by rememberInfiniteTransition(label = "beat").animateFloat( + 1f, 1f, + infiniteRepeatable(keyframes { + durationMillis = 1400 + 1f at 0; 1.16f at 210; 1f at 420; 1.1f at 630; 1f at 840 + }), + label = "beat", + ) + Canvas(Modifier.fillMaxSize()) { + inIconBox { + scale(beat, beat, Offset(24f, 24f)) { + val heart = Path().apply { + moveTo(24f, 41f) + cubicTo(10f, 31f, 5f, 23f, 5f, 16f) + cubicTo(5f, 10f, 10f, 6f, 15f, 6f) + cubicTo(19f, 6f, 22f, 8f, 24f, 12f) + cubicTo(26f, 8f, 29f, 6f, 33f, 6f) + cubicTo(38f, 6f, 43f, 10f, 43f, 16f) + cubicTo(43f, 23f, 38f, 31f, 24f, 41f) + close() + } + drawPath(heart, color) + } + } + } +} + +@Composable +private fun CheckIcon(color: Color) { + val p = entrance(IntroStep.DRIVE, 300, 500) + Canvas(Modifier.fillMaxSize()) { + inIconBox { drawPartialStroke(listOf(Offset(10f, 26f), Offset(20f, 35f), Offset(38f, 14f)), p, color, 5.5f) } + } +} + +/** A polyline drawn up to [progress] of its length, so a tick can sweep in. */ +private fun DrawScope.drawPartialStroke(points: List, progress: Float, color: Color, width: Float) { + if (progress <= 0f) return + val lengths = points.zipWithNext { a, b -> (b - a).getDistance() } + var left = lengths.sum() * FastOutSlowInEasing.transform(progress.coerceAtMost(1f)) + val path = Path().apply { moveTo(points[0].x, points[0].y) } + for (i in lengths.indices) { + if (left <= 0f) break + val f = (left / lengths[i]).coerceAtMost(1f) + val to = points[i] + (points[i + 1] - points[i]) * f + path.lineTo(to.x, to.y) + left -= lengths[i] + } + drawPath(path, color, style = Stroke(width, cap = StrokeCap.Round, join = StrokeJoin.Round)) +} + +@Composable +private fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) { + Row(modifier, horizontalArrangement = Arrangement.spacedBy(cq(2f))) { + repeat(count) { i -> + val on = i == current + val width by animateDpAsState(if (on) cq(7f) else cq(2f), tween(300), label = "dot") + Box( + Modifier + .size(width, cq(2f)) + .clip(RoundedCornerShape(50)) + .background(if (on) c.primary else c.outline.copy(alpha = 0.45f)), + ) + } + } +} + +/** "Tap to continue", once the step has finished arriving, breathing softly. */ +@Composable +private fun Hint(c: IntroColors, cqSp: (Float) -> TextUnit, key: Any, modifier: Modifier) { + val shown = entrance(key, 1600, 600) + val breathe by rememberInfiniteTransition(label = "hint").animateFloat( + 0.75f, 0.35f, infiniteRepeatable(tween(1200), RepeatMode.Reverse), label = "hint", + ) + Text( + "Tap to continue", color = c.onVariant, + style = TextStyle(fontSize = cqSp(3.6f), fontWeight = FontWeight.SemiBold), + modifier = modifier.graphicsLayer { alpha = if (shown < 1f) shown * 0.75f else breathe }, + ) +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/RecommendedApps.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/RecommendedApps.kt index 2a84106..4ee88bf 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/RecommendedApps.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/RecommendedApps.kt @@ -54,6 +54,9 @@ private val recommended = listOf( RecommendedApp("ViVi Music", "com.vivi.vivimusic", "https://github.com/vivizzz007/vivi-music", "A YouTube Music player."), RecommendedApp("Metrolist", "com.metrolist.music", "https://github.com/MetrolistGroup/Metrolist", "A YouTube Music player."), ), + "Podcasts" to listOf( + RecommendedApp("AntennaPod", "de.danoeh.antennapod", "https://github.com/AntennaPod/AntennaPod", "Podcasts, chapters and all."), + ), ) /** Opened from Settings: the same list the setup guide ends with. */ diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt index 90dab03..53f6146 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SettingsScreen.kt @@ -73,6 +73,7 @@ fun SettingsScreen( onOpenRecommendedApps: () -> Unit, onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit, + onReplayIntro: () -> Unit, atUsageNotes: Boolean = false, ) { val scroll = rememberScrollState() @@ -105,9 +106,10 @@ fun SettingsScreen( verticalArrangement = Arrangement.spacedBy(24.dp), ) { Spacer(Modifier.height(0.dp)) - AboutSection(onOpenWhatsNew, onReplayTutorial) + AboutSection(onOpenWhatsNew, onReplayTutorial, onReplayIntro) ConnectionSection() CarAudioSection() + VoiceModelSection() if (SHOW_RECOMMENDED_APPS) AppsSection(onOpenRecommendedApps) CertificateSection() SettingsSection("Usage notes", header = USAGE_STATS_INTRO) { UsageStatsRows() } @@ -125,7 +127,7 @@ private fun Arrow() = Icon( /** What's new, updates and the setup guide, one tap from the top. */ @Composable -private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit) { +private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit, onReplayIntro: () -> Unit) { SettingsSection { SettingsRow( "What's new", @@ -148,6 +150,16 @@ private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Uni icon = Icons.Filled.Refresh, trailing = { Arrow() }, ) + SettingsDivider() + SettingsRow( + "Replay the ${Intro.release} intro", + Modifier.clickable(onClick = onReplayIntro), + subtitle = "What's in this release, in a few taps", + icon = Icons.Filled.Star, + trailing = { Arrow() }, + ) + SettingsDivider() + AboutLinkRows() } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt new file mode 100644 index 0000000..4d8f22b --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SupportViews.kt @@ -0,0 +1,161 @@ +package app.seb3thehacker.gearslip.ui + +import android.content.Context +import android.content.Intent +import android.net.Uri +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.PaddingValues +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.Spacer +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.automirrored.filled.KeyboardArrowRight +import androidx.compose.material.icons.automirrored.filled.Send +import androidx.compose.material.icons.filled.Build +import androidx.compose.material.icons.filled.Favorite +import androidx.compose.material3.Button +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp + +/** Where Gearslip lives outside the app: donations, the chat room, and the code. */ +internal object GearslipLinks { + const val SPONSORS = "https://github.com/sponsors/Seb3thehacker" + const val LIBERAPAY = "https://liberapay.com/Seb3thehacker" + const val MATRIX = "https://matrix.to/#/!mf0_mgBd5KqKpc9FfX_wHp1jFCcwAtPrndIn_oGIyMk?via=matrix.org" + const val SOURCE = "https://github.com/Seb3thehacker/gearslip" + + fun open(context: Context, url: String) { + runCatching { + context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(url)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)) + } + } +} + +/** "Hide for now" on the home card puts it away for this long; the About page keeps the links. */ +private const val HIDE_MS = 30L * 24 * 60 * 60 * 1000 +private const val PREFS = "support" +private const val KEY_HIDDEN_UNTIL = "hidden_until" + +private fun hiddenUntil(context: Context) = + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getLong(KEY_HIDDEN_UNTIL, 0L) + +/** + * The donation ask on Home: always in view, never in the way. It sits below the status and update + * cards, and "Hide for now" puts it away for a month. + */ +@Composable +internal fun SupportCard(modifier: Modifier = Modifier) { + val context = LocalContext.current + var hidden by remember { mutableStateOf(System.currentTimeMillis() < hiddenUntil(context)) } + if (hidden) return + val scheme = MaterialTheme.colorScheme + Surface( + modifier, + color = scheme.tertiaryContainer, + contentColor = scheme.onTertiaryContainer, + shape = MaterialTheme.shapes.extraLarge, + ) { + Column(Modifier.fillMaxWidth().padding(20.dp)) { + Row(verticalAlignment = Alignment.CenterVertically) { + Icon(Icons.Filled.Favorite, contentDescription = null) + Text( + "Support Gearslip", + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.SemiBold, + modifier = Modifier.padding(start = 12.dp), + ) + } + Text( + "Gearslip is free, with no ads and no tracking. If it gets you on the road, a " + + "donation keeps it going.", + style = MaterialTheme.typography.bodyMedium, + modifier = Modifier.padding(top = 8.dp), + ) + Row( + Modifier.fillMaxWidth().padding(top = 12.dp), + verticalAlignment = Alignment.CenterVertically, + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + TextButton( + onClick = { + context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit() + .putLong(KEY_HIDDEN_UNTIL, System.currentTimeMillis() + HIDE_MS).apply() + hidden = true + }, + contentPadding = PaddingValues(horizontal = 0.dp), + ) { Text("Hide for now") } + Spacer(Modifier.weight(1f)) + OutlinedButton(onClick = { GearslipLinks.open(context, GearslipLinks.LIBERAPAY) }) { Text("Liberapay") } + Button(onClick = { GearslipLinks.open(context, GearslipLinks.SPONSORS) }) { Text("GitHub") } + } + } + } +} + +/** The About section's links: donating, the chat room, and the source code. */ +@Composable +internal fun AboutLinkRows() { + val context = LocalContext.current + val arrow = @Composable { + Icon( + Icons.AutoMirrored.Filled.KeyboardArrowRight, + contentDescription = null, + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + SettingsRow( + "Donate on GitHub Sponsors", + Modifier.clickable { GearslipLinks.open(context, GearslipLinks.SPONSORS) }, + subtitle = "Give once or monthly, through GitHub", + icon = Icons.Filled.Favorite, + trailing = arrow, + ) + SettingsDivider() + SettingsRow( + "Donate on Liberapay", + Modifier.clickable { GearslipLinks.open(context, GearslipLinks.LIBERAPAY) }, + subtitle = "Give weekly or monthly, through a nonprofit", + icon = Icons.Filled.Favorite, + trailing = arrow, + ) + SettingsDivider() + SettingsRow( + "Join the discussion on Matrix", + Modifier.clickable { GearslipLinks.open(context, GearslipLinks.MATRIX) }, + subtitle = "Questions, ideas, or a car to report", + icon = Icons.AutoMirrored.Filled.Send, + trailing = arrow, + ) + SettingsDivider() + SettingsRow( + "Source code", + Modifier.clickable { GearslipLinks.open(context, GearslipLinks.SOURCE) }, + subtitle = "Read or change Gearslip's code under the AGPL", + icon = Icons.Filled.Build, + trailing = arrow, + ) + SettingsDivider() + SettingsRow( + "Voice credits", + subtitle = "Speech recognition by sherpa-onnx (Apache 2.0), with Kroko ASR by Banafo " + + "(CC BY-SA 4.0), and Parakeet and Canary by NVIDIA (CC BY 4.0)", + icon = Icons.Filled.Favorite, + ) +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt new file mode 100644 index 0000000..c11665d --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/VoiceModelViews.kt @@ -0,0 +1,125 @@ +package app.seb3thehacker.gearslip.ui + +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.selection.selectable +import androidx.compose.material3.LinearProgressIndicator +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.RadioButton +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.speech.SpeechModels +import app.seb3thehacker.gearslip.speech.SpeechModels.Model +import app.seb3thehacker.gearslip.speech.SpeechModels.State + +/** + * Which speech model the voice assistant listens with, and downloading it. Each is fetched once + * from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening. + */ +@Composable +internal fun VoiceModelSection() { + val context = LocalContext.current + remember { SpeechModels.init(context) } + val selected by SpeechModels.selected.collectAsState() + val final by SpeechModels.final.collectAsState() + SettingsSection(title = "Voice: live words") { + Text( + "The voice assistant turns speech into text on this phone, and nothing you say leaves it. " + + "It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " + + "This one writes your words as you speak.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + ) + Model.entries.filter { it.role == SpeechModels.Role.LIVE }.forEachIndexed { i, model -> + if (i > 0) SettingsDivider() + ModelRow(model, model == selected) + } + } + SettingsSection(title = "Voice: final words") { + Text( + "When you stop talking, this model reads the whole phrase again. It catches names and " + + "places the live model misses.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), + ) + val context = LocalContext.current + Row( + Modifier + .fillMaxWidth() + .selectable(final == null, role = Role.RadioButton) { SpeechModels.selectFinal(context, null) } + .padding(horizontal = 8.dp, vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + RadioButton(final == null, onClick = null, modifier = Modifier.padding(8.dp)) + Column { + Text("None", style = MaterialTheme.typography.titleMedium) + Text( + "Keep the live words. Fastest, least accurate", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + } + Model.entries.filter { it.role == SpeechModels.Role.FINAL }.forEach { model -> + SettingsDivider() + ModelRow(model, model == final) + } + } +} + +@Composable +private fun ModelRow(model: Model, chosen: Boolean) { + val context = LocalContext.current + val state by SpeechModels.state(model).collectAsState() + Column( + Modifier + .fillMaxWidth() + .selectable(chosen, role = Role.RadioButton) { SpeechModels.select(context, model) } + .padding(horizontal = 8.dp, vertical = 8.dp), + ) { + Row(verticalAlignment = Alignment.CenterVertically) { + RadioButton(chosen, onClick = null, modifier = Modifier.padding(8.dp)) + Column(Modifier.weight(1f)) { + Text("${model.label} · ${model.downloadMb} MB", style = MaterialTheme.typography.titleMedium) + Text( + when (val s = state) { + State.Ready -> "Downloaded. ${model.detail}" + is State.Downloading -> "Downloading, ${s.percent}%" + is State.Failed -> "Download failed: ${s.reason}" + State.Missing -> model.detail + }, + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + Row(horizontalArrangement = Arrangement.spacedBy(4.dp)) { + when (state) { + State.Ready -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Delete") } + is State.Downloading -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Cancel") } + else -> OutlinedButton(onClick = { SpeechModels.download(context, model) }) { Text("Download") } + } + } + } + (state as? State.Downloading)?.let { + LinearProgressIndicator( + progress = { it.percent / 100f }, + modifier = Modifier.fillMaxWidth().padding(start = 48.dp, end = 8.dp, top = 4.dp), + ) + } + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt new file mode 100644 index 0000000..44491c7 --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/FeatureConfig.kt @@ -0,0 +1,12 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +data class FeatureConfig( + var sampleRate: Int = 16000, + var featureDim: Int = 80, + var dither: Float = 0.0f +) + +fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig { + return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim) +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt new file mode 100644 index 0000000..d282f0d --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.kt @@ -0,0 +1,8 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +data class HomophoneReplacerConfig( + var dictDir: String = "", // unused + var lexicon: String = "", + var ruleFsts: String = "", +) diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt new file mode 100644 index 0000000..ac229ac --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineRecognizer.kt @@ -0,0 +1,255 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out. +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class OfflineRecognizerResult( + val text: String, + val tokens: Array, + val timestamps: FloatArray, + val lang: String, + val emotion: String, + val event: String, + + // valid only for TDT models + val durations: FloatArray, +) + +data class OfflineTransducerModelConfig( + var encoder: String = "", + var decoder: String = "", + var joiner: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineParaformerModelConfig( + var model: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineNemoEncDecCtcModelConfig( + var model: String = "", +) + +data class OfflineDolphinModelConfig( + var model: String = "", +) + +data class OfflineZipformerCtcModelConfig( + var model: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineWenetCtcModelConfig( + var model: String = "", +) + +data class OfflineOmnilingualAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineMedAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineFireRedAsrCtcModelConfig( + var model: String = "", +) + +data class OfflineFunAsrNanoModelConfig( + var encoderAdaptor: String = "", + var llm: String = "", + var embedding: String = "", + var tokenizer: String = "", + var systemPrompt: String = "You are a helpful assistant.", + var userPrompt: String = "语音转写:", + var maxNewTokens: Int = 512, + var temperature: Float = 1e-6f, + var topP: Float = 0.8f, + var seed: Int = 42, + var language: String = "", + var itn: Boolean = true, + var hotwords: String = "", +) + +data class OfflineQwen3AsrModelConfig( + var convFrontend: String = "", + var encoder: String = "", + var decoder: String = "", + var tokenizer: String = "", + var maxTotalLen: Int = 512, + var maxNewTokens: Int = 128, + var temperature: Float = 1e-6f, + var topP: Float = 0.8f, + var seed: Int = 42, + var hotwords: String = "", +) + +data class OfflineWhisperModelConfig( + var encoder: String = "", + var decoder: String = "", + var language: String = "en", // Used with multilingual model + var task: String = "transcribe", // transcribe or translate + var tailPaddings: Int = 1000, // Padding added at the end of the samples + var enableTokenTimestamps: Boolean = false, + var enableSegmentTimestamps: Boolean = false, +) + +data class OfflineCanaryModelConfig( + var encoder: String = "", + var decoder: String = "", + var srcLang: String = "en", + var tgtLang: String = "en", + var usePnc: Boolean = true, +) + +data class OfflineCohereTranscribeModelConfig( + var encoder: String = "", + var decoder: String = "", + var language: String = "", + var usePunct: Boolean = true, + var useItn: Boolean = true, +) + +data class OfflineFireRedAsrModelConfig( + var encoder: String = "", + var decoder: String = "", +) + +// For moonshine v1, you need four models. +// For moonshine v2, you need two models. +// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder +// - v2: encoder, mergedDecoder +data class OfflineMoonshineModelConfig( + var preprocessor: String = "", + var encoder: String = "", + var uncachedDecoder: String = "", + var cachedDecoder: String = "", + var mergedDecoder: String = "", +) + +data class OfflineSenseVoiceModelConfig( + var model: String = "", + var language: String = "", + var useInverseTextNormalization: Boolean = true, + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OfflineModelConfig( + var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(), + var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(), + var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(), + var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(), + var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(), + var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(), + var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(), + var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(), + var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(), + var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(), + var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(), + var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(), + var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(), + var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(), + var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(), + var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(), + var cohereTranscribe: OfflineCohereTranscribeModelConfig = + OfflineCohereTranscribeModelConfig(), + var teleSpeech: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", + var modelType: String = "", + var tokens: String = "", + var modelingUnit: String = "", + var bpeVocab: String = "", +) + +data class OfflineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OfflineModelConfig = OfflineModelConfig(), + // var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it + var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(), + var decodingMethod: String = "greedy_search", + var maxActivePaths: Int = 4, + var hotwordsFile: String = "", + var hotwordsScore: Float = 1.5f, + var ruleFsts: String = "", + var ruleFars: String = "", + var blankPenalty: Float = 0.0f, +) + +class OfflineRecognizer( + assetManager: AssetManager? = null, + val config: OfflineRecognizerConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(): OfflineStream { + val p = createStream(ptr) + return OfflineStream(p) + } + + fun createStream(hotwords: String): OfflineStream { + val p = createStreamWithHotwords(ptr, hotwords) + return OfflineStream(p) + } + + fun getResult(stream: OfflineStream): OfflineRecognizerResult { + return getResult(stream.ptr) + } + + fun decode(stream: OfflineStream) = decode(ptr, stream.ptr) + + fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config) + + private external fun delete(ptr: Long) + + private external fun createStream(ptr: Long): Long + + private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long + + private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OfflineRecognizerConfig, + ): Long + + private external fun newFromFile( + config: OfflineRecognizerConfig, + ): Long + + private external fun decode(ptr: Long, streamPtr: Long) + + private external fun getResult(streamPtr: Long): OfflineRecognizerResult + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + + @JvmStatic + external fun prependAdspLibraryPath(newPath: String) // for qnn + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt new file mode 100644 index 0000000..12ac836 --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OfflineStream.kt @@ -0,0 +1,43 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +class OfflineStream(var ptr: Long) { + init { + require(ptr != 0L) { "Failed to create native OfflineStream" } + } + + fun acceptWaveform(samples: FloatArray, sampleRate: Int) = + acceptWaveform(ptr, samples, sampleRate) + + fun setOption(key: String, value: String) = setOption(ptr, key, value) + + fun getOption(key: String): String = getOption(ptr, key) + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun use(block: (OfflineStream) -> Unit) { + try { + block(this) + } finally { + release() + } + } + + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) + private external fun setOption(ptr: Long, key: String, value: String) + private external fun getOption(ptr: Long, key: String): String + private external fun delete(ptr: Long) + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt new file mode 100644 index 0000000..3cd4a16 --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.kt @@ -0,0 +1,158 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out. +package com.k2fsa.sherpa.onnx + +import android.content.res.AssetManager + +data class EndpointRule( + var mustContainNonSilence: Boolean, + var minTrailingSilence: Float, + var minUtteranceLength: Float, +) + +data class EndpointConfig( + var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f), + var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f), + var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f) +) + +data class OnlineTransducerModelConfig( + var encoder: String = "", + var decoder: String = "", + var joiner: String = "", + var qnnConfig: QnnConfig = QnnConfig(), +) + +data class OnlineParaformerModelConfig( + var encoder: String = "", + var decoder: String = "", +) + +data class OnlineZipformer2CtcModelConfig( + var model: String = "", +) + +data class OnlineNeMoCtcModelConfig( + var model: String = "", +) + +data class OnlineToneCtcModelConfig( + var model: String = "", +) + +data class OnlineModelConfig( + var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(), + var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(), + var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(), + var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(), + var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(), + var tokens: String = "", + var numThreads: Int = 1, + var debug: Boolean = false, + var provider: String = "cpu", + var modelType: String = "", + var modelingUnit: String = "", + var bpeVocab: String = "", +) + +data class OnlineLMConfig( + var model: String = "", + var scale: Float = 0.5f, +) + +data class OnlineCtcFstDecoderConfig( + var graph: String = "", + var maxActive: Int = 3000, +) + +data class OnlineRecognizerConfig( + var featConfig: FeatureConfig = FeatureConfig(), + var modelConfig: OnlineModelConfig = OnlineModelConfig(), + var lmConfig: OnlineLMConfig = OnlineLMConfig(), + var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(), + var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(), + var endpointConfig: EndpointConfig = EndpointConfig(), + var enableEndpoint: Boolean = true, + var decodingMethod: String = "greedy_search", + var maxActivePaths: Int = 4, + var hotwordsFile: String = "", + var hotwordsScore: Float = 1.5f, + var ruleFsts: String = "", + var ruleFars: String = "", + var blankPenalty: Float = 0.0f, +) + +data class OnlineRecognizerResult( + val text: String, + val tokens: Array, + val timestamps: FloatArray, + val ysProbs: FloatArray, + // TODO(fangjun): Add more fields +) + +class OnlineRecognizer( + assetManager: AssetManager? = null, + val config: OnlineRecognizerConfig, +) { + private var ptr: Long + + init { + ptr = if (assetManager != null) { + newFromAsset(assetManager, config) + } else { + newFromFile(config) + } + require(ptr != 0L) { + "Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer" + } + } + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun createStream(hotwords: String = ""): OnlineStream { + val p = createStream(ptr, hotwords) + return OnlineStream(p) + } + + fun reset(stream: OnlineStream) = reset(ptr, stream.ptr) + fun decode(stream: OnlineStream) = decode(ptr, stream.ptr) + fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr) + fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr) + fun getResult(stream: OnlineStream): OnlineRecognizerResult { + return getResult(ptr, stream.ptr) + } + + private external fun delete(ptr: Long) + + private external fun newFromAsset( + assetManager: AssetManager, + config: OnlineRecognizerConfig, + ): Long + + private external fun newFromFile( + config: OnlineRecognizerConfig, + ): Long + + private external fun createStream(ptr: Long, hotwords: String): Long + private external fun reset(ptr: Long, streamPtr: Long) + private external fun decode(ptr: Long, streamPtr: Long) + private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean + private external fun isReady(ptr: Long, streamPtr: Long): Boolean + private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + + @JvmStatic + external fun prependAdspLibraryPath(newPath: String) // for qnn + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt new file mode 100644 index 0000000..6eebf5e --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/OnlineStream.kt @@ -0,0 +1,47 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +class OnlineStream(var ptr: Long = 0) { + init { + require(ptr != 0L) { "Failed to create native OnlineStream" } + } + + fun acceptWaveform(samples: FloatArray, sampleRate: Int) = + acceptWaveform(ptr, samples, sampleRate) + + fun inputFinished() = inputFinished(ptr) + + fun setOption(key: String, value: String) = setOption(ptr, key, value) + + fun getOption(key: String): String = getOption(ptr, key) + + protected fun finalize() { + if (ptr != 0L) { + delete(ptr) + ptr = 0 + } + } + + fun release() = finalize() + + fun use(block: (OnlineStream) -> Unit) { + try { + block(this) + } finally { + release() + } + } + + private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int) + private external fun inputFinished(ptr: Long) + private external fun setOption(ptr: Long, key: String, value: String) + private external fun getOption(ptr: Long, key: String): String + private external fun delete(ptr: Long) + + + companion object { + init { + System.loadLibrary("sherpa-onnx-jni") + } + } +} diff --git a/gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt new file mode 100644 index 0000000..95be41c --- /dev/null +++ b/gearslip/src/main/java/com/k2fsa/sherpa/onnx/QnnConfig.kt @@ -0,0 +1,8 @@ +// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api. +package com.k2fsa.sherpa.onnx + +data class QnnConfig( + var backendLib: String = "", + var contextBinary: String = "", + var systemLib: String = "", +) diff --git a/gearslip/src/main/assets/speech/tiny_acft_q5_1.bin b/gearslip/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so similarity index 59% rename from gearslip/src/main/assets/speech/tiny_acft_q5_1.bin rename to gearslip/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so index 541f1df..b2f18d1 100644 Binary files a/gearslip/src/main/assets/speech/tiny_acft_q5_1.bin and b/gearslip/src/main/jniLibs/arm64-v8a/libsherpa-onnx-jni.so differ diff --git a/gearslip/src/main/res/drawable/ic_launcher_monochrome.xml b/gearslip/src/main/res/drawable/ic_launcher_monochrome.xml index e173db8..d6da12a 100644 --- a/gearslip/src/main/res/drawable/ic_launcher_monochrome.xml +++ b/gearslip/src/main/res/drawable/ic_launcher_monochrome.xml @@ -9,12 +9,10 @@ android:fillColor="#FF000000" android:fillAlpha="0.35"/> - diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index f967a0d..0075398 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -14,6 +14,7 @@ material = "1.14.0" constraintlayout = "2.2.1" zxing = "3.5.4" okhttp = "5.4.0" +commonsCompress = "1.28.0" kotlinxSerializationJson = "1.11.0" ossLicenses = "17.5.1" webkit = "1.16.0" @@ -49,6 +50,7 @@ androidx-constraintlayout = { group = "androidx.constraintlayout", name = "const google-material = { group = "com.google.android.material", name = "material", version.ref = "material" } zxing-core = { group = "com.google.zxing", name = "core", version.ref = "zxing" } okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" } +commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commonsCompress" } kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" } google-oss-licenses = { group = "com.google.android.gms", name = "play-services-oss-licenses", version.ref = "ossLicenses" } androidx-webkit = { group = "androidx.webkit", name = "webkit", version.ref = "webkit" }