mirror of
https://github.com/twinksoftware/Gearslip.git
synced 2026-11-10 16:02:25 +01:00
Merge branch 'experimental' into wireless
# Conflicts: # gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt
This commit is contained in:
commit
9fe463c4e7
50 files changed
+2801
-534
No files matched your search
@@ -0,0 +1,15 @@
|
|||||||
|
# Weekly pull requests for newer Gradle libraries and GitHub Actions, aimed at experimental,
|
||||||
|
# where unreleased code lands. Security fixes come separately, as soon as an alert opens.
|
||||||
|
version: 2
|
||||||
|
updates:
|
||||||
|
- package-ecosystem: gradle
|
||||||
|
directory: /
|
||||||
|
target-branch: experimental
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
|
open-pull-requests-limit: 5
|
||||||
|
- package-ecosystem: github-actions
|
||||||
|
directory: /
|
||||||
|
target-branch: experimental
|
||||||
|
schedule:
|
||||||
|
interval: weekly
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
name: CodeQL
|
||||||
|
|
||||||
|
# Kotlin can only be scanned while it compiles, so this builds the debug app, the same way
|
||||||
|
# release.yml does but without a signing key. Workflow files need no build.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, experimental]
|
||||||
|
pull_request:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 9 * * 1"
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
analyze:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
security-events: write
|
||||||
|
actions: read
|
||||||
|
strategy:
|
||||||
|
fail-fast: false
|
||||||
|
matrix:
|
||||||
|
include:
|
||||||
|
- language: actions
|
||||||
|
build-mode: none
|
||||||
|
- language: java-kotlin
|
||||||
|
build-mode: manual
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: github/codeql-action/init@v4
|
||||||
|
with:
|
||||||
|
languages: ${{ matrix.language }}
|
||||||
|
build-mode: ${{ matrix.build-mode }}
|
||||||
|
|
||||||
|
- if: matrix.build-mode == 'manual'
|
||||||
|
uses: actions/setup-java@v4
|
||||||
|
with:
|
||||||
|
distribution: temurin
|
||||||
|
java-version: "21"
|
||||||
|
|
||||||
|
- if: matrix.build-mode == 'manual'
|
||||||
|
uses: gradle/actions/setup-gradle@v4
|
||||||
|
|
||||||
|
- if: matrix.build-mode == 'manual'
|
||||||
|
run: ./gradlew :gearslip:assembleDebug --no-daemon
|
||||||
|
|
||||||
|
- uses: github/codeql-action/analyze@v4
|
||||||
|
with:
|
||||||
|
category: "/language:${{ matrix.language }}"
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
name: Security scan
|
||||||
|
|
||||||
|
# Trivy looks through the repository for committed secrets and risky workflow or build
|
||||||
|
# settings. Findings show under the Security tab, next to CodeQL's and Dependabot's.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main, experimental]
|
||||||
|
pull_request:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 9 * * 1"
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
trivy:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
security-events: write
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: aquasecurity/trivy-action@v0.36.0
|
||||||
|
with:
|
||||||
|
scan-type: fs
|
||||||
|
scanners: vuln,secret,misconfig
|
||||||
|
format: sarif
|
||||||
|
output: trivy.sarif
|
||||||
|
severity: MEDIUM,HIGH,CRITICAL
|
||||||
|
|
||||||
|
- uses: github/codeql-action/upload-sarif@v4
|
||||||
|
if: always()
|
||||||
|
with:
|
||||||
|
sarif_file: trivy.sarif
|
||||||
|
category: trivy
|
||||||
@@ -83,21 +83,10 @@ android {
|
|||||||
}
|
}
|
||||||
|
|
||||||
defaultConfig {
|
defaultConfig {
|
||||||
// Speech-to-text ships for 64-bit ARM only: every phone Gearslip runs on, and one copy
|
// Speech-to-text (sherpa-onnx, in jniLibs) ships for 64-bit ARM only: every phone
|
||||||
// of the native library instead of four.
|
// Gearslip runs on, and one copy of the native library instead of four.
|
||||||
ndk { abiFilters += "arm64-v8a" }
|
ndk { abiFilters += "arm64-v8a" }
|
||||||
}
|
}
|
||||||
ndkVersion = "27.1.12297006"
|
|
||||||
externalNativeBuild {
|
|
||||||
cmake {
|
|
||||||
path = file("src/main/cpp/CMakeLists.txt")
|
|
||||||
version = "3.22.1"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
androidResources {
|
|
||||||
// Model weights barely compress, and stored plainly they stream straight out of the APK.
|
|
||||||
noCompress += "bin"
|
|
||||||
}
|
|
||||||
|
|
||||||
testOptions {
|
testOptions {
|
||||||
// GearslipLog calls android.util.Log, which is a stub on the local JVM.
|
// GearslipLog calls android.util.Log, which is a stub on the local JVM.
|
||||||
@@ -130,6 +119,7 @@ dependencies {
|
|||||||
implementation(platform(libs.androidx.compose.bom))
|
implementation(platform(libs.androidx.compose.bom))
|
||||||
implementation(libs.androidx.compose.ui)
|
implementation(libs.androidx.compose.ui)
|
||||||
implementation(libs.androidx.compose.material3)
|
implementation(libs.androidx.compose.material3)
|
||||||
|
implementation(libs.commons.compress)
|
||||||
implementation(libs.androidx.compose.material.icons.core)
|
implementation(libs.androidx.compose.material.icons.core)
|
||||||
implementation(libs.androidx.activity.compose)
|
implementation(libs.androidx.activity.compose)
|
||||||
implementation(libs.androidx.lifecycle.runtime.compose)
|
implementation(libs.androidx.lifecycle.runtime.compose)
|
||||||
|
|||||||
@@ -1,36 +0,0 @@
|
|||||||
# Speech-to-text for voice replies and the assistant: whisper.cpp, pinned to one commit and
|
|
||||||
# fetched at build time, so nothing outside this folder (cloned-repos included) is needed.
|
|
||||||
cmake_minimum_required(VERSION 3.22)
|
|
||||||
project(gearslip_speech CXX C)
|
|
||||||
|
|
||||||
set(CMAKE_CXX_STANDARD 17)
|
|
||||||
|
|
||||||
# Optimised in debug builds too. Unoptimised, Whisper runs several times slower, and a dev build
|
|
||||||
# would make voice look far worse than what ships.
|
|
||||||
foreach(lang C CXX)
|
|
||||||
set(CMAKE_${lang}_FLAGS_DEBUG "-O3 -DNDEBUG")
|
|
||||||
endforeach()
|
|
||||||
|
|
||||||
include(FetchContent)
|
|
||||||
FetchContent_Declare(
|
|
||||||
whisper
|
|
||||||
URL https://github.com/ggml-org/whisper.cpp/archive/6e4ab854f67f743900934a703d5603419384c961.tar.gz
|
|
||||||
URL_HASH SHA256=55b585aec5e1bc03e2a5cea61da97c6406fa833996cf7daeed704972273cc039
|
|
||||||
)
|
|
||||||
|
|
||||||
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
|
|
||||||
set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
|
||||||
set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE)
|
|
||||||
set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE)
|
|
||||||
set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
|
|
||||||
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
|
|
||||||
# ARMv8.2 with half floats and dot products: every phone that runs Android 12 or later in
|
|
||||||
# practice, and what makes Whisper fast enough to answer in under a second.
|
|
||||||
set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE)
|
|
||||||
FetchContent_MakeAvailable(whisper)
|
|
||||||
|
|
||||||
add_library(gearslip_speech SHARED speech_jni.cpp)
|
|
||||||
target_compile_options(gearslip_speech PRIVATE -O3)
|
|
||||||
find_library(LOG_LIB log)
|
|
||||||
target_link_libraries(gearslip_speech whisper android ${LOG_LIB})
|
|
||||||
target_link_options(gearslip_speech PRIVATE -Wl,--gc-sections -Wl,-z,max-page-size=16384)
|
|
||||||
@@ -1,86 +0,0 @@
|
|||||||
// The bridge between Kotlin's SpeechEngine and whisper.cpp: load a model, turn one recorded
|
|
||||||
// phrase into text, free the model. Recording and deciding when a phrase ends stay in Kotlin.
|
|
||||||
|
|
||||||
#include <jni.h>
|
|
||||||
#include <android/asset_manager.h>
|
|
||||||
#include <android/asset_manager_jni.h>
|
|
||||||
#include <android/log.h>
|
|
||||||
#include <algorithm>
|
|
||||||
#include <cmath>
|
|
||||||
#include <string>
|
|
||||||
#include "whisper.h"
|
|
||||||
|
|
||||||
#define TAG "GearslipSpeech"
|
|
||||||
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
|
|
||||||
|
|
||||||
static whisper_context_params context_params() {
|
|
||||||
whisper_context_params p = whisper_context_default_params();
|
|
||||||
p.use_gpu = false;
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Streams a model straight out of the APK, so the bundled one is never copied to storage.
|
|
||||||
static size_t asset_read(void *ctx, void *output, size_t size) {
|
|
||||||
int n = AAsset_read(static_cast<AAsset *>(ctx), output, size);
|
|
||||||
return n > 0 ? static_cast<size_t>(n) : 0;
|
|
||||||
}
|
|
||||||
static bool asset_eof(void *ctx) { return AAsset_getRemainingLength64(static_cast<AAsset *>(ctx)) <= 0; }
|
|
||||||
static void asset_close(void *ctx) { AAsset_close(static_cast<AAsset *>(ctx)); }
|
|
||||||
|
|
||||||
extern "C" JNIEXPORT jlong JNICALL
|
|
||||||
Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeInitAsset(JNIEnv *env, jobject, jobject assets, jstring jpath) {
|
|
||||||
AAssetManager *manager = AAssetManager_fromJava(env, assets);
|
|
||||||
const char *path = env->GetStringUTFChars(jpath, nullptr);
|
|
||||||
AAsset *asset = AAssetManager_open(manager, path, AASSET_MODE_STREAMING);
|
|
||||||
env->ReleaseStringUTFChars(jpath, path);
|
|
||||||
if (!asset) return 0;
|
|
||||||
whisper_model_loader loader = {asset, asset_read, asset_eof, asset_close};
|
|
||||||
return reinterpret_cast<jlong>(whisper_init_with_params(&loader, context_params()));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Bytes, not a jstring: NewStringUTF wants modified UTF-8 and aborts on some of what Whisper can
|
|
||||||
// write (four-byte characters), so Kotlin decodes the plain UTF-8 instead.
|
|
||||||
extern "C" JNIEXPORT jbyteArray JNICALL
|
|
||||||
Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeTranscribe(
|
|
||||||
JNIEnv *env, jobject, jlong handle, jfloatArray jsamples, jstring jlang, jint threads) {
|
|
||||||
auto *ctx = reinterpret_cast<whisper_context *>(handle);
|
|
||||||
if (!ctx) return env->NewByteArray(0);
|
|
||||||
jsize n = env->GetArrayLength(jsamples);
|
|
||||||
jfloat *samples = env->GetFloatArrayElements(jsamples, nullptr);
|
|
||||||
const char *lang = env->GetStringUTFChars(jlang, nullptr);
|
|
||||||
|
|
||||||
whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
|
|
||||||
p.n_threads = threads;
|
|
||||||
p.language = lang;
|
|
||||||
p.translate = false;
|
|
||||||
p.no_context = true;
|
|
||||||
p.no_timestamps = true;
|
|
||||||
p.single_segment = true;
|
|
||||||
p.print_progress = false;
|
|
||||||
p.print_realtime = false;
|
|
||||||
p.print_special = false;
|
|
||||||
p.print_timestamps = false;
|
|
||||||
p.suppress_blank = true;
|
|
||||||
// The ACFT models are trained for a context cut to the audio's own length (1500 frames is
|
|
||||||
// 30 s, so 50 per second), which is what makes a three-second phrase fast. A little extra
|
|
||||||
// keeps the last word from being clipped.
|
|
||||||
p.audio_ctx = std::min(1500, static_cast<int>(std::ceil(n / 320.0)) + 32);
|
|
||||||
|
|
||||||
int rc = whisper_full(ctx, p, samples, n);
|
|
||||||
env->ReleaseStringUTFChars(jlang, lang);
|
|
||||||
env->ReleaseFloatArrayElements(jsamples, samples, JNI_ABORT);
|
|
||||||
if (rc != 0) {
|
|
||||||
LOGI("whisper_full failed: %d", rc);
|
|
||||||
return env->NewByteArray(0);
|
|
||||||
}
|
|
||||||
std::string text;
|
|
||||||
for (int i = 0; i < whisper_full_n_segments(ctx); i++) text += whisper_full_get_segment_text(ctx, i);
|
|
||||||
jbyteArray out = env->NewByteArray(static_cast<jsize>(text.size()));
|
|
||||||
env->SetByteArrayRegion(out, 0, static_cast<jsize>(text.size()), reinterpret_cast<const jbyte *>(text.data()));
|
|
||||||
return out;
|
|
||||||
}
|
|
||||||
|
|
||||||
extern "C" JNIEXPORT void JNICALL
|
|
||||||
Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeFree(JNIEnv *, jobject, jlong handle) {
|
|
||||||
if (handle) whisper_free(reinterpret_cast<whisper_context *>(handle));
|
|
||||||
}
|
|
||||||
@@ -9,7 +9,6 @@ object AppSettings {
|
|||||||
|
|
||||||
private const val PREFS = "gearslip_settings"
|
private const val PREFS = "gearslip_settings"
|
||||||
private const val KEY_STARTUP_URL = "startup_url"
|
private const val KEY_STARTUP_URL = "startup_url"
|
||||||
private const val KEY_SEEN_COMPAT_WARNING = "seen_compat_warning"
|
|
||||||
private const val KEY_SKIPPED_CERT_SETUP = "skipped_cert_setup"
|
private const val KEY_SKIPPED_CERT_SETUP = "skipped_cert_setup"
|
||||||
private const val KEY_SEEN_PERMISSIONS_SETUP = "seen_permissions_setup"
|
private const val KEY_SEEN_PERMISSIONS_SETUP = "seen_permissions_setup"
|
||||||
private const val KEY_GEARSLIP_ENABLED = "gearslip_enabled"
|
private const val KEY_GEARSLIP_ENABLED = "gearslip_enabled"
|
||||||
@@ -32,14 +31,6 @@ object AppSettings {
|
|||||||
prefs(context).edit().putString(KEY_STARTUP_URL, value.trim()).apply()
|
prefs(context).edit().putString(KEY_STARTUP_URL, value.trim()).apply()
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Whether the newer-car compatibility warning has already been shown and dismissed once. */
|
|
||||||
fun hasSeenCompatWarning(context: Context): Boolean =
|
|
||||||
prefs(context).getBoolean(KEY_SEEN_COMPAT_WARNING, false)
|
|
||||||
|
|
||||||
fun setSeenCompatWarning(context: Context) {
|
|
||||||
prefs(context).edit().putBoolean(KEY_SEEN_COMPAT_WARNING, true).apply()
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Whether the certificate setup screen has been deliberately skipped. */
|
/** Whether the certificate setup screen has been deliberately skipped. */
|
||||||
fun hasSkippedCertSetup(context: Context): Boolean =
|
fun hasSkippedCertSetup(context: Context): Boolean =
|
||||||
prefs(context).getBoolean(KEY_SKIPPED_CERT_SETUP, false)
|
prefs(context).getBoolean(KEY_SKIPPED_CERT_SETUP, false)
|
||||||
|
|||||||
@@ -8,8 +8,10 @@ import android.hardware.usb.UsbAccessory
|
|||||||
import android.hardware.usb.UsbManager
|
import android.hardware.usb.UsbManager
|
||||||
import android.media.projection.MediaProjection
|
import android.media.projection.MediaProjection
|
||||||
import android.media.projection.MediaProjectionManager
|
import android.media.projection.MediaProjectionManager
|
||||||
|
import android.net.Uri
|
||||||
import android.os.Bundle
|
import android.os.Bundle
|
||||||
import android.os.ParcelFileDescriptor
|
import android.os.ParcelFileDescriptor
|
||||||
|
import android.provider.Settings
|
||||||
import android.view.MotionEvent
|
import android.view.MotionEvent
|
||||||
import android.view.Surface
|
import android.view.Surface
|
||||||
import android.view.WindowManager
|
import android.view.WindowManager
|
||||||
@@ -76,7 +78,13 @@ class GearslipActivity : ComponentActivity(), Projection {
|
|||||||
private val audioPermission = registerForActivityResult(
|
private val audioPermission = registerForActivityResult(
|
||||||
androidx.activity.result.contract.ActivityResultContracts.RequestPermission(),
|
androidx.activity.result.contract.ActivityResultContracts.RequestPermission(),
|
||||||
) { granted ->
|
) { granted ->
|
||||||
if (granted) askAudioCapture() else {
|
if (granted) askAudioCapture()
|
||||||
|
else if (!shouldShowRequestPermissionRationale(android.Manifest.permission.RECORD_AUDIO)) {
|
||||||
|
// Refused twice: Android now says no without showing anything, so the phone must.
|
||||||
|
GearslipLog.w("audio: record permission blocked - Android no longer asks")
|
||||||
|
CarAudio.micBlocked()
|
||||||
|
micBlockedNotice.value = true
|
||||||
|
} else {
|
||||||
GearslipLog.w("audio: record permission declined")
|
GearslipLog.w("audio: record permission declined")
|
||||||
CarAudio.denied()
|
CarAudio.denied()
|
||||||
}
|
}
|
||||||
@@ -93,6 +101,9 @@ class GearslipActivity : ComponentActivity(), Projection {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Raised when the microphone permission is blocked; see [MicBlockedDialog]. */
|
||||||
|
private val micBlockedNotice = kotlinx.coroutines.flow.MutableStateFlow(false)
|
||||||
|
|
||||||
/** Raised once, after the first "no" to the capture dialog; see [AudioChoiceDialog]. */
|
/** Raised once, after the first "no" to the capture dialog; see [AudioChoiceDialog]. */
|
||||||
private val offerBluetooth = kotlinx.coroutines.flow.MutableStateFlow(false)
|
private val offerBluetooth = kotlinx.coroutines.flow.MutableStateFlow(false)
|
||||||
|
|
||||||
@@ -154,6 +165,18 @@ class GearslipActivity : ComponentActivity(), Projection {
|
|||||||
onKeepAsking = ::close,
|
onKeepAsking = ::close,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
val micNotice by micBlockedNotice.collectAsState()
|
||||||
|
if (micNotice) {
|
||||||
|
app.seb3thehacker.gearslip.ui.MicBlockedDialog(
|
||||||
|
onOpenSettings = {
|
||||||
|
micBlockedNotice.value = false
|
||||||
|
startActivity(
|
||||||
|
Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", packageName, null)),
|
||||||
|
)
|
||||||
|
},
|
||||||
|
onDismiss = { micBlockedNotice.value = false },
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -179,6 +202,10 @@ class GearslipActivity : ComponentActivity(), Projection {
|
|||||||
Prefetch.warm(this)
|
Prefetch.warm(this)
|
||||||
UsageStats.onAppOpened(this)
|
UsageStats.onAppOpened(this)
|
||||||
app.seb3thehacker.gearslip.update.UpdateChecker.onAppOpened(this)
|
app.seb3thehacker.gearslip.update.UpdateChecker.onAppOpened(this)
|
||||||
|
// Back from the app's settings with the microphone turned on: try car audio again.
|
||||||
|
if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) == android.content.pm.PackageManager.PERMISSION_GRANTED) {
|
||||||
|
CarAudio.micAllowed()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun onNewIntent(intent: Intent) {
|
override fun onNewIntent(intent: Intent) {
|
||||||
|
|||||||
@@ -45,7 +45,10 @@ class GearslipRunner(
|
|||||||
private lateinit var identityTask: FutureTask<PhoneTls>
|
private lateinit var identityTask: FutureTask<PhoneTls>
|
||||||
|
|
||||||
/** Sends every message, in order, so no other thread ever blocks on the USB link. See [send]. */
|
/** Sends every message, in order, so no other thread ever blocks on the USB link. See [send]. */
|
||||||
private val writer = java.util.concurrent.Executors.newSingleThreadExecutor { Thread(it, "gearslip-writer") }
|
private val writer = java.util.concurrent.Executors.newSingleThreadExecutor {
|
||||||
|
// Sound goes out through here too, so it gets audio's priority: late writes are skips.
|
||||||
|
Thread({ android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); it.run() }, "gearslip-writer")
|
||||||
|
}
|
||||||
|
|
||||||
/** Waits for the certificate only when TLS is first needed; see [run]. */
|
/** Waits for the certificate only when TLS is first needed; see [run]. */
|
||||||
private val tls: PhoneTls by lazy { identityTask.get() }
|
private val tls: PhoneTls by lazy { identityTask.get() }
|
||||||
@@ -88,6 +91,8 @@ class GearslipRunner(
|
|||||||
@Volatile private var lastKeyframeSentAt = 0L
|
@Volatile private var lastKeyframeSentAt = 0L
|
||||||
|
|
||||||
fun run() {
|
fun run() {
|
||||||
|
// This thread reads the car's acks; audio waits on them, so it can't be left at the back.
|
||||||
|
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
|
||||||
SessionReport.begin()
|
SessionReport.begin()
|
||||||
log.i("--- session starting ---")
|
log.i("--- session starting ---")
|
||||||
SessionStatus.connecting("Connected to the head unit")
|
SessionStatus.connecting("Connected to the head unit")
|
||||||
@@ -260,11 +265,10 @@ class GearslipRunner(
|
|||||||
log.i("<- VersionRequest: head unit speaks $major.$minor")
|
log.i("<- VersionRequest: head unit speaks $major.$minor")
|
||||||
SessionReport.protocolVersion(major, minor)
|
SessionReport.protocolVersion(major, minor)
|
||||||
|
|
||||||
// PROTOCOL-DOWNGRADE EXPERIMENT. The 2018 Uconnect speaks 1.3 and ACCEPTS the JVC cert;
|
// Always 1.3, on purpose. Every car that accepts Gearslip was tested this way, 1.5 units
|
||||||
// the 2025 unit speaks 4.3 and rejects it at AuthComplete (-3). If the 4.3 identity check
|
// included. Echoing the car's own version was tried too: a 4.3 unit rejected the
|
||||||
// is tied to the *negotiated* protocol version rather than baked into the firmware,
|
// certificate either way, so the version doesn't decide acceptance. The log says so, since
|
||||||
// claiming an older version makes the newer unit skip the check entirely. CLAIM_VERSION
|
// a car offering 1.5 and getting 1.3 back looks like a bug to anyone reading it.
|
||||||
// = null restores the previous echo-the-head-unit behaviour.
|
|
||||||
val claim = CLAIM_VERSION ?: (major to minor)
|
val claim = CLAIM_VERSION ?: (major to minor)
|
||||||
val response = byteArrayOf(
|
val response = byteArrayOf(
|
||||||
((claim.first shr 8) and 0xFF).toByte(), (claim.first and 0xFF).toByte(),
|
((claim.first shr 8) and 0xFF).toByte(), (claim.first and 0xFF).toByte(),
|
||||||
@@ -274,7 +278,10 @@ class GearslipRunner(
|
|||||||
send(MSG_VERSION_RESPONSE, response, encrypted = false)
|
send(MSG_VERSION_RESPONSE, response, encrypted = false)
|
||||||
log.i(
|
log.i(
|
||||||
"-> VersionResponse: ${claim.first}.${claim.second} status=0 (STATUS_SUCCESS)" +
|
"-> VersionResponse: ${claim.first}.${claim.second} status=0 (STATUS_SUCCESS)" +
|
||||||
if (CLAIM_VERSION != null) " [FORCED; head unit offered $major.$minor]" else "",
|
if (CLAIM_VERSION != null && claim != (major to minor)) {
|
||||||
|
" - intended: Gearslip always answers ${claim.first}.${claim.second}, which cars " +
|
||||||
|
"offering $major.$minor accept too"
|
||||||
|
} else "",
|
||||||
)
|
)
|
||||||
state = State.TLS_HANDSHAKE
|
state = State.TLS_HANDSHAKE
|
||||||
SessionStatus.connecting("Securing the link")
|
SessionStatus.connecting("Securing the link")
|
||||||
@@ -1209,9 +1216,9 @@ class GearslipRunner(
|
|||||||
|
|
||||||
private companion object {
|
private companion object {
|
||||||
/**
|
/**
|
||||||
* Force the protocol version Gearslip claims in VersionResponse, as (major, minor), or
|
* The protocol version Gearslip answers every car with, as (major, minor), or null to echo
|
||||||
* null to echo the head unit. 1.3 is what the 2018 Uconnect (which accepts the JVC cert)
|
* the head unit's. Kept at 1.3 because every working car was tested with it; see
|
||||||
* speaks. This is the 2025 identity-check downgrade experiment: see onVersionRequest.
|
* onVersionRequest.
|
||||||
*/
|
*/
|
||||||
val CLAIM_VERSION: Pair<Int, Int>? = 1 to 3
|
val CLAIM_VERSION: Pair<Int, Int>? = 1 to 3
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import android.media.AudioRecord
|
|||||||
import android.media.projection.MediaProjection
|
import android.media.projection.MediaProjection
|
||||||
import android.media.projection.MediaProjectionManager
|
import android.media.projection.MediaProjectionManager
|
||||||
import android.os.IBinder
|
import android.os.IBinder
|
||||||
|
import android.os.Process
|
||||||
import app.seb3thehacker.gearslip.GearslipLog
|
import app.seb3thehacker.gearslip.GearslipLog
|
||||||
import kotlin.math.abs
|
import kotlin.math.abs
|
||||||
|
|
||||||
@@ -147,6 +148,9 @@ class AudioCaptureService : Service() {
|
|||||||
|
|
||||||
reading = true
|
reading = true
|
||||||
thread = Thread({
|
thread = Thread({
|
||||||
|
// A busy phone (a map app drawing, the video encoder) would otherwise starve this
|
||||||
|
// loop, and the car hears every late read as a skip.
|
||||||
|
Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO)
|
||||||
val buffer = ByteArray(CHUNK_BYTES)
|
val buffer = ByteArray(CHUNK_BYTES)
|
||||||
var peak = 0
|
var peak = 0
|
||||||
var heardAt = System.currentTimeMillis()
|
var heardAt = System.currentTimeMillis()
|
||||||
|
|||||||
@@ -147,7 +147,8 @@ class AudioLink(
|
|||||||
fun onFocus(state: Int) {
|
fun onFocus(state: Int) {
|
||||||
val name = FOCUS_STATES[state] ?: "unknown($state)"
|
val name = FOCUS_STATES[state] ?: "unknown($state)"
|
||||||
GearslipLog.i("audio: <- AudioFocusResponse state=$name")
|
GearslipLog.i("audio: <- AudioFocusResponse state=$name")
|
||||||
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT
|
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT ||
|
||||||
|
state == FOCUS_STATE_GAIN_MEDIA_ONLY
|
||||||
if (focusGranted) startStream() else stop()
|
if (focusGranted) startStream() else stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -257,13 +258,16 @@ class AudioLink(
|
|||||||
private const val FOCUS_RELEASE = 4
|
private const val FOCUS_RELEASE = 4
|
||||||
private const val FOCUS_STATE_GAIN = 1
|
private const val FOCUS_STATE_GAIN = 1
|
||||||
private const val FOCUS_STATE_GAIN_TRANSIENT = 2
|
private const val FOCUS_STATE_GAIN_TRANSIENT = 2
|
||||||
|
private const val FOCUS_STATE_GAIN_MEDIA_ONLY = 6
|
||||||
private const val SESSION_ID = 2
|
private const val SESSION_ID = 2
|
||||||
private const val STALL_MS = 2_000L
|
private const val STALL_MS = 2_000L
|
||||||
private const val CHUNK_LOG_EVERY = 200 // about every four seconds
|
private const val CHUNK_LOG_EVERY = 200 // about every four seconds
|
||||||
private const val CONFIG_WAIT_MS = 1_000L
|
private const val CONFIG_WAIT_MS = 1_000L
|
||||||
|
|
||||||
|
// AudioFocusStateNotification.AudioFocusState in aasdk.
|
||||||
private val FOCUS_STATES = mapOf(
|
private val FOCUS_STATES = mapOf(
|
||||||
1 to "GAIN", 2 to "GAIN_TRANSIENT", 3 to "LOSS", 4 to "LOSS_TRANSIENT",
|
1 to "GAIN", 2 to "GAIN_TRANSIENT", 3 to "LOSS", 4 to "LOSS_TRANSIENT_CAN_DUCK",
|
||||||
|
5 to "LOSS_TRANSIENT", 6 to "GAIN_MEDIA_ONLY", 7 to "GAIN_TRANSIENT_GUIDANCE_ONLY",
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -100,6 +100,26 @@ object CarAudio {
|
|||||||
_capture.value = Capture.DENIED
|
_capture.value = Capture.DENIED
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private val _micBlocked = MutableStateFlow(false)
|
||||||
|
|
||||||
|
/** The microphone permission is off for good: Android refuses without asking, so say why. */
|
||||||
|
val micBlocked: StateFlow<Boolean> = _micBlocked.asStateFlow()
|
||||||
|
|
||||||
|
fun micBlocked() {
|
||||||
|
_micBlocked.value = true
|
||||||
|
denied()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The driver turned the permission back on mid-drive: ask for capture again. */
|
||||||
|
fun micAllowed() {
|
||||||
|
if (!_micBlocked.value) return
|
||||||
|
_micBlocked.value = false
|
||||||
|
if (_capture.value == Capture.DENIED) {
|
||||||
|
_capture.value = Capture.OFF
|
||||||
|
if (wanted) request()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
internal fun captureStarted() {
|
internal fun captureStarted() {
|
||||||
_capture.value = Capture.ON
|
_capture.value = Capture.ON
|
||||||
link?.start()
|
link?.start()
|
||||||
|
|||||||
@@ -1,10 +1,13 @@
|
|||||||
package app.seb3thehacker.gearslip.car
|
package app.seb3thehacker.gearslip.car
|
||||||
|
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
|
import app.seb3thehacker.gearslip.media.CarMedia
|
||||||
import java.text.DateFormat
|
import java.text.DateFormat
|
||||||
import java.util.Date
|
import java.util.Date
|
||||||
import kotlinx.coroutines.Dispatchers
|
import kotlinx.coroutines.Dispatchers
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
import kotlinx.coroutines.withContext
|
import kotlinx.coroutines.withContext
|
||||||
|
import kotlinx.coroutines.withTimeoutOrNull
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Turns what [CarAssistant] heard into an action plus something to say back.
|
* Turns what [CarAssistant] heard into an action plus something to say back.
|
||||||
@@ -17,9 +20,15 @@ import kotlinx.coroutines.withContext
|
|||||||
object AssistantCommands {
|
object AssistantCommands {
|
||||||
|
|
||||||
suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String {
|
suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String {
|
||||||
val heard = heardRaw.lowercase().trim()
|
val heard = natural(heardRaw)
|
||||||
|
VoiceHelp.hide()
|
||||||
|
if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) {
|
||||||
|
VoiceHelp.show()
|
||||||
|
return "Here's what you can say."
|
||||||
|
}
|
||||||
|
|
||||||
navigateCommand(context, navigator, frame, heard)?.let { return it }
|
navigateCommand(context, navigator, frame, heard)?.let { return it }
|
||||||
|
playSearchCommand(context, navigator, heard)?.let { return it }
|
||||||
mediaCommand(heard)?.let { return it }
|
mediaCommand(heard)?.let { return it }
|
||||||
navCommand(navigator, heard)?.let { return it }
|
navCommand(navigator, heard)?.let { return it }
|
||||||
infoCommand(heard)?.let { return it }
|
infoCommand(heard)?.let { return it }
|
||||||
@@ -29,6 +38,42 @@ object AssistantCommands {
|
|||||||
return "I don't know how to do that yet."
|
return "I don't know how to do that yet."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* What was said, minus the padding people wrap a command in: "hey", "can you", "I want to",
|
||||||
|
* "please", "thanks". Each command below then only has to know its own words.
|
||||||
|
*/
|
||||||
|
private fun natural(raw: String): String {
|
||||||
|
var t = raw.lowercase().replace(Regex("[.,!?]"), " ").replace(Regex("\\s+"), " ").trim()
|
||||||
|
var changed = true
|
||||||
|
while (changed) {
|
||||||
|
changed = false
|
||||||
|
for (lead in LEAD_FILLERS) if (t.startsWith("$lead ")) { t = t.removePrefix("$lead ").trim(); changed = true }
|
||||||
|
for (tail in TAIL_FILLERS) if (t.endsWith(" $tail")) { t = t.removeSuffix(" $tail").trim(); changed = true }
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The part after the first of [triggers] that [text] starts with, or null. */
|
||||||
|
private fun after(text: String, triggers: List<String>): String? =
|
||||||
|
triggers.firstOrNull { text.startsWith(it) }?.let { text.removePrefix(it).trim() }
|
||||||
|
|
||||||
|
private suspend fun allEntries(context: Context): List<Entry> {
|
||||||
|
val vehicleData = CarSettings.experimentalFeaturesEnabled.value && CarSettings.carSensorsEnabled.value
|
||||||
|
val builtIn = BuiltInApps.entries.filter { it.builtIn !== BuiltInApps.VehicleData || vehicleData }
|
||||||
|
return builtIn + (LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) })
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Best label match: exact, then a label starting with [name], then one containing it. */
|
||||||
|
private fun <T> bestMatch(items: List<T>, name: String, label: (T) -> String): T? {
|
||||||
|
val n = name.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
|
||||||
|
if (n.isEmpty()) return null
|
||||||
|
items.firstOrNull { label(it).lowercase() == n }?.let { return it }
|
||||||
|
// Loose matches need a real word: "pizza in a" mustn't pick "Organic Maps" for its "a".
|
||||||
|
if (n.length < 3) return null
|
||||||
|
return items.firstOrNull { label(it).lowercase().startsWith(n) }
|
||||||
|
?: items.firstOrNull { label(it).lowercase().contains(n) }
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it,
|
* "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it,
|
||||||
* whichever nav app was used last, or the only one installed, or the first one found. Starts
|
* whichever nav app was used last, or the only one installed, or the first one found. Starts
|
||||||
@@ -42,29 +87,45 @@ object AssistantCommands {
|
|||||||
frame: CarEnvironment.Frame,
|
frame: CarEnvironment.Frame,
|
||||||
heard: String,
|
heard: String,
|
||||||
): String? {
|
): String? {
|
||||||
val trigger = NAVIGATE_TRIGGERS.firstOrNull { heard.startsWith(it) } ?: return null
|
// "use Organic Maps to get to X", "open OsmAnd and navigate to X": the app comes first.
|
||||||
var destination = heard.removePrefix(trigger).trim()
|
var text = heard
|
||||||
|
var leadingApp: String? = null
|
||||||
|
Regex("^(?:use|using|with|open|in|on) (.+?) (?:to |and )(.+)$").find(text)?.let { m ->
|
||||||
|
if (after(m.groupValues[2], NAVIGATE_TRIGGERS) != null) {
|
||||||
|
leadingApp = m.groupValues[1]
|
||||||
|
text = m.groupValues[2]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var destination = after(text, NAVIGATE_TRIGGERS) ?: return null
|
||||||
if (destination.isBlank()) return "Where to?"
|
if (destination.isBlank()) return "Where to?"
|
||||||
|
|
||||||
var appName: String? = null
|
val entries = allEntries(context)
|
||||||
for (sep in listOf(" with ", " using ", " on ", " via ")) {
|
// "go to Spotify" means the app, and "go to settings" the screen; "go to Target" is a place.
|
||||||
val at = destination.lastIndexOf(sep)
|
val asApp = destination.removePrefix("the ").removeSuffix(" app")
|
||||||
if (at > 0) {
|
if (text.startsWith("go to ") &&
|
||||||
appName = destination.substring(at + sep.length).trim()
|
(asApp in SCREEN_WORDS || asApp in APP_NICKNAMES || entries.any { it.label.lowercase() == asApp })
|
||||||
|
) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
val navApps = entries.filter { it.template?.isNavigation == true }
|
||||||
|
if (navApps.isEmpty()) return "You don't have any navigation apps installed."
|
||||||
|
|
||||||
|
// "X with Organic Maps", "X on OsmAnd": split only when the end names a nav app, so a
|
||||||
|
// place like "Starbucks on Main Street" stays whole.
|
||||||
|
var chosenApp = leadingApp?.let { name -> bestMatch(navApps, name) { it.label } ?: return "I couldn't find a navigation app called $name." }
|
||||||
|
if (chosenApp == null) {
|
||||||
|
for (sep in listOf(" with ", " using ", " on ", " via ", " in ", " through ")) {
|
||||||
|
val at = destination.lastIndexOf(sep)
|
||||||
|
if (at <= 0) continue
|
||||||
|
val app = bestMatch(navApps, destination.substring(at + sep.length)) { it.label } ?: continue
|
||||||
|
chosenApp = app
|
||||||
destination = destination.substring(0, at).trim()
|
destination = destination.substring(0, at).trim()
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (destination.isBlank()) return "Where to?"
|
if (destination.isBlank()) return "Where to?"
|
||||||
|
|
||||||
val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
|
val chosen = chosenApp ?: when {
|
||||||
val navApps = entries.filter { it.template?.isNavigation == true }
|
|
||||||
if (navApps.isEmpty()) return "You don't have any navigation apps installed."
|
|
||||||
|
|
||||||
val requestedApp = appName
|
|
||||||
val chosen = when {
|
|
||||||
requestedApp != null -> navApps.firstOrNull { it.label.lowercase().contains(requestedApp) }
|
|
||||||
?: return "I couldn't find a navigation app called $requestedApp."
|
|
||||||
navApps.size == 1 -> navApps.first()
|
navApps.size == 1 -> navApps.first()
|
||||||
else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first()
|
else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first()
|
||||||
}
|
}
|
||||||
@@ -75,26 +136,84 @@ object AssistantCommands {
|
|||||||
return "Navigating to $destination with ${chosen.label}."
|
return "Navigating to $destination with ${chosen.label}."
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* "play X" with something after it: a song, artist, album or playlist, asked of the music app
|
||||||
|
* that's open through its own search ([CarMedia.search]). A bare "play", or "play music",
|
||||||
|
* falls through to resume.
|
||||||
|
*/
|
||||||
|
private suspend fun playSearchCommand(context: Context, navigator: CarNavigator, heard: String): String? {
|
||||||
|
var query = after(heard, PLAY_TRIGGERS) ?: return null
|
||||||
|
query = query.removePrefix("some ").trim()
|
||||||
|
if (query.isEmpty() || query in RESUME_WORDS) return null
|
||||||
|
val media = CarServices.media
|
||||||
|
|
||||||
|
// "play Daft Punk on Metrolist": switch to that app first, if it's one we know.
|
||||||
|
for (sep in listOf(" on ", " in ", " from ", " with ", " using ")) {
|
||||||
|
val at = query.lastIndexOf(sep)
|
||||||
|
if (at <= 0) continue
|
||||||
|
val apps = withContext(Dispatchers.IO) { app.seb3thehacker.gearslip.media.MediaCatalog.installed(context) }
|
||||||
|
val wanted = bestMatch(apps, query.substring(at + sep.length)) { it.label } ?: continue
|
||||||
|
query = query.substring(0, at).trim()
|
||||||
|
if (query.isEmpty() || query in RESUME_WORDS) {
|
||||||
|
CarServices.openMedia(wanted, autoplay = true)
|
||||||
|
navigator.media()
|
||||||
|
return "Opening ${wanted.label}."
|
||||||
|
}
|
||||||
|
if (CarServices.mediaApp.value?.component != wanted.component) {
|
||||||
|
CarServices.openMedia(wanted)
|
||||||
|
navigator.media()
|
||||||
|
// Give it a moment to connect before asking it to search.
|
||||||
|
withTimeoutOrNull(APP_CONNECT_MS) {
|
||||||
|
while (media.appLabel == null || (!media.canSearchLibrary && !media.now.value.canSearch)) delay(200)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// No music app open, or its connection is gone (the app was closed): use the one played
|
||||||
|
// last, the way "play X" works on Android Auto.
|
||||||
|
val phase = media.phase.value
|
||||||
|
if (media.appLabel == null || phase == CarMedia.Phase.IDLE || phase == CarMedia.Phase.REJECTED) {
|
||||||
|
if (!CarServices.reopenLastMedia()) return "Open a music app first, then ask again."
|
||||||
|
navigator.media()
|
||||||
|
}
|
||||||
|
withTimeoutOrNull(APP_CONNECT_MS) {
|
||||||
|
while (media.phase.value != CarMedia.Phase.READY || media.appLabel == null ||
|
||||||
|
(!media.canSearchLibrary && !media.now.value.canSearch)
|
||||||
|
) delay(200)
|
||||||
|
}
|
||||||
|
val app = media.appLabel ?: return "Your music app didn't open in time. Try again."
|
||||||
|
if (!media.canSearchLibrary && !media.now.value.canSearch) {
|
||||||
|
return "$app ignores voice searches. Pick something on its screen instead."
|
||||||
|
}
|
||||||
|
media.playBestMatch(query)
|
||||||
|
return "Looking for $query on $app."
|
||||||
|
}
|
||||||
|
|
||||||
private fun mediaCommand(heard: String): String? {
|
private fun mediaCommand(heard: String): String? {
|
||||||
val media = CarServices.media
|
val media = CarServices.media
|
||||||
return when {
|
return when {
|
||||||
"pause" in heard || heard.startsWith("stop") -> {
|
"pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> {
|
||||||
if (media.now.value.playing) media.togglePlay()
|
if (media.now.value.playing) media.togglePlay()
|
||||||
"Paused."
|
"Paused."
|
||||||
}
|
}
|
||||||
"resume" in heard || (heard.startsWith("play") && "playing" !in heard) -> {
|
"resume" in heard || "unpause" in heard || heard in RESUME_PHRASES ||
|
||||||
|
(heard.startsWith("play") && "playing" !in heard) -> {
|
||||||
if (!media.now.value.playing) media.togglePlay()
|
if (!media.now.value.playing) media.togglePlay()
|
||||||
"Playing."
|
"Playing."
|
||||||
}
|
}
|
||||||
"next" in heard || "skip" in heard -> {
|
"next" in heard || "skip" in heard || heard.startsWith("change the song") -> {
|
||||||
media.next()
|
media.next()
|
||||||
"Skipping."
|
"Skipping."
|
||||||
}
|
}
|
||||||
"previous" in heard || "last song" in heard || "back song" in heard -> {
|
"previous" in heard || "last song" in heard || "back song" in heard || "go back a song" in heard ||
|
||||||
|
"play that again" in heard || "replay" in heard -> {
|
||||||
media.previous()
|
media.previous()
|
||||||
"Playing the previous track."
|
"Playing the previous track."
|
||||||
}
|
}
|
||||||
"what" in heard && "playing" in heard -> {
|
("what" in heard && "playing" in heard) || "what song" in heard || "who sings" in heard ||
|
||||||
|
"who's singing" in heard || "who is singing" in heard || "what's this song" in heard ||
|
||||||
|
"name of this song" in heard -> {
|
||||||
val now = media.now.value
|
val now = media.now.value
|
||||||
if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ")
|
if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ")
|
||||||
else "Nothing's playing."
|
else "Nothing's playing."
|
||||||
@@ -104,11 +223,19 @@ object AssistantCommands {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun navCommand(navigator: CarNavigator, heard: String): String? = when {
|
private fun navCommand(navigator: CarNavigator, heard: String): String? = when {
|
||||||
"go home" in heard || heard == "home" -> {
|
heard == "go back" || heard == "back" -> {
|
||||||
|
navigator.back()
|
||||||
|
"Going back."
|
||||||
|
}
|
||||||
|
heard in EXIT_WORDS -> {
|
||||||
|
BuiltInApps.Exit.open(navigator)
|
||||||
|
"Back to the car's screen."
|
||||||
|
}
|
||||||
|
"go home" in heard || heard == "home" || heard == "home screen" || "go to the home screen" in heard -> {
|
||||||
navigator.home()
|
navigator.home()
|
||||||
"Heading home."
|
"Heading home."
|
||||||
}
|
}
|
||||||
"open apps" in heard || heard == "apps" || "app list" in heard -> {
|
"open apps" in heard || heard == "apps" || "app list" in heard || "all apps" in heard || "my apps" in heard || heard == "go to apps" -> {
|
||||||
navigator.apps()
|
navigator.apps()
|
||||||
"Here are your apps."
|
"Here are your apps."
|
||||||
}
|
}
|
||||||
@@ -128,9 +255,10 @@ object AssistantCommands {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private fun infoCommand(heard: String): String? = when {
|
private fun infoCommand(heard: String): String? = when {
|
||||||
"what time" in heard || "what's the time" in heard ->
|
"what time" in heard || "what's the time" in heard || heard == "time" || "the time" in heard ->
|
||||||
"It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}."
|
"It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}."
|
||||||
"weather" in heard -> {
|
"weather" in heard || "going to rain" in heard || "gonna rain" in heard || "how hot" in heard ||
|
||||||
|
"how cold" in heard || "temperature outside" in heard -> {
|
||||||
val data = (Weather.state.value as? WeatherState.Ready)?.data
|
val data = (Weather.state.value as? WeatherState.Ready)?.data
|
||||||
if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}."
|
if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}."
|
||||||
else "I don't have the weather yet."
|
else "I don't have the weather yet."
|
||||||
@@ -139,12 +267,11 @@ object AssistantCommands {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private suspend fun callCommand(context: Context, heard: String): String? {
|
private suspend fun callCommand(context: Context, heard: String): String? {
|
||||||
val name = when {
|
val name = (
|
||||||
heard.startsWith("call ") -> heard.removePrefix("call ")
|
after(heard, CALL_TRIGGERS)
|
||||||
heard.startsWith("phone ") -> heard.removePrefix("phone ")
|
?: Regex("^give (.+) a (?:call|ring)$").find(heard)?.groupValues?.get(1)
|
||||||
heard.startsWith("dial ") -> heard.removePrefix("dial ")
|
?: return null
|
||||||
else -> return null
|
).replace(Regex(" (?:on|at) (?:their |his |her )?(?:mobile|cell|phone|home|work)$"), "").trim()
|
||||||
}.trim()
|
|
||||||
if (name.isBlank()) return "Who should I call?"
|
if (name.isBlank()) return "Who should I call?"
|
||||||
if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen."
|
if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen."
|
||||||
|
|
||||||
@@ -162,21 +289,59 @@ object AssistantCommands {
|
|||||||
frame: CarEnvironment.Frame,
|
frame: CarEnvironment.Frame,
|
||||||
heard: String,
|
heard: String,
|
||||||
): String? {
|
): String? {
|
||||||
val name = when {
|
val name = (after(heard, OPEN_TRIGGERS) ?: return null)
|
||||||
heard.startsWith("open ") -> heard.removePrefix("open ")
|
.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
|
||||||
heard.startsWith("launch ") -> heard.removePrefix("launch ")
|
|
||||||
heard.startsWith("start ") -> heard.removePrefix("start ")
|
|
||||||
else -> return null
|
|
||||||
}.trim()
|
|
||||||
if (name.isBlank()) return null
|
if (name.isBlank()) return null
|
||||||
|
val wanted = APP_NICKNAMES[name] ?: name
|
||||||
|
|
||||||
val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
|
// Gearslip's own apps come first in the list, so "phone" opens its dialer, not the phone's.
|
||||||
val match = entries.firstOrNull { it.label.lowercase().contains(name) }
|
val match = bestMatch(allEntries(context), wanted) { it.label }
|
||||||
?: return "I couldn't find an app called $name."
|
?: return "I couldn't find an app called $name."
|
||||||
|
|
||||||
launchEntry(match, navigator, frame)
|
launchEntry(match, navigator, frame)
|
||||||
return "Opening ${match.label}."
|
return "Opening ${match.label}."
|
||||||
}
|
}
|
||||||
|
|
||||||
private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ")
|
private val EXIT_WORDS = setOf("exit", "exit gearslip", "close gearslip", "quit", "quit gearslip", "open exit")
|
||||||
|
|
||||||
|
/** Other names people use for Gearslip's own apps. */
|
||||||
|
private val APP_NICKNAMES = mapOf(
|
||||||
|
"web" to "browser", "internet" to "browser", "the web" to "browser", "web browser" to "browser",
|
||||||
|
"dialer" to "phone", "the dialer" to "phone", "keypad" to "phone",
|
||||||
|
"screen share" to "screen sharing", "screen mirroring" to "screen sharing", "mirroring" to "screen sharing",
|
||||||
|
)
|
||||||
|
|
||||||
|
private val NAVIGATE_TRIGGERS = listOf(
|
||||||
|
"navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ",
|
||||||
|
"take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ",
|
||||||
|
"how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ",
|
||||||
|
)
|
||||||
|
|
||||||
|
private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ")
|
||||||
|
|
||||||
|
private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ")
|
||||||
|
|
||||||
|
private val OPEN_TRIGGERS = listOf("open ", "launch ", "start ", "go to ", "switch to ", "show me ", "bring up ", "pull up ")
|
||||||
|
|
||||||
|
/** Screens "go to X" means rather than a place. */
|
||||||
|
private val SCREEN_WORDS = setOf("settings", "home", "the home screen", "home screen", "apps", "my apps", "all apps")
|
||||||
|
|
||||||
|
/** Padding stripped from the front of a command, longest first where they overlap. */
|
||||||
|
private val LEAD_FILLERS = listOf(
|
||||||
|
"hey gearslip", "ok gearslip", "okay gearslip", "gearslip", "hey", "ok", "okay", "um", "uh", "so",
|
||||||
|
"please", "can you please", "can you", "could you", "would you", "will you", "can i", "could i",
|
||||||
|
"i want you to", "i want to", "i'd like to", "i would like to", "i wanna", "i need to", "i need you to",
|
||||||
|
"let's", "lets", "go ahead and", "just", "now", "quickly",
|
||||||
|
)
|
||||||
|
|
||||||
|
/** Longest first, so "right now" goes whole rather than leaving "right". */
|
||||||
|
private val TAIL_FILLERS = listOf("thank you very much", "thank you", "right now", "for me", "please", "thanks", "now")
|
||||||
|
|
||||||
|
private const val APP_CONNECT_MS = 5_000L
|
||||||
|
|
||||||
|
private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands")
|
||||||
|
|
||||||
|
private val RESUME_WORDS = setOf("music", "the music", "some music", "something", "it", "again", "that", "my music")
|
||||||
|
|
||||||
|
private val RESUME_PHRASES = setOf("keep playing", "continue", "keep going", "start the music", "music on", "turn the music back on")
|
||||||
}
|
}
|
||||||
@@ -4,7 +4,21 @@ import app.seb3thehacker.gearslip.car.theme.*
|
|||||||
import android.content.Intent
|
import android.content.Intent
|
||||||
import android.net.Uri
|
import android.net.Uri
|
||||||
import android.provider.Settings
|
import android.provider.Settings
|
||||||
|
import androidx.compose.animation.core.FastOutSlowInEasing
|
||||||
|
import androidx.compose.animation.core.RepeatMode
|
||||||
|
import androidx.compose.animation.core.animateFloat
|
||||||
|
import androidx.compose.animation.core.infiniteRepeatable
|
||||||
|
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||||
|
import androidx.compose.animation.core.tween
|
||||||
import androidx.compose.foundation.layout.Arrangement
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
|
import androidx.compose.foundation.layout.Box
|
||||||
|
import androidx.compose.ui.draw.drawBehind
|
||||||
|
import androidx.compose.ui.geometry.Offset
|
||||||
|
import androidx.compose.ui.graphics.Brush
|
||||||
|
import androidx.compose.ui.graphics.Color
|
||||||
|
import androidx.compose.ui.graphics.drawscope.scale
|
||||||
|
import androidx.compose.ui.text.style.TextAlign
|
||||||
|
import androidx.compose.ui.text.style.TextOverflow
|
||||||
import androidx.compose.foundation.layout.Column
|
import androidx.compose.foundation.layout.Column
|
||||||
import androidx.compose.foundation.layout.Row
|
import androidx.compose.foundation.layout.Row
|
||||||
import androidx.compose.foundation.layout.fillMaxWidth
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
@@ -30,47 +44,66 @@ import androidx.compose.ui.text.font.FontWeight
|
|||||||
import androidx.compose.ui.unit.dp
|
import androidx.compose.ui.unit.dp
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The band that appears while [CarAssistant] is listening, thinking or speaking - a thin strip
|
* What the assistant hears and says, laid over the bottom of whatever screen is up: a dark shade
|
||||||
* across the top, same idea as [CallOverlay]'s active-call banner, so it never blocks the screen
|
* so white words read over a bright map, and a glow along the bottom edge in the phone's own
|
||||||
* the driver was already looking at.
|
* colours that breathes while listening. The shade passes touches through; only the buttons take them.
|
||||||
*/
|
*/
|
||||||
@Composable
|
@Composable
|
||||||
fun AssistantOverlay(modifier: Modifier = Modifier) {
|
fun AssistantOverlay(modifier: Modifier = Modifier) {
|
||||||
val state by CarAssistant.state.collectAsState()
|
val state by CarAssistant.state.collectAsState()
|
||||||
if (state.phase == CarAssistant.Phase.IDLE) return
|
if (state.phase == CarAssistant.Phase.IDLE) return
|
||||||
val context = LocalContext.current
|
val context = LocalContext.current
|
||||||
|
val listening = state.phase == CarAssistant.Phase.LISTENING && state.ready
|
||||||
|
val glow = MaterialTheme.colorScheme.primary
|
||||||
|
val glow2 = MaterialTheme.colorScheme.tertiary
|
||||||
|
val breathe by rememberInfiniteTransition(label = "glow").animateFloat(
|
||||||
|
0.55f, 1f, infiniteRepeatable(tween(900, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "glow",
|
||||||
|
)
|
||||||
|
val strength = when (state.phase) {
|
||||||
|
CarAssistant.Phase.LISTENING -> if (state.ready) breathe else 0.3f
|
||||||
|
CarAssistant.Phase.ERROR -> 0.3f
|
||||||
|
else -> 0.6f
|
||||||
|
}
|
||||||
|
|
||||||
Surface(
|
Box(
|
||||||
color = MaterialTheme.colorScheme.primaryContainer,
|
modifier
|
||||||
contentColor = MaterialTheme.colorScheme.onPrimaryContainer,
|
.fillMaxWidth()
|
||||||
modifier = modifier.fillMaxWidth(),
|
.drawBehind {
|
||||||
) {
|
drawRect(Brush.verticalGradient(0f to Color.Transparent, 0.45f to Color.Black.copy(alpha = 0.6f), 1f to Color.Black.copy(alpha = 0.85f)))
|
||||||
Row(
|
// A wide, flat oval of light rising from the bottom edge.
|
||||||
Modifier.padding(horizontal = 16.dp, vertical = 10.dp),
|
scale(1f, 0.22f, Offset(size.width / 2, size.height)) {
|
||||||
verticalAlignment = Alignment.CenterVertically,
|
drawCircle(
|
||||||
horizontalArrangement = Arrangement.spacedBy(12.dp),
|
Brush.radialGradient(
|
||||||
) {
|
0f to glow.copy(alpha = 0.9f * strength), 0.5f to glow2.copy(alpha = 0.35f * strength), 1f to Color.Transparent,
|
||||||
PulsingMic(state.phase == CarAssistant.Phase.LISTENING)
|
center = Offset(size.width / 2, size.height), radius = size.width * 0.55f,
|
||||||
Column(Modifier.weight(1f)) {
|
),
|
||||||
Text(
|
radius = size.width * 0.55f, center = Offset(size.width / 2, size.height),
|
||||||
when (state.phase) {
|
|
||||||
CarAssistant.Phase.LISTENING -> "Listening…"
|
|
||||||
CarAssistant.Phase.THINKING -> "…"
|
|
||||||
CarAssistant.Phase.SPEAKING -> state.reply
|
|
||||||
CarAssistant.Phase.ERROR -> state.reply
|
|
||||||
CarAssistant.Phase.IDLE -> ""
|
|
||||||
},
|
|
||||||
style = MaterialTheme.typography.titleSmall,
|
|
||||||
fontWeight = FontWeight.SemiBold,
|
|
||||||
maxLines = 2,
|
|
||||||
)
|
|
||||||
if (state.heard.isNotBlank() && state.phase != CarAssistant.Phase.SPEAKING) {
|
|
||||||
Text(
|
|
||||||
"\"${state.heard}\"",
|
|
||||||
style = MaterialTheme.typography.bodySmall,
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
.padding(start = 16.dp, end = 16.dp, top = 56.dp, bottom = 20.dp),
|
||||||
|
) {
|
||||||
|
Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(16.dp)) {
|
||||||
|
PulsingMic(listening)
|
||||||
|
Column(Modifier.weight(1f), horizontalAlignment = Alignment.CenterHorizontally) {
|
||||||
|
val (small, big) = when (state.phase) {
|
||||||
|
CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { if (state.ready) "Listening…" else "One moment…" }
|
||||||
|
CarAssistant.Phase.THINKING -> "" to state.heard.ifBlank { "…" }
|
||||||
|
else -> state.heard to state.reply
|
||||||
|
}
|
||||||
|
if (small.isNotBlank()) {
|
||||||
|
Text(small, color = Color.White.copy(alpha = 0.7f), style = MaterialTheme.typography.bodyLarge, maxLines = 1, overflow = TextOverflow.Ellipsis)
|
||||||
|
}
|
||||||
|
Text(
|
||||||
|
big,
|
||||||
|
color = if (big == "Listening…" || big == "One moment…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White,
|
||||||
|
style = MaterialTheme.typography.headlineSmall,
|
||||||
|
fontWeight = FontWeight.SemiBold,
|
||||||
|
textAlign = TextAlign.Center,
|
||||||
|
maxLines = 2,
|
||||||
|
overflow = TextOverflow.Ellipsis,
|
||||||
|
)
|
||||||
|
}
|
||||||
if (state.missingPermission) {
|
if (state.missingPermission) {
|
||||||
GsButton(tone = GsTone.Tonal, onClick = {
|
GsButton(tone = GsTone.Tonal, onClick = {
|
||||||
context.startActivity(
|
context.startActivity(
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ class CarApp(
|
|||||||
object CarApps {
|
object CarApps {
|
||||||
|
|
||||||
val all: List<CarApp> = listOf(
|
val all: List<CarApp> = listOf(
|
||||||
CarApp("web", "Web", Icons.Filled.Search) {
|
CarApp("web", "Browser", Icons.Filled.Search) {
|
||||||
WebApp(content = ProjectedContent.resolve(LocalContext.current))
|
WebApp(content = ProjectedContent.resolve(LocalContext.current))
|
||||||
},
|
},
|
||||||
CarApp("phone", "Screen sharing", Icons.Filled.Share) {
|
CarApp("phone", "Screen sharing", Icons.Filled.Share) {
|
||||||
|
|||||||
@@ -38,6 +38,8 @@ object CarAssistant {
|
|||||||
val heard: String = "",
|
val heard: String = "",
|
||||||
val reply: String = "",
|
val reply: String = "",
|
||||||
val missingPermission: Boolean = false,
|
val missingPermission: Boolean = false,
|
||||||
|
/** False from the press until the microphone is actually open, so the overlay doesn't claim to listen early. */
|
||||||
|
val ready: Boolean = false,
|
||||||
)
|
)
|
||||||
|
|
||||||
private val _state = MutableStateFlow(State())
|
private val _state = MutableStateFlow(State())
|
||||||
@@ -89,6 +91,7 @@ object CarAssistant {
|
|||||||
tts?.stop()
|
tts?.stop()
|
||||||
_state.value = State(Phase.LISTENING)
|
_state.value = State(Phase.LISTENING)
|
||||||
listenOnce(
|
listenOnce(
|
||||||
|
onListening = { _state.value = _state.value.copy(ready = true) },
|
||||||
onPartial = { text -> _state.value = _state.value.copy(heard = text) },
|
onPartial = { text -> _state.value = _state.value.copy(heard = text) },
|
||||||
onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) },
|
onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) },
|
||||||
onResult = { text ->
|
onResult = { text ->
|
||||||
@@ -174,15 +177,16 @@ object CarAssistant {
|
|||||||
onSpeaking: () -> Unit = {},
|
onSpeaking: () -> Unit = {},
|
||||||
) {
|
) {
|
||||||
cancel()
|
cancel()
|
||||||
listenOnce(onPartial, onHeard = {}, onResult, onError, cue, onSpeaking)
|
listenOnce(onPartial, onHeard = {}, onResult = onResult, onError = onError, cue = cue, onSpeaking = onSpeaking)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* One phrase, through Gearslip's own [SpeechEngine]. The phone's recognizer is only a fallback
|
* One phrase, through Gearslip's own [SpeechEngine], or the phone's recognizer when the driver
|
||||||
* for when the engine itself can't run, since plenty of phones (GrapheneOS among them) have none.
|
* chose it in settings. Never the phone's on its own: on most phones that's Google's.
|
||||||
*/
|
*/
|
||||||
private fun listenOnce(
|
private fun listenOnce(
|
||||||
onPartial: (String) -> Unit,
|
onPartial: (String) -> Unit,
|
||||||
|
onListening: () -> Unit = {},
|
||||||
onHeard: () -> Unit,
|
onHeard: () -> Unit,
|
||||||
onResult: (String) -> Unit,
|
onResult: (String) -> Unit,
|
||||||
onError: (String) -> Unit,
|
onError: (String) -> Unit,
|
||||||
@@ -190,24 +194,24 @@ object CarAssistant {
|
|||||||
onSpeaking: () -> Unit = {},
|
onSpeaking: () -> Unit = {},
|
||||||
) {
|
) {
|
||||||
val context = appContext ?: return onError("Speech isn't ready yet.")
|
val context = appContext ?: return onError("Speech isn't ready yet.")
|
||||||
|
if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) {
|
||||||
|
return systemListen(context, onListening, onPartial, onHeard, onResult, onError)
|
||||||
|
}
|
||||||
SpeechEngine.listen(
|
SpeechEngine.listen(
|
||||||
context,
|
context,
|
||||||
cue = cue,
|
cue = cue,
|
||||||
onSpeaking = onSpeaking,
|
onSpeaking = onSpeaking,
|
||||||
|
onPartial = onPartial,
|
||||||
|
onListening = onListening,
|
||||||
onHeard = onHeard,
|
onHeard = onHeard,
|
||||||
onResult = onResult,
|
onResult = onResult,
|
||||||
onError = { message ->
|
onError = onError,
|
||||||
if (message == SpeechEngine.NOT_LOADED && SpeechRecognizer.isRecognitionAvailable(context)) {
|
|
||||||
systemListen(context, onPartial, onHeard, onResult, onError)
|
|
||||||
} else {
|
|
||||||
onError(message)
|
|
||||||
}
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun systemListen(
|
private fun systemListen(
|
||||||
context: Context,
|
context: Context,
|
||||||
|
onListening: () -> Unit,
|
||||||
onPartial: (String) -> Unit,
|
onPartial: (String) -> Unit,
|
||||||
onHeard: () -> Unit,
|
onHeard: () -> Unit,
|
||||||
onResult: (String) -> Unit,
|
onResult: (String) -> Unit,
|
||||||
@@ -216,7 +220,7 @@ object CarAssistant {
|
|||||||
val r = SpeechRecognizer.createSpeechRecognizer(context)
|
val r = SpeechRecognizer.createSpeechRecognizer(context)
|
||||||
recognizer = r
|
recognizer = r
|
||||||
r.setRecognitionListener(object : RecognitionListener {
|
r.setRecognitionListener(object : RecognitionListener {
|
||||||
override fun onReadyForSpeech(params: Bundle?) {}
|
override fun onReadyForSpeech(params: Bundle?) = onListening()
|
||||||
override fun onBeginningOfSpeech() {}
|
override fun onBeginningOfSpeech() {}
|
||||||
override fun onRmsChanged(rmsdB: Float) {}
|
override fun onRmsChanged(rmsdB: Float) {}
|
||||||
override fun onBufferReceived(buffer: ByteArray?) {}
|
override fun onBufferReceived(buffer: ByteArray?) {}
|
||||||
|
|||||||
@@ -138,18 +138,18 @@ fun CarKeyboard(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Control row: no backspace here - it stays with the letters above, not next to the
|
// Control row: no backspace here - it stays with the letters above, not next to the
|
||||||
// key most likely to be hit right after it by mistake. Dismiss (when offered) is the
|
// key most likely to be hit right after it by mistake. Search/enter takes the
|
||||||
// last key, bottom-right corner of the keyboard.
|
// bottom-right corner, where a thumb expects it; dismiss (when offered) sits left of it.
|
||||||
Row(
|
Row(
|
||||||
Modifier.fillMaxWidth(),
|
Modifier.fillMaxWidth(),
|
||||||
horizontalArrangement = Arrangement.spacedBy(4.dp),
|
horizontalArrangement = Arrangement.spacedBy(4.dp),
|
||||||
) {
|
) {
|
||||||
Key(if (symbols) "abc" else "?123", Modifier.weight(1.4f)) { symbols = !symbols }
|
Key(if (symbols) "abc" else "?123", Modifier.weight(1.4f)) { symbols = !symbols }
|
||||||
Key("space", Modifier.weight(if (onDismiss != null) 3.2f else 4f)) { onTextChange("$text ") }
|
Key("space", Modifier.weight(if (onDismiss != null) 3.2f else 4f)) { onTextChange("$text ") }
|
||||||
SubmitKey(Modifier.weight(1.6f), submitIcon, submitImage, onSubmit)
|
|
||||||
if (onDismiss != null) {
|
if (onDismiss != null) {
|
||||||
DismissKey(onDismiss, Modifier.weight(1.4f))
|
DismissKey(onDismiss, Modifier.weight(1.4f))
|
||||||
}
|
}
|
||||||
|
SubmitKey(Modifier.weight(1.6f), submitIcon, submitImage, onSubmit)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -222,9 +222,9 @@ private fun SubmitKey(modifier: Modifier, search: Boolean, image: ImageVector?,
|
|||||||
* Big enough to hit on a bumpy road, small enough that four rows plus a field plus a few results
|
* Big enough to hit on a bumpy road, small enough that four rows plus a field plus a few results
|
||||||
* still fit. A 480-tall screen sets the floor; a taller one (LIVI's 1280x720) gets bigger keys.
|
* still fit. A 480-tall screen sets the floor; a taller one (LIVI's 1280x720) gets bigger keys.
|
||||||
*/
|
*/
|
||||||
private fun keyHeight(screenHeightDp: Int): Dp = (screenHeightDp * 0.075f).dp.coerceIn(36.dp, 52.dp)
|
private fun keyHeight(screenHeightDp: Int): Dp = (screenHeightDp * 0.09f).dp.coerceIn(44.dp, 64.dp)
|
||||||
|
|
||||||
private val LocalKeyHeight = compositionLocalOf { 36.dp }
|
private val LocalKeyHeight = compositionLocalOf { 44.dp }
|
||||||
|
|
||||||
private val LETTER_ROWS = listOf(
|
private val LETTER_ROWS = listOf(
|
||||||
"qwertyuiop".map { it.toString() },
|
"qwertyuiop".map { it.toString() },
|
||||||
|
|||||||
@@ -102,15 +102,10 @@ object CarKeys {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* The voice key: answers the message on screen if there is one, otherwise starts the
|
* The voice key: answers the message on screen if there is one, otherwise starts the
|
||||||
* assistant when it's turned on.
|
* assistant.
|
||||||
*/
|
*/
|
||||||
private fun voice() = main.post {
|
private fun voice() = main.post {
|
||||||
if (VoiceReply.onVoiceKey()) return@post
|
if (!VoiceReply.onVoiceKey()) CarAssistant.start()
|
||||||
if (CarSettings.experimentalFeaturesEnabled.value && CarSettings.voiceAssistantEnabled.value) {
|
|
||||||
CarAssistant.start()
|
|
||||||
} else {
|
|
||||||
GearslipLog.i("keys: voice pressed, but the assistant is off in settings")
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Answers a ringing call; with nothing ringing, opens the phone screen. */
|
/** Answers a ringing call; with nothing ringing, opens the phone screen. */
|
||||||
|
|||||||
@@ -247,6 +247,8 @@ object CarServices {
|
|||||||
runCatching { media.disconnect() }
|
runCatching { media.disconnect() }
|
||||||
runCatching { browse.disconnect() }
|
runCatching { browse.disconnect() }
|
||||||
CarAssistant.cancel()
|
CarAssistant.cancel()
|
||||||
|
// About 900 MB that nothing needs once the car's gone.
|
||||||
|
app.seb3thehacker.gearslip.speech.SpeechEngine.unload()
|
||||||
CarAudio.release()
|
CarAudio.release()
|
||||||
_mediaApp.value = null
|
_mediaApp.value = null
|
||||||
navStopped = false
|
navStopped = false
|
||||||
|
|||||||
@@ -38,9 +38,8 @@ object CarSettings {
|
|||||||
private const val KEY_PINNED = "car_pinned_apps"
|
private const val KEY_PINNED = "car_pinned_apps"
|
||||||
private const val KEY_APP_ORDER = "car_app_order"
|
private const val KEY_APP_ORDER = "car_app_order"
|
||||||
private const val KEY_EXPERIMENTAL_FEATURES = "car_experimental_features"
|
private const val KEY_EXPERIMENTAL_FEATURES = "car_experimental_features"
|
||||||
private const val KEY_VOICE_ASSISTANT = "car_voice_assistant"
|
private const val KEY_SYSTEM_SPEECH = "car_system_speech"
|
||||||
private const val KEY_CAR_SENSORS = "car_sensors_enabled"
|
private const val KEY_CAR_SENSORS = "car_sensors_enabled"
|
||||||
private const val KEY_MEDIA_SEARCH = "car_media_search_enabled"
|
|
||||||
private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning"
|
private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning"
|
||||||
private const val KEY_UNITS = "car_units"
|
private const val KEY_UNITS = "car_units"
|
||||||
private const val KEY_WHATS_NEW = "car_whats_new_after_updates"
|
private const val KEY_WHATS_NEW = "car_whats_new_after_updates"
|
||||||
@@ -72,9 +71,8 @@ object CarSettings {
|
|||||||
private val pinnedFlow = MutableStateFlow<Set<String>>(emptySet())
|
private val pinnedFlow = MutableStateFlow<Set<String>>(emptySet())
|
||||||
private val appOrderFlow = MutableStateFlow<List<String>>(emptyList())
|
private val appOrderFlow = MutableStateFlow<List<String>>(emptyList())
|
||||||
private val experimentalFeaturesFlow = MutableStateFlow(false)
|
private val experimentalFeaturesFlow = MutableStateFlow(false)
|
||||||
private val voiceAssistantFlow = MutableStateFlow(false)
|
private val systemSpeechFlow = MutableStateFlow(false)
|
||||||
private val carSensorsFlow = MutableStateFlow(false)
|
private val carSensorsFlow = MutableStateFlow(false)
|
||||||
private val mediaSearchFlow = MutableStateFlow(false)
|
|
||||||
private val seenSafetyWarningFlow = MutableStateFlow(false)
|
private val seenSafetyWarningFlow = MutableStateFlow(false)
|
||||||
private val unitsFlow = MutableStateFlow(Units.AUTO)
|
private val unitsFlow = MutableStateFlow(Units.AUTO)
|
||||||
|
|
||||||
@@ -170,12 +168,8 @@ object CarSettings {
|
|||||||
*/
|
*/
|
||||||
val experimentalFeaturesEnabled: StateFlow<Boolean> = experimentalFeaturesFlow
|
val experimentalFeaturesEnabled: StateFlow<Boolean> = experimentalFeaturesFlow
|
||||||
|
|
||||||
/**
|
/** Listen with the phone's own speech service instead of Gearslip's on-device model. */
|
||||||
* The voice assistant is experimental and off by default; a driver has to turn it on
|
val systemSpeech: StateFlow<Boolean> = systemSpeechFlow
|
||||||
* knowingly, from the settings screen, after reading the warning shown there. This flag
|
|
||||||
* alone does not run the assistant - [experimentalFeaturesEnabled] must also be on.
|
|
||||||
*/
|
|
||||||
val voiceAssistantEnabled: StateFlow<Boolean> = voiceAssistantFlow
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Vehicle data is experimental and off by default: the launcher's tile for it, and the
|
* Vehicle data is experimental and off by default: the launcher's tile for it, and the
|
||||||
@@ -185,13 +179,6 @@ object CarSettings {
|
|||||||
*/
|
*/
|
||||||
val carSensorsEnabled: StateFlow<Boolean> = carSensorsFlow
|
val carSensorsEnabled: StateFlow<Boolean> = carSensorsFlow
|
||||||
|
|
||||||
/**
|
|
||||||
* Searching inside a media app is experimental and off by default: many apps claim to take
|
|
||||||
* searches and then ignore them or play something else. The media screen's search button
|
|
||||||
* shows only once this and [experimentalFeaturesEnabled] are both on.
|
|
||||||
*/
|
|
||||||
val mediaSearchEnabled: StateFlow<Boolean> = mediaSearchFlow
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Whether the car screen's own "drive safely" warning has already been shown and dismissed
|
* Whether the car screen's own "drive safely" warning has already been shown and dismissed
|
||||||
* once, from this same install. Separate from the phone side's own compat/safety dialogs -
|
* once, from this same install. Separate from the phone side's own compat/safety dialogs -
|
||||||
@@ -226,9 +213,8 @@ object CarSettings {
|
|||||||
appOrderFlow.value = AppSettings.getString(app, KEY_APP_ORDER, "")
|
appOrderFlow.value = AppSettings.getString(app, KEY_APP_ORDER, "")
|
||||||
.split(",").filter { it.isNotEmpty() }
|
.split(",").filter { it.isNotEmpty() }
|
||||||
experimentalFeaturesFlow.value = AppSettings.getString(app, KEY_EXPERIMENTAL_FEATURES, "false") == "true"
|
experimentalFeaturesFlow.value = AppSettings.getString(app, KEY_EXPERIMENTAL_FEATURES, "false") == "true"
|
||||||
voiceAssistantFlow.value = AppSettings.getString(app, KEY_VOICE_ASSISTANT, "false") == "true"
|
systemSpeechFlow.value = AppSettings.getString(app, KEY_SYSTEM_SPEECH, "false") == "true"
|
||||||
carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true"
|
carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true"
|
||||||
mediaSearchFlow.value = AppSettings.getString(app, KEY_MEDIA_SEARCH, "false") == "true"
|
|
||||||
seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true"
|
seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true"
|
||||||
unitsFlow.value = runCatching {
|
unitsFlow.value = runCatching {
|
||||||
Units.valueOf(AppSettings.getString(app, KEY_UNITS, Units.AUTO.name))
|
Units.valueOf(AppSettings.getString(app, KEY_UNITS, Units.AUTO.name))
|
||||||
@@ -293,9 +279,9 @@ object CarSettings {
|
|||||||
AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
|
AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
|
||||||
}
|
}
|
||||||
|
|
||||||
fun setVoiceAssistantEnabled(on: Boolean) {
|
fun setSystemSpeech(on: Boolean) {
|
||||||
voiceAssistantFlow.value = on
|
systemSpeechFlow.value = on
|
||||||
AppSettings.putString(app, KEY_VOICE_ASSISTANT, on.toString())
|
AppSettings.putString(app, KEY_SYSTEM_SPEECH, on.toString())
|
||||||
}
|
}
|
||||||
|
|
||||||
fun setCarSensorsEnabled(on: Boolean) {
|
fun setCarSensorsEnabled(on: Boolean) {
|
||||||
@@ -303,11 +289,6 @@ object CarSettings {
|
|||||||
AppSettings.putString(app, KEY_CAR_SENSORS, on.toString())
|
AppSettings.putString(app, KEY_CAR_SENSORS, on.toString())
|
||||||
}
|
}
|
||||||
|
|
||||||
fun setMediaSearchEnabled(on: Boolean) {
|
|
||||||
mediaSearchFlow.value = on
|
|
||||||
AppSettings.putString(app, KEY_MEDIA_SEARCH, on.toString())
|
|
||||||
}
|
|
||||||
|
|
||||||
fun setNightMode(mode: NightMode) {
|
fun setNightMode(mode: NightMode) {
|
||||||
nightFlow.value = mode
|
nightFlow.value = mode
|
||||||
AppSettings.putString(app, KEY_NIGHT, mode.name)
|
AppSettings.putString(app, KEY_NIGHT, mode.name)
|
||||||
|
|||||||
@@ -58,9 +58,8 @@ fun CarSettingsScreen() {
|
|||||||
val theme by CarSettings.appTheme.collectAsState()
|
val theme by CarSettings.appTheme.collectAsState()
|
||||||
val uiTheme by CarSettings.uiTheme.collectAsState()
|
val uiTheme by CarSettings.uiTheme.collectAsState()
|
||||||
val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState()
|
val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState()
|
||||||
val voiceAssistant by CarSettings.voiceAssistantEnabled.collectAsState()
|
|
||||||
val carSensors by CarSettings.carSensorsEnabled.collectAsState()
|
val carSensors by CarSettings.carSensorsEnabled.collectAsState()
|
||||||
val mediaSearch by CarSettings.mediaSearchEnabled.collectAsState()
|
val systemSpeech by CarSettings.systemSpeech.collectAsState()
|
||||||
val display by CarEnvironment.display.collectAsState()
|
val display by CarEnvironment.display.collectAsState()
|
||||||
val vehicle by CarEnvironment.vehicle.collectAsState()
|
val vehicle by CarEnvironment.vehicle.collectAsState()
|
||||||
val context = LocalContext.current
|
val context = LocalContext.current
|
||||||
@@ -138,6 +137,14 @@ fun CarSettingsScreen() {
|
|||||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
ExperimentalFeatureRow(
|
||||||
|
title = "Use the phone's speech recognition",
|
||||||
|
description = "Listens through the phone's own speech service instead of Gearslip's. On " +
|
||||||
|
"most phones that's Google's, which may send your voice to Google. GrapheneOS has " +
|
||||||
|
"none, so Gearslip keeps using its own.",
|
||||||
|
checked = systemSpeech,
|
||||||
|
) { CarSettings.setSystemSpeech(it) }
|
||||||
|
|
||||||
WebPageSetting()
|
WebPageSetting()
|
||||||
|
|
||||||
ScreenFit(vehicle)
|
ScreenFit(vehicle)
|
||||||
@@ -148,7 +155,7 @@ fun CarSettingsScreen() {
|
|||||||
experimentalFeatures,
|
experimentalFeatures,
|
||||||
) { CarSettings.setExperimentalFeaturesEnabled(it) }
|
) { CarSettings.setExperimentalFeaturesEnabled(it) }
|
||||||
if (experimentalFeatures) {
|
if (experimentalFeatures) {
|
||||||
ExperimentalFeaturesList(voiceAssistant, carSensors, mediaSearch)
|
ExperimentalFeaturesList(carSensors)
|
||||||
}
|
}
|
||||||
|
|
||||||
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||||
@@ -205,21 +212,11 @@ private fun <T> ChoiceRow(title: String, options: List<Pair<T, String>>, selecte
|
|||||||
* so the list reads fine whether there's one entry or several.
|
* so the list reads fine whether there's one entry or several.
|
||||||
*/
|
*/
|
||||||
@Composable
|
@Composable
|
||||||
private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolean, mediaSearch: Boolean) {
|
private fun ExperimentalFeaturesList(carSensors: Boolean) {
|
||||||
Column(
|
Column(
|
||||||
Modifier.fillMaxWidth(),
|
Modifier.fillMaxWidth(),
|
||||||
verticalArrangement = Arrangement.spacedBy(10.dp),
|
verticalArrangement = Arrangement.spacedBy(10.dp),
|
||||||
) {
|
) {
|
||||||
ExperimentalFeatureRow(
|
|
||||||
title = "Voice assistant",
|
|
||||||
description = "Push-to-talk and the car's voice button start listening: navigate, " +
|
|
||||||
"play media, call, or open an app by saying so. Can mishear you or act on the " +
|
|
||||||
"wrong command - keep your eyes on the road, and confirm anything it does " +
|
|
||||||
"before you rely on it.",
|
|
||||||
warning = true,
|
|
||||||
checked = voiceAssistant,
|
|
||||||
) { CarSettings.setVoiceAssistantEnabled(it) }
|
|
||||||
|
|
||||||
ExperimentalFeatureRow(
|
ExperimentalFeatureRow(
|
||||||
title = "Vehicle data",
|
title = "Vehicle data",
|
||||||
description = "Adds a Vehicle data tile: a live, raw dump of the head unit's sensor " +
|
description = "Adds a Vehicle data tile: a live, raw dump of the head unit's sensor " +
|
||||||
@@ -228,13 +225,6 @@ private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolea
|
|||||||
"guess, not a fact, until you've watched one change for yourself.",
|
"guess, not a fact, until you've watched one change for yourself.",
|
||||||
checked = carSensors,
|
checked = carSensors,
|
||||||
) { CarSettings.setCarSensorsEnabled(it) }
|
) { CarSettings.setCarSensorsEnabled(it) }
|
||||||
|
|
||||||
ExperimentalFeatureRow(
|
|
||||||
title = "Media search",
|
|
||||||
description = "Adds a search button to the media screen. Some apps ignore the search " +
|
|
||||||
"or play something else.",
|
|
||||||
checked = mediaSearch,
|
|
||||||
) { CarSettings.setMediaSearchEnabled(it) }
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -272,7 +262,7 @@ private fun ExperimentalFeatureRow(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** The URL (or raw HTML) the Web app shows. Edited with [CarKeyboard] - no phone in hand needed. */
|
/** The URL (or raw HTML) the Browser app shows. Edited with [CarKeyboard] - no phone in hand needed. */
|
||||||
@Composable
|
@Composable
|
||||||
private fun WebPageSetting() {
|
private fun WebPageSetting() {
|
||||||
val context = LocalContext.current
|
val context = LocalContext.current
|
||||||
@@ -297,7 +287,7 @@ private fun WebPageSetting() {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
Text(
|
Text(
|
||||||
"Shown by the Web app on the launcher. Applies on the next connection.",
|
"Shown by the Browser app on the launcher. Applies on the next connection.",
|
||||||
style = MaterialTheme.typography.bodySmall,
|
style = MaterialTheme.typography.bodySmall,
|
||||||
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -256,8 +256,9 @@ fun CarUi() {
|
|||||||
modifier = Modifier.align(Alignment.TopEnd),
|
modifier = Modifier.align(Alignment.TopEnd),
|
||||||
)
|
)
|
||||||
// Above the popup: a call is more urgent than any notification.
|
// Above the popup: a call is more urgent than any notification.
|
||||||
|
VoiceHelpOverlay(Modifier.align(Alignment.Center))
|
||||||
CallOverlay(Modifier.align(Alignment.TopCenter))
|
CallOverlay(Modifier.align(Alignment.TopCenter))
|
||||||
AssistantOverlay(Modifier.align(Alignment.TopCenter))
|
AssistantOverlay(Modifier.align(Alignment.BottomCenter))
|
||||||
if (app.seb3thehacker.gearslip.BuildConfig.DEBUG) {
|
if (app.seb3thehacker.gearslip.BuildConfig.DEBUG) {
|
||||||
// Bottom corner, not top: a screen's own header almost always puts
|
// Bottom corner, not top: a screen's own header almost always puts
|
||||||
// something in the top-right (settings, search, a Liked Songs
|
// something in the top-right (settings, search, a Liked Songs
|
||||||
@@ -469,12 +470,8 @@ private fun CarNavBar(navigator: CarNavigator) {
|
|||||||
NavHomeButton(navigator, entries, running)
|
NavHomeButton(navigator, entries, running)
|
||||||
Spacer(Modifier.width(6.dp))
|
Spacer(Modifier.width(6.dp))
|
||||||
NavItem(MediaIcons.Apps, navigator.current == CarScreen.Apps) { navigator.apps() }
|
NavItem(MediaIcons.Apps, navigator.current == CarScreen.Apps) { navigator.apps() }
|
||||||
val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState()
|
Spacer(Modifier.width(6.dp))
|
||||||
val voiceEnabled by CarSettings.voiceAssistantEnabled.collectAsState()
|
NavAssistantIcon()
|
||||||
if (experimentalFeatures && voiceEnabled) {
|
|
||||||
Spacer(Modifier.width(6.dp))
|
|
||||||
NavAssistantIcon()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pinned shortcuts, then whatever else is open right now. A pinned app that's also
|
// Pinned shortcuts, then whatever else is open right now. A pinned app that's also
|
||||||
// open stays in its pinned spot, marked with a dot, instead of showing up twice.
|
// open stays in its pinned spot, marked with a dot, instead of showing up twice.
|
||||||
|
|||||||
@@ -61,7 +61,6 @@ import androidx.compose.ui.text.font.FontWeight
|
|||||||
import androidx.compose.ui.text.style.TextOverflow
|
import androidx.compose.ui.text.style.TextOverflow
|
||||||
import androidx.compose.ui.unit.dp
|
import androidx.compose.ui.unit.dp
|
||||||
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
import androidx.lifecycle.compose.collectAsStateWithLifecycle
|
||||||
import app.seb3thehacker.gearslip.audio.CarAudio
|
|
||||||
import app.seb3thehacker.gearslip.media.CarMedia
|
import app.seb3thehacker.gearslip.media.CarMedia
|
||||||
import app.seb3thehacker.gearslip.media.MediaApp
|
import app.seb3thehacker.gearslip.media.MediaApp
|
||||||
import app.seb3thehacker.gearslip.media.MediaArt
|
import app.seb3thehacker.gearslip.media.MediaArt
|
||||||
@@ -88,11 +87,9 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) {
|
|||||||
val queue by media.queue.collectAsStateWithLifecycle()
|
val queue by media.queue.collectAsStateWithLifecycle()
|
||||||
val browse by media.browse.collectAsStateWithLifecycle()
|
val browse by media.browse.collectAsStateWithLifecycle()
|
||||||
val rejection by media.rejection.collectAsStateWithLifecycle()
|
val rejection by media.rejection.collectAsStateWithLifecycle()
|
||||||
val capture by CarAudio.capture.collectAsStateWithLifecycle()
|
|
||||||
val search by media.search.collectAsStateWithLifecycle()
|
val search by media.search.collectAsStateWithLifecycle()
|
||||||
var tab by remember { mutableStateOf(Tab.NOW_PLAYING) }
|
var tab by remember { mutableStateOf(Tab.NOW_PLAYING) }
|
||||||
var searching by remember { mutableStateOf(false) }
|
var searching by remember { mutableStateOf(false) }
|
||||||
val searchAllowed = CarSettings.experimentalFeaturesEnabled.collectAsState().value && CarSettings.mediaSearchEnabled.collectAsState().value
|
|
||||||
|
|
||||||
// The connection outlives this screen: the home screen's player is the same one.
|
// The connection outlives this screen: the home screen's player is the same one.
|
||||||
LaunchedEffect(app) { CarServices.openMedia(app) }
|
LaunchedEffect(app) { CarServices.openMedia(app) }
|
||||||
@@ -174,7 +171,6 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) {
|
|||||||
} else {
|
} else {
|
||||||
CompactPlayer(now, media, Modifier.weight(1f).fillMaxWidth())
|
CompactPlayer(now, media, Modifier.weight(1f).fillMaxWidth())
|
||||||
}
|
}
|
||||||
AudioBadge(capture)
|
|
||||||
}
|
}
|
||||||
Column(Modifier.weight(1.2f).fillMaxHeight()) {
|
Column(Modifier.weight(1.2f).fillMaxHeight()) {
|
||||||
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) {
|
||||||
@@ -182,7 +178,7 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) {
|
|||||||
MediaTab("Browse", tab == Tab.BROWSE, Modifier.weight(1f)) { tab = Tab.BROWSE }
|
MediaTab("Browse", tab == Tab.BROWSE, Modifier.weight(1f)) { tab = Tab.BROWSE }
|
||||||
MediaTab("Up next", tab == Tab.QUEUE, Modifier.weight(1f)) { tab = Tab.QUEUE }
|
MediaTab("Up next", tab == Tab.QUEUE, Modifier.weight(1f)) { tab = Tab.QUEUE }
|
||||||
MediaTab("Lyrics", tab == Tab.LYRICS, Modifier.weight(1f)) { tab = Tab.LYRICS }
|
MediaTab("Lyrics", tab == Tab.LYRICS, Modifier.weight(1f)) { tab = Tab.LYRICS }
|
||||||
if (searchAllowed && (now.canSearch || media.canSearchLibrary)) {
|
if (now.canSearch || media.canSearchLibrary) {
|
||||||
GsIconButton(
|
GsIconButton(
|
||||||
Icons.Filled.Search,
|
Icons.Filled.Search,
|
||||||
"Search ${app.label}",
|
"Search ${app.label}",
|
||||||
@@ -447,17 +443,6 @@ private sealed interface LyricsState {
|
|||||||
class Found(val result: LyricsResult) : LyricsState
|
class Found(val result: LyricsResult) : LyricsState
|
||||||
}
|
}
|
||||||
|
|
||||||
@Composable
|
|
||||||
private fun AudioBadge(capture: CarAudio.Capture) {
|
|
||||||
val text = when (capture) {
|
|
||||||
CarAudio.Capture.ON -> "Audio to car"
|
|
||||||
CarAudio.Capture.ASKING -> "Allow audio capture on the phone"
|
|
||||||
CarAudio.Capture.DENIED -> "Audio stays on the phone"
|
|
||||||
CarAudio.Capture.OFF -> return
|
|
||||||
}
|
|
||||||
Text(text, style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurfaceVariant)
|
|
||||||
}
|
|
||||||
|
|
||||||
@Composable
|
@Composable
|
||||||
private fun Notice(title: String, detail: String?, action: Pair<String, () -> Unit>? = null) {
|
private fun Notice(title: String, detail: String?, action: Pair<String, () -> Unit>? = null) {
|
||||||
Column(
|
Column(
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ internal class BuiltInApp(
|
|||||||
)
|
)
|
||||||
|
|
||||||
internal object BuiltInApps {
|
internal object BuiltInApps {
|
||||||
val Web = BuiltInApp("web", "Web", Icons.Filled.Search, CarScreen.App("web")) { it.open("web") }
|
val Web = BuiltInApp("web", "Browser", Icons.Filled.Search, CarScreen.App("web")) { it.open("web") }
|
||||||
val ScreenSharing = BuiltInApp("phone", "Screen sharing", Icons.Filled.Share, CarScreen.App("phone")) { it.open("phone") }
|
val ScreenSharing = BuiltInApp("phone", "Screen sharing", Icons.Filled.Share, CarScreen.App("phone")) { it.open("phone") }
|
||||||
val Phone = BuiltInApp("dialer", "Phone", Icons.Filled.Call, CarScreen.App("dialer")) { it.open("dialer") }
|
val Phone = BuiltInApp("dialer", "Phone", Icons.Filled.Call, CarScreen.App("dialer")) { it.open("dialer") }
|
||||||
val VehicleData = BuiltInApp("vehicle", "Vehicle data", Icons.Filled.Info, CarScreen.VehicleData) { it.vehicleData() }
|
val VehicleData = BuiltInApp("vehicle", "Vehicle data", Icons.Filled.Info, CarScreen.VehicleData) { it.vehicleData() }
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ import androidx.car.app.model.MessageTemplate
|
|||||||
import androidx.car.app.model.PaneTemplate
|
import androidx.car.app.model.PaneTemplate
|
||||||
import androidx.car.app.model.Row as CarRow
|
import androidx.car.app.model.Row as CarRow
|
||||||
import androidx.car.app.model.SearchTemplate
|
import androidx.car.app.model.SearchTemplate
|
||||||
|
import androidx.compose.foundation.clickable
|
||||||
import androidx.car.app.model.Section
|
import androidx.car.app.model.Section
|
||||||
import androidx.car.app.model.SectionedItemTemplate
|
import androidx.car.app.model.SectionedItemTemplate
|
||||||
import androidx.car.app.model.TabTemplate
|
import androidx.car.app.model.TabTemplate
|
||||||
@@ -238,6 +239,9 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) {
|
|||||||
// initialSearchText would fight whatever is being typed.
|
// initialSearchText would fight whatever is being typed.
|
||||||
var query by remember { mutableStateOf(template.initialSearchText.orEmpty()) }
|
var query by remember { mutableStateOf(template.initialSearchText.orEmpty()) }
|
||||||
val delegate = template.searchCallbackDelegate
|
val delegate = template.searchCallbackDelegate
|
||||||
|
// An app that opens on results ("navigate to Starbucks") asks for the keyboard to stay down,
|
||||||
|
// so the results aren't hidden under it. Tapping the field brings it up.
|
||||||
|
var typing by remember { mutableStateOf(template.isShowKeyboardByDefault) }
|
||||||
|
|
||||||
ContentSurface(modifier) {
|
ContentSurface(modifier) {
|
||||||
// Back button and field share a row: on a 480px screen the keyboard and the results
|
// Back button and field share a row: on a 480px screen the keyboard and the results
|
||||||
@@ -251,7 +255,7 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) {
|
|||||||
SearchField(
|
SearchField(
|
||||||
query,
|
query,
|
||||||
template.searchHint.orEmpty(),
|
template.searchHint.orEmpty(),
|
||||||
Modifier.weight(1f),
|
Modifier.weight(1f).clickable { typing = true },
|
||||||
onClear = {
|
onClear = {
|
||||||
query = ""
|
query = ""
|
||||||
delegate.textChanged("")
|
delegate.textChanged("")
|
||||||
@@ -265,14 +269,17 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) {
|
|||||||
else ItemListColumn(template.itemList, modifier = Modifier.fillMaxSize())
|
else ItemListColumn(template.itemList, modifier = Modifier.fillMaxSize())
|
||||||
}
|
}
|
||||||
|
|
||||||
CarKeyboard(
|
if (typing) {
|
||||||
text = query,
|
CarKeyboard(
|
||||||
onTextChange = {
|
text = query,
|
||||||
query = it
|
onTextChange = {
|
||||||
delegate.textChanged(it)
|
query = it
|
||||||
},
|
delegate.textChanged(it)
|
||||||
onSubmit = { delegate.submitted(query) },
|
},
|
||||||
)
|
onSubmit = { delegate.submitted(query) },
|
||||||
|
onDismiss = { typing = false },
|
||||||
|
)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package app.seb3thehacker.gearslip.car
|
||||||
|
|
||||||
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
|
import androidx.compose.foundation.layout.Column
|
||||||
|
import androidx.compose.foundation.layout.Row
|
||||||
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
|
import androidx.compose.foundation.layout.padding
|
||||||
|
import androidx.compose.foundation.layout.widthIn
|
||||||
|
import androidx.compose.foundation.rememberScrollState
|
||||||
|
import androidx.compose.foundation.verticalScroll
|
||||||
|
import androidx.compose.material.icons.Icons
|
||||||
|
import androidx.compose.material.icons.filled.Close
|
||||||
|
import androidx.compose.material3.MaterialTheme
|
||||||
|
import androidx.compose.material3.Surface
|
||||||
|
import androidx.compose.material3.Text
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.LaunchedEffect
|
||||||
|
import androidx.compose.runtime.collectAsState
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.text.font.FontWeight
|
||||||
|
import androidx.compose.ui.unit.dp
|
||||||
|
import app.seb3thehacker.gearslip.car.theme.GsIconButton
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
|
|
||||||
|
/** What the assistant understands, shown on the car screen when the driver says "help". */
|
||||||
|
object VoiceHelp {
|
||||||
|
private val _shown = MutableStateFlow(false)
|
||||||
|
val shown: StateFlow<Boolean> = _shown.asStateFlow()
|
||||||
|
|
||||||
|
fun show() { _shown.value = true }
|
||||||
|
fun hide() { _shown.value = false }
|
||||||
|
|
||||||
|
/** Say this, and this happens. Kept in step with [AssistantCommands]. */
|
||||||
|
val COMMANDS = listOf(
|
||||||
|
"Navigate to…" to "Directions there. Add \"with Organic Maps\" to pick the app",
|
||||||
|
"Play…" to "A song, artist, or album in the open music app",
|
||||||
|
"Pause · Resume · Next · Previous" to "Control the music",
|
||||||
|
"What's playing" to "The song and artist",
|
||||||
|
"Call…" to "A contact by name",
|
||||||
|
"Open…" to "Any app, like Browser, Phone, or Screen sharing",
|
||||||
|
"Exit" to "Back to the car's own screen",
|
||||||
|
"Go home · Open apps · Settings" to "Move around Gearslip",
|
||||||
|
"What time is it · Weather" to "The time, or the forecast",
|
||||||
|
"Help" to "This list",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The command list over the car screen: read it at a glance, tap the cross or wait to dismiss. */
|
||||||
|
@Composable
|
||||||
|
fun VoiceHelpOverlay(modifier: Modifier = Modifier) {
|
||||||
|
val shown by VoiceHelp.shown.collectAsState()
|
||||||
|
if (!shown) return
|
||||||
|
// A list left up would cover the map for the rest of the drive.
|
||||||
|
LaunchedEffect(Unit) {
|
||||||
|
delay(30_000)
|
||||||
|
VoiceHelp.hide()
|
||||||
|
}
|
||||||
|
Surface(
|
||||||
|
color = MaterialTheme.colorScheme.surfaceContainerHigh,
|
||||||
|
contentColor = MaterialTheme.colorScheme.onSurface,
|
||||||
|
shape = MaterialTheme.shapes.extraLarge,
|
||||||
|
tonalElevation = 6.dp,
|
||||||
|
shadowElevation = 8.dp,
|
||||||
|
modifier = modifier.padding(16.dp).widthIn(max = 640.dp),
|
||||||
|
) {
|
||||||
|
Column(Modifier.padding(20.dp)) {
|
||||||
|
Row(Modifier.fillMaxWidth()) {
|
||||||
|
Text(
|
||||||
|
"You can say",
|
||||||
|
style = MaterialTheme.typography.titleLarge,
|
||||||
|
fontWeight = FontWeight.Bold,
|
||||||
|
modifier = Modifier.weight(1f).padding(top = 6.dp),
|
||||||
|
)
|
||||||
|
GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide)
|
||||||
|
}
|
||||||
|
// Full width, so a drag anywhere on the card scrolls, not just one on the words.
|
||||||
|
Column(
|
||||||
|
Modifier.fillMaxWidth().verticalScroll(rememberScrollState()).padding(top = 8.dp),
|
||||||
|
verticalArrangement = Arrangement.spacedBy(10.dp),
|
||||||
|
) {
|
||||||
|
VoiceHelp.COMMANDS.forEach { (say, does) ->
|
||||||
|
Column(Modifier.fillMaxWidth()) {
|
||||||
|
Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold)
|
||||||
|
Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -59,7 +59,7 @@ fun WebApp(content: String) {
|
|||||||
|
|
||||||
Column(Modifier.fillMaxSize()) {
|
Column(Modifier.fillMaxSize()) {
|
||||||
// Back steps through the page's history first (the interceptor above), then leaves.
|
// Back steps through the page's history first (the interceptor above), then leaves.
|
||||||
ScreenHeader("Web", Modifier.padding(horizontal = 16.dp, vertical = 8.dp))
|
ScreenHeader("Browser", Modifier.padding(horizontal = 16.dp, vertical = 8.dp))
|
||||||
AndroidView(
|
AndroidView(
|
||||||
modifier = Modifier.weight(1f).fillMaxSize(),
|
modifier = Modifier.weight(1f).fillMaxSize(),
|
||||||
factory = { context ->
|
factory = { context ->
|
||||||
|
|||||||
@@ -301,11 +301,14 @@ class CarAppConnection(private val context: Context) {
|
|||||||
/**
|
/**
|
||||||
* "navigate to X" deep link, the same one Android Auto's own Assistant integration sends -
|
* "navigate to X" deep link, the same one Android Auto's own Assistant integration sends -
|
||||||
* [ACTION_NAVIGATE] with the destination as a `geo:` search query, rather than anything
|
* [ACTION_NAVIGATE] with the destination as a `geo:` search query, rather than anything
|
||||||
* app-specific.
|
* app-specific. Centred on the phone when there's a fix: `geo:0,0` made Organic Maps pick the
|
||||||
|
* first Del Taco it found anywhere, not the nearest. The location goes only to the map app.
|
||||||
*/
|
*/
|
||||||
private fun navigateIntent(destination: String) =
|
private fun navigateIntent(destination: String): Intent {
|
||||||
Intent(CarContext.ACTION_NAVIGATE, Uri.parse("geo:0,0?q=" + Uri.encode(destination)))
|
val near = app.seb3thehacker.gearslip.notify.CarMotion.nearHere(context) ?: "0,0"
|
||||||
|
return Intent(CarContext.ACTION_NAVIGATE, Uri.parse("geo:$near?q=" + Uri.encode(destination)))
|
||||||
.setComponent(component)
|
.setComponent(component)
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Sends a fresh destination to an app that's already running, instead of tearing it down
|
* Sends a fresh destination to an app that's already running, instead of tearing it down
|
||||||
|
|||||||
@@ -156,6 +156,9 @@ class CarMedia(private val context: Context) {
|
|||||||
private val main = Handler(Looper.getMainLooper())
|
private val main = Handler(Looper.getMainLooper())
|
||||||
private var connecting: MediaApp? = null
|
private var connecting: MediaApp? = null
|
||||||
|
|
||||||
|
/** The music app Gearslip is attached to, for the assistant to name; null when there's none. */
|
||||||
|
val appLabel: String? get() = connecting?.label?.takeIf { controller != null }
|
||||||
|
|
||||||
private val _phase = MutableStateFlow(Phase.IDLE)
|
private val _phase = MutableStateFlow(Phase.IDLE)
|
||||||
val phase: StateFlow<Phase> = _phase.asStateFlow()
|
val phase: StateFlow<Phase> = _phase.asStateFlow()
|
||||||
|
|
||||||
@@ -236,6 +239,12 @@ class CarMedia(private val context: Context) {
|
|||||||
connecting = app
|
connecting = app
|
||||||
_phase.value = Phase.CONNECTING
|
_phase.value = Phase.CONNECTING
|
||||||
_browse.value = BrowseState()
|
_browse.value = BrowseState()
|
||||||
|
// Known to refuse its library (Spotify): asking only costs the seconds it takes to say no,
|
||||||
|
// so go straight to its playback session.
|
||||||
|
if (KnownApps.playerWorks(app.component.packageName)) {
|
||||||
|
attachToSession(app)
|
||||||
|
return
|
||||||
|
}
|
||||||
val mb = MediaBrowser(context, app.component, object : MediaBrowser.ConnectionCallback() {
|
val mb = MediaBrowser(context, app.component, object : MediaBrowser.ConnectionCallback() {
|
||||||
override fun onConnected() = this@CarMedia.onConnected(app)
|
override fun onConnected() = this@CarMedia.onConnected(app)
|
||||||
override fun onConnectionFailed() {
|
override fun onConnectionFailed() {
|
||||||
@@ -273,7 +282,12 @@ class CarMedia(private val context: Context) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if (session == null) {
|
if (session == null) {
|
||||||
val limit = if (launched) SESSION_RETRIES_LAUNCHED else SESSION_RETRIES
|
// A known session-only app that isn't running won't start by waiting: say so at once.
|
||||||
|
val limit = when {
|
||||||
|
launched -> SESSION_RETRIES_LAUNCHED
|
||||||
|
KnownApps.playerWorks(app.component.packageName) -> 0
|
||||||
|
else -> SESSION_RETRIES
|
||||||
|
}
|
||||||
// The app only publishes its session once its service has started; give it a moment.
|
// The app only publishes its session once its service has started; give it a moment.
|
||||||
if (attempt < limit) {
|
if (attempt < limit) {
|
||||||
main.postDelayed({ attachToSession(app, attempt + 1, launched) }, SESSION_RETRY_MS)
|
main.postDelayed({ attachToSession(app, attempt + 1, launched) }, SESSION_RETRY_MS)
|
||||||
@@ -454,7 +468,7 @@ class CarMedia(private val context: Context) {
|
|||||||
}
|
}
|
||||||
val component = searchComponent
|
val component = searchComponent
|
||||||
if (component == null) {
|
if (component == null) {
|
||||||
_results.value = Results(q, emptyList(), supported = false)
|
noLibrarySearch(q)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
resultsQuery = q
|
resultsQuery = q
|
||||||
@@ -476,13 +490,13 @@ class CarMedia(private val context: Context) {
|
|||||||
override fun onConnectionFailed() {
|
override fun onConnectionFailed() {
|
||||||
GearslipLog.w("media: no library connection for search")
|
GearslipLog.w("media: no library connection for search")
|
||||||
searchBrowser = null
|
searchBrowser = null
|
||||||
resultsQuery?.let { _results.value = Results(it, emptyList(), supported = false) }
|
resultsQuery?.let { noLibrarySearch(it) }
|
||||||
}
|
}
|
||||||
}, null)
|
}, null)
|
||||||
searchBrowser = created
|
searchBrowser = created
|
||||||
runCatching { created.connect() }.onFailure {
|
runCatching { created.connect() }.onFailure {
|
||||||
searchBrowser = null
|
searchBrowser = null
|
||||||
_results.value = Results(q, emptyList(), supported = false)
|
noLibrarySearch(q)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -492,18 +506,42 @@ class CarMedia(private val context: Context) {
|
|||||||
override fun onSearchResult(query: String, extras: Bundle?, items: List<MediaBrowserCompat.MediaItem>) {
|
override fun onSearchResult(query: String, extras: Bundle?, items: List<MediaBrowserCompat.MediaItem>) {
|
||||||
if (query != resultsQuery) return
|
if (query != resultsQuery) return
|
||||||
GearslipLog.i("media: library search found ${items.size}")
|
GearslipLog.i("media: library search found ${items.size}")
|
||||||
_results.value = Results(query, items.map(::entryOfCompat))
|
val entries = items.map(::entryOfCompat)
|
||||||
|
if (query == playFirstFor) {
|
||||||
|
playFirstFor = null
|
||||||
|
val first = entries.firstOrNull { it.playable } ?: entries.firstOrNull()
|
||||||
|
if (first == null) {
|
||||||
|
clearResults()
|
||||||
|
search(query)
|
||||||
|
} else {
|
||||||
|
GearslipLog.i("media: playing the library's best match")
|
||||||
|
clearResults()
|
||||||
|
searchBefore = _now.value
|
||||||
|
_search.value = Search(query)
|
||||||
|
main.removeCallbacks(searchTimeout)
|
||||||
|
controller?.transportControls?.playFromMediaId(first.id, null)
|
||||||
|
main.postDelayed(searchTimeout, SEARCH_WAIT_MS)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_results.value = Results(query, entries)
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun onError(query: String, extras: Bundle?) {
|
override fun onError(query: String, extras: Bundle?) {
|
||||||
if (query != resultsQuery) return
|
if (query != resultsQuery) return
|
||||||
|
if (query == playFirstFor) {
|
||||||
|
playFirstFor = null
|
||||||
|
clearResults()
|
||||||
|
search(query)
|
||||||
|
return
|
||||||
|
}
|
||||||
GearslipLog.i("media: the app has no library search")
|
GearslipLog.i("media: the app has no library search")
|
||||||
_results.value = Results(query, emptyList(), supported = false)
|
_results.value = Results(query, emptyList(), supported = false)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}.onFailure {
|
}.onFailure {
|
||||||
GearslipLog.w("media: library search failed: ${it.message}")
|
GearslipLog.w("media: library search failed: ${it.message}")
|
||||||
_results.value = Results(q, emptyList(), supported = false)
|
noLibrarySearch(q)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -520,6 +558,36 @@ class CarMedia(private val context: Context) {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The library can't search: a voice "play X" falls back to the app's own search; a typed one says so. */
|
||||||
|
private fun noLibrarySearch(q: String) {
|
||||||
|
if (q == playFirstFor) {
|
||||||
|
playFirstFor = null
|
||||||
|
clearResults()
|
||||||
|
search(q)
|
||||||
|
} else {
|
||||||
|
_results.value = Results(q, emptyList(), supported = false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A voice "play X" waiting on the library search, to play its first match instead of listing them. */
|
||||||
|
private var playFirstFor: String? = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Plays the best match for [query], for voice. Asks the app's library first and plays its top
|
||||||
|
* result, which apps like ViVi honour where they ignore [search]; without a library search,
|
||||||
|
* falls back to [search].
|
||||||
|
*/
|
||||||
|
fun playBestMatch(query: String) {
|
||||||
|
val q = query.trim()
|
||||||
|
if (q.isEmpty()) return
|
||||||
|
if (!canSearchLibrary) {
|
||||||
|
search(q)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
playFirstFor = q
|
||||||
|
searchLibrary(q)
|
||||||
|
}
|
||||||
|
|
||||||
/** The connected app's service, for the search connection; null when nothing is connected. */
|
/** The connected app's service, for the search connection; null when nothing is connected. */
|
||||||
private var searchComponent: android.content.ComponentName? = null
|
private var searchComponent: android.content.ComponentName? = null
|
||||||
private var searchBrowser: MediaBrowserCompat? = null
|
private var searchBrowser: MediaBrowserCompat? = null
|
||||||
|
|||||||
@@ -22,6 +22,33 @@ object CarMotion {
|
|||||||
private var listener: LocationListener? = null
|
private var listener: LocationListener? = null
|
||||||
private var speed = 0f
|
private var speed = 0f
|
||||||
private var fixAt = 0L
|
private var fixAt = 0L
|
||||||
|
@Volatile private var last: Location? = null
|
||||||
|
|
||||||
|
/** A fix older than this is somewhere the car has left. */
|
||||||
|
private const val NEAR_STALE_MS = 10 * 60_000L
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where the phone is, as "lat,lng", for "navigate to Del Taco" to find the nearest one. From
|
||||||
|
* this drive's GPS, or Android's last known fix; null with no permission or nothing recent.
|
||||||
|
*/
|
||||||
|
fun nearHere(context: Context): String? {
|
||||||
|
val fix = last?.takeIf { SystemClock.elapsedRealtime() - fixAt < NEAR_STALE_MS }
|
||||||
|
?: lastKnown(context)
|
||||||
|
?: return null
|
||||||
|
return String.format(java.util.Locale.US, "%.5f,%.5f", fix.latitude, fix.longitude)
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun lastKnown(context: Context): Location? {
|
||||||
|
if (context.checkSelfPermission(Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED &&
|
||||||
|
context.checkSelfPermission(Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED
|
||||||
|
) return null
|
||||||
|
val manager = context.getSystemService(LocationManager::class.java) ?: return null
|
||||||
|
val now = System.currentTimeMillis()
|
||||||
|
return manager.getProviders(true)
|
||||||
|
.mapNotNull { runCatching { manager.getLastKnownLocation(it) }.getOrNull() }
|
||||||
|
.filter { now - it.time < NEAR_STALE_MS }
|
||||||
|
.maxByOrNull { it.time }
|
||||||
|
}
|
||||||
|
|
||||||
val moving: Boolean
|
val moving: Boolean
|
||||||
get() = SystemClock.elapsedRealtime() - fixAt < STALE_MS && speed > MOVING_MS
|
get() = SystemClock.elapsedRealtime() - fixAt < STALE_MS && speed > MOVING_MS
|
||||||
@@ -34,6 +61,7 @@ object CarMotion {
|
|||||||
override fun onLocationChanged(location: Location) {
|
override fun onLocationChanged(location: Location) {
|
||||||
speed = if (location.hasSpeed()) location.speed else 0f
|
speed = if (location.hasSpeed()) location.speed else 0f
|
||||||
fixAt = SystemClock.elapsedRealtime()
|
fixAt = SystemClock.elapsedRealtime()
|
||||||
|
last = location
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
listener = l
|
listener = l
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ import android.Manifest
|
|||||||
import android.annotation.SuppressLint
|
import android.annotation.SuppressLint
|
||||||
import android.content.Context
|
import android.content.Context
|
||||||
import android.content.pm.PackageManager
|
import android.content.pm.PackageManager
|
||||||
import android.content.res.AssetManager
|
|
||||||
import android.media.AudioFormat
|
import android.media.AudioFormat
|
||||||
import android.media.AudioManager
|
import android.media.AudioManager
|
||||||
import android.media.AudioRecord
|
import android.media.AudioRecord
|
||||||
@@ -15,38 +14,50 @@ import android.os.Looper
|
|||||||
import android.os.SystemClock
|
import android.os.SystemClock
|
||||||
import app.seb3thehacker.gearslip.GearslipLog
|
import app.seb3thehacker.gearslip.GearslipLog
|
||||||
import app.seb3thehacker.gearslip.audio.CarMic
|
import app.seb3thehacker.gearslip.audio.CarMic
|
||||||
import java.util.Locale
|
import com.k2fsa.sherpa.onnx.EndpointConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.EndpointRule
|
||||||
|
import com.k2fsa.sherpa.onnx.FeatureConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OfflineCanaryModelConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OfflineModelConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OfflineRecognizer
|
||||||
|
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineModelConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineRecognizer
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig
|
||||||
|
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig
|
||||||
|
import java.io.File
|
||||||
import java.util.concurrent.Executors
|
import java.util.concurrent.Executors
|
||||||
import kotlin.math.log10
|
|
||||||
import kotlin.math.max
|
import kotlin.math.max
|
||||||
import kotlin.math.sqrt
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships
|
* Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships
|
||||||
* none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply
|
* none). sherpa-onnx runs a streaming model ([SpeechModels]) on the phone: words come out while
|
||||||
* of a few seconds comes back in well under a second on a recent phone.
|
* the driver is still talking, and the model itself decides when they've finished. A larger model
|
||||||
|
* then reads the whole phrase once more for the final words, since a model that waits for the end
|
||||||
|
* hears names and places far better than one that has to keep up. The phrase is held in memory
|
||||||
|
* for that and nowhere else, and dropped as soon as it's read.
|
||||||
*
|
*
|
||||||
* Whisper needs the whole phrase before it starts, so this records from the car's microphone (the
|
* Audio comes from the car's microphone, or the phone's when the car has none or sends nothing.
|
||||||
* phone's when the car has none or sends nothing),
|
* Everything runs on one worker thread; callbacks arrive on the main thread.
|
||||||
* decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes
|
|
||||||
* once. Everything runs on one worker thread; callbacks arrive on the main thread.
|
|
||||||
*/
|
*/
|
||||||
object SpeechEngine {
|
object SpeechEngine {
|
||||||
|
|
||||||
/** FUTO's ACFT Whisper tiny (Apache-2.0), re-quantized to 5 bits, inside the APK. */
|
|
||||||
private const val MODEL = "speech/tiny_acft_q5_1.bin"
|
|
||||||
|
|
||||||
/** The error for an engine that can't run at all, which callers may answer with a fallback. */
|
/** The error for an engine that can't run at all, which callers may answer with a fallback. */
|
||||||
const val NOT_LOADED = "The speech model didn't load."
|
const val NOT_LOADED = "The speech model didn't load."
|
||||||
|
|
||||||
|
/** Said when the chosen model hasn't been downloaded yet. */
|
||||||
|
const val NOT_DOWNLOADED = "Download a voice model in Gearslip's settings on the phone first."
|
||||||
|
|
||||||
private const val RATE = 16_000
|
private const val RATE = 16_000
|
||||||
private const val FRAME = 480 // 30 ms
|
private const val CHUNK = 1_600 // 100 ms
|
||||||
private const val MAX_MS = 15_000L
|
private const val MAX_S = 15f
|
||||||
private const val NO_SPEECH_MS = 6_000L
|
/** Quiet before the first word that counts as "nobody spoke". */
|
||||||
private const val END_SILENCE_MS = 1_100L
|
private const val NO_SPEECH_S = 6f
|
||||||
private const val CALIBRATE_MS = 300L
|
/** Quiet after the last word that ends the phrase. */
|
||||||
private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word
|
private const val END_SILENCE_S = 1.0f
|
||||||
private const val CUE_VOLUME = 80
|
private const val CUE_VOLUME = 80
|
||||||
|
private const val IDLE_UNLOAD_MS = 5 * 60_000L
|
||||||
/** How long the car gets to start sending once asked, before the phone's microphone takes over. */
|
/** How long the car gets to start sending once asked, before the phone's microphone takes over. */
|
||||||
private const val CAR_FIRST_AUDIO_MS = 1_500L
|
private const val CAR_FIRST_AUDIO_MS = 1_500L
|
||||||
private const val CAR_GAP_MS = 1_000L
|
private const val CAR_GAP_MS = 1_000L
|
||||||
@@ -55,27 +66,79 @@ object SpeechEngine {
|
|||||||
private const val CUE_TAIL_MS = 250L
|
private const val CUE_TAIL_MS = 250L
|
||||||
|
|
||||||
private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") }
|
private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") }
|
||||||
|
/**
|
||||||
|
* Parakeet large takes seconds to load. On [worker], the first press after connecting
|
||||||
|
* waited behind it before listening at all; here it loads while the driver talks.
|
||||||
|
*/
|
||||||
|
private val finalWorker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-final") }
|
||||||
|
private val finalLock = Object()
|
||||||
private val main = Handler(Looper.getMainLooper())
|
private val main = Handler(Looper.getMainLooper())
|
||||||
|
|
||||||
private var libraryLoaded = false
|
private var recognizer: OnlineRecognizer? = null
|
||||||
private var handle = 0L
|
private var loadedModel: SpeechModels.Model? = null
|
||||||
|
@Volatile private var finalRecognizer: OfflineRecognizer? = null
|
||||||
|
@Volatile private var loadedFinal: SpeechModels.Model? = null
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parakeet large holds most of the ~900 MB the two models take, so it goes after a few
|
||||||
|
* minutes without a command. Kroko stays: it's small, and keeps listening instant.
|
||||||
|
*/
|
||||||
|
private val idleUnload = Runnable { unloadFinal("after ${IDLE_UNLOAD_MS / 60_000} minutes without use") }
|
||||||
@Volatile private var run = 0
|
@Volatile private var run = 0
|
||||||
|
|
||||||
/** Loads the model ahead of time, so the first reply doesn't wait for it. */
|
/** Loads the model ahead of time, so the first command doesn't wait for it. */
|
||||||
fun warm(context: Context) {
|
fun warm(context: Context) {
|
||||||
val app = context.applicationContext
|
val app = context.applicationContext
|
||||||
worker.execute { ensureLoaded(app) }
|
SpeechModels.init(app)
|
||||||
|
// Kroko first, alone: loading both at once slowed it from a fraction of a second to six.
|
||||||
|
worker.execute {
|
||||||
|
ensureLoaded(app)
|
||||||
|
finalWorker.execute { ensureFinalLoaded(app) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun unloadFinal(why: String) {
|
||||||
|
finalWorker.execute {
|
||||||
|
synchronized(finalLock) {
|
||||||
|
val model = loadedFinal ?: return@execute
|
||||||
|
finalRecognizer?.release()
|
||||||
|
finalRecognizer = null
|
||||||
|
loadedFinal = null
|
||||||
|
GearslipLog.i("speech: unloaded ${model.label} $why")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drops the loaded models: the driver picked another or deleted one, or the car went away. */
|
||||||
|
fun unload() {
|
||||||
|
main.removeCallbacks(idleUnload)
|
||||||
|
run++
|
||||||
|
worker.execute {
|
||||||
|
recognizer?.release()
|
||||||
|
recognizer = null
|
||||||
|
loadedModel = null
|
||||||
|
}
|
||||||
|
finalWorker.execute {
|
||||||
|
synchronized(finalLock) {
|
||||||
|
finalRecognizer?.release()
|
||||||
|
finalRecognizer = null
|
||||||
|
loadedFinal = null
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk.
|
* Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk.
|
||||||
* [onSpeaking] fires at the first word, [onHeard] when the driver stops and the words are being
|
* [onPartial] gets the words so far as they're recognized, [onSpeaking] fires at the first
|
||||||
* worked out; [onResult] gets the text, [onError] a sentence fit to show on the car screen.
|
* word, [onHeard] when the driver stops; [onResult] gets the text, [onError] a sentence fit to
|
||||||
|
* show on the car screen.
|
||||||
*/
|
*/
|
||||||
fun listen(
|
fun listen(
|
||||||
context: Context,
|
context: Context,
|
||||||
cue: Boolean = true,
|
cue: Boolean = true,
|
||||||
onSpeaking: () -> Unit = {},
|
onSpeaking: () -> Unit = {},
|
||||||
|
onPartial: (String) -> Unit = {},
|
||||||
|
onListening: () -> Unit = {},
|
||||||
onHeard: () -> Unit,
|
onHeard: () -> Unit,
|
||||||
onResult: (String) -> Unit,
|
onResult: (String) -> Unit,
|
||||||
onError: (String) -> Unit,
|
onError: (String) -> Unit,
|
||||||
@@ -87,28 +150,29 @@ object SpeechEngine {
|
|||||||
}
|
}
|
||||||
val id = ++run
|
val id = ++run
|
||||||
fun post(block: () -> Unit) { main.post { if (id == run) block() } }
|
fun post(block: () -> Unit) { main.post { if (id == run) block() } }
|
||||||
|
// Brings Parakeet back if it was unloaded, while the driver talks.
|
||||||
|
main.removeCallbacks(idleUnload)
|
||||||
|
finalWorker.execute { ensureFinalLoaded(app) }
|
||||||
worker.execute {
|
worker.execute {
|
||||||
if (id != run) return@execute
|
if (id != run) return@execute
|
||||||
if (!ensureLoaded(app)) return@execute post { onError(NOT_LOADED) }
|
if (!SpeechModels.isReady(app, SpeechModels.selected.value)) return@execute post { onError(NOT_DOWNLOADED) }
|
||||||
|
val r = ensureLoaded(app) ?: return@execute post { onError(NOT_LOADED) }
|
||||||
VoiceFocus.acquire(app)
|
VoiceFocus.acquire(app)
|
||||||
val audio = try {
|
val text = try {
|
||||||
if (cue) cue()
|
if (cue) cue()
|
||||||
if (id != run) return@execute
|
if (id != run) return@execute
|
||||||
record(id) { post(onSpeaking) }
|
post(onListening)
|
||||||
|
stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) })
|
||||||
} catch (t: Throwable) {
|
} catch (t: Throwable) {
|
||||||
GearslipLog.e("speech: recording failed", t)
|
GearslipLog.e("speech: listening failed", t)
|
||||||
return@execute post { onError("The microphone isn't available right now.") }
|
return@execute post { onError("The microphone isn't available right now.") }
|
||||||
} finally {
|
} finally {
|
||||||
VoiceFocus.release()
|
VoiceFocus.release()
|
||||||
}
|
}
|
||||||
|
main.removeCallbacks(idleUnload)
|
||||||
|
main.postDelayed(idleUnload, IDLE_UNLOAD_MS)
|
||||||
if (id != run) return@execute
|
if (id != run) return@execute
|
||||||
if (audio == null) return@execute post { onError("Didn't catch that.") }
|
|
||||||
post(onHeard)
|
post(onHeard)
|
||||||
val started = SystemClock.uptimeMillis()
|
|
||||||
val text = clean(transcribe(audio))
|
|
||||||
GearslipLog.i(
|
|
||||||
"speech: ${audio.size * 1000L / RATE} ms of audio took ${SystemClock.uptimeMillis() - started} ms",
|
|
||||||
)
|
|
||||||
post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) }
|
post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -116,6 +180,185 @@ object SpeechEngine {
|
|||||||
/** Stops listening; a phrase already being worked out is dropped. */
|
/** Stops listening; a phrase already being worked out is dropped. */
|
||||||
fun cancel() { run++ }
|
fun cancel() { run++ }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Feeds the microphone to the recognizer in 100 ms pieces, reporting the words as they firm
|
||||||
|
* up, until the model hears the end of the phrase (or nobody speaks, or it runs too long).
|
||||||
|
*/
|
||||||
|
private fun stream(
|
||||||
|
context: Context,
|
||||||
|
r: OnlineRecognizer,
|
||||||
|
id: Int,
|
||||||
|
onSpeaking: () -> Unit,
|
||||||
|
onPartial: (String) -> Unit,
|
||||||
|
onStopped: () -> Unit,
|
||||||
|
): String {
|
||||||
|
val mic = carMic() ?: phoneMic()
|
||||||
|
val s = r.createStream()
|
||||||
|
val frame = ShortArray(CHUNK)
|
||||||
|
val samples = FloatArray(CHUNK)
|
||||||
|
// The whole phrase, for the final read; [MAX_S] and a little over is the most it can hold.
|
||||||
|
val phrase = FloatArray((RATE * (MAX_S + 1)).toInt())
|
||||||
|
var held = 0
|
||||||
|
var last = ""
|
||||||
|
var spoke = false
|
||||||
|
var audioMs = 0L
|
||||||
|
var decodeMs = 0L
|
||||||
|
try {
|
||||||
|
while (id == run) {
|
||||||
|
val n = mic.read(frame)
|
||||||
|
if (n < 0) break
|
||||||
|
if (n == 0) continue
|
||||||
|
for (i in 0 until n) samples[i] = frame[i] / 32768f
|
||||||
|
val keep = minOf(n, phrase.size - held)
|
||||||
|
System.arraycopy(samples, 0, phrase, held, keep)
|
||||||
|
held += keep
|
||||||
|
s.acceptWaveform(if (n == CHUNK) samples else samples.copyOf(n), RATE)
|
||||||
|
audioMs += n * 1000L / RATE
|
||||||
|
val started = SystemClock.uptimeMillis()
|
||||||
|
while (r.isReady(s)) r.decode(s)
|
||||||
|
decodeMs += SystemClock.uptimeMillis() - started
|
||||||
|
val text = clean(r.getResult(s).text)
|
||||||
|
if (text != last) {
|
||||||
|
last = text
|
||||||
|
if (text.isNotEmpty() && !spoke) {
|
||||||
|
spoke = true
|
||||||
|
onSpeaking()
|
||||||
|
}
|
||||||
|
onPartial(text)
|
||||||
|
}
|
||||||
|
if (r.isEndpoint(s)) break
|
||||||
|
}
|
||||||
|
s.inputFinished()
|
||||||
|
while (r.isReady(s)) r.decode(s)
|
||||||
|
last = clean(r.getResult(s).text)
|
||||||
|
} finally {
|
||||||
|
mic.close()
|
||||||
|
s.release()
|
||||||
|
}
|
||||||
|
// Under 1.0 keeps up with the driver; over it, the words fall behind the voice.
|
||||||
|
GearslipLog.i(
|
||||||
|
"speech: $audioMs ms of audio, decoding took $decodeMs ms " +
|
||||||
|
"(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)",
|
||||||
|
)
|
||||||
|
if (last.isEmpty() || id != run) { phrase.fill(0f); return last }
|
||||||
|
// Still loading (just connected, or back after going idle): the live words now beat
|
||||||
|
// Parakeet's in several seconds.
|
||||||
|
val wanted = SpeechModels.final.value
|
||||||
|
if (wanted == null || finalRecognizer == null || loadedFinal != wanted) {
|
||||||
|
if (wanted != null) GearslipLog.i("speech: ${wanted.label} isn't loaded yet, keeping the live words")
|
||||||
|
phrase.fill(0f)
|
||||||
|
return last
|
||||||
|
}
|
||||||
|
onStopped()
|
||||||
|
val started = SystemClock.uptimeMillis()
|
||||||
|
// Held for the whole read, so a model swap or the idle unload can't free it mid-decode.
|
||||||
|
val reread = synchronized(finalLock) {
|
||||||
|
val final = finalRecognizer?.takeIf { loadedFinal == wanted } ?: run { phrase.fill(0f); return last }
|
||||||
|
final.createStream().run {
|
||||||
|
try {
|
||||||
|
acceptWaveform(phrase.copyOf(held), RATE)
|
||||||
|
final.decode(this)
|
||||||
|
clean(final.getResult(this).text)
|
||||||
|
} finally {
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
phrase.fill(0f)
|
||||||
|
GearslipLog.i("speech: the final read took ${SystemClock.uptimeMillis() - started} ms")
|
||||||
|
if (reread.isEmpty()) return last
|
||||||
|
if (reread != last) onPartial(reread)
|
||||||
|
return reread
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */
|
||||||
|
private fun ensureFinalLoaded(context: Context): OfflineRecognizer? = synchronized(finalLock) {
|
||||||
|
val model = SpeechModels.final.value ?: return null
|
||||||
|
if (finalRecognizer != null && loadedFinal == model) return finalRecognizer
|
||||||
|
finalRecognizer?.release()
|
||||||
|
finalRecognizer = null
|
||||||
|
if (!SpeechModels.isReady(context, model)) return null
|
||||||
|
val dir = SpeechModels.dir(context, model)
|
||||||
|
fun path(name: String) = File(dir, name).absolutePath
|
||||||
|
val started = SystemClock.uptimeMillis()
|
||||||
|
return runCatching {
|
||||||
|
OfflineRecognizer(
|
||||||
|
config = OfflineRecognizerConfig(
|
||||||
|
featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80),
|
||||||
|
modelConfig = if (model.modelType == "canary") {
|
||||||
|
// English in, English out, and no punctuation: clean() strips it anyway.
|
||||||
|
OfflineModelConfig(
|
||||||
|
canary = OfflineCanaryModelConfig(path(model.encoder), path(model.decoder), srcLang = "en", tgtLang = "en", usePnc = false),
|
||||||
|
tokens = path("tokens.txt"),
|
||||||
|
numThreads = threads(),
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
OfflineModelConfig(
|
||||||
|
transducer = OfflineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)),
|
||||||
|
tokens = path("tokens.txt"),
|
||||||
|
numThreads = threads(),
|
||||||
|
modelType = model.modelType,
|
||||||
|
)
|
||||||
|
},
|
||||||
|
decodingMethod = "greedy_search",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}.onSuccess {
|
||||||
|
finalRecognizer = it
|
||||||
|
loadedFinal = model
|
||||||
|
GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms")
|
||||||
|
}.onFailure {
|
||||||
|
GearslipLog.e("speech: couldn't load ${model.label}", it)
|
||||||
|
}.getOrNull()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun ensureLoaded(context: Context): OnlineRecognizer? {
|
||||||
|
val model = SpeechModels.selected.value
|
||||||
|
if (recognizer != null && loadedModel == model) return recognizer
|
||||||
|
recognizer?.release()
|
||||||
|
recognizer = null
|
||||||
|
if (!SpeechModels.isReady(context, model)) return null
|
||||||
|
val dir = SpeechModels.dir(context, model)
|
||||||
|
fun path(name: String) = File(dir, name).absolutePath
|
||||||
|
val started = SystemClock.uptimeMillis()
|
||||||
|
return runCatching {
|
||||||
|
OnlineRecognizer(
|
||||||
|
config = OnlineRecognizerConfig(
|
||||||
|
featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80),
|
||||||
|
modelConfig = OnlineModelConfig(
|
||||||
|
transducer = OnlineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)),
|
||||||
|
tokens = path("tokens.txt"),
|
||||||
|
numThreads = threads(),
|
||||||
|
modelType = model.modelType,
|
||||||
|
),
|
||||||
|
endpointConfig = EndpointConfig(
|
||||||
|
rule1 = EndpointRule(false, NO_SPEECH_S, 0f),
|
||||||
|
rule2 = EndpointRule(true, END_SILENCE_S, 0f),
|
||||||
|
rule3 = EndpointRule(false, 0f, MAX_S),
|
||||||
|
),
|
||||||
|
enableEndpoint = true,
|
||||||
|
decodingMethod = "greedy_search",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
}.onSuccess {
|
||||||
|
recognizer = it
|
||||||
|
loadedModel = model
|
||||||
|
GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms")
|
||||||
|
}.onFailure {
|
||||||
|
GearslipLog.e("speech: couldn't load ${model.label}", it)
|
||||||
|
}.getOrNull()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The big cores only: more threads than that slow the model down on a phone. */
|
||||||
|
private fun threads() = Runtime.getRuntime().availableProcessors().let { if (it >= 8) 4 else max(1, it / 2) }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Models write in upper case, or with punctuation and capitals of their own; commands are
|
||||||
|
* matched in plain lower case, so only words and apostrophes are kept.
|
||||||
|
*/
|
||||||
|
private fun clean(text: String): String =
|
||||||
|
text.lowercase().replace(Regex("[^\\p{L}\\p{N}' ]"), " ").replace(Regex("\\s+"), " ").trim()
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A short beep on the media stream, so it plays through the car like everything else, then a
|
* A short beep on the media stream, so it plays through the car like everything else, then a
|
||||||
* pause long enough for it to finish there before the microphone opens and hears it.
|
* pause long enough for it to finish there before the microphone opens and hears it.
|
||||||
@@ -130,14 +373,9 @@ object SpeechEngine {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- recording --------------------------------------------------------------------------
|
// --- microphones -----------------------------------------------------------------------
|
||||||
|
|
||||||
/**
|
/** Where a phrase is heard from: [read] fills [frame] and returns how many samples it got, -1 at the end. */
|
||||||
* Records until the driver stops talking. Returns the phrase from just before the first word,
|
|
||||||
* or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
|
|
||||||
* loud cabin needs a louder voice than a quiet one to count as talking.
|
|
||||||
*/
|
|
||||||
/** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */
|
|
||||||
private interface Mic {
|
private interface Mic {
|
||||||
fun read(frame: ShortArray): Int
|
fun read(frame: ShortArray): Int
|
||||||
fun close()
|
fun close()
|
||||||
@@ -148,10 +386,11 @@ object SpeechEngine {
|
|||||||
val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
|
val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
|
||||||
val recorder = AudioRecord(
|
val recorder = AudioRecord(
|
||||||
MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE,
|
MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE,
|
||||||
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8),
|
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, CHUNK * 4),
|
||||||
)
|
)
|
||||||
check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" }
|
check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" }
|
||||||
recorder.startRecording()
|
recorder.startRecording()
|
||||||
|
GearslipLog.i("speech: listening through the phone's microphone")
|
||||||
return object : Mic {
|
return object : Mic {
|
||||||
override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size)
|
override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size)
|
||||||
override fun close() {
|
override fun close() {
|
||||||
@@ -163,7 +402,7 @@ object SpeechEngine {
|
|||||||
|
|
||||||
/**
|
/**
|
||||||
* The car's microphone, which hears the driver better than a phone in a cup holder. The car
|
* The car's microphone, which hears the driver better than a phone in a cup holder. The car
|
||||||
* sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car
|
* sends audio in its own sized pieces; they're cut to [CHUNK]s here. Null when there's no car
|
||||||
* microphone, or it sends nothing at first, so the phone's is used instead.
|
* microphone, or it sends nothing at first, so the phone's is used instead.
|
||||||
*/
|
*/
|
||||||
private fun carMic(): Mic? {
|
private fun carMic(): Mic? {
|
||||||
@@ -190,105 +429,10 @@ object SpeechEngine {
|
|||||||
at += n
|
at += n
|
||||||
if (at == chunk.size) { pending = null; at = 0 }
|
if (at == chunk.size) { pending = null; at = 0 }
|
||||||
}
|
}
|
||||||
// The car stopped sending mid-phrase: end the recording rather than wait forever.
|
// The car stopped sending mid-phrase: end the phrase rather than wait forever.
|
||||||
return if (filled == 0) -1 else filled
|
return if (filled == 0) -1 else filled
|
||||||
}
|
}
|
||||||
override fun close() = car.stop()
|
override fun close() = car.stop()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Records until the driver stops talking. Returns the phrase from just before the first word,
|
|
||||||
* or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
|
|
||||||
* loud cabin needs a louder voice than a quiet one to count as talking.
|
|
||||||
*/
|
|
||||||
private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? {
|
|
||||||
val mic = carMic() ?: phoneMic()
|
|
||||||
val samples = FloatArray((RATE * MAX_MS / 1000).toInt())
|
|
||||||
var count = 0
|
|
||||||
val frame = ShortArray(FRAME)
|
|
||||||
var noise = 0.0
|
|
||||||
var noiseFrames = 0
|
|
||||||
var speechStart = -1
|
|
||||||
var lastSpeech = 0
|
|
||||||
try {
|
|
||||||
while (id == run && count + FRAME <= samples.size) {
|
|
||||||
val n = mic.read(frame)
|
|
||||||
if (n < 0) break
|
|
||||||
if (n == 0) continue
|
|
||||||
var sum = 0.0
|
|
||||||
for (i in 0 until n) {
|
|
||||||
val v = frame[i] / 32768f
|
|
||||||
samples[count + i] = v
|
|
||||||
sum += v * v
|
|
||||||
}
|
|
||||||
count += n
|
|
||||||
val db = 20 * log10(sqrt(sum / n) + 1e-9)
|
|
||||||
val ms = count * 1000L / RATE
|
|
||||||
if (ms <= CALIBRATE_MS) {
|
|
||||||
noise += db; noiseFrames++
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
val floor = if (noiseFrames > 0) noise / noiseFrames else -60.0
|
|
||||||
val talking = db > max(floor + 12, -45.0)
|
|
||||||
if (talking) {
|
|
||||||
if (speechStart < 0) {
|
|
||||||
speechStart = max(0, count - n - PRE_ROLL)
|
|
||||||
onSpeaking()
|
|
||||||
}
|
|
||||||
lastSpeech = count
|
|
||||||
}
|
|
||||||
if (speechStart < 0 && ms > NO_SPEECH_MS) return null
|
|
||||||
if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
mic.close()
|
|
||||||
}
|
|
||||||
if (speechStart < 0) return null
|
|
||||||
return samples.copyOfRange(speechStart, count)
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- the model --------------------------------------------------------------------------
|
|
||||||
|
|
||||||
private fun ensureLoaded(context: Context): Boolean {
|
|
||||||
if (handle != 0L) return true
|
|
||||||
if (!libraryLoaded) {
|
|
||||||
runCatching { System.loadLibrary("gearslip_speech") }
|
|
||||||
.onFailure { GearslipLog.e("speech: native library missing", it); return false }
|
|
||||||
libraryLoaded = true
|
|
||||||
}
|
|
||||||
val started = SystemClock.uptimeMillis()
|
|
||||||
handle = nativeInitAsset(context.assets, MODEL)
|
|
||||||
if (handle == 0L) {
|
|
||||||
GearslipLog.w("speech: couldn't load the model")
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
GearslipLog.i("speech: loaded the model in ${SystemClock.uptimeMillis() - started} ms")
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun transcribe(audio: FloatArray): String {
|
|
||||||
val bytes = nativeTranscribe(handle, audio, language(), threads())
|
|
||||||
return String(bytes, Charsets.UTF_8)
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The phone's language, as Whisper names it. Older Java codes differ for a few. */
|
|
||||||
private fun language(): String = when (val lang = Locale.getDefault().language) {
|
|
||||||
"iw" -> "he"
|
|
||||||
"in" -> "id"
|
|
||||||
"ji" -> "yi"
|
|
||||||
"" -> "en"
|
|
||||||
else -> lang
|
|
||||||
}
|
|
||||||
|
|
||||||
/** The big cores only: more threads than that slow Whisper down on a phone. */
|
|
||||||
private fun threads() = Runtime.getRuntime().availableProcessors().coerceIn(1, 8).let { if (it >= 8) 4 else max(1, it / 2) }
|
|
||||||
|
|
||||||
/** Whisper marks silence and noise as "[BLANK_AUDIO]" or "(wind blowing)"; none of it is words. */
|
|
||||||
private fun clean(text: String): String =
|
|
||||||
text.replace(Regex("""\[[^\]]*\]|\([^)]*\)|\*[^*]*\*"""), " ").replace(Regex("\\s+"), " ").trim()
|
|
||||||
|
|
||||||
private external fun nativeInitAsset(assets: AssetManager, path: String): Long
|
|
||||||
private external fun nativeTranscribe(handle: Long, samples: FloatArray, language: String, threads: Int): ByteArray
|
|
||||||
private external fun nativeFree(handle: Long)
|
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,193 @@
|
|||||||
|
package app.seb3thehacker.gearslip.speech
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import app.seb3thehacker.gearslip.GearslipLog
|
||||||
|
import java.io.File
|
||||||
|
import java.net.HttpURLConnection
|
||||||
|
import java.net.URL
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import kotlinx.coroutines.flow.MutableStateFlow
|
||||||
|
import kotlinx.coroutines.flow.StateFlow
|
||||||
|
import kotlinx.coroutines.flow.asStateFlow
|
||||||
|
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
|
||||||
|
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The speech models Gearslip listens with, in two roles: a streaming one that writes the words
|
||||||
|
* while the driver talks, and an optional final one that reads the whole phrase again once they
|
||||||
|
* stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's
|
||||||
|
* GitHub releases, where its authors publish it, and kept in the app's own storage.
|
||||||
|
*/
|
||||||
|
object SpeechModels {
|
||||||
|
|
||||||
|
enum class Role { LIVE, FINAL }
|
||||||
|
|
||||||
|
enum class Model(
|
||||||
|
val role: Role,
|
||||||
|
val id: String,
|
||||||
|
val label: String,
|
||||||
|
val detail: String,
|
||||||
|
val downloadMb: Int,
|
||||||
|
private val archive: String,
|
||||||
|
val encoder: String,
|
||||||
|
val decoder: String,
|
||||||
|
/** Empty for Canary, which decodes with attention and has no joiner. */
|
||||||
|
val joiner: String,
|
||||||
|
val modelType: String = "",
|
||||||
|
) {
|
||||||
|
KROKO(
|
||||||
|
Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54,
|
||||||
|
"sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06",
|
||||||
|
"encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2",
|
||||||
|
),
|
||||||
|
PARAKEET_SMALL(
|
||||||
|
Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103,
|
||||||
|
"sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8",
|
||||||
|
"encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
|
||||||
|
),
|
||||||
|
CANARY(
|
||||||
|
Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 154,
|
||||||
|
"sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8",
|
||||||
|
"encoder.int8.onnx", "decoder.int8.onnx", "", modelType = "canary",
|
||||||
|
),
|
||||||
|
PARAKEET_LARGE(
|
||||||
|
Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460,
|
||||||
|
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
|
||||||
|
"encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
|
||||||
|
),
|
||||||
|
;
|
||||||
|
|
||||||
|
val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2"
|
||||||
|
val files get() = listOf(encoder, decoder, joiner, "tokens.txt").filter { it.isNotEmpty() }
|
||||||
|
}
|
||||||
|
|
||||||
|
sealed interface State {
|
||||||
|
data object Missing : State
|
||||||
|
data class Downloading(val percent: Int) : State
|
||||||
|
data object Ready : State
|
||||||
|
data class Failed(val reason: String) : State
|
||||||
|
}
|
||||||
|
|
||||||
|
private const val PREFS = "speech"
|
||||||
|
private const val KEY_MODEL = "model"
|
||||||
|
private const val KEY_FINAL = "final_model"
|
||||||
|
|
||||||
|
private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-download") }
|
||||||
|
private val states = Model.entries.associateWith { MutableStateFlow<State>(State.Missing) }
|
||||||
|
private val _selected = MutableStateFlow(Model.KROKO)
|
||||||
|
val selected: StateFlow<Model> = _selected.asStateFlow()
|
||||||
|
private val _final = MutableStateFlow<Model?>(Model.PARAKEET_LARGE)
|
||||||
|
|
||||||
|
/** The model that rereads the whole phrase at the end; null to keep the live model's words. */
|
||||||
|
val final: StateFlow<Model?> = _final.asStateFlow()
|
||||||
|
@Volatile private var cancelled: Model? = null
|
||||||
|
|
||||||
|
fun init(context: Context) {
|
||||||
|
val id = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_MODEL, null)
|
||||||
|
_selected.value = Model.entries.firstOrNull { it.id == id && it.role == Role.LIVE } ?: Model.KROKO
|
||||||
|
val finalId = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_FINAL, Model.PARAKEET_LARGE.id)
|
||||||
|
_final.value = Model.entries.firstOrNull { it.id == finalId && it.role == Role.FINAL }
|
||||||
|
Model.entries.forEach { m -> if (isReady(context, m)) states.getValue(m).value = State.Ready }
|
||||||
|
// Models an older build offered and this one doesn't: hundreds of MB nobody can use.
|
||||||
|
val known = Model.entries.map { it.id }.toSet()
|
||||||
|
File(context.filesDir, "speech").listFiles()?.filter { it.isDirectory && it.name !in known }?.forEach {
|
||||||
|
GearslipLog.i("speech: removing ${it.name}, no longer offered")
|
||||||
|
it.deleteRecursively()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun state(model: Model): StateFlow<State> = states.getValue(model).asStateFlow()
|
||||||
|
|
||||||
|
fun dir(context: Context, model: Model) = File(File(context.filesDir, "speech"), model.id)
|
||||||
|
|
||||||
|
fun isReady(context: Context, model: Model) = model.files.all { File(dir(context, model), it).isFile }
|
||||||
|
|
||||||
|
fun select(context: Context, model: Model) {
|
||||||
|
if (model.role == Role.FINAL) return selectFinal(context, model)
|
||||||
|
_selected.value = model
|
||||||
|
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_MODEL, model.id).apply()
|
||||||
|
SpeechEngine.unload()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** [model] null keeps the live words as they are. */
|
||||||
|
fun selectFinal(context: Context, model: Model?) {
|
||||||
|
_final.value = model
|
||||||
|
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_FINAL, model?.id ?: "none").apply()
|
||||||
|
SpeechEngine.unload()
|
||||||
|
}
|
||||||
|
|
||||||
|
fun delete(context: Context, model: Model) {
|
||||||
|
cancelled = model
|
||||||
|
dir(context, model).deleteRecursively()
|
||||||
|
states.getValue(model).value = State.Missing
|
||||||
|
if (model == _selected.value || model == _final.value) SpeechEngine.unload()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */
|
||||||
|
fun download(context: Context, model: Model) {
|
||||||
|
val app = context.applicationContext
|
||||||
|
val state = states.getValue(model)
|
||||||
|
if (state.value is State.Downloading || state.value == State.Ready) return
|
||||||
|
state.value = State.Downloading(0)
|
||||||
|
cancelled = null
|
||||||
|
worker.execute {
|
||||||
|
val target = dir(app, model)
|
||||||
|
val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() }
|
||||||
|
try {
|
||||||
|
val conn = (URL(model.url).openConnection() as HttpURLConnection).apply {
|
||||||
|
connectTimeout = 15_000
|
||||||
|
readTimeout = 30_000
|
||||||
|
instanceFollowRedirects = true
|
||||||
|
}
|
||||||
|
check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" }
|
||||||
|
val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L)
|
||||||
|
val counting = CountingStream(conn.inputStream) { read ->
|
||||||
|
if (cancelled == model) throw InterruptedException("cancelled")
|
||||||
|
state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99))
|
||||||
|
}
|
||||||
|
TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar ->
|
||||||
|
while (true) {
|
||||||
|
val entry = tar.nextEntry ?: break
|
||||||
|
val name = entry.name.substringAfterLast('/')
|
||||||
|
if (entry.isDirectory || name !in model.files) continue
|
||||||
|
File(partial, name).outputStream().use { tar.copyTo(it) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" }
|
||||||
|
target.deleteRecursively()
|
||||||
|
check(partial.renameTo(target)) { "couldn't store the model" }
|
||||||
|
GearslipLog.i("speech: downloaded ${model.label}")
|
||||||
|
state.value = State.Ready
|
||||||
|
} catch (e: Exception) {
|
||||||
|
partial.deleteRecursively()
|
||||||
|
if (cancelled == model) {
|
||||||
|
state.value = State.Missing
|
||||||
|
} else {
|
||||||
|
GearslipLog.w("speech: download of ${model.label} failed: ${e.message}")
|
||||||
|
state.value = State.Failed(e.message ?: e.javaClass.simpleName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Counts bytes as they pass, reporting about every megabyte. */
|
||||||
|
private class CountingStream(
|
||||||
|
private val inner: java.io.InputStream,
|
||||||
|
private val onProgress: (Long) -> Unit,
|
||||||
|
) : java.io.InputStream() {
|
||||||
|
private var read = 0L
|
||||||
|
private var reported = 0L
|
||||||
|
|
||||||
|
override fun read(): Int = inner.read().also { if (it >= 0) count(1) }
|
||||||
|
override fun read(b: ByteArray, off: Int, len: Int): Int = inner.read(b, off, len).also { if (it > 0) count(it.toLong()) }
|
||||||
|
override fun close() = inner.close()
|
||||||
|
|
||||||
|
private fun count(n: Long) {
|
||||||
|
read += n
|
||||||
|
if (read - reported >= 1_048_576) {
|
||||||
|
reported = read
|
||||||
|
onProgress(read)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -47,3 +47,25 @@ fun AudioChoiceDialog(onUseBluetooth: () -> Unit, onKeepAsking: () -> Unit) {
|
|||||||
dismissButton = { TextButton(onClick = onKeepAsking) { Text("Keep asking") } },
|
dismissButton = { TextButton(onClick = onKeepAsking) { Text("Keep asking") } },
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Android lets an app capture other apps' audio only while it holds the microphone permission.
|
||||||
|
* Once the driver has refused that twice, Android stops asking, and music silently stays on the
|
||||||
|
* phone; this says why and where to turn it back on.
|
||||||
|
*/
|
||||||
|
@Composable
|
||||||
|
fun MicBlockedDialog(onOpenSettings: () -> Unit, onDismiss: () -> Unit) {
|
||||||
|
AlertDialog(
|
||||||
|
onDismissRequest = onDismiss,
|
||||||
|
title = { Text("Car audio needs the microphone permission") },
|
||||||
|
text = {
|
||||||
|
Text(
|
||||||
|
"Android lets an app send other apps' sound to the car only if it has the " +
|
||||||
|
"microphone permission. It's turned off for Gearslip, so music plays on the " +
|
||||||
|
"phone. Turn it on under Permissions, then Microphone.",
|
||||||
|
)
|
||||||
|
},
|
||||||
|
confirmButton = { TextButton(onClick = onOpenSettings) { Text("Open app settings") } },
|
||||||
|
dismissButton = { TextButton(onClick = onDismiss) { Text("Not now") } },
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -16,11 +16,9 @@ import androidx.compose.foundation.layout.fillMaxWidth
|
|||||||
import androidx.compose.foundation.layout.height
|
import androidx.compose.foundation.layout.height
|
||||||
import androidx.compose.foundation.layout.statusBars
|
import androidx.compose.foundation.layout.statusBars
|
||||||
import androidx.compose.foundation.layout.windowInsetsPadding
|
import androidx.compose.foundation.layout.windowInsetsPadding
|
||||||
import androidx.compose.material3.AlertDialog
|
|
||||||
import androidx.compose.material3.MaterialTheme
|
import androidx.compose.material3.MaterialTheme
|
||||||
import androidx.compose.material3.Surface
|
import androidx.compose.material3.Surface
|
||||||
import androidx.compose.material3.Text
|
import androidx.compose.material3.Text
|
||||||
import androidx.compose.material3.TextButton
|
|
||||||
import androidx.compose.runtime.Composable
|
import androidx.compose.runtime.Composable
|
||||||
import androidx.compose.runtime.LaunchedEffect
|
import androidx.compose.runtime.LaunchedEffect
|
||||||
import androidx.compose.runtime.getValue
|
import androidx.compose.runtime.getValue
|
||||||
@@ -39,7 +37,7 @@ import app.seb3thehacker.gearslip.BuildConfig
|
|||||||
import app.seb3thehacker.gearslip.CertProvider
|
import app.seb3thehacker.gearslip.CertProvider
|
||||||
import app.seb3thehacker.gearslip.stats.UsageStats
|
import app.seb3thehacker.gearslip.stats.UsageStats
|
||||||
|
|
||||||
private enum class Screen { HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, USAGE_NOTES, HELP, WHATS_NEW, RECOMMENDED_APPS }
|
private enum class Screen { INTRO, HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, USAGE_NOTES, HELP, WHATS_NEW, RECOMMENDED_APPS }
|
||||||
|
|
||||||
/** Three screens and a back stack of depth one: no navigation library needed. */
|
/** Three screens and a back stack of depth one: no navigation library needed. */
|
||||||
@Composable
|
@Composable
|
||||||
@@ -49,20 +47,22 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
|
|||||||
!AppSettings.hasSkippedCertSetup(context) && !CertProvider.hasAnyCert(context)
|
!AppSettings.hasSkippedCertSetup(context) && !CertProvider.hasAnyCert(context)
|
||||||
}
|
}
|
||||||
val needsSetupGuide = remember { !AppSettings.hasSeenPermissionsSetup(context) }
|
val needsSetupGuide = remember { !AppSettings.hasSeenPermissionsSetup(context) }
|
||||||
var screen by rememberSaveable {
|
// Where launch goes once the intro (if any) is done.
|
||||||
mutableStateOf(
|
val afterIntro = remember {
|
||||||
when {
|
when {
|
||||||
needsCertSetup -> Screen.CERT_SETUP
|
needsCertSetup -> Screen.CERT_SETUP
|
||||||
needsSetupGuide -> Screen.SETUP_GUIDE
|
needsSetupGuide -> Screen.SETUP_GUIDE
|
||||||
// Set up before usage notes existed: ask once, on the setup guide's own page.
|
// Set up before usage notes existed: ask once, on the setup guide's own page.
|
||||||
!UsageStats.asked(context) -> Screen.USAGE_NOTES
|
!UsageStats.asked(context) -> Screen.USAGE_NOTES
|
||||||
else -> Screen.HOME
|
else -> Screen.HOME
|
||||||
},
|
}
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
var screen by rememberSaveable { mutableStateOf(if (Intro.due(context)) Screen.INTRO else afterIntro) }
|
||||||
|
// A replay from About goes back to Settings; the launch one carries on into setup.
|
||||||
|
var introReturn by rememberSaveable { mutableStateOf(afterIntro) }
|
||||||
// Home's usage notes row opens Settings at that section rather than at the top.
|
// Home's usage notes row opens Settings at that section rather than at the top.
|
||||||
var settingsAtUsageNotes by rememberSaveable { mutableStateOf(false) }
|
var settingsAtUsageNotes by rememberSaveable { mutableStateOf(false) }
|
||||||
BackHandler(enabled = screen != Screen.HOME) {
|
BackHandler(enabled = screen != Screen.HOME && screen != Screen.INTRO) {
|
||||||
screen = if (screen == Screen.RECOMMENDED_APPS) Screen.SETTINGS else Screen.HOME
|
screen = if (screen == Screen.RECOMMENDED_APPS) Screen.SETTINGS else Screen.HOME
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -95,10 +95,11 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
|
|||||||
.windowInsetsPadding(WindowInsets.statusBars)
|
.windowInsetsPadding(WindowInsets.statusBars)
|
||||||
.consumeWindowInsets(WindowInsets.statusBars),
|
.consumeWindowInsets(WindowInsets.statusBars),
|
||||||
) {
|
) {
|
||||||
if (BuildConfig.DEBUG && screen != Screen.CAR_PREVIEW) DevBuildBanner()
|
if (BuildConfig.DEBUG && screen != Screen.CAR_PREVIEW && screen != Screen.INTRO) DevBuildBanner()
|
||||||
|
|
||||||
Column(Modifier.weight(1f)) {
|
Column(Modifier.weight(1f)) {
|
||||||
when (screen) {
|
when (screen) {
|
||||||
|
Screen.INTRO -> IntroScreen(onDone = { screen = introReturn })
|
||||||
Screen.HOME -> HomeScreen(
|
Screen.HOME -> HomeScreen(
|
||||||
onOpenLogs = { screen = Screen.LOGS },
|
onOpenLogs = { screen = Screen.LOGS },
|
||||||
onOpenSettings = { settingsAtUsageNotes = false; screen = Screen.SETTINGS },
|
onOpenSettings = { settingsAtUsageNotes = false; screen = Screen.SETTINGS },
|
||||||
@@ -114,6 +115,7 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
|
|||||||
onOpenRecommendedApps = { settingsAtUsageNotes = false; screen = Screen.RECOMMENDED_APPS },
|
onOpenRecommendedApps = { settingsAtUsageNotes = false; screen = Screen.RECOMMENDED_APPS },
|
||||||
onOpenWhatsNew = { settingsAtUsageNotes = false; screen = Screen.WHATS_NEW },
|
onOpenWhatsNew = { settingsAtUsageNotes = false; screen = Screen.WHATS_NEW },
|
||||||
onReplayTutorial = { screen = Screen.SETUP_GUIDE },
|
onReplayTutorial = { screen = Screen.SETUP_GUIDE },
|
||||||
|
onReplayIntro = { introReturn = Screen.SETTINGS; screen = Screen.INTRO },
|
||||||
atUsageNotes = settingsAtUsageNotes,
|
atUsageNotes = settingsAtUsageNotes,
|
||||||
)
|
)
|
||||||
Screen.RECOMMENDED_APPS -> RecommendedAppsScreen(onBack = { screen = Screen.SETTINGS })
|
Screen.RECOMMENDED_APPS -> RecommendedAppsScreen(onBack = { screen = Screen.SETTINGS })
|
||||||
@@ -141,8 +143,6 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
|
|||||||
// A car keyboard is open: offer the phone's own, for a passenger.
|
// A car keyboard is open: offer the phone's own, for a passenger.
|
||||||
if (screen != Screen.CAR_PREVIEW) PhoneTypingBar()
|
if (screen != Screen.CAR_PREVIEW) PhoneTypingBar()
|
||||||
}
|
}
|
||||||
|
|
||||||
CompatWarning()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -167,29 +167,3 @@ private fun DevBuildBanner() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Shown once, on first run: the certificate is known to fail on head units newer than ~2020. */
|
|
||||||
@Composable
|
|
||||||
private fun CompatWarning() {
|
|
||||||
val context = LocalContext.current
|
|
||||||
var visible by rememberSaveable { mutableStateOf(!AppSettings.hasSeenCompatWarning(context)) }
|
|
||||||
|
|
||||||
if (visible) {
|
|
||||||
AlertDialog(
|
|
||||||
onDismissRequest = {},
|
|
||||||
title = { Text("Before you plug in") },
|
|
||||||
text = {
|
|
||||||
Text(
|
|
||||||
"Gearslip works with cars built before 2020. " +
|
|
||||||
"Newer firmware often blocks the connection.",
|
|
||||||
)
|
|
||||||
},
|
|
||||||
confirmButton = {
|
|
||||||
TextButton(onClick = {
|
|
||||||
AppSettings.setSeenCompatWarning(context)
|
|
||||||
visible = false
|
|
||||||
}) { Text("Got it") }
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -112,6 +112,10 @@ fun HomeScreen(
|
|||||||
Spacer(Modifier.height(16.dp))
|
Spacer(Modifier.height(16.dp))
|
||||||
WirelessCard()
|
WirelessCard()
|
||||||
}
|
}
|
||||||
|
if (!live) {
|
||||||
|
Spacer(Modifier.height(16.dp))
|
||||||
|
SupportCard()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Sits at the bottom, where a thumb reaches it.
|
// Sits at the bottom, where a thumb reaches it.
|
||||||
|
|||||||
@@ -0,0 +1,726 @@
|
|||||||
|
package app.seb3thehacker.gearslip.ui
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.content.Intent
|
||||||
|
import android.net.Uri
|
||||||
|
import android.provider.Settings
|
||||||
|
import androidx.activity.compose.BackHandler
|
||||||
|
import androidx.compose.animation.core.Animatable
|
||||||
|
import androidx.compose.animation.core.CubicBezierEasing
|
||||||
|
import androidx.compose.animation.core.FastOutSlowInEasing
|
||||||
|
import androidx.compose.animation.core.LinearEasing
|
||||||
|
import androidx.compose.animation.core.RepeatMode
|
||||||
|
import androidx.compose.animation.core.animateDpAsState
|
||||||
|
import androidx.compose.animation.core.animateFloat
|
||||||
|
import androidx.compose.animation.core.infiniteRepeatable
|
||||||
|
import androidx.compose.animation.core.keyframes
|
||||||
|
import androidx.compose.animation.core.rememberInfiniteTransition
|
||||||
|
import androidx.compose.animation.core.tween
|
||||||
|
import androidx.compose.foundation.Canvas
|
||||||
|
import androidx.compose.foundation.BorderStroke
|
||||||
|
import androidx.compose.foundation.background
|
||||||
|
import androidx.compose.foundation.border
|
||||||
|
import androidx.compose.foundation.clickable
|
||||||
|
import androidx.compose.foundation.interaction.MutableInteractionSource
|
||||||
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
|
import androidx.compose.foundation.layout.Box
|
||||||
|
import androidx.compose.foundation.layout.BoxWithConstraints
|
||||||
|
import androidx.compose.foundation.layout.Column
|
||||||
|
import androidx.compose.foundation.layout.Row
|
||||||
|
import androidx.compose.foundation.layout.Spacer
|
||||||
|
import androidx.compose.foundation.layout.fillMaxSize
|
||||||
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
|
import androidx.compose.foundation.layout.height
|
||||||
|
import androidx.compose.foundation.layout.offset
|
||||||
|
import androidx.compose.foundation.layout.padding
|
||||||
|
import androidx.compose.foundation.layout.size
|
||||||
|
import androidx.compose.foundation.layout.width
|
||||||
|
import androidx.compose.foundation.layout.wrapContentWidth
|
||||||
|
import androidx.compose.foundation.shape.RoundedCornerShape
|
||||||
|
import androidx.compose.material3.MaterialTheme
|
||||||
|
import androidx.compose.material3.Text
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.LaunchedEffect
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.runtime.mutableIntStateOf
|
||||||
|
import androidx.compose.runtime.mutableStateOf
|
||||||
|
import androidx.compose.runtime.remember
|
||||||
|
import androidx.compose.runtime.setValue
|
||||||
|
import androidx.compose.ui.Alignment
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.draw.clip
|
||||||
|
import androidx.compose.ui.geometry.Offset
|
||||||
|
import androidx.compose.ui.geometry.Size
|
||||||
|
import androidx.compose.ui.graphics.Color
|
||||||
|
import androidx.compose.ui.graphics.Path
|
||||||
|
import androidx.compose.ui.graphics.StrokeCap
|
||||||
|
import androidx.compose.ui.graphics.StrokeJoin
|
||||||
|
import androidx.compose.ui.graphics.drawscope.DrawScope
|
||||||
|
import androidx.compose.ui.graphics.drawscope.Stroke
|
||||||
|
import androidx.compose.ui.graphics.drawscope.rotate
|
||||||
|
import androidx.compose.ui.graphics.drawscope.scale
|
||||||
|
import androidx.compose.ui.graphics.graphicsLayer
|
||||||
|
import androidx.compose.ui.platform.LocalContext
|
||||||
|
import androidx.compose.ui.platform.LocalDensity
|
||||||
|
import androidx.compose.ui.text.TextStyle
|
||||||
|
import androidx.compose.ui.text.font.FontWeight
|
||||||
|
import androidx.compose.ui.unit.Dp
|
||||||
|
import androidx.compose.ui.unit.TextUnit
|
||||||
|
import androidx.compose.ui.unit.dp
|
||||||
|
import app.seb3thehacker.gearslip.BuildConfig
|
||||||
|
import androidx.compose.runtime.collectAsState
|
||||||
|
import androidx.compose.ui.draw.clipToBounds
|
||||||
|
import androidx.compose.ui.draw.drawWithContent
|
||||||
|
import androidx.compose.ui.graphics.BlendMode
|
||||||
|
import androidx.compose.ui.graphics.Brush
|
||||||
|
import androidx.compose.ui.graphics.CompositingStrategy
|
||||||
|
import androidx.compose.foundation.layout.wrapContentHeight
|
||||||
|
import app.seb3thehacker.gearslip.speech.SpeechModels
|
||||||
|
import app.seb3thehacker.gearslip.stats.UsageStats
|
||||||
|
import kotlin.math.PI
|
||||||
|
import kotlin.math.cos
|
||||||
|
import kotlin.math.atan2
|
||||||
|
import kotlin.math.sqrt
|
||||||
|
import kotlin.math.sin
|
||||||
|
import kotlin.random.Random
|
||||||
|
import kotlinx.coroutines.delay
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The release intro: a few tap-through screens shown once after each new release (and on a new
|
||||||
|
* install), before setup. Its look follows the 0.3 mockup: Material shapes in the phone's own colours,
|
||||||
|
* with every size taken from the screen's width so it scales from small phones to tablets.
|
||||||
|
*/
|
||||||
|
object Intro {
|
||||||
|
private const val PREFS = "intro"
|
||||||
|
private const val KEY_SEEN = "seen_release"
|
||||||
|
|
||||||
|
/** "0.3" from "0.3.00-alpha": the intro belongs to a release, not to each patch. */
|
||||||
|
val release: String = BuildConfig.VERSION_NAME.split('.').take(2).joinToString(".")
|
||||||
|
|
||||||
|
fun due(context: Context): Boolean =
|
||||||
|
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_SEEN, null) != release
|
||||||
|
|
||||||
|
fun markSeen(context: Context) {
|
||||||
|
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_SEEN, release).apply()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The intro's tones, taken from the phone's own Material You colours like the rest of the app. */
|
||||||
|
private class IntroColors(
|
||||||
|
val surface: Color, val onSurface: Color, val onVariant: Color, val outline: Color,
|
||||||
|
val primary: Color, val onPrimary: Color, val primaryC: Color, val onPrimaryC: Color,
|
||||||
|
val tertiaryC: Color, val onTertiaryC: Color, val secondaryC: Color, val onSecondaryC: Color,
|
||||||
|
)
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun introColors(): IntroColors {
|
||||||
|
val m = MaterialTheme.colorScheme
|
||||||
|
return IntroColors(
|
||||||
|
surface = m.surface, onSurface = m.onSurface, onVariant = m.onSurfaceVariant, outline = m.outline,
|
||||||
|
primary = m.primary, onPrimary = m.onPrimary, primaryC = m.primaryContainer, onPrimaryC = m.onPrimaryContainer,
|
||||||
|
tertiaryC = m.tertiaryContainer, onTertiaryC = m.onTertiaryContainer,
|
||||||
|
secondaryC = m.secondaryContainer, onSecondaryC = m.onSecondaryContainer,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, DRIVE }
|
||||||
|
|
||||||
|
private val Pop = CubicBezierEasing(0.3f, 1.6f, 0.5f, 1f)
|
||||||
|
private val Rise = CubicBezierEasing(0.2f, 0.8f, 0.2f, 1f)
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
fun IntroScreen(onDone: () -> Unit) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
val c = introColors()
|
||||||
|
// The GrapheneOS screen is only news to people running it.
|
||||||
|
val steps = remember {
|
||||||
|
IntroStep.entries.filter { it != IntroStep.GRAPHENE || UsageStats.osName(context) == "GrapheneOS" }
|
||||||
|
}
|
||||||
|
var index by remember { mutableIntStateOf(0) }
|
||||||
|
val step = steps[index]
|
||||||
|
val finish = {
|
||||||
|
Intro.markSeen(context)
|
||||||
|
onDone()
|
||||||
|
}
|
||||||
|
BackHandler(enabled = index > 0) { index-- }
|
||||||
|
|
||||||
|
BoxWithConstraints(
|
||||||
|
Modifier
|
||||||
|
.fillMaxSize()
|
||||||
|
.background(c.surface)
|
||||||
|
.clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) {
|
||||||
|
if (index < steps.lastIndex) index++
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
val w = maxWidth
|
||||||
|
// The mockup's cqw: one percent of the screen's width.
|
||||||
|
fun cq(n: Float): Dp = w * (n / 100f)
|
||||||
|
val density = LocalDensity.current
|
||||||
|
fun cqSp(n: Float): TextUnit = with(density) { cq(n).toSp() }
|
||||||
|
|
||||||
|
when (step) {
|
||||||
|
IntroStep.WELCOME -> Welcome(c, ::cq, ::cqSp, maxHeight)
|
||||||
|
IntroStep.WIRELESS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Wireless.",
|
||||||
|
"If your car does wireless Android Auto, pair once and leave the cable at home.", step) { WifiIcon(c.onSecondaryC) }
|
||||||
|
IntroStep.KNOBS -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Knobs work.",
|
||||||
|
"Turn, push, done. No touchscreen? Gearslip doesn't need one anymore.", step) { DialIcon(c.onTertiaryC) }
|
||||||
|
IntroStep.VOICE -> VoiceStep(c, ::cq, ::cqSp, step)
|
||||||
|
IntroStep.GRAPHENE -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Protections back on.",
|
||||||
|
"On GrapheneOS, Gearslip no longer needs exploit protection compatibility mode. If you " +
|
||||||
|
"turned it on for Gearslip, turn it off to get full protection back.", step,
|
||||||
|
action = "Open app settings" to {
|
||||||
|
runCatching {
|
||||||
|
context.startActivity(
|
||||||
|
Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", context.packageName, null)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
outlined = true,
|
||||||
|
) { ShieldIcon(c.onSecondaryC, c.secondaryC) }
|
||||||
|
IntroStep.THANKS -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Thank you.",
|
||||||
|
"To everyone sending usage notes: they show me which cars Gearslip works in and which " +
|
||||||
|
"ones still fail, so I know what to fix next. Every note helps.", step) { HeartIcon(c.onTertiaryC) }
|
||||||
|
IntroStep.APPS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Good company.",
|
||||||
|
"Free, open-source apps that work well in the car. Tap one to see it on GitHub.", step,
|
||||||
|
extra = { AppPicks(c, ::cq, ::cqSp, step) }) { StarIcon(c.onSecondaryC) }
|
||||||
|
IntroStep.DRIVE -> BadgeStep(c, ::cq, ::cqSp, c.primaryC, "Let's drive.",
|
||||||
|
"That's ${Intro.release}. Plug in, or just get in.", step,
|
||||||
|
action = "Start" to finish) { CheckIcon(c.onPrimaryC) }
|
||||||
|
}
|
||||||
|
|
||||||
|
Dots(c, steps.size, index, ::cq, Modifier.align(Alignment.BottomCenter).padding(bottom = maxHeight * 0.05f))
|
||||||
|
if (step != IntroStep.DRIVE) Hint(c, ::cqSp, step, Modifier.align(Alignment.BottomCenter).padding(bottom = maxHeight * 0.09f))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 0 to 1 once each time [key] changes, after [delayMs]: drives the entrance of one element. */
|
||||||
|
@Composable
|
||||||
|
private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float {
|
||||||
|
val a = remember(key) { Animatable(0f) }
|
||||||
|
LaunchedEffect(key) {
|
||||||
|
delay(delayMs.toLong())
|
||||||
|
a.animateTo(1f, tween(durationMs, easing = LinearEasing))
|
||||||
|
}
|
||||||
|
return a.value
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Slides up and fades in, like the mockup's `up`. */
|
||||||
|
private fun Modifier.up(p: Float, distance: Dp) = graphicsLayer {
|
||||||
|
val e = Rise.transform(p)
|
||||||
|
alpha = e
|
||||||
|
translationY = (1 - e) * distance.toPx()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Grows in with a bounce, like the mockup's `bounce`. */
|
||||||
|
private fun Modifier.pop(p: Float) = graphicsLayer {
|
||||||
|
val e = Pop.transform(p)
|
||||||
|
alpha = p.coerceAtMost(1f)
|
||||||
|
scaleX = 0.5f + 0.5f * e
|
||||||
|
scaleY = 0.5f + 0.5f * e
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun Welcome(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, height: Dp) {
|
||||||
|
val key = IntroStep.WELCOME
|
||||||
|
val spin by rememberInfiniteTransition(label = "spin").animateFloat(
|
||||||
|
0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin",
|
||||||
|
)
|
||||||
|
Box(Modifier.fillMaxSize()) {
|
||||||
|
Cookie(c.primaryC, Modifier.size(cq(120f)).offset(x = cq(28f), y = -cq(22f)).align(Alignment.TopEnd).graphicsLayer { rotationZ = spin; alpha = 0.9f })
|
||||||
|
Ticker(c.secondaryC, c.onSecondaryC, "OPEN SOURCE ✦ NO ACCOUNT ✦ YOUR CAR, YOUR PHONE ✦ ", cq, cqSp, reverse = true, tilt = 3f,
|
||||||
|
key = key, modifier = Modifier.align(Alignment.TopCenter).padding(top = height * 0.11f))
|
||||||
|
|
||||||
|
Column(Modifier.align(Alignment.CenterStart).padding(horizontal = cq(8f))) {
|
||||||
|
Text("Welcome to", color = c.onVariant, style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.SemiBold),
|
||||||
|
modifier = Modifier.up(entrance(key, 100), cq(7f)))
|
||||||
|
Text(
|
||||||
|
Intro.release, color = c.primary,
|
||||||
|
style = TextStyle(fontSize = cqSp(44f), fontWeight = FontWeight.Black, letterSpacing = cqSp(-2.4f), lineHeight = cqSp(38f)),
|
||||||
|
modifier = Modifier.offset(x = -cq(1.5f)).pop(entrance(key, 250, 900)),
|
||||||
|
)
|
||||||
|
Text(
|
||||||
|
"alpha · the big one", color = c.onTertiaryC,
|
||||||
|
style = TextStyle(fontSize = cqSp(4f), fontWeight = FontWeight.Bold),
|
||||||
|
modifier = Modifier.padding(top = cq(4f)).up(entrance(key, 700), cq(7f))
|
||||||
|
.clip(RoundedCornerShape(50)).background(c.tertiaryC).padding(horizontal = cq(4.4f), vertical = cq(2.2f)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
Ticker(c.primary, c.onPrimary, "WIRELESS ✦ KNOB SUPPORT ✦ OFFLINE VOICE ✦ FASTER VIDEO ✦ AUTO UPDATES ✦ ", cq, cqSp, reverse = false, tilt = -4f,
|
||||||
|
key = key, modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = height * 0.15f))
|
||||||
|
Confetti(c, key, Modifier.fillMaxSize())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A tilted band of words scrolling past, slid in from the side when the step opens. */
|
||||||
|
@Composable
|
||||||
|
private fun Ticker(
|
||||||
|
bg: Color, fg: Color, words: String, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit,
|
||||||
|
reverse: Boolean, tilt: Float, key: Any, modifier: Modifier,
|
||||||
|
) {
|
||||||
|
val enter = entrance(key, 400, 800)
|
||||||
|
val scroll by rememberInfiniteTransition(label = "ticker").animateFloat(
|
||||||
|
0f, 1f, infiniteRepeatable(tween(if (reverse) 20_000 else 16_000, easing = LinearEasing)), label = "ticker",
|
||||||
|
)
|
||||||
|
Box(
|
||||||
|
modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.graphicsLayer {
|
||||||
|
rotationZ = tilt
|
||||||
|
scaleX = 1.2f
|
||||||
|
val e = Rise.transform(enter)
|
||||||
|
alpha = e
|
||||||
|
translationX = (1 - e) * -cq(30f).toPx()
|
||||||
|
}
|
||||||
|
.background(bg)
|
||||||
|
.padding(vertical = cq(2.6f)),
|
||||||
|
) {
|
||||||
|
Text(
|
||||||
|
words.repeat(4),
|
||||||
|
color = fg, maxLines = 1, softWrap = false,
|
||||||
|
style = TextStyle(fontSize = cqSp(5f), fontWeight = FontWeight.ExtraBold),
|
||||||
|
modifier = Modifier
|
||||||
|
.wrapContentWidth(Alignment.Start, unbounded = true)
|
||||||
|
.graphicsLayer {
|
||||||
|
// The text holds the words four times; sliding by half loops cleanly.
|
||||||
|
val shift = size.width / 2f * (if (reverse) 1 - scroll else scroll)
|
||||||
|
translationX = -shift
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Material shapes thrown once from behind the version number. */
|
||||||
|
@Composable
|
||||||
|
private fun Confetti(c: IntroColors, key: Any, modifier: Modifier) {
|
||||||
|
val p = entrance(key, 450, 1300)
|
||||||
|
val bits = remember {
|
||||||
|
val r = Random(7)
|
||||||
|
List(22) { i ->
|
||||||
|
val a = i / 22f * 2 * PI.toFloat() + r.nextFloat() * 0.4f
|
||||||
|
val d = 30f + r.nextFloat() * 45f
|
||||||
|
Bit(cos(a) * d, sin(a) * d * 1.2f, r.nextInt(-270, 270).toFloat(), i % 4, i % 3)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (p <= 0f || p >= 1f) return
|
||||||
|
val tones = listOf(c.primary, c.primaryC, c.tertiaryC, c.secondaryC)
|
||||||
|
Canvas(modifier) {
|
||||||
|
val unit = size.width / 100f
|
||||||
|
val e = CubicBezierEasing(0.15f, 0.7f, 0.3f, 1f).transform(p)
|
||||||
|
val alpha = if (p < 0.7f) 1f else 1f - (p - 0.7f) / 0.3f
|
||||||
|
val origin = Offset(size.width * 0.3f, size.height * 0.42f)
|
||||||
|
bits.forEach { b ->
|
||||||
|
val at = origin + Offset(b.dx * unit * e, b.dy * unit * e)
|
||||||
|
val s = 3.6f * unit * (0.3f + 0.7f * e)
|
||||||
|
rotate(b.spin * e, at) {
|
||||||
|
val color = tones[b.tone].copy(alpha = alpha)
|
||||||
|
when (b.shape) {
|
||||||
|
0 -> drawCircle(color, s / 2, at)
|
||||||
|
1 -> drawRoundRect(color, at - Offset(s / 2, s / 2), Size(s, s), androidx.compose.ui.geometry.CornerRadius(unit))
|
||||||
|
else -> drawRoundRect(color, at - Offset(s / 2, s / 4), Size(s, s / 2), androidx.compose.ui.geometry.CornerRadius(s / 4))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private class Bit(val dx: Float, val dy: Float, val spin: Float, val tone: Int, val shape: Int)
|
||||||
|
|
||||||
|
/** One idea per screen: a spinning cookie with an icon, a headline, a line of text, maybe a button. */
|
||||||
|
@Composable
|
||||||
|
private fun BadgeStep(
|
||||||
|
c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, badge: Color,
|
||||||
|
title: String, body: String, key: Any,
|
||||||
|
action: Pair<String, () -> Unit>? = null, outlined: Boolean = false,
|
||||||
|
extra: (@Composable () -> Unit)? = null,
|
||||||
|
icon: @Composable () -> Unit,
|
||||||
|
) {
|
||||||
|
val spin by rememberInfiniteTransition(label = "spin").animateFloat(
|
||||||
|
0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin",
|
||||||
|
)
|
||||||
|
Column(
|
||||||
|
Modifier.fillMaxSize().padding(horizontal = cq(8f)),
|
||||||
|
verticalArrangement = Arrangement.Center,
|
||||||
|
) {
|
||||||
|
Box(Modifier.size(cq(34f)).pop(entrance(key, 50, 800)), contentAlignment = Alignment.Center) {
|
||||||
|
Cookie(badge, Modifier.fillMaxSize().graphicsLayer { rotationZ = spin })
|
||||||
|
Box(Modifier.size(cq(15f))) { icon() }
|
||||||
|
}
|
||||||
|
Spacer(Modifier.height(cq(8f)))
|
||||||
|
Text(
|
||||||
|
title, color = c.onSurface,
|
||||||
|
style = TextStyle(fontSize = cqSp(13.5f), fontWeight = FontWeight.Black, letterSpacing = cqSp(-0.6f), lineHeight = cqSp(12.5f)),
|
||||||
|
modifier = Modifier.up(entrance(key, 200, 700), cq(7f)),
|
||||||
|
)
|
||||||
|
Spacer(Modifier.height(cq(4f)))
|
||||||
|
Text(
|
||||||
|
body, color = c.onVariant,
|
||||||
|
style = TextStyle(fontSize = cqSp(4.5f), lineHeight = cqSp(6.5f)),
|
||||||
|
modifier = Modifier.up(entrance(key, 350, 700), cq(7f)),
|
||||||
|
)
|
||||||
|
extra?.invoke()
|
||||||
|
if (action != null) {
|
||||||
|
val (label, onClick) = action
|
||||||
|
Spacer(Modifier.height(if (outlined) cq(7f) else cq(9f)))
|
||||||
|
Text(
|
||||||
|
label,
|
||||||
|
color = if (outlined) c.primary else c.onPrimary,
|
||||||
|
style = TextStyle(fontSize = cqSp(if (outlined) 4.4f else 4.6f), fontWeight = FontWeight.Bold),
|
||||||
|
modifier = Modifier
|
||||||
|
.up(entrance(key, 500, 700), cq(7f))
|
||||||
|
.clip(RoundedCornerShape(50))
|
||||||
|
.then(
|
||||||
|
if (outlined) Modifier.border(BorderStroke(2.dp, c.outline), RoundedCornerShape(50))
|
||||||
|
else Modifier.background(c.primary),
|
||||||
|
)
|
||||||
|
.clickable(onClick = onClick)
|
||||||
|
.padding(horizontal = if (outlined) cq(7f) else cq(9f), vertical = if (outlined) cq(3.6f) else cq(4.2f)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Voice: a few things to say, drifting past, and a button that fetches both speech models so the
|
||||||
|
* assistant works the first time someone tries it in the car.
|
||||||
|
*/
|
||||||
|
@Composable
|
||||||
|
private fun VoiceStep(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
remember { SpeechModels.init(context) }
|
||||||
|
val needed = listOf(SpeechModels.Model.KROKO, SpeechModels.Model.PARAKEET_LARGE)
|
||||||
|
val states = needed.map { SpeechModels.state(it).collectAsState().value }
|
||||||
|
val ready = states.all { it == SpeechModels.State.Ready }
|
||||||
|
val label = when {
|
||||||
|
states.any { it is SpeechModels.State.Downloading } ->
|
||||||
|
"Downloading, ${needed.zip(states).sumOf { (m, st) -> m.downloadMb * percentOf(st) } / needed.sumOf { it.downloadMb }}%"
|
||||||
|
states.any { it is SpeechModels.State.Failed } -> "Download failed. Try again"
|
||||||
|
else -> "Download voice, ${needed.sumOf { it.downloadMb }} MB"
|
||||||
|
}
|
||||||
|
BadgeStep(
|
||||||
|
c, cq, cqSp, c.primaryC, "Just say it.",
|
||||||
|
"Tap the mic, or press the talk button on your wheel, and speak. Your phone turns speech " +
|
||||||
|
"into text by itself, and nothing you say leaves it. Say \"help\" in the car for the full list.",
|
||||||
|
key,
|
||||||
|
action = if (ready) null else label to { needed.forEach { SpeechModels.download(context, it) } },
|
||||||
|
outlined = true,
|
||||||
|
extra = { CommandDrift(c, cq, cqSp, key) },
|
||||||
|
) { MicIcon(c.onPrimaryC) }
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun percentOf(state: SpeechModels.State) = when (state) {
|
||||||
|
SpeechModels.State.Ready -> 100
|
||||||
|
is SpeechModels.State.Downloading -> state.percent
|
||||||
|
else -> 0
|
||||||
|
}
|
||||||
|
|
||||||
|
private val SAMPLE_COMMANDS = listOf(
|
||||||
|
"Play Daft Punk", "What's playing?", "Next song",
|
||||||
|
"Call Mom", "Open Organic Maps", "Pause", "Help",
|
||||||
|
)
|
||||||
|
|
||||||
|
/** Example commands rising slowly through a window that fades at both edges. */
|
||||||
|
@Composable
|
||||||
|
private fun CommandDrift(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: Any) {
|
||||||
|
val scroll by rememberInfiniteTransition(label = "drift").animateFloat(
|
||||||
|
0f, 1f, infiniteRepeatable(tween(SAMPLE_COMMANDS.size * 2_200, easing = LinearEasing)), label = "drift",
|
||||||
|
)
|
||||||
|
Box(
|
||||||
|
Modifier
|
||||||
|
.padding(top = cq(6f))
|
||||||
|
.fillMaxWidth()
|
||||||
|
.height(cq(34f))
|
||||||
|
.up(entrance(key, 500, 700), cq(7f))
|
||||||
|
.graphicsLayer { compositingStrategy = CompositingStrategy.Offscreen }
|
||||||
|
.drawWithContent {
|
||||||
|
drawContent()
|
||||||
|
drawRect(
|
||||||
|
Brush.verticalGradient(0f to Color.Transparent, 0.25f to Color.Black, 0.75f to Color.Black, 1f to Color.Transparent),
|
||||||
|
blendMode = BlendMode.DstIn,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
.clipToBounds(),
|
||||||
|
) {
|
||||||
|
Column(
|
||||||
|
Modifier
|
||||||
|
.wrapContentHeight(Alignment.Top, unbounded = true)
|
||||||
|
// The list is there twice; rising by half loops cleanly.
|
||||||
|
.graphicsLayer { translationY = -size.height / 2f * scroll },
|
||||||
|
verticalArrangement = Arrangement.spacedBy(cq(2.4f)),
|
||||||
|
) {
|
||||||
|
repeat(2) {
|
||||||
|
SAMPLE_COMMANDS.forEach { say ->
|
||||||
|
Text(
|
||||||
|
"\u201c$say\u201d", color = c.onSecondaryC,
|
||||||
|
style = TextStyle(fontSize = cqSp(4.4f), fontWeight = FontWeight.SemiBold),
|
||||||
|
modifier = Modifier
|
||||||
|
.clip(RoundedCornerShape(50))
|
||||||
|
.background(c.secondaryC)
|
||||||
|
.padding(horizontal = cq(4.4f), vertical = cq(2f)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private val APP_PICKS = listOf(
|
||||||
|
Triple("Music", "Metrolist", "https://github.com/MetrolistGroup/Metrolist"),
|
||||||
|
Triple("Maps", "Vela", "https://github.com/PimpinPumpkin/Vela"),
|
||||||
|
Triple("Podcasts", "AntennaPod", "https://github.com/AntennaPod/AntennaPod"),
|
||||||
|
)
|
||||||
|
|
||||||
|
/** One row per pick, each opening the app's GitHub page; they arrive one after another. */
|
||||||
|
@Composable
|
||||||
|
private fun AppPicks(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
Column(Modifier.padding(top = cq(6f)), verticalArrangement = Arrangement.spacedBy(cq(2.6f))) {
|
||||||
|
APP_PICKS.forEachIndexed { i, (kind, name, url) ->
|
||||||
|
Row(
|
||||||
|
Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.up(entrance(key, 500 + i * 120, 700), cq(7f))
|
||||||
|
.clip(RoundedCornerShape(cq(5f)))
|
||||||
|
.background(c.secondaryC)
|
||||||
|
.clickable {
|
||||||
|
runCatching {
|
||||||
|
context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(url)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
.padding(horizontal = cq(5f), vertical = cq(3.4f)),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
) {
|
||||||
|
Column(Modifier.weight(1f)) {
|
||||||
|
Text(kind.uppercase(), color = c.onSecondaryC.copy(alpha = 0.7f),
|
||||||
|
style = TextStyle(fontSize = cqSp(3.2f), fontWeight = FontWeight.Bold, letterSpacing = cqSp(0.3f)))
|
||||||
|
Text(name, color = c.onSecondaryC, style = TextStyle(fontSize = cqSp(5.2f), fontWeight = FontWeight.ExtraBold))
|
||||||
|
}
|
||||||
|
Text("GitHub ↗", color = c.onSecondaryC, style = TextStyle(fontSize = cqSp(3.8f), fontWeight = FontWeight.SemiBold))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Material's scalloped "cookie": a circle whose radius waves around its edge. */
|
||||||
|
@Composable
|
||||||
|
private fun Cookie(color: Color, modifier: Modifier) {
|
||||||
|
Canvas(modifier) {
|
||||||
|
val unit = size.minDimension / 100f
|
||||||
|
val path = Path()
|
||||||
|
var deg = 0
|
||||||
|
while (deg <= 360) {
|
||||||
|
val a = deg * PI / 180
|
||||||
|
val r = (44 + 4.5 * cos(9 * a)) * unit
|
||||||
|
val x = center.x + (r * cos(a)).toFloat()
|
||||||
|
val y = center.y + (r * sin(a)).toFloat()
|
||||||
|
if (deg == 0) path.moveTo(x, y) else path.lineTo(x, y)
|
||||||
|
deg += 3
|
||||||
|
}
|
||||||
|
path.close()
|
||||||
|
drawPath(path, color)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Draws in the mockup's 48-unit icon box. */
|
||||||
|
private inline fun DrawScope.inIconBox(block: DrawScope.() -> Unit) {
|
||||||
|
val s = size.minDimension / 48f
|
||||||
|
scale(s, s, Offset.Zero) { block() }
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun WifiIcon(color: Color) {
|
||||||
|
val t = rememberInfiniteTransition(label = "wifi")
|
||||||
|
val waves = (0..2).map { i ->
|
||||||
|
t.animateFloat(
|
||||||
|
0.35f, 1f,
|
||||||
|
infiniteRepeatable(tween(800, delayMillis = 0, easing = FastOutSlowInEasing), RepeatMode.Reverse,
|
||||||
|
initialStartOffset = androidx.compose.animation.core.StartOffset(i * 200)),
|
||||||
|
label = "wave$i",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox {
|
||||||
|
val stroke = Stroke(5f, cap = StrokeCap.Round)
|
||||||
|
// The mockup's three arcs, inner to outer: radius r through (24 ± half, y).
|
||||||
|
listOf(Triple(8f, 6f, 34f), Triple(18f, 13f, 26f), Triple(28f, 20f, 18f)).forEachIndexed { i, (r, half, y) ->
|
||||||
|
val cy = y + sqrt(r * r - half * half)
|
||||||
|
val start = Math.toDegrees(atan2((y - cy).toDouble(), (-half).toDouble())).toFloat()
|
||||||
|
val end = Math.toDegrees(atan2((y - cy).toDouble(), half.toDouble())).toFloat()
|
||||||
|
drawArc(color.copy(alpha = waves[i].value), start, (end - start + 360f) % 360f, false,
|
||||||
|
Offset(24f - r, cy - r), Size(2 * r, 2 * r), style = stroke)
|
||||||
|
}
|
||||||
|
drawCircle(color, 3.6f, Offset(24f, 41f))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun DialIcon(color: Color) {
|
||||||
|
val angle by rememberInfiniteTransition(label = "dial").animateFloat(
|
||||||
|
-70f, -70f,
|
||||||
|
infiniteRepeatable(keyframes {
|
||||||
|
durationMillis = 2600
|
||||||
|
-70f at 0 using CubicBezierEasing(0.5f, 0f, 0.3f, 1f)
|
||||||
|
70f at 1040
|
||||||
|
70f at 1430 using CubicBezierEasing(0.5f, 0f, 0.3f, 1f)
|
||||||
|
-70f at 2470
|
||||||
|
}),
|
||||||
|
label = "dial",
|
||||||
|
)
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox {
|
||||||
|
drawCircle(color, 19f, Offset(24f, 24f), style = Stroke(5f))
|
||||||
|
rotate(angle, Offset(24f, 24f)) {
|
||||||
|
drawRoundRect(color, Offset(21.5f, 7f), Size(5f, 13f), androidx.compose.ui.geometry.CornerRadius(2.5f))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun MicIcon(color: Color) {
|
||||||
|
val t = rememberInfiniteTransition(label = "mic")
|
||||||
|
val waves = (0..1).map { i ->
|
||||||
|
t.animateFloat(
|
||||||
|
0.25f, 1f,
|
||||||
|
infiniteRepeatable(tween(700, easing = FastOutSlowInEasing), RepeatMode.Reverse,
|
||||||
|
initialStartOffset = androidx.compose.animation.core.StartOffset(i * 250)),
|
||||||
|
label = "mic$i",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox {
|
||||||
|
drawRoundRect(color, Offset(18f, 5f), Size(12f, 22f), androidx.compose.ui.geometry.CornerRadius(6f))
|
||||||
|
val stroke = Stroke(4f, cap = StrokeCap.Round)
|
||||||
|
drawArc(color, 0f, 180f, false, Offset(12f, 14f), Size(24f, 22f), style = stroke)
|
||||||
|
drawLine(color, Offset(24f, 36f), Offset(24f, 42f), 4f, StrokeCap.Round)
|
||||||
|
// Sound waves either side, pulsing as if someone's talking.
|
||||||
|
listOf(1f to 0, -1f to 0, 1f to 1, -1f to 1).forEach { (side, i) ->
|
||||||
|
val r = if (i == 0) 15f else 21f
|
||||||
|
drawArc(color.copy(alpha = waves[i].value), if (side > 0) -35f else 145f, 70f, false,
|
||||||
|
Offset(24f - r, 16f - r), Size(2 * r, 2 * r), style = stroke)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun StarIcon(color: Color) {
|
||||||
|
val turn by rememberInfiniteTransition(label = "star").animateFloat(
|
||||||
|
-8f, 8f, infiniteRepeatable(tween(1600, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "star",
|
||||||
|
)
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox {
|
||||||
|
val star = Path()
|
||||||
|
for (i in 0 until 10) {
|
||||||
|
val r = if (i % 2 == 0) 20f else 8.5f
|
||||||
|
val a = (-90 + i * 36) * PI / 180
|
||||||
|
val x = 24f + (r * cos(a)).toFloat()
|
||||||
|
val y = 25f + (r * sin(a)).toFloat()
|
||||||
|
if (i == 0) star.moveTo(x, y) else star.lineTo(x, y)
|
||||||
|
}
|
||||||
|
star.close()
|
||||||
|
rotate(turn, Offset(24f, 25f)) { drawPath(star, color) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun ShieldIcon(color: Color, tick: Color) {
|
||||||
|
val p = entrance(IntroStep.GRAPHENE, 800, 500)
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox {
|
||||||
|
val shield = Path().apply {
|
||||||
|
moveTo(24f, 4f); lineTo(40f, 10f); lineTo(40f, 22f)
|
||||||
|
cubicTo(40f, 32f, 33f, 40f, 24f, 44f)
|
||||||
|
cubicTo(15f, 40f, 8f, 32f, 8f, 22f)
|
||||||
|
lineTo(8f, 10f); close()
|
||||||
|
}
|
||||||
|
drawPath(shield, color)
|
||||||
|
drawPartialStroke(listOf(Offset(16f, 24f), Offset(22f, 30f), Offset(33f, 18f)), p, tick, 4.5f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun HeartIcon(color: Color) {
|
||||||
|
val beat by rememberInfiniteTransition(label = "beat").animateFloat(
|
||||||
|
1f, 1f,
|
||||||
|
infiniteRepeatable(keyframes {
|
||||||
|
durationMillis = 1400
|
||||||
|
1f at 0; 1.16f at 210; 1f at 420; 1.1f at 630; 1f at 840
|
||||||
|
}),
|
||||||
|
label = "beat",
|
||||||
|
)
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox {
|
||||||
|
scale(beat, beat, Offset(24f, 24f)) {
|
||||||
|
val heart = Path().apply {
|
||||||
|
moveTo(24f, 41f)
|
||||||
|
cubicTo(10f, 31f, 5f, 23f, 5f, 16f)
|
||||||
|
cubicTo(5f, 10f, 10f, 6f, 15f, 6f)
|
||||||
|
cubicTo(19f, 6f, 22f, 8f, 24f, 12f)
|
||||||
|
cubicTo(26f, 8f, 29f, 6f, 33f, 6f)
|
||||||
|
cubicTo(38f, 6f, 43f, 10f, 43f, 16f)
|
||||||
|
cubicTo(43f, 23f, 38f, 31f, 24f, 41f)
|
||||||
|
close()
|
||||||
|
}
|
||||||
|
drawPath(heart, color)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun CheckIcon(color: Color) {
|
||||||
|
val p = entrance(IntroStep.DRIVE, 300, 500)
|
||||||
|
Canvas(Modifier.fillMaxSize()) {
|
||||||
|
inIconBox { drawPartialStroke(listOf(Offset(10f, 26f), Offset(20f, 35f), Offset(38f, 14f)), p, color, 5.5f) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A polyline drawn up to [progress] of its length, so a tick can sweep in. */
|
||||||
|
private fun DrawScope.drawPartialStroke(points: List<Offset>, progress: Float, color: Color, width: Float) {
|
||||||
|
if (progress <= 0f) return
|
||||||
|
val lengths = points.zipWithNext { a, b -> (b - a).getDistance() }
|
||||||
|
var left = lengths.sum() * FastOutSlowInEasing.transform(progress.coerceAtMost(1f))
|
||||||
|
val path = Path().apply { moveTo(points[0].x, points[0].y) }
|
||||||
|
for (i in lengths.indices) {
|
||||||
|
if (left <= 0f) break
|
||||||
|
val f = (left / lengths[i]).coerceAtMost(1f)
|
||||||
|
val to = points[i] + (points[i + 1] - points[i]) * f
|
||||||
|
path.lineTo(to.x, to.y)
|
||||||
|
left -= lengths[i]
|
||||||
|
}
|
||||||
|
drawPath(path, color, style = Stroke(width, cap = StrokeCap.Round, join = StrokeJoin.Round))
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) {
|
||||||
|
Row(modifier, horizontalArrangement = Arrangement.spacedBy(cq(2f))) {
|
||||||
|
repeat(count) { i ->
|
||||||
|
val on = i == current
|
||||||
|
val width by animateDpAsState(if (on) cq(7f) else cq(2f), tween(300), label = "dot")
|
||||||
|
Box(
|
||||||
|
Modifier
|
||||||
|
.size(width, cq(2f))
|
||||||
|
.clip(RoundedCornerShape(50))
|
||||||
|
.background(if (on) c.primary else c.outline.copy(alpha = 0.45f)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "Tap to continue", once the step has finished arriving, breathing softly. */
|
||||||
|
@Composable
|
||||||
|
private fun Hint(c: IntroColors, cqSp: (Float) -> TextUnit, key: Any, modifier: Modifier) {
|
||||||
|
val shown = entrance(key, 1600, 600)
|
||||||
|
val breathe by rememberInfiniteTransition(label = "hint").animateFloat(
|
||||||
|
0.75f, 0.35f, infiniteRepeatable(tween(1200), RepeatMode.Reverse), label = "hint",
|
||||||
|
)
|
||||||
|
Text(
|
||||||
|
"Tap to continue", color = c.onVariant,
|
||||||
|
style = TextStyle(fontSize = cqSp(3.6f), fontWeight = FontWeight.SemiBold),
|
||||||
|
modifier = modifier.graphicsLayer { alpha = if (shown < 1f) shown * 0.75f else breathe },
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -54,6 +54,9 @@ private val recommended = listOf(
|
|||||||
RecommendedApp("ViVi Music", "com.vivi.vivimusic", "https://github.com/vivizzz007/vivi-music", "A YouTube Music player."),
|
RecommendedApp("ViVi Music", "com.vivi.vivimusic", "https://github.com/vivizzz007/vivi-music", "A YouTube Music player."),
|
||||||
RecommendedApp("Metrolist", "com.metrolist.music", "https://github.com/MetrolistGroup/Metrolist", "A YouTube Music player."),
|
RecommendedApp("Metrolist", "com.metrolist.music", "https://github.com/MetrolistGroup/Metrolist", "A YouTube Music player."),
|
||||||
),
|
),
|
||||||
|
"Podcasts" to listOf(
|
||||||
|
RecommendedApp("AntennaPod", "de.danoeh.antennapod", "https://github.com/AntennaPod/AntennaPod", "Podcasts, chapters and all."),
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
/** Opened from Settings: the same list the setup guide ends with. */
|
/** Opened from Settings: the same list the setup guide ends with. */
|
||||||
|
|||||||
@@ -73,6 +73,7 @@ fun SettingsScreen(
|
|||||||
onOpenRecommendedApps: () -> Unit,
|
onOpenRecommendedApps: () -> Unit,
|
||||||
onOpenWhatsNew: () -> Unit,
|
onOpenWhatsNew: () -> Unit,
|
||||||
onReplayTutorial: () -> Unit,
|
onReplayTutorial: () -> Unit,
|
||||||
|
onReplayIntro: () -> Unit,
|
||||||
atUsageNotes: Boolean = false,
|
atUsageNotes: Boolean = false,
|
||||||
) {
|
) {
|
||||||
val scroll = rememberScrollState()
|
val scroll = rememberScrollState()
|
||||||
@@ -105,9 +106,10 @@ fun SettingsScreen(
|
|||||||
verticalArrangement = Arrangement.spacedBy(24.dp),
|
verticalArrangement = Arrangement.spacedBy(24.dp),
|
||||||
) {
|
) {
|
||||||
Spacer(Modifier.height(0.dp))
|
Spacer(Modifier.height(0.dp))
|
||||||
AboutSection(onOpenWhatsNew, onReplayTutorial)
|
AboutSection(onOpenWhatsNew, onReplayTutorial, onReplayIntro)
|
||||||
ConnectionSection()
|
ConnectionSection()
|
||||||
CarAudioSection()
|
CarAudioSection()
|
||||||
|
VoiceModelSection()
|
||||||
if (SHOW_RECOMMENDED_APPS) AppsSection(onOpenRecommendedApps)
|
if (SHOW_RECOMMENDED_APPS) AppsSection(onOpenRecommendedApps)
|
||||||
CertificateSection()
|
CertificateSection()
|
||||||
SettingsSection("Usage notes", header = USAGE_STATS_INTRO) { UsageStatsRows() }
|
SettingsSection("Usage notes", header = USAGE_STATS_INTRO) { UsageStatsRows() }
|
||||||
@@ -125,7 +127,7 @@ private fun Arrow() = Icon(
|
|||||||
|
|
||||||
/** What's new, updates and the setup guide, one tap from the top. */
|
/** What's new, updates and the setup guide, one tap from the top. */
|
||||||
@Composable
|
@Composable
|
||||||
private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit) {
|
private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit, onReplayIntro: () -> Unit) {
|
||||||
SettingsSection {
|
SettingsSection {
|
||||||
SettingsRow(
|
SettingsRow(
|
||||||
"What's new",
|
"What's new",
|
||||||
@@ -148,6 +150,16 @@ private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Uni
|
|||||||
icon = Icons.Filled.Refresh,
|
icon = Icons.Filled.Refresh,
|
||||||
trailing = { Arrow() },
|
trailing = { Arrow() },
|
||||||
)
|
)
|
||||||
|
SettingsDivider()
|
||||||
|
SettingsRow(
|
||||||
|
"Replay the ${Intro.release} intro",
|
||||||
|
Modifier.clickable(onClick = onReplayIntro),
|
||||||
|
subtitle = "What's in this release, in a few taps",
|
||||||
|
icon = Icons.Filled.Star,
|
||||||
|
trailing = { Arrow() },
|
||||||
|
)
|
||||||
|
SettingsDivider()
|
||||||
|
AboutLinkRows()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,161 @@
|
|||||||
|
package app.seb3thehacker.gearslip.ui
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import android.content.Intent
|
||||||
|
import android.net.Uri
|
||||||
|
import androidx.compose.foundation.clickable
|
||||||
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
|
import androidx.compose.foundation.layout.Column
|
||||||
|
import androidx.compose.foundation.layout.PaddingValues
|
||||||
|
import androidx.compose.foundation.layout.Row
|
||||||
|
import androidx.compose.foundation.layout.Spacer
|
||||||
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
|
import androidx.compose.foundation.layout.padding
|
||||||
|
import androidx.compose.material.icons.Icons
|
||||||
|
import androidx.compose.material.icons.automirrored.filled.KeyboardArrowRight
|
||||||
|
import androidx.compose.material.icons.automirrored.filled.Send
|
||||||
|
import androidx.compose.material.icons.filled.Build
|
||||||
|
import androidx.compose.material.icons.filled.Favorite
|
||||||
|
import androidx.compose.material3.Button
|
||||||
|
import androidx.compose.material3.Icon
|
||||||
|
import androidx.compose.material3.MaterialTheme
|
||||||
|
import androidx.compose.material3.OutlinedButton
|
||||||
|
import androidx.compose.material3.Surface
|
||||||
|
import androidx.compose.material3.Text
|
||||||
|
import androidx.compose.material3.TextButton
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.runtime.mutableStateOf
|
||||||
|
import androidx.compose.runtime.remember
|
||||||
|
import androidx.compose.runtime.setValue
|
||||||
|
import androidx.compose.ui.Alignment
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.platform.LocalContext
|
||||||
|
import androidx.compose.ui.text.font.FontWeight
|
||||||
|
import androidx.compose.ui.unit.dp
|
||||||
|
|
||||||
|
/** Where Gearslip lives outside the app: donations, the chat room, and the code. */
|
||||||
|
internal object GearslipLinks {
|
||||||
|
const val SPONSORS = "https://github.com/sponsors/Seb3thehacker"
|
||||||
|
const val LIBERAPAY = "https://liberapay.com/Seb3thehacker"
|
||||||
|
const val MATRIX = "https://matrix.to/#/!mf0_mgBd5KqKpc9FfX_wHp1jFCcwAtPrndIn_oGIyMk?via=matrix.org"
|
||||||
|
const val SOURCE = "https://github.com/Seb3thehacker/gearslip"
|
||||||
|
|
||||||
|
fun open(context: Context, url: String) {
|
||||||
|
runCatching {
|
||||||
|
context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(url)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** "Hide for now" on the home card puts it away for this long; the About page keeps the links. */
|
||||||
|
private const val HIDE_MS = 30L * 24 * 60 * 60 * 1000
|
||||||
|
private const val PREFS = "support"
|
||||||
|
private const val KEY_HIDDEN_UNTIL = "hidden_until"
|
||||||
|
|
||||||
|
private fun hiddenUntil(context: Context) =
|
||||||
|
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getLong(KEY_HIDDEN_UNTIL, 0L)
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The donation ask on Home: always in view, never in the way. It sits below the status and update
|
||||||
|
* cards, and "Hide for now" puts it away for a month.
|
||||||
|
*/
|
||||||
|
@Composable
|
||||||
|
internal fun SupportCard(modifier: Modifier = Modifier) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
var hidden by remember { mutableStateOf(System.currentTimeMillis() < hiddenUntil(context)) }
|
||||||
|
if (hidden) return
|
||||||
|
val scheme = MaterialTheme.colorScheme
|
||||||
|
Surface(
|
||||||
|
modifier,
|
||||||
|
color = scheme.tertiaryContainer,
|
||||||
|
contentColor = scheme.onTertiaryContainer,
|
||||||
|
shape = MaterialTheme.shapes.extraLarge,
|
||||||
|
) {
|
||||||
|
Column(Modifier.fillMaxWidth().padding(20.dp)) {
|
||||||
|
Row(verticalAlignment = Alignment.CenterVertically) {
|
||||||
|
Icon(Icons.Filled.Favorite, contentDescription = null)
|
||||||
|
Text(
|
||||||
|
"Support Gearslip",
|
||||||
|
style = MaterialTheme.typography.titleLarge,
|
||||||
|
fontWeight = FontWeight.SemiBold,
|
||||||
|
modifier = Modifier.padding(start = 12.dp),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Text(
|
||||||
|
"Gearslip is free, with no ads and no tracking. If it gets you on the road, a " +
|
||||||
|
"donation keeps it going.",
|
||||||
|
style = MaterialTheme.typography.bodyMedium,
|
||||||
|
modifier = Modifier.padding(top = 8.dp),
|
||||||
|
)
|
||||||
|
Row(
|
||||||
|
Modifier.fillMaxWidth().padding(top = 12.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
horizontalArrangement = Arrangement.spacedBy(8.dp),
|
||||||
|
) {
|
||||||
|
TextButton(
|
||||||
|
onClick = {
|
||||||
|
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit()
|
||||||
|
.putLong(KEY_HIDDEN_UNTIL, System.currentTimeMillis() + HIDE_MS).apply()
|
||||||
|
hidden = true
|
||||||
|
},
|
||||||
|
contentPadding = PaddingValues(horizontal = 0.dp),
|
||||||
|
) { Text("Hide for now") }
|
||||||
|
Spacer(Modifier.weight(1f))
|
||||||
|
OutlinedButton(onClick = { GearslipLinks.open(context, GearslipLinks.LIBERAPAY) }) { Text("Liberapay") }
|
||||||
|
Button(onClick = { GearslipLinks.open(context, GearslipLinks.SPONSORS) }) { Text("GitHub") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The About section's links: donating, the chat room, and the source code. */
|
||||||
|
@Composable
|
||||||
|
internal fun AboutLinkRows() {
|
||||||
|
val context = LocalContext.current
|
||||||
|
val arrow = @Composable {
|
||||||
|
Icon(
|
||||||
|
Icons.AutoMirrored.Filled.KeyboardArrowRight,
|
||||||
|
contentDescription = null,
|
||||||
|
tint = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
SettingsRow(
|
||||||
|
"Donate on GitHub Sponsors",
|
||||||
|
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.SPONSORS) },
|
||||||
|
subtitle = "Give once or monthly, through GitHub",
|
||||||
|
icon = Icons.Filled.Favorite,
|
||||||
|
trailing = arrow,
|
||||||
|
)
|
||||||
|
SettingsDivider()
|
||||||
|
SettingsRow(
|
||||||
|
"Donate on Liberapay",
|
||||||
|
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.LIBERAPAY) },
|
||||||
|
subtitle = "Give weekly or monthly, through a nonprofit",
|
||||||
|
icon = Icons.Filled.Favorite,
|
||||||
|
trailing = arrow,
|
||||||
|
)
|
||||||
|
SettingsDivider()
|
||||||
|
SettingsRow(
|
||||||
|
"Join the discussion on Matrix",
|
||||||
|
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.MATRIX) },
|
||||||
|
subtitle = "Questions, ideas, or a car to report",
|
||||||
|
icon = Icons.AutoMirrored.Filled.Send,
|
||||||
|
trailing = arrow,
|
||||||
|
)
|
||||||
|
SettingsDivider()
|
||||||
|
SettingsRow(
|
||||||
|
"Source code",
|
||||||
|
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.SOURCE) },
|
||||||
|
subtitle = "Read or change Gearslip's code under the AGPL",
|
||||||
|
icon = Icons.Filled.Build,
|
||||||
|
trailing = arrow,
|
||||||
|
)
|
||||||
|
SettingsDivider()
|
||||||
|
SettingsRow(
|
||||||
|
"Voice credits",
|
||||||
|
subtitle = "Speech recognition by sherpa-onnx (Apache 2.0), with Kroko ASR by Banafo " +
|
||||||
|
"(CC BY-SA 4.0), and Parakeet and Canary by NVIDIA (CC BY 4.0)",
|
||||||
|
icon = Icons.Filled.Favorite,
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package app.seb3thehacker.gearslip.ui
|
||||||
|
|
||||||
|
import androidx.compose.foundation.layout.Arrangement
|
||||||
|
import androidx.compose.foundation.layout.Column
|
||||||
|
import androidx.compose.foundation.layout.Row
|
||||||
|
import androidx.compose.foundation.layout.fillMaxWidth
|
||||||
|
import androidx.compose.foundation.layout.padding
|
||||||
|
import androidx.compose.foundation.selection.selectable
|
||||||
|
import androidx.compose.material3.LinearProgressIndicator
|
||||||
|
import androidx.compose.material3.MaterialTheme
|
||||||
|
import androidx.compose.material3.OutlinedButton
|
||||||
|
import androidx.compose.material3.RadioButton
|
||||||
|
import androidx.compose.material3.Text
|
||||||
|
import androidx.compose.material3.TextButton
|
||||||
|
import androidx.compose.runtime.Composable
|
||||||
|
import androidx.compose.runtime.collectAsState
|
||||||
|
import androidx.compose.runtime.getValue
|
||||||
|
import androidx.compose.runtime.remember
|
||||||
|
import androidx.compose.ui.Alignment
|
||||||
|
import androidx.compose.ui.Modifier
|
||||||
|
import androidx.compose.ui.platform.LocalContext
|
||||||
|
import androidx.compose.ui.semantics.Role
|
||||||
|
import androidx.compose.ui.unit.dp
|
||||||
|
import app.seb3thehacker.gearslip.speech.SpeechModels
|
||||||
|
import app.seb3thehacker.gearslip.speech.SpeechModels.Model
|
||||||
|
import app.seb3thehacker.gearslip.speech.SpeechModels.State
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which speech model the voice assistant listens with, and downloading it. Each is fetched once
|
||||||
|
* from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening.
|
||||||
|
*/
|
||||||
|
@Composable
|
||||||
|
internal fun VoiceModelSection() {
|
||||||
|
val context = LocalContext.current
|
||||||
|
remember { SpeechModels.init(context) }
|
||||||
|
val selected by SpeechModels.selected.collectAsState()
|
||||||
|
val final by SpeechModels.final.collectAsState()
|
||||||
|
SettingsSection(title = "Voice: live words") {
|
||||||
|
Text(
|
||||||
|
"The voice assistant turns speech into text on this phone, and nothing you say leaves it. " +
|
||||||
|
"It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " +
|
||||||
|
"This one writes your words as you speak.",
|
||||||
|
style = MaterialTheme.typography.bodyMedium,
|
||||||
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
|
||||||
|
)
|
||||||
|
Model.entries.filter { it.role == SpeechModels.Role.LIVE }.forEachIndexed { i, model ->
|
||||||
|
if (i > 0) SettingsDivider()
|
||||||
|
ModelRow(model, model == selected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
SettingsSection(title = "Voice: final words") {
|
||||||
|
Text(
|
||||||
|
"When you stop talking, this model reads the whole phrase again. It catches names and " +
|
||||||
|
"places the live model misses.",
|
||||||
|
style = MaterialTheme.typography.bodyMedium,
|
||||||
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
|
||||||
|
)
|
||||||
|
val context = LocalContext.current
|
||||||
|
Row(
|
||||||
|
Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.selectable(final == null, role = Role.RadioButton) { SpeechModels.selectFinal(context, null) }
|
||||||
|
.padding(horizontal = 8.dp, vertical = 8.dp),
|
||||||
|
verticalAlignment = Alignment.CenterVertically,
|
||||||
|
) {
|
||||||
|
RadioButton(final == null, onClick = null, modifier = Modifier.padding(8.dp))
|
||||||
|
Column {
|
||||||
|
Text("None", style = MaterialTheme.typography.titleMedium)
|
||||||
|
Text(
|
||||||
|
"Keep the live words. Fastest, least accurate",
|
||||||
|
style = MaterialTheme.typography.bodySmall,
|
||||||
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Model.entries.filter { it.role == SpeechModels.Role.FINAL }.forEach { model ->
|
||||||
|
SettingsDivider()
|
||||||
|
ModelRow(model, model == final)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Composable
|
||||||
|
private fun ModelRow(model: Model, chosen: Boolean) {
|
||||||
|
val context = LocalContext.current
|
||||||
|
val state by SpeechModels.state(model).collectAsState()
|
||||||
|
Column(
|
||||||
|
Modifier
|
||||||
|
.fillMaxWidth()
|
||||||
|
.selectable(chosen, role = Role.RadioButton) { SpeechModels.select(context, model) }
|
||||||
|
.padding(horizontal = 8.dp, vertical = 8.dp),
|
||||||
|
) {
|
||||||
|
Row(verticalAlignment = Alignment.CenterVertically) {
|
||||||
|
RadioButton(chosen, onClick = null, modifier = Modifier.padding(8.dp))
|
||||||
|
Column(Modifier.weight(1f)) {
|
||||||
|
Text("${model.label} · ${model.downloadMb} MB", style = MaterialTheme.typography.titleMedium)
|
||||||
|
Text(
|
||||||
|
when (val s = state) {
|
||||||
|
State.Ready -> "Downloaded. ${model.detail}"
|
||||||
|
is State.Downloading -> "Downloading, ${s.percent}%"
|
||||||
|
is State.Failed -> "Download failed: ${s.reason}"
|
||||||
|
State.Missing -> model.detail
|
||||||
|
},
|
||||||
|
style = MaterialTheme.typography.bodySmall,
|
||||||
|
color = MaterialTheme.colorScheme.onSurfaceVariant,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Row(horizontalArrangement = Arrangement.spacedBy(4.dp)) {
|
||||||
|
when (state) {
|
||||||
|
State.Ready -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Delete") }
|
||||||
|
is State.Downloading -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Cancel") }
|
||||||
|
else -> OutlinedButton(onClick = { SpeechModels.download(context, model) }) { Text("Download") }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(state as? State.Downloading)?.let {
|
||||||
|
LinearProgressIndicator(
|
||||||
|
progress = { it.percent / 100f },
|
||||||
|
modifier = Modifier.fillMaxWidth().padding(start = 48.dp, end = 8.dp, top = 4.dp),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
data class FeatureConfig(
|
||||||
|
var sampleRate: Int = 16000,
|
||||||
|
var featureDim: Int = 80,
|
||||||
|
var dither: Float = 0.0f
|
||||||
|
)
|
||||||
|
|
||||||
|
fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig {
|
||||||
|
return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim)
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
data class HomophoneReplacerConfig(
|
||||||
|
var dictDir: String = "", // unused
|
||||||
|
var lexicon: String = "",
|
||||||
|
var ruleFsts: String = "",
|
||||||
|
)
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
import android.content.res.AssetManager
|
||||||
|
|
||||||
|
data class OfflineRecognizerResult(
|
||||||
|
val text: String,
|
||||||
|
val tokens: Array<String>,
|
||||||
|
val timestamps: FloatArray,
|
||||||
|
val lang: String,
|
||||||
|
val emotion: String,
|
||||||
|
val event: String,
|
||||||
|
|
||||||
|
// valid only for TDT models
|
||||||
|
val durations: FloatArray,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineTransducerModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
var joiner: String = "",
|
||||||
|
var qnnConfig: QnnConfig = QnnConfig(),
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineParaformerModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
var qnnConfig: QnnConfig = QnnConfig(),
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineNemoEncDecCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineDolphinModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineZipformerCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
var qnnConfig: QnnConfig = QnnConfig(),
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineWenetCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineOmnilingualAsrCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineMedAsrCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineFireRedAsrCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineFunAsrNanoModelConfig(
|
||||||
|
var encoderAdaptor: String = "",
|
||||||
|
var llm: String = "",
|
||||||
|
var embedding: String = "",
|
||||||
|
var tokenizer: String = "",
|
||||||
|
var systemPrompt: String = "You are a helpful assistant.",
|
||||||
|
var userPrompt: String = "语音转写:",
|
||||||
|
var maxNewTokens: Int = 512,
|
||||||
|
var temperature: Float = 1e-6f,
|
||||||
|
var topP: Float = 0.8f,
|
||||||
|
var seed: Int = 42,
|
||||||
|
var language: String = "",
|
||||||
|
var itn: Boolean = true,
|
||||||
|
var hotwords: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineQwen3AsrModelConfig(
|
||||||
|
var convFrontend: String = "",
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
var tokenizer: String = "",
|
||||||
|
var maxTotalLen: Int = 512,
|
||||||
|
var maxNewTokens: Int = 128,
|
||||||
|
var temperature: Float = 1e-6f,
|
||||||
|
var topP: Float = 0.8f,
|
||||||
|
var seed: Int = 42,
|
||||||
|
var hotwords: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineWhisperModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
var language: String = "en", // Used with multilingual model
|
||||||
|
var task: String = "transcribe", // transcribe or translate
|
||||||
|
var tailPaddings: Int = 1000, // Padding added at the end of the samples
|
||||||
|
var enableTokenTimestamps: Boolean = false,
|
||||||
|
var enableSegmentTimestamps: Boolean = false,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineCanaryModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
var srcLang: String = "en",
|
||||||
|
var tgtLang: String = "en",
|
||||||
|
var usePnc: Boolean = true,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineCohereTranscribeModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
var language: String = "",
|
||||||
|
var usePunct: Boolean = true,
|
||||||
|
var useItn: Boolean = true,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineFireRedAsrModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
// For moonshine v1, you need four models.
|
||||||
|
// For moonshine v2, you need two models.
|
||||||
|
// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder
|
||||||
|
// - v2: encoder, mergedDecoder
|
||||||
|
data class OfflineMoonshineModelConfig(
|
||||||
|
var preprocessor: String = "",
|
||||||
|
var encoder: String = "",
|
||||||
|
var uncachedDecoder: String = "",
|
||||||
|
var cachedDecoder: String = "",
|
||||||
|
var mergedDecoder: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineSenseVoiceModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
var language: String = "",
|
||||||
|
var useInverseTextNormalization: Boolean = true,
|
||||||
|
var qnnConfig: QnnConfig = QnnConfig(),
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineModelConfig(
|
||||||
|
var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
|
||||||
|
var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
|
||||||
|
var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
|
||||||
|
var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(),
|
||||||
|
var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(),
|
||||||
|
var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(),
|
||||||
|
var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
|
||||||
|
var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
|
||||||
|
var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
|
||||||
|
var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(),
|
||||||
|
var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(),
|
||||||
|
var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(),
|
||||||
|
var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(),
|
||||||
|
var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(),
|
||||||
|
var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(),
|
||||||
|
var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
|
||||||
|
var cohereTranscribe: OfflineCohereTranscribeModelConfig =
|
||||||
|
OfflineCohereTranscribeModelConfig(),
|
||||||
|
var teleSpeech: String = "",
|
||||||
|
var numThreads: Int = 1,
|
||||||
|
var debug: Boolean = false,
|
||||||
|
var provider: String = "cpu",
|
||||||
|
var modelType: String = "",
|
||||||
|
var tokens: String = "",
|
||||||
|
var modelingUnit: String = "",
|
||||||
|
var bpeVocab: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OfflineRecognizerConfig(
|
||||||
|
var featConfig: FeatureConfig = FeatureConfig(),
|
||||||
|
var modelConfig: OfflineModelConfig = OfflineModelConfig(),
|
||||||
|
// var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it
|
||||||
|
var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
|
||||||
|
var decodingMethod: String = "greedy_search",
|
||||||
|
var maxActivePaths: Int = 4,
|
||||||
|
var hotwordsFile: String = "",
|
||||||
|
var hotwordsScore: Float = 1.5f,
|
||||||
|
var ruleFsts: String = "",
|
||||||
|
var ruleFars: String = "",
|
||||||
|
var blankPenalty: Float = 0.0f,
|
||||||
|
)
|
||||||
|
|
||||||
|
class OfflineRecognizer(
|
||||||
|
assetManager: AssetManager? = null,
|
||||||
|
val config: OfflineRecognizerConfig,
|
||||||
|
) {
|
||||||
|
private var ptr: Long
|
||||||
|
|
||||||
|
init {
|
||||||
|
ptr = if (assetManager != null) {
|
||||||
|
newFromAsset(assetManager, config)
|
||||||
|
} else {
|
||||||
|
newFromFile(config)
|
||||||
|
}
|
||||||
|
require(ptr != 0L) {
|
||||||
|
"Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun finalize() {
|
||||||
|
if (ptr != 0L) {
|
||||||
|
delete(ptr)
|
||||||
|
ptr = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun release() = finalize()
|
||||||
|
|
||||||
|
fun createStream(): OfflineStream {
|
||||||
|
val p = createStream(ptr)
|
||||||
|
return OfflineStream(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun createStream(hotwords: String): OfflineStream {
|
||||||
|
val p = createStreamWithHotwords(ptr, hotwords)
|
||||||
|
return OfflineStream(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun getResult(stream: OfflineStream): OfflineRecognizerResult {
|
||||||
|
return getResult(stream.ptr)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
|
||||||
|
|
||||||
|
fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config)
|
||||||
|
|
||||||
|
private external fun delete(ptr: Long)
|
||||||
|
|
||||||
|
private external fun createStream(ptr: Long): Long
|
||||||
|
|
||||||
|
private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long
|
||||||
|
|
||||||
|
private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig)
|
||||||
|
|
||||||
|
private external fun newFromAsset(
|
||||||
|
assetManager: AssetManager,
|
||||||
|
config: OfflineRecognizerConfig,
|
||||||
|
): Long
|
||||||
|
|
||||||
|
private external fun newFromFile(
|
||||||
|
config: OfflineRecognizerConfig,
|
||||||
|
): Long
|
||||||
|
|
||||||
|
private external fun decode(ptr: Long, streamPtr: Long)
|
||||||
|
|
||||||
|
private external fun getResult(streamPtr: Long): OfflineRecognizerResult
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
init {
|
||||||
|
System.loadLibrary("sherpa-onnx-jni")
|
||||||
|
}
|
||||||
|
|
||||||
|
@JvmStatic
|
||||||
|
external fun prependAdspLibraryPath(newPath: String) // for qnn
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
class OfflineStream(var ptr: Long) {
|
||||||
|
init {
|
||||||
|
require(ptr != 0L) { "Failed to create native OfflineStream" }
|
||||||
|
}
|
||||||
|
|
||||||
|
fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
|
||||||
|
acceptWaveform(ptr, samples, sampleRate)
|
||||||
|
|
||||||
|
fun setOption(key: String, value: String) = setOption(ptr, key, value)
|
||||||
|
|
||||||
|
fun getOption(key: String): String = getOption(ptr, key)
|
||||||
|
|
||||||
|
protected fun finalize() {
|
||||||
|
if (ptr != 0L) {
|
||||||
|
delete(ptr)
|
||||||
|
ptr = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun release() = finalize()
|
||||||
|
|
||||||
|
fun use(block: (OfflineStream) -> Unit) {
|
||||||
|
try {
|
||||||
|
block(this)
|
||||||
|
} finally {
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
|
||||||
|
private external fun setOption(ptr: Long, key: String, value: String)
|
||||||
|
private external fun getOption(ptr: Long, key: String): String
|
||||||
|
private external fun delete(ptr: Long)
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
init {
|
||||||
|
System.loadLibrary("sherpa-onnx-jni")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
import android.content.res.AssetManager
|
||||||
|
|
||||||
|
data class EndpointRule(
|
||||||
|
var mustContainNonSilence: Boolean,
|
||||||
|
var minTrailingSilence: Float,
|
||||||
|
var minUtteranceLength: Float,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class EndpointConfig(
|
||||||
|
var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f),
|
||||||
|
var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f),
|
||||||
|
var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f)
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineTransducerModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
var joiner: String = "",
|
||||||
|
var qnnConfig: QnnConfig = QnnConfig(),
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineParaformerModelConfig(
|
||||||
|
var encoder: String = "",
|
||||||
|
var decoder: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineZipformer2CtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineNeMoCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineToneCtcModelConfig(
|
||||||
|
var model: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineModelConfig(
|
||||||
|
var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
|
||||||
|
var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
|
||||||
|
var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
|
||||||
|
var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(),
|
||||||
|
var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(),
|
||||||
|
var tokens: String = "",
|
||||||
|
var numThreads: Int = 1,
|
||||||
|
var debug: Boolean = false,
|
||||||
|
var provider: String = "cpu",
|
||||||
|
var modelType: String = "",
|
||||||
|
var modelingUnit: String = "",
|
||||||
|
var bpeVocab: String = "",
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineLMConfig(
|
||||||
|
var model: String = "",
|
||||||
|
var scale: Float = 0.5f,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineCtcFstDecoderConfig(
|
||||||
|
var graph: String = "",
|
||||||
|
var maxActive: Int = 3000,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineRecognizerConfig(
|
||||||
|
var featConfig: FeatureConfig = FeatureConfig(),
|
||||||
|
var modelConfig: OnlineModelConfig = OnlineModelConfig(),
|
||||||
|
var lmConfig: OnlineLMConfig = OnlineLMConfig(),
|
||||||
|
var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(),
|
||||||
|
var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
|
||||||
|
var endpointConfig: EndpointConfig = EndpointConfig(),
|
||||||
|
var enableEndpoint: Boolean = true,
|
||||||
|
var decodingMethod: String = "greedy_search",
|
||||||
|
var maxActivePaths: Int = 4,
|
||||||
|
var hotwordsFile: String = "",
|
||||||
|
var hotwordsScore: Float = 1.5f,
|
||||||
|
var ruleFsts: String = "",
|
||||||
|
var ruleFars: String = "",
|
||||||
|
var blankPenalty: Float = 0.0f,
|
||||||
|
)
|
||||||
|
|
||||||
|
data class OnlineRecognizerResult(
|
||||||
|
val text: String,
|
||||||
|
val tokens: Array<String>,
|
||||||
|
val timestamps: FloatArray,
|
||||||
|
val ysProbs: FloatArray,
|
||||||
|
// TODO(fangjun): Add more fields
|
||||||
|
)
|
||||||
|
|
||||||
|
class OnlineRecognizer(
|
||||||
|
assetManager: AssetManager? = null,
|
||||||
|
val config: OnlineRecognizerConfig,
|
||||||
|
) {
|
||||||
|
private var ptr: Long
|
||||||
|
|
||||||
|
init {
|
||||||
|
ptr = if (assetManager != null) {
|
||||||
|
newFromAsset(assetManager, config)
|
||||||
|
} else {
|
||||||
|
newFromFile(config)
|
||||||
|
}
|
||||||
|
require(ptr != 0L) {
|
||||||
|
"Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected fun finalize() {
|
||||||
|
if (ptr != 0L) {
|
||||||
|
delete(ptr)
|
||||||
|
ptr = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun release() = finalize()
|
||||||
|
|
||||||
|
fun createStream(hotwords: String = ""): OnlineStream {
|
||||||
|
val p = createStream(ptr, hotwords)
|
||||||
|
return OnlineStream(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
fun reset(stream: OnlineStream) = reset(ptr, stream.ptr)
|
||||||
|
fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
|
||||||
|
fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr)
|
||||||
|
fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
|
||||||
|
fun getResult(stream: OnlineStream): OnlineRecognizerResult {
|
||||||
|
return getResult(ptr, stream.ptr)
|
||||||
|
}
|
||||||
|
|
||||||
|
private external fun delete(ptr: Long)
|
||||||
|
|
||||||
|
private external fun newFromAsset(
|
||||||
|
assetManager: AssetManager,
|
||||||
|
config: OnlineRecognizerConfig,
|
||||||
|
): Long
|
||||||
|
|
||||||
|
private external fun newFromFile(
|
||||||
|
config: OnlineRecognizerConfig,
|
||||||
|
): Long
|
||||||
|
|
||||||
|
private external fun createStream(ptr: Long, hotwords: String): Long
|
||||||
|
private external fun reset(ptr: Long, streamPtr: Long)
|
||||||
|
private external fun decode(ptr: Long, streamPtr: Long)
|
||||||
|
private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean
|
||||||
|
private external fun isReady(ptr: Long, streamPtr: Long): Boolean
|
||||||
|
private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
init {
|
||||||
|
System.loadLibrary("sherpa-onnx-jni")
|
||||||
|
}
|
||||||
|
|
||||||
|
@JvmStatic
|
||||||
|
external fun prependAdspLibraryPath(newPath: String) // for qnn
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,47 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
class OnlineStream(var ptr: Long = 0) {
|
||||||
|
init {
|
||||||
|
require(ptr != 0L) { "Failed to create native OnlineStream" }
|
||||||
|
}
|
||||||
|
|
||||||
|
fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
|
||||||
|
acceptWaveform(ptr, samples, sampleRate)
|
||||||
|
|
||||||
|
fun inputFinished() = inputFinished(ptr)
|
||||||
|
|
||||||
|
fun setOption(key: String, value: String) = setOption(ptr, key, value)
|
||||||
|
|
||||||
|
fun getOption(key: String): String = getOption(ptr, key)
|
||||||
|
|
||||||
|
protected fun finalize() {
|
||||||
|
if (ptr != 0L) {
|
||||||
|
delete(ptr)
|
||||||
|
ptr = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun release() = finalize()
|
||||||
|
|
||||||
|
fun use(block: (OnlineStream) -> Unit) {
|
||||||
|
try {
|
||||||
|
block(this)
|
||||||
|
} finally {
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
|
||||||
|
private external fun inputFinished(ptr: Long)
|
||||||
|
private external fun setOption(ptr: Long, key: String, value: String)
|
||||||
|
private external fun getOption(ptr: Long, key: String): String
|
||||||
|
private external fun delete(ptr: Long)
|
||||||
|
|
||||||
|
|
||||||
|
companion object {
|
||||||
|
init {
|
||||||
|
System.loadLibrary("sherpa-onnx-jni")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
|
||||||
|
package com.k2fsa.sherpa.onnx
|
||||||
|
|
||||||
|
data class QnnConfig(
|
||||||
|
var backendLib: String = "",
|
||||||
|
var contextBinary: String = "",
|
||||||
|
var systemLib: String = "",
|
||||||
|
)
|
||||||
BIN
Binary file not shown.
@@ -9,12 +9,10 @@
|
|||||||
android:fillColor="#FF000000"
|
android:fillColor="#FF000000"
|
||||||
android:fillAlpha="0.35"/>
|
android:fillAlpha="0.35"/>
|
||||||
<path
|
<path
|
||||||
android:pathData="M 54.00,36.18 L 58.64,30.70 L 67.20,34.24 L 66.60,41.40 L 71.82,54.00 L 77.30,58.64 L 73.76,67.20 L 66.60,66.60 L 66.60,66.60 L 67.20,73.76 L 58.64,77.30 L 54.00,71.82 L 54.00,71.82 L 49.36,77.30 L 40.80,73.76 L 41.40,66.60 L 41.40,66.60 L 34.24,67.20 L 30.70,58.64 L 36.18,54.00 L 36.18,54.00 L 30.70,49.36 L 34.24,40.80 L 41.40,41.40 L 41.40,41.40 L 40.80,34.24 L 49.36,30.70 L 54.00,36.18 Z"
|
android:pathData="M 54.00,36.18 L 58.64,30.70 L 67.20,34.24 L 66.60,41.40 L 71.82,54.00 L 77.30,58.64 L 73.76,67.20 L 66.60,66.60 L 66.60,66.60 L 67.20,73.76 L 58.64,77.30 L 54.00,71.82 L 54.00,71.82 L 49.36,77.30 L 40.80,73.76 L 41.40,66.60 L 41.40,66.60 L 34.24,67.20 L 30.70,58.64 L 36.18,54.00 L 36.18,54.00 L 30.70,49.36 L 34.24,40.80 L 41.40,41.40 L 41.40,41.40 L 40.80,34.24 L 49.36,30.70 L 54.00,36.18 Z M54,46.08 A7.92,7.92 0 1,1 53.99,46.08 Z"
|
||||||
|
android:fillType="evenOdd"
|
||||||
android:fillColor="#FF000000"/>
|
android:fillColor="#FF000000"/>
|
||||||
<path
|
<path
|
||||||
android:pathData="M 78.61,40.70 L 85.31,43.29 L 84.74,52.55 L 77.78,54.31 Z"
|
android:pathData="M 78.61,40.70 L 85.31,43.29 L 84.74,52.55 L 77.78,54.31 Z"
|
||||||
android:fillColor="#FF000000"/>
|
android:fillColor="#FF000000"/>
|
||||||
<path
|
|
||||||
android:pathData="M54,46.08 A7.92,7.92 0 1,1 53.99,46.08 Z"
|
|
||||||
android:fillColor="#00000000"/>
|
|
||||||
</vector>
|
</vector>
|
||||||
@@ -14,6 +14,7 @@ material = "1.14.0"
|
|||||||
constraintlayout = "2.2.1"
|
constraintlayout = "2.2.1"
|
||||||
zxing = "3.5.4"
|
zxing = "3.5.4"
|
||||||
okhttp = "5.4.0"
|
okhttp = "5.4.0"
|
||||||
|
commonsCompress = "1.28.0"
|
||||||
kotlinxSerializationJson = "1.11.0"
|
kotlinxSerializationJson = "1.11.0"
|
||||||
ossLicenses = "17.5.1"
|
ossLicenses = "17.5.1"
|
||||||
webkit = "1.16.0"
|
webkit = "1.16.0"
|
||||||
@@ -49,6 +50,7 @@ androidx-constraintlayout = { group = "androidx.constraintlayout", name = "const
|
|||||||
google-material = { group = "com.google.android.material", name = "material", version.ref = "material" }
|
google-material = { group = "com.google.android.material", name = "material", version.ref = "material" }
|
||||||
zxing-core = { group = "com.google.zxing", name = "core", version.ref = "zxing" }
|
zxing-core = { group = "com.google.zxing", name = "core", version.ref = "zxing" }
|
||||||
okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
|
okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
|
||||||
|
commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commonsCompress" }
|
||||||
kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
|
kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
|
||||||
google-oss-licenses = { group = "com.google.android.gms", name = "play-services-oss-licenses", version.ref = "ossLicenses" }
|
google-oss-licenses = { group = "com.google.android.gms", name = "play-services-oss-licenses", version.ref = "ossLicenses" }
|
||||||
androidx-webkit = { group = "androidx.webkit", name = "webkit", version.ref = "webkit" }
|
androidx-webkit = { group = "androidx.webkit", name = "webkit", version.ref = "webkit" }
|
||||||
|
|||||||
Reference in new issue
Block a user