Merge branch 'experimental' into wireless

# Conflicts:
#	gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt
This commit is contained in:
Seb3thehacker committed 2026-10-12 22:09:43 -07:00
commit 9fe463c4e7
50 files changed
+2801 -534

No files matched your search

+15
View File
@@ -0,0 +1,15 @@
# Weekly pull requests for newer Gradle libraries and GitHub Actions, aimed at experimental,
# where unreleased code lands. Security fixes come separately, as soon as an alert opens.
version: 2
updates:
- package-ecosystem: gradle
directory: /
target-branch: experimental
schedule:
interval: weekly
open-pull-requests-limit: 5
- package-ecosystem: github-actions
directory: /
target-branch: experimental
schedule:
interval: weekly
+50
View File
@@ -0,0 +1,50 @@
name: CodeQL
# Kotlin can only be scanned while it compiles, so this builds the debug app, the same way
# release.yml does but without a signing key. Workflow files need no build.
on:
push:
branches: [main, experimental]
pull_request:
schedule:
- cron: "0 9 * * 1"
workflow_dispatch:
jobs:
analyze:
runs-on: ubuntu-latest
permissions:
contents: read
security-events: write
actions: read
strategy:
fail-fast: false
matrix:
include:
- language: actions
build-mode: none
- language: java-kotlin
build-mode: manual
steps:
- uses: actions/checkout@v4
- uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
- if: matrix.build-mode == 'manual'
uses: actions/setup-java@v4
with:
distribution: temurin
java-version: "21"
- if: matrix.build-mode == 'manual'
uses: gradle/actions/setup-gradle@v4
- if: matrix.build-mode == 'manual'
run: ./gradlew :gearslip:assembleDebug --no-daemon
- uses: github/codeql-action/analyze@v4
with:
category: "/language:${{ matrix.language }}"
+34
View File
@@ -0,0 +1,34 @@
name: Security scan
# Trivy looks through the repository for committed secrets and risky workflow or build
# settings. Findings show under the Security tab, next to CodeQL's and Dependabot's.
on:
push:
branches: [main, experimental]
pull_request:
schedule:
- cron: "0 9 * * 1"
workflow_dispatch:
jobs:
trivy:
runs-on: ubuntu-latest
permissions:
contents: read
security-events: write
steps:
- uses: actions/checkout@v4
- uses: aquasecurity/trivy-action@v0.36.0
with:
scan-type: fs
scanners: vuln,secret,misconfig
format: sarif
output: trivy.sarif
severity: MEDIUM,HIGH,CRITICAL
- uses: github/codeql-action/upload-sarif@v4
if: always()
with:
sarif_file: trivy.sarif
category: trivy
+3 -13
View File
@@ -83,21 +83,10 @@ android {
}
defaultConfig {
// Speech-to-text ships for 64-bit ARM only: every phone Gearslip runs on, and one copy
// of the native library instead of four.
// Speech-to-text (sherpa-onnx, in jniLibs) ships for 64-bit ARM only: every phone
// Gearslip runs on, and one copy of the native library instead of four.
ndk { abiFilters += "arm64-v8a" }
}
ndkVersion = "27.1.12297006"
externalNativeBuild {
cmake {
path = file("src/main/cpp/CMakeLists.txt")
version = "3.22.1"
}
}
androidResources {
// Model weights barely compress, and stored plainly they stream straight out of the APK.
noCompress += "bin"
}
testOptions {
// GearslipLog calls android.util.Log, which is a stub on the local JVM.
@@ -130,6 +119,7 @@ dependencies {
implementation(platform(libs.androidx.compose.bom))
implementation(libs.androidx.compose.ui)
implementation(libs.androidx.compose.material3)
implementation(libs.commons.compress)
implementation(libs.androidx.compose.material.icons.core)
implementation(libs.androidx.activity.compose)
implementation(libs.androidx.lifecycle.runtime.compose)
-36
View File
@@ -1,36 +0,0 @@
# Speech-to-text for voice replies and the assistant: whisper.cpp, pinned to one commit and
# fetched at build time, so nothing outside this folder (cloned-repos included) is needed.
cmake_minimum_required(VERSION 3.22)
project(gearslip_speech CXX C)
set(CMAKE_CXX_STANDARD 17)
# Optimised in debug builds too. Unoptimised, Whisper runs several times slower, and a dev build
# would make voice look far worse than what ships.
foreach(lang C CXX)
set(CMAKE_${lang}_FLAGS_DEBUG "-O3 -DNDEBUG")
endforeach()
include(FetchContent)
FetchContent_Declare(
whisper
URL https://github.com/ggml-org/whisper.cpp/archive/6e4ab854f67f743900934a703d5603419384c961.tar.gz
URL_HASH SHA256=55b585aec5e1bc03e2a5cea61da97c6406fa833996cf7daeed704972273cc039
)
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
set(WHISPER_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
set(WHISPER_BUILD_TESTS OFF CACHE BOOL "" FORCE)
set(WHISPER_BUILD_SERVER OFF CACHE BOOL "" FORCE)
set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
set(GGML_NATIVE OFF CACHE BOOL "" FORCE)
# ARMv8.2 with half floats and dot products: every phone that runs Android 12 or later in
# practice, and what makes Whisper fast enough to answer in under a second.
set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE)
FetchContent_MakeAvailable(whisper)
add_library(gearslip_speech SHARED speech_jni.cpp)
target_compile_options(gearslip_speech PRIVATE -O3)
find_library(LOG_LIB log)
target_link_libraries(gearslip_speech whisper android ${LOG_LIB})
target_link_options(gearslip_speech PRIVATE -Wl,--gc-sections -Wl,-z,max-page-size=16384)
-86
View File
@@ -1,86 +0,0 @@
// The bridge between Kotlin's SpeechEngine and whisper.cpp: load a model, turn one recorded
// phrase into text, free the model. Recording and deciding when a phrase ends stay in Kotlin.
#include <jni.h>
#include <android/asset_manager.h>
#include <android/asset_manager_jni.h>
#include <android/log.h>
#include <algorithm>
#include <cmath>
#include <string>
#include "whisper.h"
#define TAG "GearslipSpeech"
#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
static whisper_context_params context_params() {
whisper_context_params p = whisper_context_default_params();
p.use_gpu = false;
return p;
}
// Streams a model straight out of the APK, so the bundled one is never copied to storage.
static size_t asset_read(void *ctx, void *output, size_t size) {
int n = AAsset_read(static_cast<AAsset *>(ctx), output, size);
return n > 0 ? static_cast<size_t>(n) : 0;
}
static bool asset_eof(void *ctx) { return AAsset_getRemainingLength64(static_cast<AAsset *>(ctx)) <= 0; }
static void asset_close(void *ctx) { AAsset_close(static_cast<AAsset *>(ctx)); }
extern "C" JNIEXPORT jlong JNICALL
Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeInitAsset(JNIEnv *env, jobject, jobject assets, jstring jpath) {
AAssetManager *manager = AAssetManager_fromJava(env, assets);
const char *path = env->GetStringUTFChars(jpath, nullptr);
AAsset *asset = AAssetManager_open(manager, path, AASSET_MODE_STREAMING);
env->ReleaseStringUTFChars(jpath, path);
if (!asset) return 0;
whisper_model_loader loader = {asset, asset_read, asset_eof, asset_close};
return reinterpret_cast<jlong>(whisper_init_with_params(&loader, context_params()));
}
// Bytes, not a jstring: NewStringUTF wants modified UTF-8 and aborts on some of what Whisper can
// write (four-byte characters), so Kotlin decodes the plain UTF-8 instead.
extern "C" JNIEXPORT jbyteArray JNICALL
Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeTranscribe(
JNIEnv *env, jobject, jlong handle, jfloatArray jsamples, jstring jlang, jint threads) {
auto *ctx = reinterpret_cast<whisper_context *>(handle);
if (!ctx) return env->NewByteArray(0);
jsize n = env->GetArrayLength(jsamples);
jfloat *samples = env->GetFloatArrayElements(jsamples, nullptr);
const char *lang = env->GetStringUTFChars(jlang, nullptr);
whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY);
p.n_threads = threads;
p.language = lang;
p.translate = false;
p.no_context = true;
p.no_timestamps = true;
p.single_segment = true;
p.print_progress = false;
p.print_realtime = false;
p.print_special = false;
p.print_timestamps = false;
p.suppress_blank = true;
// The ACFT models are trained for a context cut to the audio's own length (1500 frames is
// 30 s, so 50 per second), which is what makes a three-second phrase fast. A little extra
// keeps the last word from being clipped.
p.audio_ctx = std::min(1500, static_cast<int>(std::ceil(n / 320.0)) + 32);
int rc = whisper_full(ctx, p, samples, n);
env->ReleaseStringUTFChars(jlang, lang);
env->ReleaseFloatArrayElements(jsamples, samples, JNI_ABORT);
if (rc != 0) {
LOGI("whisper_full failed: %d", rc);
return env->NewByteArray(0);
}
std::string text;
for (int i = 0; i < whisper_full_n_segments(ctx); i++) text += whisper_full_get_segment_text(ctx, i);
jbyteArray out = env->NewByteArray(static_cast<jsize>(text.size()));
env->SetByteArrayRegion(out, 0, static_cast<jsize>(text.size()), reinterpret_cast<const jbyte *>(text.data()));
return out;
}
extern "C" JNIEXPORT void JNICALL
Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeFree(JNIEnv *, jobject, jlong handle) {
if (handle) whisper_free(reinterpret_cast<whisper_context *>(handle));
}
@@ -9,7 +9,6 @@ object AppSettings {
private const val PREFS = "gearslip_settings"
private const val KEY_STARTUP_URL = "startup_url"
private const val KEY_SEEN_COMPAT_WARNING = "seen_compat_warning"
private const val KEY_SKIPPED_CERT_SETUP = "skipped_cert_setup"
private const val KEY_SEEN_PERMISSIONS_SETUP = "seen_permissions_setup"
private const val KEY_GEARSLIP_ENABLED = "gearslip_enabled"
@@ -32,14 +31,6 @@ object AppSettings {
prefs(context).edit().putString(KEY_STARTUP_URL, value.trim()).apply()
}
/** Whether the newer-car compatibility warning has already been shown and dismissed once. */
fun hasSeenCompatWarning(context: Context): Boolean =
prefs(context).getBoolean(KEY_SEEN_COMPAT_WARNING, false)
fun setSeenCompatWarning(context: Context) {
prefs(context).edit().putBoolean(KEY_SEEN_COMPAT_WARNING, true).apply()
}
/** Whether the certificate setup screen has been deliberately skipped. */
fun hasSkippedCertSetup(context: Context): Boolean =
prefs(context).getBoolean(KEY_SKIPPED_CERT_SETUP, false)
@@ -8,8 +8,10 @@ import android.hardware.usb.UsbAccessory
import android.hardware.usb.UsbManager
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.net.Uri
import android.os.Bundle
import android.os.ParcelFileDescriptor
import android.provider.Settings
import android.view.MotionEvent
import android.view.Surface
import android.view.WindowManager
@@ -76,7 +78,13 @@ class GearslipActivity : ComponentActivity(), Projection {
private val audioPermission = registerForActivityResult(
androidx.activity.result.contract.ActivityResultContracts.RequestPermission(),
) { granted ->
if (granted) askAudioCapture() else {
if (granted) askAudioCapture()
else if (!shouldShowRequestPermissionRationale(android.Manifest.permission.RECORD_AUDIO)) {
// Refused twice: Android now says no without showing anything, so the phone must.
GearslipLog.w("audio: record permission blocked - Android no longer asks")
CarAudio.micBlocked()
micBlockedNotice.value = true
} else {
GearslipLog.w("audio: record permission declined")
CarAudio.denied()
}
@@ -93,6 +101,9 @@ class GearslipActivity : ComponentActivity(), Projection {
}
}
/** Raised when the microphone permission is blocked; see [MicBlockedDialog]. */
private val micBlockedNotice = kotlinx.coroutines.flow.MutableStateFlow(false)
/** Raised once, after the first "no" to the capture dialog; see [AudioChoiceDialog]. */
private val offerBluetooth = kotlinx.coroutines.flow.MutableStateFlow(false)
@@ -154,6 +165,18 @@ class GearslipActivity : ComponentActivity(), Projection {
onKeepAsking = ::close,
)
}
val micNotice by micBlockedNotice.collectAsState()
if (micNotice) {
app.seb3thehacker.gearslip.ui.MicBlockedDialog(
onOpenSettings = {
micBlockedNotice.value = false
startActivity(
Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", packageName, null)),
)
},
onDismiss = { micBlockedNotice.value = false },
)
}
}
}
@@ -179,6 +202,10 @@ class GearslipActivity : ComponentActivity(), Projection {
Prefetch.warm(this)
UsageStats.onAppOpened(this)
app.seb3thehacker.gearslip.update.UpdateChecker.onAppOpened(this)
// Back from the app's settings with the microphone turned on: try car audio again.
if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) == android.content.pm.PackageManager.PERMISSION_GRANTED) {
CarAudio.micAllowed()
}
}
override fun onNewIntent(intent: Intent) {
@@ -45,7 +45,10 @@ class GearslipRunner(
private lateinit var identityTask: FutureTask<PhoneTls>
/** Sends every message, in order, so no other thread ever blocks on the USB link. See [send]. */
private val writer = java.util.concurrent.Executors.newSingleThreadExecutor { Thread(it, "gearslip-writer") }
private val writer = java.util.concurrent.Executors.newSingleThreadExecutor {
// Sound goes out through here too, so it gets audio's priority: late writes are skips.
Thread({ android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO); it.run() }, "gearslip-writer")
}
/** Waits for the certificate only when TLS is first needed; see [run]. */
private val tls: PhoneTls by lazy { identityTask.get() }
@@ -88,6 +91,8 @@ class GearslipRunner(
@Volatile private var lastKeyframeSentAt = 0L
fun run() {
// This thread reads the car's acks; audio waits on them, so it can't be left at the back.
android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO)
SessionReport.begin()
log.i("--- session starting ---")
SessionStatus.connecting("Connected to the head unit")
@@ -260,11 +265,10 @@ class GearslipRunner(
log.i("<- VersionRequest: head unit speaks $major.$minor")
SessionReport.protocolVersion(major, minor)
// PROTOCOL-DOWNGRADE EXPERIMENT. The 2018 Uconnect speaks 1.3 and ACCEPTS the JVC cert;
// the 2025 unit speaks 4.3 and rejects it at AuthComplete (-3). If the 4.3 identity check
// is tied to the *negotiated* protocol version rather than baked into the firmware,
// claiming an older version makes the newer unit skip the check entirely. CLAIM_VERSION
// = null restores the previous echo-the-head-unit behaviour.
// Always 1.3, on purpose. Every car that accepts Gearslip was tested this way, 1.5 units
// included. Echoing the car's own version was tried too: a 4.3 unit rejected the
// certificate either way, so the version doesn't decide acceptance. The log says so, since
// a car offering 1.5 and getting 1.3 back looks like a bug to anyone reading it.
val claim = CLAIM_VERSION ?: (major to minor)
val response = byteArrayOf(
((claim.first shr 8) and 0xFF).toByte(), (claim.first and 0xFF).toByte(),
@@ -274,7 +278,10 @@ class GearslipRunner(
send(MSG_VERSION_RESPONSE, response, encrypted = false)
log.i(
"-> VersionResponse: ${claim.first}.${claim.second} status=0 (STATUS_SUCCESS)" +
if (CLAIM_VERSION != null) " [FORCED; head unit offered $major.$minor]" else "",
if (CLAIM_VERSION != null && claim != (major to minor)) {
" - intended: Gearslip always answers ${claim.first}.${claim.second}, which cars " +
"offering $major.$minor accept too"
} else "",
)
state = State.TLS_HANDSHAKE
SessionStatus.connecting("Securing the link")
@@ -1209,9 +1216,9 @@ class GearslipRunner(
private companion object {
/**
* Force the protocol version Gearslip claims in VersionResponse, as (major, minor), or
* null to echo the head unit. 1.3 is what the 2018 Uconnect (which accepts the JVC cert)
* speaks. This is the 2025 identity-check downgrade experiment: see onVersionRequest.
* The protocol version Gearslip answers every car with, as (major, minor), or null to echo
* the head unit's. Kept at 1.3 because every working car was tested with it; see
* onVersionRequest.
*/
val CLAIM_VERSION: Pair<Int, Int>? = 1 to 3
@@ -16,6 +16,7 @@ import android.media.AudioRecord
import android.media.projection.MediaProjection
import android.media.projection.MediaProjectionManager
import android.os.IBinder
import android.os.Process
import app.seb3thehacker.gearslip.GearslipLog
import kotlin.math.abs
@@ -147,6 +148,9 @@ class AudioCaptureService : Service() {
reading = true
thread = Thread({
// A busy phone (a map app drawing, the video encoder) would otherwise starve this
// loop, and the car hears every late read as a skip.
Process.setThreadPriority(Process.THREAD_PRIORITY_URGENT_AUDIO)
val buffer = ByteArray(CHUNK_BYTES)
var peak = 0
var heardAt = System.currentTimeMillis()
@@ -147,7 +147,8 @@ class AudioLink(
fun onFocus(state: Int) {
val name = FOCUS_STATES[state] ?: "unknown($state)"
GearslipLog.i("audio: <- AudioFocusResponse state=$name")
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT
focusGranted = state == FOCUS_STATE_GAIN || state == FOCUS_STATE_GAIN_TRANSIENT ||
state == FOCUS_STATE_GAIN_MEDIA_ONLY
if (focusGranted) startStream() else stop()
}
@@ -257,13 +258,16 @@ class AudioLink(
private const val FOCUS_RELEASE = 4
private const val FOCUS_STATE_GAIN = 1
private const val FOCUS_STATE_GAIN_TRANSIENT = 2
private const val FOCUS_STATE_GAIN_MEDIA_ONLY = 6
private const val SESSION_ID = 2
private const val STALL_MS = 2_000L
private const val CHUNK_LOG_EVERY = 200 // about every four seconds
private const val CONFIG_WAIT_MS = 1_000L
// AudioFocusStateNotification.AudioFocusState in aasdk.
private val FOCUS_STATES = mapOf(
1 to "GAIN", 2 to "GAIN_TRANSIENT", 3 to "LOSS", 4 to "LOSS_TRANSIENT",
1 to "GAIN", 2 to "GAIN_TRANSIENT", 3 to "LOSS", 4 to "LOSS_TRANSIENT_CAN_DUCK",
5 to "LOSS_TRANSIENT", 6 to "GAIN_MEDIA_ONLY", 7 to "GAIN_TRANSIENT_GUIDANCE_ONLY",
)
}
}
@@ -100,6 +100,26 @@ object CarAudio {
_capture.value = Capture.DENIED
}
private val _micBlocked = MutableStateFlow(false)
/** The microphone permission is off for good: Android refuses without asking, so say why. */
val micBlocked: StateFlow<Boolean> = _micBlocked.asStateFlow()
fun micBlocked() {
_micBlocked.value = true
denied()
}
/** The driver turned the permission back on mid-drive: ask for capture again. */
fun micAllowed() {
if (!_micBlocked.value) return
_micBlocked.value = false
if (_capture.value == Capture.DENIED) {
_capture.value = Capture.OFF
if (wanted) request()
}
}
internal fun captureStarted() {
_capture.value = Capture.ON
link?.start()
@@ -1,10 +1,13 @@
package app.seb3thehacker.gearslip.car
import android.content.Context
import app.seb3thehacker.gearslip.media.CarMedia
import java.text.DateFormat
import java.util.Date
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.delay
import kotlinx.coroutines.withContext
import kotlinx.coroutines.withTimeoutOrNull
/**
* Turns what [CarAssistant] heard into an action plus something to say back.
@@ -17,9 +20,15 @@ import kotlinx.coroutines.withContext
object AssistantCommands {
suspend fun handle(context: Context, navigator: CarNavigator, frame: CarEnvironment.Frame, heardRaw: String): String {
val heard = heardRaw.lowercase().trim()
val heard = natural(heardRaw)
VoiceHelp.hide()
if (heard in HELP_WORDS || heard.startsWith("what can i say") || heard.startsWith("what can you do")) {
VoiceHelp.show()
return "Here's what you can say."
}
navigateCommand(context, navigator, frame, heard)?.let { return it }
playSearchCommand(context, navigator, heard)?.let { return it }
mediaCommand(heard)?.let { return it }
navCommand(navigator, heard)?.let { return it }
infoCommand(heard)?.let { return it }
@@ -29,6 +38,42 @@ object AssistantCommands {
return "I don't know how to do that yet."
}
/**
* What was said, minus the padding people wrap a command in: "hey", "can you", "I want to",
* "please", "thanks". Each command below then only has to know its own words.
*/
private fun natural(raw: String): String {
var t = raw.lowercase().replace(Regex("[.,!?]"), " ").replace(Regex("\\s+"), " ").trim()
var changed = true
while (changed) {
changed = false
for (lead in LEAD_FILLERS) if (t.startsWith("$lead ")) { t = t.removePrefix("$lead ").trim(); changed = true }
for (tail in TAIL_FILLERS) if (t.endsWith(" $tail")) { t = t.removeSuffix(" $tail").trim(); changed = true }
}
return t
}
/** The part after the first of [triggers] that [text] starts with, or null. */
private fun after(text: String, triggers: List<String>): String? =
triggers.firstOrNull { text.startsWith(it) }?.let { text.removePrefix(it).trim() }
private suspend fun allEntries(context: Context): List<Entry> {
val vehicleData = CarSettings.experimentalFeaturesEnabled.value && CarSettings.carSensorsEnabled.value
val builtIn = BuiltInApps.entries.filter { it.builtIn !== BuiltInApps.VehicleData || vehicleData }
return builtIn + (LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) })
}
/** Best label match: exact, then a label starting with [name], then one containing it. */
private fun <T> bestMatch(items: List<T>, name: String, label: (T) -> String): T? {
val n = name.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
if (n.isEmpty()) return null
items.firstOrNull { label(it).lowercase() == n }?.let { return it }
// Loose matches need a real word: "pizza in a" mustn't pick "Organic Maps" for its "a".
if (n.length < 3) return null
return items.firstOrNull { label(it).lowercase().startsWith(n) }
?: items.firstOrNull { label(it).lowercase().contains(n) }
}
/**
* "navigate to X [with|using|on|via Y]" - Y picks the nav app by (partial) label; without it,
* whichever nav app was used last, or the only one installed, or the first one found. Starts
@@ -42,29 +87,45 @@ object AssistantCommands {
frame: CarEnvironment.Frame,
heard: String,
): String? {
val trigger = NAVIGATE_TRIGGERS.firstOrNull { heard.startsWith(it) } ?: return null
var destination = heard.removePrefix(trigger).trim()
// "use Organic Maps to get to X", "open OsmAnd and navigate to X": the app comes first.
var text = heard
var leadingApp: String? = null
Regex("^(?:use|using|with|open|in|on) (.+?) (?:to |and )(.+)$").find(text)?.let { m ->
if (after(m.groupValues[2], NAVIGATE_TRIGGERS) != null) {
leadingApp = m.groupValues[1]
text = m.groupValues[2]
}
}
var destination = after(text, NAVIGATE_TRIGGERS) ?: return null
if (destination.isBlank()) return "Where to?"
var appName: String? = null
for (sep in listOf(" with ", " using ", " on ", " via ")) {
val at = destination.lastIndexOf(sep)
if (at > 0) {
appName = destination.substring(at + sep.length).trim()
val entries = allEntries(context)
// "go to Spotify" means the app, and "go to settings" the screen; "go to Target" is a place.
val asApp = destination.removePrefix("the ").removeSuffix(" app")
if (text.startsWith("go to ") &&
(asApp in SCREEN_WORDS || asApp in APP_NICKNAMES || entries.any { it.label.lowercase() == asApp })
) {
return null
}
val navApps = entries.filter { it.template?.isNavigation == true }
if (navApps.isEmpty()) return "You don't have any navigation apps installed."
// "X with Organic Maps", "X on OsmAnd": split only when the end names a nav app, so a
// place like "Starbucks on Main Street" stays whole.
var chosenApp = leadingApp?.let { name -> bestMatch(navApps, name) { it.label } ?: return "I couldn't find a navigation app called $name." }
if (chosenApp == null) {
for (sep in listOf(" with ", " using ", " on ", " via ", " in ", " through ")) {
val at = destination.lastIndexOf(sep)
if (at <= 0) continue
val app = bestMatch(navApps, destination.substring(at + sep.length)) { it.label } ?: continue
chosenApp = app
destination = destination.substring(0, at).trim()
break
}
}
if (destination.isBlank()) return "Where to?"
val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
val navApps = entries.filter { it.template?.isNavigation == true }
if (navApps.isEmpty()) return "You don't have any navigation apps installed."
val requestedApp = appName
val chosen = when {
requestedApp != null -> navApps.firstOrNull { it.label.lowercase().contains(requestedApp) }
?: return "I couldn't find a navigation app called $requestedApp."
val chosen = chosenApp ?: when {
navApps.size == 1 -> navApps.first()
else -> navApps.firstOrNull { it.componentId == CarSettings.lastNav.value } ?: navApps.first()
}
@@ -75,26 +136,84 @@ object AssistantCommands {
return "Navigating to $destination with ${chosen.label}."
}
/**
* "play X" with something after it: a song, artist, album or playlist, asked of the music app
* that's open through its own search ([CarMedia.search]). A bare "play", or "play music",
* falls through to resume.
*/
private suspend fun playSearchCommand(context: Context, navigator: CarNavigator, heard: String): String? {
var query = after(heard, PLAY_TRIGGERS) ?: return null
query = query.removePrefix("some ").trim()
if (query.isEmpty() || query in RESUME_WORDS) return null
val media = CarServices.media
// "play Daft Punk on Metrolist": switch to that app first, if it's one we know.
for (sep in listOf(" on ", " in ", " from ", " with ", " using ")) {
val at = query.lastIndexOf(sep)
if (at <= 0) continue
val apps = withContext(Dispatchers.IO) { app.seb3thehacker.gearslip.media.MediaCatalog.installed(context) }
val wanted = bestMatch(apps, query.substring(at + sep.length)) { it.label } ?: continue
query = query.substring(0, at).trim()
if (query.isEmpty() || query in RESUME_WORDS) {
CarServices.openMedia(wanted, autoplay = true)
navigator.media()
return "Opening ${wanted.label}."
}
if (CarServices.mediaApp.value?.component != wanted.component) {
CarServices.openMedia(wanted)
navigator.media()
// Give it a moment to connect before asking it to search.
withTimeoutOrNull(APP_CONNECT_MS) {
while (media.appLabel == null || (!media.canSearchLibrary && !media.now.value.canSearch)) delay(200)
}
}
break
}
// No music app open, or its connection is gone (the app was closed): use the one played
// last, the way "play X" works on Android Auto.
val phase = media.phase.value
if (media.appLabel == null || phase == CarMedia.Phase.IDLE || phase == CarMedia.Phase.REJECTED) {
if (!CarServices.reopenLastMedia()) return "Open a music app first, then ask again."
navigator.media()
}
withTimeoutOrNull(APP_CONNECT_MS) {
while (media.phase.value != CarMedia.Phase.READY || media.appLabel == null ||
(!media.canSearchLibrary && !media.now.value.canSearch)
) delay(200)
}
val app = media.appLabel ?: return "Your music app didn't open in time. Try again."
if (!media.canSearchLibrary && !media.now.value.canSearch) {
return "$app ignores voice searches. Pick something on its screen instead."
}
media.playBestMatch(query)
return "Looking for $query on $app."
}
private fun mediaCommand(heard: String): String? {
val media = CarServices.media
return when {
"pause" in heard || heard.startsWith("stop") -> {
"pause" in heard || heard.startsWith("stop") || heard == "quiet" || heard == "shut up" -> {
if (media.now.value.playing) media.togglePlay()
"Paused."
}
"resume" in heard || (heard.startsWith("play") && "playing" !in heard) -> {
"resume" in heard || "unpause" in heard || heard in RESUME_PHRASES ||
(heard.startsWith("play") && "playing" !in heard) -> {
if (!media.now.value.playing) media.togglePlay()
"Playing."
}
"next" in heard || "skip" in heard -> {
"next" in heard || "skip" in heard || heard.startsWith("change the song") -> {
media.next()
"Skipping."
}
"previous" in heard || "last song" in heard || "back song" in heard -> {
"previous" in heard || "last song" in heard || "back song" in heard || "go back a song" in heard ||
"play that again" in heard || "replay" in heard -> {
media.previous()
"Playing the previous track."
}
"what" in heard && "playing" in heard -> {
("what" in heard && "playing" in heard) || "what song" in heard || "who sings" in heard ||
"who's singing" in heard || "who is singing" in heard || "what's this song" in heard ||
"name of this song" in heard -> {
val now = media.now.value
if (now.isActive) listOf(now.title, now.artist).filter { it.isNotBlank() }.joinToString(" by ")
else "Nothing's playing."
@@ -104,11 +223,19 @@ object AssistantCommands {
}
private fun navCommand(navigator: CarNavigator, heard: String): String? = when {
"go home" in heard || heard == "home" -> {
heard == "go back" || heard == "back" -> {
navigator.back()
"Going back."
}
heard in EXIT_WORDS -> {
BuiltInApps.Exit.open(navigator)
"Back to the car's screen."
}
"go home" in heard || heard == "home" || heard == "home screen" || "go to the home screen" in heard -> {
navigator.home()
"Heading home."
}
"open apps" in heard || heard == "apps" || "app list" in heard -> {
"open apps" in heard || heard == "apps" || "app list" in heard || "all apps" in heard || "my apps" in heard || heard == "go to apps" -> {
navigator.apps()
"Here are your apps."
}
@@ -128,9 +255,10 @@ object AssistantCommands {
}
private fun infoCommand(heard: String): String? = when {
"what time" in heard || "what's the time" in heard ->
"what time" in heard || "what's the time" in heard || heard == "time" || "the time" in heard ->
"It's ${DateFormat.getTimeInstance(DateFormat.SHORT).format(Date())}."
"weather" in heard -> {
"weather" in heard || "going to rain" in heard || "gonna rain" in heard || "how hot" in heard ||
"how cold" in heard || "temperature outside" in heard -> {
val data = (Weather.state.value as? WeatherState.Ready)?.data
if (data != null) "${data.temp}${data.tempUnit} and ${Weather.describe(data.code)} in ${data.place}."
else "I don't have the weather yet."
@@ -139,12 +267,11 @@ object AssistantCommands {
}
private suspend fun callCommand(context: Context, heard: String): String? {
val name = when {
heard.startsWith("call ") -> heard.removePrefix("call ")
heard.startsWith("phone ") -> heard.removePrefix("phone ")
heard.startsWith("dial ") -> heard.removePrefix("dial ")
else -> return null
}.trim()
val name = (
after(heard, CALL_TRIGGERS)
?: Regex("^give (.+) a (?:call|ring)$").find(heard)?.groupValues?.get(1)
?: return null
).replace(Regex(" (?:on|at) (?:their |his |her )?(?:mobile|cell|phone|home|work)$"), "").trim()
if (name.isBlank()) return "Who should I call?"
if (!hasCallPermissions(context)) return "Gearslip needs the phone and contacts permissions first, from the phone screen."
@@ -162,21 +289,59 @@ object AssistantCommands {
frame: CarEnvironment.Frame,
heard: String,
): String? {
val name = when {
heard.startsWith("open ") -> heard.removePrefix("open ")
heard.startsWith("launch ") -> heard.removePrefix("launch ")
heard.startsWith("start ") -> heard.removePrefix("start ")
else -> return null
}.trim()
val name = (after(heard, OPEN_TRIGGERS) ?: return null)
.removePrefix("the ").removePrefix("my ").removeSuffix(" app").trim()
if (name.isBlank()) return null
val wanted = APP_NICKNAMES[name] ?: name
val entries = LauncherCache.entries ?: withContext(Dispatchers.IO) { LauncherCache.load(context) }
val match = entries.firstOrNull { it.label.lowercase().contains(name) }
// Gearslip's own apps come first in the list, so "phone" opens its dialer, not the phone's.
val match = bestMatch(allEntries(context), wanted) { it.label }
?: return "I couldn't find an app called $name."
launchEntry(match, navigator, frame)
return "Opening ${match.label}."
}
private val NAVIGATE_TRIGGERS = listOf("navigate to ", "directions to ", "take me to ", "drive to ")
private val EXIT_WORDS = setOf("exit", "exit gearslip", "close gearslip", "quit", "quit gearslip", "open exit")
/** Other names people use for Gearslip's own apps. */
private val APP_NICKNAMES = mapOf(
"web" to "browser", "internet" to "browser", "the web" to "browser", "web browser" to "browser",
"dialer" to "phone", "the dialer" to "phone", "keypad" to "phone",
"screen share" to "screen sharing", "screen mirroring" to "screen sharing", "mirroring" to "screen sharing",
)
private val NAVIGATE_TRIGGERS = listOf(
"navigate to ", "navigate me to ", "directions to ", "get directions to ", "give me directions to ",
"take me to ", "drive to ", "drive me to ", "get me to ", "go to ", "head to ", "route to ",
"how do i get to ", "show me the way to ", "find a route to ", "navigation to ", "get to ",
)
private val PLAY_TRIGGERS = listOf("play me ", "play ", "put on ", "listen to ", "i want to hear ", "shuffle ")
private val CALL_TRIGGERS = listOf("call ", "phone ", "dial ", "ring ", "make a call to ", "place a call to ", "start a call with ")
private val OPEN_TRIGGERS = listOf("open ", "launch ", "start ", "go to ", "switch to ", "show me ", "bring up ", "pull up ")
/** Screens "go to X" means rather than a place. */
private val SCREEN_WORDS = setOf("settings", "home", "the home screen", "home screen", "apps", "my apps", "all apps")
/** Padding stripped from the front of a command, longest first where they overlap. */
private val LEAD_FILLERS = listOf(
"hey gearslip", "ok gearslip", "okay gearslip", "gearslip", "hey", "ok", "okay", "um", "uh", "so",
"please", "can you please", "can you", "could you", "would you", "will you", "can i", "could i",
"i want you to", "i want to", "i'd like to", "i would like to", "i wanna", "i need to", "i need you to",
"let's", "lets", "go ahead and", "just", "now", "quickly",
)
/** Longest first, so "right now" goes whole rather than leaving "right". */
private val TAIL_FILLERS = listOf("thank you very much", "thank you", "right now", "for me", "please", "thanks", "now")
private const val APP_CONNECT_MS = 5_000L
private val HELP_WORDS = setOf("help", "commands", "voice commands", "list commands", "show commands")
private val RESUME_WORDS = setOf("music", "the music", "some music", "something", "it", "again", "that", "my music")
private val RESUME_PHRASES = setOf("keep playing", "continue", "keep going", "start the music", "music on", "turn the music back on")
}
@@ -4,7 +4,21 @@ import app.seb3thehacker.gearslip.car.theme.*
import android.content.Intent
import android.net.Uri
import android.provider.Settings
import androidx.compose.animation.core.FastOutSlowInEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.ui.draw.drawBehind
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.scale
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
@@ -30,47 +44,66 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
/**
* The band that appears while [CarAssistant] is listening, thinking or speaking - a thin strip
* across the top, same idea as [CallOverlay]'s active-call banner, so it never blocks the screen
* the driver was already looking at.
* What the assistant hears and says, laid over the bottom of whatever screen is up: a dark shade
* so white words read over a bright map, and a glow along the bottom edge in the phone's own
* colours that breathes while listening. The shade passes touches through; only the buttons take them.
*/
@Composable
fun AssistantOverlay(modifier: Modifier = Modifier) {
val state by CarAssistant.state.collectAsState()
if (state.phase == CarAssistant.Phase.IDLE) return
val context = LocalContext.current
val listening = state.phase == CarAssistant.Phase.LISTENING && state.ready
val glow = MaterialTheme.colorScheme.primary
val glow2 = MaterialTheme.colorScheme.tertiary
val breathe by rememberInfiniteTransition(label = "glow").animateFloat(
0.55f, 1f, infiniteRepeatable(tween(900, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "glow",
)
val strength = when (state.phase) {
CarAssistant.Phase.LISTENING -> if (state.ready) breathe else 0.3f
CarAssistant.Phase.ERROR -> 0.3f
else -> 0.6f
}
Surface(
color = MaterialTheme.colorScheme.primaryContainer,
contentColor = MaterialTheme.colorScheme.onPrimaryContainer,
modifier = modifier.fillMaxWidth(),
) {
Row(
Modifier.padding(horizontal = 16.dp, vertical = 10.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(12.dp),
) {
PulsingMic(state.phase == CarAssistant.Phase.LISTENING)
Column(Modifier.weight(1f)) {
Text(
when (state.phase) {
CarAssistant.Phase.LISTENING -> "Listening…"
CarAssistant.Phase.THINKING -> "…"
CarAssistant.Phase.SPEAKING -> state.reply
CarAssistant.Phase.ERROR -> state.reply
CarAssistant.Phase.IDLE -> ""
},
style = MaterialTheme.typography.titleSmall,
fontWeight = FontWeight.SemiBold,
maxLines = 2,
)
if (state.heard.isNotBlank() && state.phase != CarAssistant.Phase.SPEAKING) {
Text(
"\"${state.heard}\"",
style = MaterialTheme.typography.bodySmall,
Box(
modifier
.fillMaxWidth()
.drawBehind {
drawRect(Brush.verticalGradient(0f to Color.Transparent, 0.45f to Color.Black.copy(alpha = 0.6f), 1f to Color.Black.copy(alpha = 0.85f)))
// A wide, flat oval of light rising from the bottom edge.
scale(1f, 0.22f, Offset(size.width / 2, size.height)) {
drawCircle(
Brush.radialGradient(
0f to glow.copy(alpha = 0.9f * strength), 0.5f to glow2.copy(alpha = 0.35f * strength), 1f to Color.Transparent,
center = Offset(size.width / 2, size.height), radius = size.width * 0.55f,
),
radius = size.width * 0.55f, center = Offset(size.width / 2, size.height),
)
}
}
.padding(start = 16.dp, end = 16.dp, top = 56.dp, bottom = 20.dp),
) {
Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(16.dp)) {
PulsingMic(listening)
Column(Modifier.weight(1f), horizontalAlignment = Alignment.CenterHorizontally) {
val (small, big) = when (state.phase) {
CarAssistant.Phase.LISTENING -> "" to state.heard.ifBlank { if (state.ready) "Listening…" else "One moment…" }
CarAssistant.Phase.THINKING -> "" to state.heard.ifBlank { "…" }
else -> state.heard to state.reply
}
if (small.isNotBlank()) {
Text(small, color = Color.White.copy(alpha = 0.7f), style = MaterialTheme.typography.bodyLarge, maxLines = 1, overflow = TextOverflow.Ellipsis)
}
Text(
big,
color = if (big == "Listening…" || big == "One moment…" || big == "…") Color.White.copy(alpha = 0.7f) else Color.White,
style = MaterialTheme.typography.headlineSmall,
fontWeight = FontWeight.SemiBold,
textAlign = TextAlign.Center,
maxLines = 2,
overflow = TextOverflow.Ellipsis,
)
}
if (state.missingPermission) {
GsButton(tone = GsTone.Tonal, onClick = {
context.startActivity(
@@ -22,7 +22,7 @@ class CarApp(
object CarApps {
val all: List<CarApp> = listOf(
CarApp("web", "Web", Icons.Filled.Search) {
CarApp("web", "Browser", Icons.Filled.Search) {
WebApp(content = ProjectedContent.resolve(LocalContext.current))
},
CarApp("phone", "Screen sharing", Icons.Filled.Share) {
@@ -38,6 +38,8 @@ object CarAssistant {
val heard: String = "",
val reply: String = "",
val missingPermission: Boolean = false,
/** False from the press until the microphone is actually open, so the overlay doesn't claim to listen early. */
val ready: Boolean = false,
)
private val _state = MutableStateFlow(State())
@@ -89,6 +91,7 @@ object CarAssistant {
tts?.stop()
_state.value = State(Phase.LISTENING)
listenOnce(
onListening = { _state.value = _state.value.copy(ready = true) },
onPartial = { text -> _state.value = _state.value.copy(heard = text) },
onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) },
onResult = { text ->
@@ -174,15 +177,16 @@ object CarAssistant {
onSpeaking: () -> Unit = {},
) {
cancel()
listenOnce(onPartial, onHeard = {}, onResult, onError, cue, onSpeaking)
listenOnce(onPartial, onHeard = {}, onResult = onResult, onError = onError, cue = cue, onSpeaking = onSpeaking)
}
/**
* One phrase, through Gearslip's own [SpeechEngine]. The phone's recognizer is only a fallback
* for when the engine itself can't run, since plenty of phones (GrapheneOS among them) have none.
* One phrase, through Gearslip's own [SpeechEngine], or the phone's recognizer when the driver
* chose it in settings. Never the phone's on its own: on most phones that's Google's.
*/
private fun listenOnce(
onPartial: (String) -> Unit,
onListening: () -> Unit = {},
onHeard: () -> Unit,
onResult: (String) -> Unit,
onError: (String) -> Unit,
@@ -190,24 +194,24 @@ object CarAssistant {
onSpeaking: () -> Unit = {},
) {
val context = appContext ?: return onError("Speech isn't ready yet.")
if (CarSettings.systemSpeech.value && SpeechRecognizer.isRecognitionAvailable(context)) {
return systemListen(context, onListening, onPartial, onHeard, onResult, onError)
}
SpeechEngine.listen(
context,
cue = cue,
onSpeaking = onSpeaking,
onPartial = onPartial,
onListening = onListening,
onHeard = onHeard,
onResult = onResult,
onError = { message ->
if (message == SpeechEngine.NOT_LOADED && SpeechRecognizer.isRecognitionAvailable(context)) {
systemListen(context, onPartial, onHeard, onResult, onError)
} else {
onError(message)
}
},
onError = onError,
)
}
private fun systemListen(
context: Context,
onListening: () -> Unit,
onPartial: (String) -> Unit,
onHeard: () -> Unit,
onResult: (String) -> Unit,
@@ -216,7 +220,7 @@ object CarAssistant {
val r = SpeechRecognizer.createSpeechRecognizer(context)
recognizer = r
r.setRecognitionListener(object : RecognitionListener {
override fun onReadyForSpeech(params: Bundle?) {}
override fun onReadyForSpeech(params: Bundle?) = onListening()
override fun onBeginningOfSpeech() {}
override fun onRmsChanged(rmsdB: Float) {}
override fun onBufferReceived(buffer: ByteArray?) {}
@@ -138,18 +138,18 @@ fun CarKeyboard(
}
// Control row: no backspace here - it stays with the letters above, not next to the
// key most likely to be hit right after it by mistake. Dismiss (when offered) is the
// last key, bottom-right corner of the keyboard.
// key most likely to be hit right after it by mistake. Search/enter takes the
// bottom-right corner, where a thumb expects it; dismiss (when offered) sits left of it.
Row(
Modifier.fillMaxWidth(),
horizontalArrangement = Arrangement.spacedBy(4.dp),
) {
Key(if (symbols) "abc" else "?123", Modifier.weight(1.4f)) { symbols = !symbols }
Key("space", Modifier.weight(if (onDismiss != null) 3.2f else 4f)) { onTextChange("$text ") }
SubmitKey(Modifier.weight(1.6f), submitIcon, submitImage, onSubmit)
if (onDismiss != null) {
DismissKey(onDismiss, Modifier.weight(1.4f))
}
SubmitKey(Modifier.weight(1.6f), submitIcon, submitImage, onSubmit)
}
}
}
@@ -222,9 +222,9 @@ private fun SubmitKey(modifier: Modifier, search: Boolean, image: ImageVector?,
* Big enough to hit on a bumpy road, small enough that four rows plus a field plus a few results
* still fit. A 480-tall screen sets the floor; a taller one (LIVI's 1280x720) gets bigger keys.
*/
private fun keyHeight(screenHeightDp: Int): Dp = (screenHeightDp * 0.075f).dp.coerceIn(36.dp, 52.dp)
private fun keyHeight(screenHeightDp: Int): Dp = (screenHeightDp * 0.09f).dp.coerceIn(44.dp, 64.dp)
private val LocalKeyHeight = compositionLocalOf { 36.dp }
private val LocalKeyHeight = compositionLocalOf { 44.dp }
private val LETTER_ROWS = listOf(
"qwertyuiop".map { it.toString() },
@@ -102,15 +102,10 @@ object CarKeys {
/**
* The voice key: answers the message on screen if there is one, otherwise starts the
* assistant when it's turned on.
* assistant.
*/
private fun voice() = main.post {
if (VoiceReply.onVoiceKey()) return@post
if (CarSettings.experimentalFeaturesEnabled.value && CarSettings.voiceAssistantEnabled.value) {
CarAssistant.start()
} else {
GearslipLog.i("keys: voice pressed, but the assistant is off in settings")
}
if (!VoiceReply.onVoiceKey()) CarAssistant.start()
}
/** Answers a ringing call; with nothing ringing, opens the phone screen. */
@@ -247,6 +247,8 @@ object CarServices {
runCatching { media.disconnect() }
runCatching { browse.disconnect() }
CarAssistant.cancel()
// About 900 MB that nothing needs once the car's gone.
app.seb3thehacker.gearslip.speech.SpeechEngine.unload()
CarAudio.release()
_mediaApp.value = null
navStopped = false
@@ -38,9 +38,8 @@ object CarSettings {
private const val KEY_PINNED = "car_pinned_apps"
private const val KEY_APP_ORDER = "car_app_order"
private const val KEY_EXPERIMENTAL_FEATURES = "car_experimental_features"
private const val KEY_VOICE_ASSISTANT = "car_voice_assistant"
private const val KEY_SYSTEM_SPEECH = "car_system_speech"
private const val KEY_CAR_SENSORS = "car_sensors_enabled"
private const val KEY_MEDIA_SEARCH = "car_media_search_enabled"
private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning"
private const val KEY_UNITS = "car_units"
private const val KEY_WHATS_NEW = "car_whats_new_after_updates"
@@ -72,9 +71,8 @@ object CarSettings {
private val pinnedFlow = MutableStateFlow<Set<String>>(emptySet())
private val appOrderFlow = MutableStateFlow<List<String>>(emptyList())
private val experimentalFeaturesFlow = MutableStateFlow(false)
private val voiceAssistantFlow = MutableStateFlow(false)
private val systemSpeechFlow = MutableStateFlow(false)
private val carSensorsFlow = MutableStateFlow(false)
private val mediaSearchFlow = MutableStateFlow(false)
private val seenSafetyWarningFlow = MutableStateFlow(false)
private val unitsFlow = MutableStateFlow(Units.AUTO)
@@ -170,12 +168,8 @@ object CarSettings {
*/
val experimentalFeaturesEnabled: StateFlow<Boolean> = experimentalFeaturesFlow
/**
* The voice assistant is experimental and off by default; a driver has to turn it on
* knowingly, from the settings screen, after reading the warning shown there. This flag
* alone does not run the assistant - [experimentalFeaturesEnabled] must also be on.
*/
val voiceAssistantEnabled: StateFlow<Boolean> = voiceAssistantFlow
/** Listen with the phone's own speech service instead of Gearslip's on-device model. */
val systemSpeech: StateFlow<Boolean> = systemSpeechFlow
/**
* Vehicle data is experimental and off by default: the launcher's tile for it, and the
@@ -185,13 +179,6 @@ object CarSettings {
*/
val carSensorsEnabled: StateFlow<Boolean> = carSensorsFlow
/**
* Searching inside a media app is experimental and off by default: many apps claim to take
* searches and then ignore them or play something else. The media screen's search button
* shows only once this and [experimentalFeaturesEnabled] are both on.
*/
val mediaSearchEnabled: StateFlow<Boolean> = mediaSearchFlow
/**
* Whether the car screen's own "drive safely" warning has already been shown and dismissed
* once, from this same install. Separate from the phone side's own compat/safety dialogs -
@@ -226,9 +213,8 @@ object CarSettings {
appOrderFlow.value = AppSettings.getString(app, KEY_APP_ORDER, "")
.split(",").filter { it.isNotEmpty() }
experimentalFeaturesFlow.value = AppSettings.getString(app, KEY_EXPERIMENTAL_FEATURES, "false") == "true"
voiceAssistantFlow.value = AppSettings.getString(app, KEY_VOICE_ASSISTANT, "false") == "true"
systemSpeechFlow.value = AppSettings.getString(app, KEY_SYSTEM_SPEECH, "false") == "true"
carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true"
mediaSearchFlow.value = AppSettings.getString(app, KEY_MEDIA_SEARCH, "false") == "true"
seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true"
unitsFlow.value = runCatching {
Units.valueOf(AppSettings.getString(app, KEY_UNITS, Units.AUTO.name))
@@ -293,9 +279,9 @@ object CarSettings {
AppSettings.putString(app, KEY_EXPERIMENTAL_FEATURES, on.toString())
}
fun setVoiceAssistantEnabled(on: Boolean) {
voiceAssistantFlow.value = on
AppSettings.putString(app, KEY_VOICE_ASSISTANT, on.toString())
fun setSystemSpeech(on: Boolean) {
systemSpeechFlow.value = on
AppSettings.putString(app, KEY_SYSTEM_SPEECH, on.toString())
}
fun setCarSensorsEnabled(on: Boolean) {
@@ -303,11 +289,6 @@ object CarSettings {
AppSettings.putString(app, KEY_CAR_SENSORS, on.toString())
}
fun setMediaSearchEnabled(on: Boolean) {
mediaSearchFlow.value = on
AppSettings.putString(app, KEY_MEDIA_SEARCH, on.toString())
}
fun setNightMode(mode: NightMode) {
nightFlow.value = mode
AppSettings.putString(app, KEY_NIGHT, mode.name)
@@ -58,9 +58,8 @@ fun CarSettingsScreen() {
val theme by CarSettings.appTheme.collectAsState()
val uiTheme by CarSettings.uiTheme.collectAsState()
val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState()
val voiceAssistant by CarSettings.voiceAssistantEnabled.collectAsState()
val carSensors by CarSettings.carSensorsEnabled.collectAsState()
val mediaSearch by CarSettings.mediaSearchEnabled.collectAsState()
val systemSpeech by CarSettings.systemSpeech.collectAsState()
val display by CarEnvironment.display.collectAsState()
val vehicle by CarEnvironment.vehicle.collectAsState()
val context = LocalContext.current
@@ -138,6 +137,14 @@ fun CarSettingsScreen() {
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
ExperimentalFeatureRow(
title = "Use the phone's speech recognition",
description = "Listens through the phone's own speech service instead of Gearslip's. On " +
"most phones that's Google's, which may send your voice to Google. GrapheneOS has " +
"none, so Gearslip keeps using its own.",
checked = systemSpeech,
) { CarSettings.setSystemSpeech(it) }
WebPageSetting()
ScreenFit(vehicle)
@@ -148,7 +155,7 @@ fun CarSettingsScreen() {
experimentalFeatures,
) { CarSettings.setExperimentalFeaturesEnabled(it) }
if (experimentalFeatures) {
ExperimentalFeaturesList(voiceAssistant, carSensors, mediaSearch)
ExperimentalFeaturesList(carSensors)
}
Column(verticalArrangement = Arrangement.spacedBy(8.dp)) {
@@ -205,21 +212,11 @@ private fun <T> ChoiceRow(title: String, options: List<Pair<T, String>>, selecte
* so the list reads fine whether there's one entry or several.
*/
@Composable
private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolean, mediaSearch: Boolean) {
private fun ExperimentalFeaturesList(carSensors: Boolean) {
Column(
Modifier.fillMaxWidth(),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
ExperimentalFeatureRow(
title = "Voice assistant",
description = "Push-to-talk and the car's voice button start listening: navigate, " +
"play media, call, or open an app by saying so. Can mishear you or act on the " +
"wrong command - keep your eyes on the road, and confirm anything it does " +
"before you rely on it.",
warning = true,
checked = voiceAssistant,
) { CarSettings.setVoiceAssistantEnabled(it) }
ExperimentalFeatureRow(
title = "Vehicle data",
description = "Adds a Vehicle data tile: a live, raw dump of the head unit's sensor " +
@@ -228,13 +225,6 @@ private fun ExperimentalFeaturesList(voiceAssistant: Boolean, carSensors: Boolea
"guess, not a fact, until you've watched one change for yourself.",
checked = carSensors,
) { CarSettings.setCarSensorsEnabled(it) }
ExperimentalFeatureRow(
title = "Media search",
description = "Adds a search button to the media screen. Some apps ignore the search " +
"or play something else.",
checked = mediaSearch,
) { CarSettings.setMediaSearchEnabled(it) }
}
}
@@ -272,7 +262,7 @@ private fun ExperimentalFeatureRow(
}
}
/** The URL (or raw HTML) the Web app shows. Edited with [CarKeyboard] - no phone in hand needed. */
/** The URL (or raw HTML) the Browser app shows. Edited with [CarKeyboard] - no phone in hand needed. */
@Composable
private fun WebPageSetting() {
val context = LocalContext.current
@@ -297,7 +287,7 @@ private fun WebPageSetting() {
)
}
Text(
"Shown by the Web app on the launcher. Applies on the next connection.",
"Shown by the Browser app on the launcher. Applies on the next connection.",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
@@ -256,8 +256,9 @@ fun CarUi() {
modifier = Modifier.align(Alignment.TopEnd),
)
// Above the popup: a call is more urgent than any notification.
VoiceHelpOverlay(Modifier.align(Alignment.Center))
CallOverlay(Modifier.align(Alignment.TopCenter))
AssistantOverlay(Modifier.align(Alignment.TopCenter))
AssistantOverlay(Modifier.align(Alignment.BottomCenter))
if (app.seb3thehacker.gearslip.BuildConfig.DEBUG) {
// Bottom corner, not top: a screen's own header almost always puts
// something in the top-right (settings, search, a Liked Songs
@@ -469,12 +470,8 @@ private fun CarNavBar(navigator: CarNavigator) {
NavHomeButton(navigator, entries, running)
Spacer(Modifier.width(6.dp))
NavItem(MediaIcons.Apps, navigator.current == CarScreen.Apps) { navigator.apps() }
val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState()
val voiceEnabled by CarSettings.voiceAssistantEnabled.collectAsState()
if (experimentalFeatures && voiceEnabled) {
Spacer(Modifier.width(6.dp))
NavAssistantIcon()
}
Spacer(Modifier.width(6.dp))
NavAssistantIcon()
// Pinned shortcuts, then whatever else is open right now. A pinned app that's also
// open stays in its pinned spot, marked with a dot, instead of showing up twice.
@@ -61,7 +61,6 @@ import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import androidx.lifecycle.compose.collectAsStateWithLifecycle
import app.seb3thehacker.gearslip.audio.CarAudio
import app.seb3thehacker.gearslip.media.CarMedia
import app.seb3thehacker.gearslip.media.MediaApp
import app.seb3thehacker.gearslip.media.MediaArt
@@ -88,11 +87,9 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) {
val queue by media.queue.collectAsStateWithLifecycle()
val browse by media.browse.collectAsStateWithLifecycle()
val rejection by media.rejection.collectAsStateWithLifecycle()
val capture by CarAudio.capture.collectAsStateWithLifecycle()
val search by media.search.collectAsStateWithLifecycle()
var tab by remember { mutableStateOf(Tab.NOW_PLAYING) }
var searching by remember { mutableStateOf(false) }
val searchAllowed = CarSettings.experimentalFeaturesEnabled.collectAsState().value && CarSettings.mediaSearchEnabled.collectAsState().value
// The connection outlives this screen: the home screen's player is the same one.
LaunchedEffect(app) { CarServices.openMedia(app) }
@@ -174,7 +171,6 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) {
} else {
CompactPlayer(now, media, Modifier.weight(1f).fillMaxWidth())
}
AudioBadge(capture)
}
Column(Modifier.weight(1.2f).fillMaxHeight()) {
Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) {
@@ -182,7 +178,7 @@ fun MediaScreen(app: MediaApp, onExit: () -> Unit) {
MediaTab("Browse", tab == Tab.BROWSE, Modifier.weight(1f)) { tab = Tab.BROWSE }
MediaTab("Up next", tab == Tab.QUEUE, Modifier.weight(1f)) { tab = Tab.QUEUE }
MediaTab("Lyrics", tab == Tab.LYRICS, Modifier.weight(1f)) { tab = Tab.LYRICS }
if (searchAllowed && (now.canSearch || media.canSearchLibrary)) {
if (now.canSearch || media.canSearchLibrary) {
GsIconButton(
Icons.Filled.Search,
"Search ${app.label}",
@@ -447,17 +443,6 @@ private sealed interface LyricsState {
class Found(val result: LyricsResult) : LyricsState
}
@Composable
private fun AudioBadge(capture: CarAudio.Capture) {
val text = when (capture) {
CarAudio.Capture.ON -> "Audio to car"
CarAudio.Capture.ASKING -> "Allow audio capture on the phone"
CarAudio.Capture.DENIED -> "Audio stays on the phone"
CarAudio.Capture.OFF -> return
}
Text(text, style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
@Composable
private fun Notice(title: String, detail: String?, action: Pair<String, () -> Unit>? = null) {
Column(
@@ -72,7 +72,7 @@ internal class BuiltInApp(
)
internal object BuiltInApps {
val Web = BuiltInApp("web", "Web", Icons.Filled.Search, CarScreen.App("web")) { it.open("web") }
val Web = BuiltInApp("web", "Browser", Icons.Filled.Search, CarScreen.App("web")) { it.open("web") }
val ScreenSharing = BuiltInApp("phone", "Screen sharing", Icons.Filled.Share, CarScreen.App("phone")) { it.open("phone") }
val Phone = BuiltInApp("dialer", "Phone", Icons.Filled.Call, CarScreen.App("dialer")) { it.open("dialer") }
val VehicleData = BuiltInApp("vehicle", "Vehicle data", Icons.Filled.Info, CarScreen.VehicleData) { it.vehicleData() }
@@ -45,6 +45,7 @@ import androidx.car.app.model.MessageTemplate
import androidx.car.app.model.PaneTemplate
import androidx.car.app.model.Row as CarRow
import androidx.car.app.model.SearchTemplate
import androidx.compose.foundation.clickable
import androidx.car.app.model.Section
import androidx.car.app.model.SectionedItemTemplate
import androidx.car.app.model.TabTemplate
@@ -238,6 +239,9 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) {
// initialSearchText would fight whatever is being typed.
var query by remember { mutableStateOf(template.initialSearchText.orEmpty()) }
val delegate = template.searchCallbackDelegate
// An app that opens on results ("navigate to Starbucks") asks for the keyboard to stay down,
// so the results aren't hidden under it. Tapping the field brings it up.
var typing by remember { mutableStateOf(template.isShowKeyboardByDefault) }
ContentSurface(modifier) {
// Back button and field share a row: on a 480px screen the keyboard and the results
@@ -251,7 +255,7 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) {
SearchField(
query,
template.searchHint.orEmpty(),
Modifier.weight(1f),
Modifier.weight(1f).clickable { typing = true },
onClear = {
query = ""
delegate.textChanged("")
@@ -265,14 +269,17 @@ private fun SearchContent(template: SearchTemplate, modifier: Modifier) {
else ItemListColumn(template.itemList, modifier = Modifier.fillMaxSize())
}
CarKeyboard(
text = query,
onTextChange = {
query = it
delegate.textChanged(it)
},
onSubmit = { delegate.submitted(query) },
)
if (typing) {
CarKeyboard(
text = query,
onTextChange = {
query = it
delegate.textChanged(it)
},
onSubmit = { delegate.submitted(query) },
onDismiss = { typing = false },
)
}
}
}
@@ -0,0 +1,94 @@
package app.seb3thehacker.gearslip.car
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.widthIn
import androidx.compose.foundation.rememberScrollState
import androidx.compose.foundation.verticalScroll
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Close
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.ui.Modifier
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
import app.seb3thehacker.gearslip.car.theme.GsIconButton
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
/** What the assistant understands, shown on the car screen when the driver says "help". */
object VoiceHelp {
private val _shown = MutableStateFlow(false)
val shown: StateFlow<Boolean> = _shown.asStateFlow()
fun show() { _shown.value = true }
fun hide() { _shown.value = false }
/** Say this, and this happens. Kept in step with [AssistantCommands]. */
val COMMANDS = listOf(
"Navigate to…" to "Directions there. Add \"with Organic Maps\" to pick the app",
"Play…" to "A song, artist, or album in the open music app",
"Pause · Resume · Next · Previous" to "Control the music",
"What's playing" to "The song and artist",
"Call…" to "A contact by name",
"Open…" to "Any app, like Browser, Phone, or Screen sharing",
"Exit" to "Back to the car's own screen",
"Go home · Open apps · Settings" to "Move around Gearslip",
"What time is it · Weather" to "The time, or the forecast",
"Help" to "This list",
)
}
/** The command list over the car screen: read it at a glance, tap the cross or wait to dismiss. */
@Composable
fun VoiceHelpOverlay(modifier: Modifier = Modifier) {
val shown by VoiceHelp.shown.collectAsState()
if (!shown) return
// A list left up would cover the map for the rest of the drive.
LaunchedEffect(Unit) {
delay(30_000)
VoiceHelp.hide()
}
Surface(
color = MaterialTheme.colorScheme.surfaceContainerHigh,
contentColor = MaterialTheme.colorScheme.onSurface,
shape = MaterialTheme.shapes.extraLarge,
tonalElevation = 6.dp,
shadowElevation = 8.dp,
modifier = modifier.padding(16.dp).widthIn(max = 640.dp),
) {
Column(Modifier.padding(20.dp)) {
Row(Modifier.fillMaxWidth()) {
Text(
"You can say",
style = MaterialTheme.typography.titleLarge,
fontWeight = FontWeight.Bold,
modifier = Modifier.weight(1f).padding(top = 6.dp),
)
GsIconButton(Icons.Filled.Close, "Close", VoiceHelp::hide)
}
// Full width, so a drag anywhere on the card scrolls, not just one on the words.
Column(
Modifier.fillMaxWidth().verticalScroll(rememberScrollState()).padding(top = 8.dp),
verticalArrangement = Arrangement.spacedBy(10.dp),
) {
VoiceHelp.COMMANDS.forEach { (say, does) ->
Column(Modifier.fillMaxWidth()) {
Text(say, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold)
Text(does, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant)
}
}
}
}
}
}
@@ -59,7 +59,7 @@ fun WebApp(content: String) {
Column(Modifier.fillMaxSize()) {
// Back steps through the page's history first (the interceptor above), then leaves.
ScreenHeader("Web", Modifier.padding(horizontal = 16.dp, vertical = 8.dp))
ScreenHeader("Browser", Modifier.padding(horizontal = 16.dp, vertical = 8.dp))
AndroidView(
modifier = Modifier.weight(1f).fillMaxSize(),
factory = { context ->
@@ -301,11 +301,14 @@ class CarAppConnection(private val context: Context) {
/**
* "navigate to X" deep link, the same one Android Auto's own Assistant integration sends -
* [ACTION_NAVIGATE] with the destination as a `geo:` search query, rather than anything
* app-specific.
* app-specific. Centred on the phone when there's a fix: `geo:0,0` made Organic Maps pick the
* first Del Taco it found anywhere, not the nearest. The location goes only to the map app.
*/
private fun navigateIntent(destination: String) =
Intent(CarContext.ACTION_NAVIGATE, Uri.parse("geo:0,0?q=" + Uri.encode(destination)))
private fun navigateIntent(destination: String): Intent {
val near = app.seb3thehacker.gearslip.notify.CarMotion.nearHere(context) ?: "0,0"
return Intent(CarContext.ACTION_NAVIGATE, Uri.parse("geo:$near?q=" + Uri.encode(destination)))
.setComponent(component)
}
/**
* Sends a fresh destination to an app that's already running, instead of tearing it down
@@ -156,6 +156,9 @@ class CarMedia(private val context: Context) {
private val main = Handler(Looper.getMainLooper())
private var connecting: MediaApp? = null
/** The music app Gearslip is attached to, for the assistant to name; null when there's none. */
val appLabel: String? get() = connecting?.label?.takeIf { controller != null }
private val _phase = MutableStateFlow(Phase.IDLE)
val phase: StateFlow<Phase> = _phase.asStateFlow()
@@ -236,6 +239,12 @@ class CarMedia(private val context: Context) {
connecting = app
_phase.value = Phase.CONNECTING
_browse.value = BrowseState()
// Known to refuse its library (Spotify): asking only costs the seconds it takes to say no,
// so go straight to its playback session.
if (KnownApps.playerWorks(app.component.packageName)) {
attachToSession(app)
return
}
val mb = MediaBrowser(context, app.component, object : MediaBrowser.ConnectionCallback() {
override fun onConnected() = this@CarMedia.onConnected(app)
override fun onConnectionFailed() {
@@ -273,7 +282,12 @@ class CarMedia(private val context: Context) {
return
}
if (session == null) {
val limit = if (launched) SESSION_RETRIES_LAUNCHED else SESSION_RETRIES
// A known session-only app that isn't running won't start by waiting: say so at once.
val limit = when {
launched -> SESSION_RETRIES_LAUNCHED
KnownApps.playerWorks(app.component.packageName) -> 0
else -> SESSION_RETRIES
}
// The app only publishes its session once its service has started; give it a moment.
if (attempt < limit) {
main.postDelayed({ attachToSession(app, attempt + 1, launched) }, SESSION_RETRY_MS)
@@ -454,7 +468,7 @@ class CarMedia(private val context: Context) {
}
val component = searchComponent
if (component == null) {
_results.value = Results(q, emptyList(), supported = false)
noLibrarySearch(q)
return
}
resultsQuery = q
@@ -476,13 +490,13 @@ class CarMedia(private val context: Context) {
override fun onConnectionFailed() {
GearslipLog.w("media: no library connection for search")
searchBrowser = null
resultsQuery?.let { _results.value = Results(it, emptyList(), supported = false) }
resultsQuery?.let { noLibrarySearch(it) }
}
}, null)
searchBrowser = created
runCatching { created.connect() }.onFailure {
searchBrowser = null
_results.value = Results(q, emptyList(), supported = false)
noLibrarySearch(q)
}
}
@@ -492,18 +506,42 @@ class CarMedia(private val context: Context) {
override fun onSearchResult(query: String, extras: Bundle?, items: List<MediaBrowserCompat.MediaItem>) {
if (query != resultsQuery) return
GearslipLog.i("media: library search found ${items.size}")
_results.value = Results(query, items.map(::entryOfCompat))
val entries = items.map(::entryOfCompat)
if (query == playFirstFor) {
playFirstFor = null
val first = entries.firstOrNull { it.playable } ?: entries.firstOrNull()
if (first == null) {
clearResults()
search(query)
} else {
GearslipLog.i("media: playing the library's best match")
clearResults()
searchBefore = _now.value
_search.value = Search(query)
main.removeCallbacks(searchTimeout)
controller?.transportControls?.playFromMediaId(first.id, null)
main.postDelayed(searchTimeout, SEARCH_WAIT_MS)
}
return
}
_results.value = Results(query, entries)
}
override fun onError(query: String, extras: Bundle?) {
if (query != resultsQuery) return
if (query == playFirstFor) {
playFirstFor = null
clearResults()
search(query)
return
}
GearslipLog.i("media: the app has no library search")
_results.value = Results(query, emptyList(), supported = false)
}
})
}.onFailure {
GearslipLog.w("media: library search failed: ${it.message}")
_results.value = Results(q, emptyList(), supported = false)
noLibrarySearch(q)
}
}
@@ -520,6 +558,36 @@ class CarMedia(private val context: Context) {
)
}
/** The library can't search: a voice "play X" falls back to the app's own search; a typed one says so. */
private fun noLibrarySearch(q: String) {
if (q == playFirstFor) {
playFirstFor = null
clearResults()
search(q)
} else {
_results.value = Results(q, emptyList(), supported = false)
}
}
/** A voice "play X" waiting on the library search, to play its first match instead of listing them. */
private var playFirstFor: String? = null
/**
* Plays the best match for [query], for voice. Asks the app's library first and plays its top
* result, which apps like ViVi honour where they ignore [search]; without a library search,
* falls back to [search].
*/
fun playBestMatch(query: String) {
val q = query.trim()
if (q.isEmpty()) return
if (!canSearchLibrary) {
search(q)
return
}
playFirstFor = q
searchLibrary(q)
}
/** The connected app's service, for the search connection; null when nothing is connected. */
private var searchComponent: android.content.ComponentName? = null
private var searchBrowser: MediaBrowserCompat? = null
@@ -22,6 +22,33 @@ object CarMotion {
private var listener: LocationListener? = null
private var speed = 0f
private var fixAt = 0L
@Volatile private var last: Location? = null
/** A fix older than this is somewhere the car has left. */
private const val NEAR_STALE_MS = 10 * 60_000L
/**
* Where the phone is, as "lat,lng", for "navigate to Del Taco" to find the nearest one. From
* this drive's GPS, or Android's last known fix; null with no permission or nothing recent.
*/
fun nearHere(context: Context): String? {
val fix = last?.takeIf { SystemClock.elapsedRealtime() - fixAt < NEAR_STALE_MS }
?: lastKnown(context)
?: return null
return String.format(java.util.Locale.US, "%.5f,%.5f", fix.latitude, fix.longitude)
}
private fun lastKnown(context: Context): Location? {
if (context.checkSelfPermission(Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED &&
context.checkSelfPermission(Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED
) return null
val manager = context.getSystemService(LocationManager::class.java) ?: return null
val now = System.currentTimeMillis()
return manager.getProviders(true)
.mapNotNull { runCatching { manager.getLastKnownLocation(it) }.getOrNull() }
.filter { now - it.time < NEAR_STALE_MS }
.maxByOrNull { it.time }
}
val moving: Boolean
get() = SystemClock.elapsedRealtime() - fixAt < STALE_MS && speed > MOVING_MS
@@ -34,6 +61,7 @@ object CarMotion {
override fun onLocationChanged(location: Location) {
speed = if (location.hasSpeed()) location.speed else 0f
fixAt = SystemClock.elapsedRealtime()
last = location
}
}
listener = l
@@ -4,7 +4,6 @@ import android.Manifest
import android.annotation.SuppressLint
import android.content.Context
import android.content.pm.PackageManager
import android.content.res.AssetManager
import android.media.AudioFormat
import android.media.AudioManager
import android.media.AudioRecord
@@ -15,38 +14,50 @@ import android.os.Looper
import android.os.SystemClock
import app.seb3thehacker.gearslip.GearslipLog
import app.seb3thehacker.gearslip.audio.CarMic
import java.util.Locale
import com.k2fsa.sherpa.onnx.EndpointConfig
import com.k2fsa.sherpa.onnx.EndpointRule
import com.k2fsa.sherpa.onnx.FeatureConfig
import com.k2fsa.sherpa.onnx.OfflineCanaryModelConfig
import com.k2fsa.sherpa.onnx.OfflineModelConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizer
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
import com.k2fsa.sherpa.onnx.OfflineTransducerModelConfig
import com.k2fsa.sherpa.onnx.OnlineModelConfig
import com.k2fsa.sherpa.onnx.OnlineRecognizer
import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig
import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig
import java.io.File
import java.util.concurrent.Executors
import kotlin.math.log10
import kotlin.math.max
import kotlin.math.sqrt
/**
* Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships
* none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply
* of a few seconds comes back in well under a second on a recent phone.
* none). sherpa-onnx runs a streaming model ([SpeechModels]) on the phone: words come out while
* the driver is still talking, and the model itself decides when they've finished. A larger model
* then reads the whole phrase once more for the final words, since a model that waits for the end
* hears names and places far better than one that has to keep up. The phrase is held in memory
* for that and nowhere else, and dropped as soon as it's read.
*
* Whisper needs the whole phrase before it starts, so this records from the car's microphone (the
* phone's when the car has none or sends nothing),
* decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes
* once. Everything runs on one worker thread; callbacks arrive on the main thread.
* Audio comes from the car's microphone, or the phone's when the car has none or sends nothing.
* Everything runs on one worker thread; callbacks arrive on the main thread.
*/
object SpeechEngine {
/** FUTO's ACFT Whisper tiny (Apache-2.0), re-quantized to 5 bits, inside the APK. */
private const val MODEL = "speech/tiny_acft_q5_1.bin"
/** The error for an engine that can't run at all, which callers may answer with a fallback. */
const val NOT_LOADED = "The speech model didn't load."
/** Said when the chosen model hasn't been downloaded yet. */
const val NOT_DOWNLOADED = "Download a voice model in Gearslip's settings on the phone first."
private const val RATE = 16_000
private const val FRAME = 480 // 30 ms
private const val MAX_MS = 15_000L
private const val NO_SPEECH_MS = 6_000L
private const val END_SILENCE_MS = 1_100L
private const val CALIBRATE_MS = 300L
private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word
private const val CHUNK = 1_600 // 100 ms
private const val MAX_S = 15f
/** Quiet before the first word that counts as "nobody spoke". */
private const val NO_SPEECH_S = 6f
/** Quiet after the last word that ends the phrase. */
private const val END_SILENCE_S = 1.0f
private const val CUE_VOLUME = 80
private const val IDLE_UNLOAD_MS = 5 * 60_000L
/** How long the car gets to start sending once asked, before the phone's microphone takes over. */
private const val CAR_FIRST_AUDIO_MS = 1_500L
private const val CAR_GAP_MS = 1_000L
@@ -55,27 +66,79 @@ object SpeechEngine {
private const val CUE_TAIL_MS = 250L
private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") }
/**
* Parakeet large takes seconds to load. On [worker], the first press after connecting
* waited behind it before listening at all; here it loads while the driver talks.
*/
private val finalWorker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-final") }
private val finalLock = Object()
private val main = Handler(Looper.getMainLooper())
private var libraryLoaded = false
private var handle = 0L
private var recognizer: OnlineRecognizer? = null
private var loadedModel: SpeechModels.Model? = null
@Volatile private var finalRecognizer: OfflineRecognizer? = null
@Volatile private var loadedFinal: SpeechModels.Model? = null
/**
* Parakeet large holds most of the ~900 MB the two models take, so it goes after a few
* minutes without a command. Kroko stays: it's small, and keeps listening instant.
*/
private val idleUnload = Runnable { unloadFinal("after ${IDLE_UNLOAD_MS / 60_000} minutes without use") }
@Volatile private var run = 0
/** Loads the model ahead of time, so the first reply doesn't wait for it. */
/** Loads the model ahead of time, so the first command doesn't wait for it. */
fun warm(context: Context) {
val app = context.applicationContext
worker.execute { ensureLoaded(app) }
SpeechModels.init(app)
// Kroko first, alone: loading both at once slowed it from a fraction of a second to six.
worker.execute {
ensureLoaded(app)
finalWorker.execute { ensureFinalLoaded(app) }
}
}
private fun unloadFinal(why: String) {
finalWorker.execute {
synchronized(finalLock) {
val model = loadedFinal ?: return@execute
finalRecognizer?.release()
finalRecognizer = null
loadedFinal = null
GearslipLog.i("speech: unloaded ${model.label} $why")
}
}
}
/** Drops the loaded models: the driver picked another or deleted one, or the car went away. */
fun unload() {
main.removeCallbacks(idleUnload)
run++
worker.execute {
recognizer?.release()
recognizer = null
loadedModel = null
}
finalWorker.execute {
synchronized(finalLock) {
finalRecognizer?.release()
finalRecognizer = null
loadedFinal = null
}
}
}
/**
* Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk.
* [onSpeaking] fires at the first word, [onHeard] when the driver stops and the words are being
* worked out; [onResult] gets the text, [onError] a sentence fit to show on the car screen.
* [onPartial] gets the words so far as they're recognized, [onSpeaking] fires at the first
* word, [onHeard] when the driver stops; [onResult] gets the text, [onError] a sentence fit to
* show on the car screen.
*/
fun listen(
context: Context,
cue: Boolean = true,
onSpeaking: () -> Unit = {},
onPartial: (String) -> Unit = {},
onListening: () -> Unit = {},
onHeard: () -> Unit,
onResult: (String) -> Unit,
onError: (String) -> Unit,
@@ -87,28 +150,29 @@ object SpeechEngine {
}
val id = ++run
fun post(block: () -> Unit) { main.post { if (id == run) block() } }
// Brings Parakeet back if it was unloaded, while the driver talks.
main.removeCallbacks(idleUnload)
finalWorker.execute { ensureFinalLoaded(app) }
worker.execute {
if (id != run) return@execute
if (!ensureLoaded(app)) return@execute post { onError(NOT_LOADED) }
if (!SpeechModels.isReady(app, SpeechModels.selected.value)) return@execute post { onError(NOT_DOWNLOADED) }
val r = ensureLoaded(app) ?: return@execute post { onError(NOT_LOADED) }
VoiceFocus.acquire(app)
val audio = try {
val text = try {
if (cue) cue()
if (id != run) return@execute
record(id) { post(onSpeaking) }
post(onListening)
stream(app, r, id, onSpeaking = { post(onSpeaking) }, onPartial = { post { onPartial(it) } }, onStopped = { post(onHeard) })
} catch (t: Throwable) {
GearslipLog.e("speech: recording failed", t)
GearslipLog.e("speech: listening failed", t)
return@execute post { onError("The microphone isn't available right now.") }
} finally {
VoiceFocus.release()
}
main.removeCallbacks(idleUnload)
main.postDelayed(idleUnload, IDLE_UNLOAD_MS)
if (id != run) return@execute
if (audio == null) return@execute post { onError("Didn't catch that.") }
post(onHeard)
val started = SystemClock.uptimeMillis()
val text = clean(transcribe(audio))
GearslipLog.i(
"speech: ${audio.size * 1000L / RATE} ms of audio took ${SystemClock.uptimeMillis() - started} ms",
)
post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) }
}
}
@@ -116,6 +180,185 @@ object SpeechEngine {
/** Stops listening; a phrase already being worked out is dropped. */
fun cancel() { run++ }
/**
* Feeds the microphone to the recognizer in 100 ms pieces, reporting the words as they firm
* up, until the model hears the end of the phrase (or nobody speaks, or it runs too long).
*/
private fun stream(
context: Context,
r: OnlineRecognizer,
id: Int,
onSpeaking: () -> Unit,
onPartial: (String) -> Unit,
onStopped: () -> Unit,
): String {
val mic = carMic() ?: phoneMic()
val s = r.createStream()
val frame = ShortArray(CHUNK)
val samples = FloatArray(CHUNK)
// The whole phrase, for the final read; [MAX_S] and a little over is the most it can hold.
val phrase = FloatArray((RATE * (MAX_S + 1)).toInt())
var held = 0
var last = ""
var spoke = false
var audioMs = 0L
var decodeMs = 0L
try {
while (id == run) {
val n = mic.read(frame)
if (n < 0) break
if (n == 0) continue
for (i in 0 until n) samples[i] = frame[i] / 32768f
val keep = minOf(n, phrase.size - held)
System.arraycopy(samples, 0, phrase, held, keep)
held += keep
s.acceptWaveform(if (n == CHUNK) samples else samples.copyOf(n), RATE)
audioMs += n * 1000L / RATE
val started = SystemClock.uptimeMillis()
while (r.isReady(s)) r.decode(s)
decodeMs += SystemClock.uptimeMillis() - started
val text = clean(r.getResult(s).text)
if (text != last) {
last = text
if (text.isNotEmpty() && !spoke) {
spoke = true
onSpeaking()
}
onPartial(text)
}
if (r.isEndpoint(s)) break
}
s.inputFinished()
while (r.isReady(s)) r.decode(s)
last = clean(r.getResult(s).text)
} finally {
mic.close()
s.release()
}
// Under 1.0 keeps up with the driver; over it, the words fall behind the voice.
GearslipLog.i(
"speech: $audioMs ms of audio, decoding took $decodeMs ms " +
"(${"%.2f".format(decodeMs.toFloat() / max(1L, audioMs))} of real time)",
)
if (last.isEmpty() || id != run) { phrase.fill(0f); return last }
// Still loading (just connected, or back after going idle): the live words now beat
// Parakeet's in several seconds.
val wanted = SpeechModels.final.value
if (wanted == null || finalRecognizer == null || loadedFinal != wanted) {
if (wanted != null) GearslipLog.i("speech: ${wanted.label} isn't loaded yet, keeping the live words")
phrase.fill(0f)
return last
}
onStopped()
val started = SystemClock.uptimeMillis()
// Held for the whole read, so a model swap or the idle unload can't free it mid-decode.
val reread = synchronized(finalLock) {
val final = finalRecognizer?.takeIf { loadedFinal == wanted } ?: run { phrase.fill(0f); return last }
final.createStream().run {
try {
acceptWaveform(phrase.copyOf(held), RATE)
final.decode(this)
clean(final.getResult(this).text)
} finally {
release()
}
}
}
phrase.fill(0f)
GearslipLog.i("speech: the final read took ${SystemClock.uptimeMillis() - started} ms")
if (reread.isEmpty()) return last
if (reread != last) onPartial(reread)
return reread
}
/** The model that rereads the phrase, when one is chosen and downloaded; null otherwise. */
private fun ensureFinalLoaded(context: Context): OfflineRecognizer? = synchronized(finalLock) {
val model = SpeechModels.final.value ?: return null
if (finalRecognizer != null && loadedFinal == model) return finalRecognizer
finalRecognizer?.release()
finalRecognizer = null
if (!SpeechModels.isReady(context, model)) return null
val dir = SpeechModels.dir(context, model)
fun path(name: String) = File(dir, name).absolutePath
val started = SystemClock.uptimeMillis()
return runCatching {
OfflineRecognizer(
config = OfflineRecognizerConfig(
featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80),
modelConfig = if (model.modelType == "canary") {
// English in, English out, and no punctuation: clean() strips it anyway.
OfflineModelConfig(
canary = OfflineCanaryModelConfig(path(model.encoder), path(model.decoder), srcLang = "en", tgtLang = "en", usePnc = false),
tokens = path("tokens.txt"),
numThreads = threads(),
)
} else {
OfflineModelConfig(
transducer = OfflineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)),
tokens = path("tokens.txt"),
numThreads = threads(),
modelType = model.modelType,
)
},
decodingMethod = "greedy_search",
),
)
}.onSuccess {
finalRecognizer = it
loadedFinal = model
GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms")
}.onFailure {
GearslipLog.e("speech: couldn't load ${model.label}", it)
}.getOrNull()
}
private fun ensureLoaded(context: Context): OnlineRecognizer? {
val model = SpeechModels.selected.value
if (recognizer != null && loadedModel == model) return recognizer
recognizer?.release()
recognizer = null
if (!SpeechModels.isReady(context, model)) return null
val dir = SpeechModels.dir(context, model)
fun path(name: String) = File(dir, name).absolutePath
val started = SystemClock.uptimeMillis()
return runCatching {
OnlineRecognizer(
config = OnlineRecognizerConfig(
featConfig = FeatureConfig(sampleRate = RATE, featureDim = 80),
modelConfig = OnlineModelConfig(
transducer = OnlineTransducerModelConfig(path(model.encoder), path(model.decoder), path(model.joiner)),
tokens = path("tokens.txt"),
numThreads = threads(),
modelType = model.modelType,
),
endpointConfig = EndpointConfig(
rule1 = EndpointRule(false, NO_SPEECH_S, 0f),
rule2 = EndpointRule(true, END_SILENCE_S, 0f),
rule3 = EndpointRule(false, 0f, MAX_S),
),
enableEndpoint = true,
decodingMethod = "greedy_search",
),
)
}.onSuccess {
recognizer = it
loadedModel = model
GearslipLog.i("speech: loaded ${model.label} in ${SystemClock.uptimeMillis() - started} ms")
}.onFailure {
GearslipLog.e("speech: couldn't load ${model.label}", it)
}.getOrNull()
}
/** The big cores only: more threads than that slow the model down on a phone. */
private fun threads() = Runtime.getRuntime().availableProcessors().let { if (it >= 8) 4 else max(1, it / 2) }
/**
* Models write in upper case, or with punctuation and capitals of their own; commands are
* matched in plain lower case, so only words and apostrophes are kept.
*/
private fun clean(text: String): String =
text.lowercase().replace(Regex("[^\\p{L}\\p{N}' ]"), " ").replace(Regex("\\s+"), " ").trim()
/**
* A short beep on the media stream, so it plays through the car like everything else, then a
* pause long enough for it to finish there before the microphone opens and hears it.
@@ -130,14 +373,9 @@ object SpeechEngine {
}
}
// --- recording --------------------------------------------------------------------------
// --- microphones -----------------------------------------------------------------------
/**
* Records until the driver stops talking. Returns the phrase from just before the first word,
* or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
* loud cabin needs a louder voice than a quiet one to count as talking.
*/
/** Where a phrase is recorded from: [read] fills [frame] and returns how many samples it got. */
/** Where a phrase is heard from: [read] fills [frame] and returns how many samples it got, -1 at the end. */
private interface Mic {
fun read(frame: ShortArray): Int
fun close()
@@ -148,10 +386,11 @@ object SpeechEngine {
val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT)
val recorder = AudioRecord(
MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE,
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8),
AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, CHUNK * 4),
)
check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" }
recorder.startRecording()
GearslipLog.i("speech: listening through the phone's microphone")
return object : Mic {
override fun read(frame: ShortArray) = recorder.read(frame, 0, frame.size)
override fun close() {
@@ -163,7 +402,7 @@ object SpeechEngine {
/**
* The car's microphone, which hears the driver better than a phone in a cup holder. The car
* sends audio in its own sized pieces; they're cut to [FRAME]s here. Null when there's no car
* sends audio in its own sized pieces; they're cut to [CHUNK]s here. Null when there's no car
* microphone, or it sends nothing at first, so the phone's is used instead.
*/
private fun carMic(): Mic? {
@@ -190,105 +429,10 @@ object SpeechEngine {
at += n
if (at == chunk.size) { pending = null; at = 0 }
}
// The car stopped sending mid-phrase: end the recording rather than wait forever.
// The car stopped sending mid-phrase: end the phrase rather than wait forever.
return if (filled == 0) -1 else filled
}
override fun close() = car.stop()
}
}
/**
* Records until the driver stops talking. Returns the phrase from just before the first word,
* or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a
* loud cabin needs a louder voice than a quiet one to count as talking.
*/
private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? {
val mic = carMic() ?: phoneMic()
val samples = FloatArray((RATE * MAX_MS / 1000).toInt())
var count = 0
val frame = ShortArray(FRAME)
var noise = 0.0
var noiseFrames = 0
var speechStart = -1
var lastSpeech = 0
try {
while (id == run && count + FRAME <= samples.size) {
val n = mic.read(frame)
if (n < 0) break
if (n == 0) continue
var sum = 0.0
for (i in 0 until n) {
val v = frame[i] / 32768f
samples[count + i] = v
sum += v * v
}
count += n
val db = 20 * log10(sqrt(sum / n) + 1e-9)
val ms = count * 1000L / RATE
if (ms <= CALIBRATE_MS) {
noise += db; noiseFrames++
continue
}
val floor = if (noiseFrames > 0) noise / noiseFrames else -60.0
val talking = db > max(floor + 12, -45.0)
if (talking) {
if (speechStart < 0) {
speechStart = max(0, count - n - PRE_ROLL)
onSpeaking()
}
lastSpeech = count
}
if (speechStart < 0 && ms > NO_SPEECH_MS) return null
if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break
}
} finally {
mic.close()
}
if (speechStart < 0) return null
return samples.copyOfRange(speechStart, count)
}
// --- the model --------------------------------------------------------------------------
private fun ensureLoaded(context: Context): Boolean {
if (handle != 0L) return true
if (!libraryLoaded) {
runCatching { System.loadLibrary("gearslip_speech") }
.onFailure { GearslipLog.e("speech: native library missing", it); return false }
libraryLoaded = true
}
val started = SystemClock.uptimeMillis()
handle = nativeInitAsset(context.assets, MODEL)
if (handle == 0L) {
GearslipLog.w("speech: couldn't load the model")
return false
}
GearslipLog.i("speech: loaded the model in ${SystemClock.uptimeMillis() - started} ms")
return true
}
private fun transcribe(audio: FloatArray): String {
val bytes = nativeTranscribe(handle, audio, language(), threads())
return String(bytes, Charsets.UTF_8)
}
/** The phone's language, as Whisper names it. Older Java codes differ for a few. */
private fun language(): String = when (val lang = Locale.getDefault().language) {
"iw" -> "he"
"in" -> "id"
"ji" -> "yi"
"" -> "en"
else -> lang
}
/** The big cores only: more threads than that slow Whisper down on a phone. */
private fun threads() = Runtime.getRuntime().availableProcessors().coerceIn(1, 8).let { if (it >= 8) 4 else max(1, it / 2) }
/** Whisper marks silence and noise as "[BLANK_AUDIO]" or "(wind blowing)"; none of it is words. */
private fun clean(text: String): String =
text.replace(Regex("""\[[^\]]*\]|\([^)]*\)|\*[^*]*\*"""), " ").replace(Regex("\\s+"), " ").trim()
private external fun nativeInitAsset(assets: AssetManager, path: String): Long
private external fun nativeTranscribe(handle: Long, samples: FloatArray, language: String, threads: Int): ByteArray
private external fun nativeFree(handle: Long)
}
@@ -0,0 +1,193 @@
package app.seb3thehacker.gearslip.speech
import android.content.Context
import app.seb3thehacker.gearslip.GearslipLog
import java.io.File
import java.net.HttpURLConnection
import java.net.URL
import java.util.concurrent.Executors
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
/**
* The speech models Gearslip listens with, in two roles: a streaming one that writes the words
* while the driver talks, and an optional final one that reads the whole phrase again once they
* stop, for a more accurate result. None ships in the APK: each is fetched once from sherpa-onnx's
* GitHub releases, where its authors publish it, and kept in the app's own storage.
*/
object SpeechModels {
enum class Role { LIVE, FINAL }
enum class Model(
val role: Role,
val id: String,
val label: String,
val detail: String,
val downloadMb: Int,
private val archive: String,
val encoder: String,
val decoder: String,
/** Empty for Canary, which decodes with attention and has no joiner. */
val joiner: String,
val modelType: String = "",
) {
KROKO(
Role.LIVE, "kroko-en", "Kroko", "Banafo's Kroko ASR, CC BY-SA 4.0", 54,
"sherpa-onnx-streaming-zipformer-en-kroko-2025-08-06",
"encoder.onnx", "decoder.onnx", "joiner.onnx", modelType = "zipformer2",
),
PARAKEET_SMALL(
Role.FINAL, "parakeet-110m-en", "Parakeet small", "Lighter and quicker, misses more names. NVIDIA NeMo, CC BY 4.0", 103,
"sherpa-onnx-nemo-parakeet_tdt_transducer_110m-en-36000-int8",
"encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
),
CANARY(
Role.FINAL, "canary-180m-en", "Canary", "In between: better than Parakeet small, quicker than large. NVIDIA NeMo, CC BY 4.0", 154,
"sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8",
"encoder.int8.onnx", "decoder.int8.onnx", "", modelType = "canary",
),
PARAKEET_LARGE(
Role.FINAL, "parakeet-0.6b-en", "Parakeet large", "Recommended. Hears names and places best. NVIDIA NeMo, CC BY 4.0", 460,
"sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8",
"encoder.int8.onnx", "decoder.int8.onnx", "joiner.int8.onnx", modelType = "nemo_transducer",
),
;
val url get() = "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/$archive.tar.bz2"
val files get() = listOf(encoder, decoder, joiner, "tokens.txt").filter { it.isNotEmpty() }
}
sealed interface State {
data object Missing : State
data class Downloading(val percent: Int) : State
data object Ready : State
data class Failed(val reason: String) : State
}
private const val PREFS = "speech"
private const val KEY_MODEL = "model"
private const val KEY_FINAL = "final_model"
private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech-download") }
private val states = Model.entries.associateWith { MutableStateFlow<State>(State.Missing) }
private val _selected = MutableStateFlow(Model.KROKO)
val selected: StateFlow<Model> = _selected.asStateFlow()
private val _final = MutableStateFlow<Model?>(Model.PARAKEET_LARGE)
/** The model that rereads the whole phrase at the end; null to keep the live model's words. */
val final: StateFlow<Model?> = _final.asStateFlow()
@Volatile private var cancelled: Model? = null
fun init(context: Context) {
val id = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_MODEL, null)
_selected.value = Model.entries.firstOrNull { it.id == id && it.role == Role.LIVE } ?: Model.KROKO
val finalId = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_FINAL, Model.PARAKEET_LARGE.id)
_final.value = Model.entries.firstOrNull { it.id == finalId && it.role == Role.FINAL }
Model.entries.forEach { m -> if (isReady(context, m)) states.getValue(m).value = State.Ready }
// Models an older build offered and this one doesn't: hundreds of MB nobody can use.
val known = Model.entries.map { it.id }.toSet()
File(context.filesDir, "speech").listFiles()?.filter { it.isDirectory && it.name !in known }?.forEach {
GearslipLog.i("speech: removing ${it.name}, no longer offered")
it.deleteRecursively()
}
}
fun state(model: Model): StateFlow<State> = states.getValue(model).asStateFlow()
fun dir(context: Context, model: Model) = File(File(context.filesDir, "speech"), model.id)
fun isReady(context: Context, model: Model) = model.files.all { File(dir(context, model), it).isFile }
fun select(context: Context, model: Model) {
if (model.role == Role.FINAL) return selectFinal(context, model)
_selected.value = model
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_MODEL, model.id).apply()
SpeechEngine.unload()
}
/** [model] null keeps the live words as they are. */
fun selectFinal(context: Context, model: Model?) {
_final.value = model
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_FINAL, model?.id ?: "none").apply()
SpeechEngine.unload()
}
fun delete(context: Context, model: Model) {
cancelled = model
dir(context, model).deleteRecursively()
states.getValue(model).value = State.Missing
if (model == _selected.value || model == _final.value) SpeechEngine.unload()
}
/** Fetches [model] and unpacks just the files the recognizer reads; the rest of the archive is skipped. */
fun download(context: Context, model: Model) {
val app = context.applicationContext
val state = states.getValue(model)
if (state.value is State.Downloading || state.value == State.Ready) return
state.value = State.Downloading(0)
cancelled = null
worker.execute {
val target = dir(app, model)
val partial = File(target.parentFile, "${model.id}.partial").apply { deleteRecursively(); mkdirs() }
try {
val conn = (URL(model.url).openConnection() as HttpURLConnection).apply {
connectTimeout = 15_000
readTimeout = 30_000
instanceFollowRedirects = true
}
check(conn.responseCode == 200) { "the server answered ${conn.responseCode}" }
val total = conn.contentLengthLong.takeIf { it > 0 } ?: (model.downloadMb * 1_048_576L)
val counting = CountingStream(conn.inputStream) { read ->
if (cancelled == model) throw InterruptedException("cancelled")
state.value = State.Downloading(((read * 100) / total).toInt().coerceIn(0, 99))
}
TarArchiveInputStream(BZip2CompressorInputStream(counting.buffered())).use { tar ->
while (true) {
val entry = tar.nextEntry ?: break
val name = entry.name.substringAfterLast('/')
if (entry.isDirectory || name !in model.files) continue
File(partial, name).outputStream().use { tar.copyTo(it) }
}
}
check(model.files.all { File(partial, it).isFile }) { "the archive was missing files" }
target.deleteRecursively()
check(partial.renameTo(target)) { "couldn't store the model" }
GearslipLog.i("speech: downloaded ${model.label}")
state.value = State.Ready
} catch (e: Exception) {
partial.deleteRecursively()
if (cancelled == model) {
state.value = State.Missing
} else {
GearslipLog.w("speech: download of ${model.label} failed: ${e.message}")
state.value = State.Failed(e.message ?: e.javaClass.simpleName)
}
}
}
}
/** Counts bytes as they pass, reporting about every megabyte. */
private class CountingStream(
private val inner: java.io.InputStream,
private val onProgress: (Long) -> Unit,
) : java.io.InputStream() {
private var read = 0L
private var reported = 0L
override fun read(): Int = inner.read().also { if (it >= 0) count(1) }
override fun read(b: ByteArray, off: Int, len: Int): Int = inner.read(b, off, len).also { if (it > 0) count(it.toLong()) }
override fun close() = inner.close()
private fun count(n: Long) {
read += n
if (read - reported >= 1_048_576) {
reported = read
onProgress(read)
}
}
}
}
@@ -47,3 +47,25 @@ fun AudioChoiceDialog(onUseBluetooth: () -> Unit, onKeepAsking: () -> Unit) {
dismissButton = { TextButton(onClick = onKeepAsking) { Text("Keep asking") } },
)
}
/**
* Android lets an app capture other apps' audio only while it holds the microphone permission.
* Once the driver has refused that twice, Android stops asking, and music silently stays on the
* phone; this says why and where to turn it back on.
*/
@Composable
fun MicBlockedDialog(onOpenSettings: () -> Unit, onDismiss: () -> Unit) {
AlertDialog(
onDismissRequest = onDismiss,
title = { Text("Car audio needs the microphone permission") },
text = {
Text(
"Android lets an app send other apps' sound to the car only if it has the " +
"microphone permission. It's turned off for Gearslip, so music plays on the " +
"phone. Turn it on under Permissions, then Microphone.",
)
},
confirmButton = { TextButton(onClick = onOpenSettings) { Text("Open app settings") } },
dismissButton = { TextButton(onClick = onDismiss) { Text("Not now") } },
)
}
@@ -16,11 +16,9 @@ import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.statusBars
import androidx.compose.foundation.layout.windowInsetsPadding
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
@@ -39,7 +37,7 @@ import app.seb3thehacker.gearslip.BuildConfig
import app.seb3thehacker.gearslip.CertProvider
import app.seb3thehacker.gearslip.stats.UsageStats
private enum class Screen { HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, USAGE_NOTES, HELP, WHATS_NEW, RECOMMENDED_APPS }
private enum class Screen { INTRO, HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, USAGE_NOTES, HELP, WHATS_NEW, RECOMMENDED_APPS }
/** Three screens and a back stack of depth one: no navigation library needed. */
@Composable
@@ -49,20 +47,22 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
!AppSettings.hasSkippedCertSetup(context) && !CertProvider.hasAnyCert(context)
}
val needsSetupGuide = remember { !AppSettings.hasSeenPermissionsSetup(context) }
var screen by rememberSaveable {
mutableStateOf(
when {
needsCertSetup -> Screen.CERT_SETUP
needsSetupGuide -> Screen.SETUP_GUIDE
// Set up before usage notes existed: ask once, on the setup guide's own page.
!UsageStats.asked(context) -> Screen.USAGE_NOTES
else -> Screen.HOME
},
)
// Where launch goes once the intro (if any) is done.
val afterIntro = remember {
when {
needsCertSetup -> Screen.CERT_SETUP
needsSetupGuide -> Screen.SETUP_GUIDE
// Set up before usage notes existed: ask once, on the setup guide's own page.
!UsageStats.asked(context) -> Screen.USAGE_NOTES
else -> Screen.HOME
}
}
var screen by rememberSaveable { mutableStateOf(if (Intro.due(context)) Screen.INTRO else afterIntro) }
// A replay from About goes back to Settings; the launch one carries on into setup.
var introReturn by rememberSaveable { mutableStateOf(afterIntro) }
// Home's usage notes row opens Settings at that section rather than at the top.
var settingsAtUsageNotes by rememberSaveable { mutableStateOf(false) }
BackHandler(enabled = screen != Screen.HOME) {
BackHandler(enabled = screen != Screen.HOME && screen != Screen.INTRO) {
screen = if (screen == Screen.RECOMMENDED_APPS) Screen.SETTINGS else Screen.HOME
}
@@ -95,10 +95,11 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
.windowInsetsPadding(WindowInsets.statusBars)
.consumeWindowInsets(WindowInsets.statusBars),
) {
if (BuildConfig.DEBUG && screen != Screen.CAR_PREVIEW) DevBuildBanner()
if (BuildConfig.DEBUG && screen != Screen.CAR_PREVIEW && screen != Screen.INTRO) DevBuildBanner()
Column(Modifier.weight(1f)) {
when (screen) {
Screen.INTRO -> IntroScreen(onDone = { screen = introReturn })
Screen.HOME -> HomeScreen(
onOpenLogs = { screen = Screen.LOGS },
onOpenSettings = { settingsAtUsageNotes = false; screen = Screen.SETTINGS },
@@ -114,6 +115,7 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
onOpenRecommendedApps = { settingsAtUsageNotes = false; screen = Screen.RECOMMENDED_APPS },
onOpenWhatsNew = { settingsAtUsageNotes = false; screen = Screen.WHATS_NEW },
onReplayTutorial = { screen = Screen.SETUP_GUIDE },
onReplayIntro = { introReturn = Screen.SETTINGS; screen = Screen.INTRO },
atUsageNotes = settingsAtUsageNotes,
)
Screen.RECOMMENDED_APPS -> RecommendedAppsScreen(onBack = { screen = Screen.SETTINGS })
@@ -141,8 +143,6 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) {
// A car keyboard is open: offer the phone's own, for a passenger.
if (screen != Screen.CAR_PREVIEW) PhoneTypingBar()
}
CompatWarning()
}
/**
@@ -167,29 +167,3 @@ private fun DevBuildBanner() {
}
}
}
/** Shown once, on first run: the certificate is known to fail on head units newer than ~2020. */
@Composable
private fun CompatWarning() {
val context = LocalContext.current
var visible by rememberSaveable { mutableStateOf(!AppSettings.hasSeenCompatWarning(context)) }
if (visible) {
AlertDialog(
onDismissRequest = {},
title = { Text("Before you plug in") },
text = {
Text(
"Gearslip works with cars built before 2020. " +
"Newer firmware often blocks the connection.",
)
},
confirmButton = {
TextButton(onClick = {
AppSettings.setSeenCompatWarning(context)
visible = false
}) { Text("Got it") }
},
)
}
}
@@ -112,6 +112,10 @@ fun HomeScreen(
Spacer(Modifier.height(16.dp))
WirelessCard()
}
if (!live) {
Spacer(Modifier.height(16.dp))
SupportCard()
}
}
// Sits at the bottom, where a thumb reaches it.
@@ -0,0 +1,726 @@
package app.seb3thehacker.gearslip.ui
import android.content.Context
import android.content.Intent
import android.net.Uri
import android.provider.Settings
import androidx.activity.compose.BackHandler
import androidx.compose.animation.core.Animatable
import androidx.compose.animation.core.CubicBezierEasing
import androidx.compose.animation.core.FastOutSlowInEasing
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateDpAsState
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.keyframes
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.BorderStroke
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.clickable
import androidx.compose.foundation.interaction.MutableInteractionSource
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.BoxWithConstraints
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.height
import androidx.compose.foundation.layout.offset
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.foundation.layout.wrapContentWidth
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.LaunchedEffect
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableIntStateOf
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.geometry.Offset
import androidx.compose.ui.geometry.Size
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.Path
import androidx.compose.ui.graphics.StrokeCap
import androidx.compose.ui.graphics.StrokeJoin
import androidx.compose.ui.graphics.drawscope.DrawScope
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.graphics.drawscope.rotate
import androidx.compose.ui.graphics.drawscope.scale
import androidx.compose.ui.graphics.graphicsLayer
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.text.TextStyle
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.TextUnit
import androidx.compose.ui.unit.dp
import app.seb3thehacker.gearslip.BuildConfig
import androidx.compose.runtime.collectAsState
import androidx.compose.ui.draw.clipToBounds
import androidx.compose.ui.draw.drawWithContent
import androidx.compose.ui.graphics.BlendMode
import androidx.compose.ui.graphics.Brush
import androidx.compose.ui.graphics.CompositingStrategy
import androidx.compose.foundation.layout.wrapContentHeight
import app.seb3thehacker.gearslip.speech.SpeechModels
import app.seb3thehacker.gearslip.stats.UsageStats
import kotlin.math.PI
import kotlin.math.cos
import kotlin.math.atan2
import kotlin.math.sqrt
import kotlin.math.sin
import kotlin.random.Random
import kotlinx.coroutines.delay
/**
* The release intro: a few tap-through screens shown once after each new release (and on a new
* install), before setup. Its look follows the 0.3 mockup: Material shapes in the phone's own colours,
* with every size taken from the screen's width so it scales from small phones to tablets.
*/
object Intro {
private const val PREFS = "intro"
private const val KEY_SEEN = "seen_release"
/** "0.3" from "0.3.00-alpha": the intro belongs to a release, not to each patch. */
val release: String = BuildConfig.VERSION_NAME.split('.').take(2).joinToString(".")
fun due(context: Context): Boolean =
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getString(KEY_SEEN, null) != release
fun markSeen(context: Context) {
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit().putString(KEY_SEEN, release).apply()
}
}
/** The intro's tones, taken from the phone's own Material You colours like the rest of the app. */
private class IntroColors(
val surface: Color, val onSurface: Color, val onVariant: Color, val outline: Color,
val primary: Color, val onPrimary: Color, val primaryC: Color, val onPrimaryC: Color,
val tertiaryC: Color, val onTertiaryC: Color, val secondaryC: Color, val onSecondaryC: Color,
)
@Composable
private fun introColors(): IntroColors {
val m = MaterialTheme.colorScheme
return IntroColors(
surface = m.surface, onSurface = m.onSurface, onVariant = m.onSurfaceVariant, outline = m.outline,
primary = m.primary, onPrimary = m.onPrimary, primaryC = m.primaryContainer, onPrimaryC = m.onPrimaryContainer,
tertiaryC = m.tertiaryContainer, onTertiaryC = m.onTertiaryContainer,
secondaryC = m.secondaryContainer, onSecondaryC = m.onSecondaryContainer,
)
}
private enum class IntroStep { WELCOME, WIRELESS, KNOBS, VOICE, GRAPHENE, THANKS, APPS, DRIVE }
private val Pop = CubicBezierEasing(0.3f, 1.6f, 0.5f, 1f)
private val Rise = CubicBezierEasing(0.2f, 0.8f, 0.2f, 1f)
@Composable
fun IntroScreen(onDone: () -> Unit) {
val context = LocalContext.current
val c = introColors()
// The GrapheneOS screen is only news to people running it.
val steps = remember {
IntroStep.entries.filter { it != IntroStep.GRAPHENE || UsageStats.osName(context) == "GrapheneOS" }
}
var index by remember { mutableIntStateOf(0) }
val step = steps[index]
val finish = {
Intro.markSeen(context)
onDone()
}
BackHandler(enabled = index > 0) { index-- }
BoxWithConstraints(
Modifier
.fillMaxSize()
.background(c.surface)
.clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) {
if (index < steps.lastIndex) index++
},
) {
val w = maxWidth
// The mockup's cqw: one percent of the screen's width.
fun cq(n: Float): Dp = w * (n / 100f)
val density = LocalDensity.current
fun cqSp(n: Float): TextUnit = with(density) { cq(n).toSp() }
when (step) {
IntroStep.WELCOME -> Welcome(c, ::cq, ::cqSp, maxHeight)
IntroStep.WIRELESS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Wireless.",
"If your car does wireless Android Auto, pair once and leave the cable at home.", step) { WifiIcon(c.onSecondaryC) }
IntroStep.KNOBS -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Knobs work.",
"Turn, push, done. No touchscreen? Gearslip doesn't need one anymore.", step) { DialIcon(c.onTertiaryC) }
IntroStep.VOICE -> VoiceStep(c, ::cq, ::cqSp, step)
IntroStep.GRAPHENE -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Protections back on.",
"On GrapheneOS, Gearslip no longer needs exploit protection compatibility mode. If you " +
"turned it on for Gearslip, turn it off to get full protection back.", step,
action = "Open app settings" to {
runCatching {
context.startActivity(
Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS, Uri.fromParts("package", context.packageName, null)),
)
}
},
outlined = true,
) { ShieldIcon(c.onSecondaryC, c.secondaryC) }
IntroStep.THANKS -> BadgeStep(c, ::cq, ::cqSp, c.tertiaryC, "Thank you.",
"To everyone sending usage notes: they show me which cars Gearslip works in and which " +
"ones still fail, so I know what to fix next. Every note helps.", step) { HeartIcon(c.onTertiaryC) }
IntroStep.APPS -> BadgeStep(c, ::cq, ::cqSp, c.secondaryC, "Good company.",
"Free, open-source apps that work well in the car. Tap one to see it on GitHub.", step,
extra = { AppPicks(c, ::cq, ::cqSp, step) }) { StarIcon(c.onSecondaryC) }
IntroStep.DRIVE -> BadgeStep(c, ::cq, ::cqSp, c.primaryC, "Let's drive.",
"That's ${Intro.release}. Plug in, or just get in.", step,
action = "Start" to finish) { CheckIcon(c.onPrimaryC) }
}
Dots(c, steps.size, index, ::cq, Modifier.align(Alignment.BottomCenter).padding(bottom = maxHeight * 0.05f))
if (step != IntroStep.DRIVE) Hint(c, ::cqSp, step, Modifier.align(Alignment.BottomCenter).padding(bottom = maxHeight * 0.09f))
}
}
/** 0 to 1 once each time [key] changes, after [delayMs]: drives the entrance of one element. */
@Composable
private fun entrance(key: Any, delayMs: Int, durationMs: Int = 650): Float {
val a = remember(key) { Animatable(0f) }
LaunchedEffect(key) {
delay(delayMs.toLong())
a.animateTo(1f, tween(durationMs, easing = LinearEasing))
}
return a.value
}
/** Slides up and fades in, like the mockup's `up`. */
private fun Modifier.up(p: Float, distance: Dp) = graphicsLayer {
val e = Rise.transform(p)
alpha = e
translationY = (1 - e) * distance.toPx()
}
/** Grows in with a bounce, like the mockup's `bounce`. */
private fun Modifier.pop(p: Float) = graphicsLayer {
val e = Pop.transform(p)
alpha = p.coerceAtMost(1f)
scaleX = 0.5f + 0.5f * e
scaleY = 0.5f + 0.5f * e
}
@Composable
private fun Welcome(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, height: Dp) {
val key = IntroStep.WELCOME
val spin by rememberInfiniteTransition(label = "spin").animateFloat(
0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin",
)
Box(Modifier.fillMaxSize()) {
Cookie(c.primaryC, Modifier.size(cq(120f)).offset(x = cq(28f), y = -cq(22f)).align(Alignment.TopEnd).graphicsLayer { rotationZ = spin; alpha = 0.9f })
Ticker(c.secondaryC, c.onSecondaryC, "OPEN SOURCE ✦ NO ACCOUNT ✦ YOUR CAR, YOUR PHONE ✦ ", cq, cqSp, reverse = true, tilt = 3f,
key = key, modifier = Modifier.align(Alignment.TopCenter).padding(top = height * 0.11f))
Column(Modifier.align(Alignment.CenterStart).padding(horizontal = cq(8f))) {
Text("Welcome to", color = c.onVariant, style = TextStyle(fontSize = cqSp(4.2f), fontWeight = FontWeight.SemiBold),
modifier = Modifier.up(entrance(key, 100), cq(7f)))
Text(
Intro.release, color = c.primary,
style = TextStyle(fontSize = cqSp(44f), fontWeight = FontWeight.Black, letterSpacing = cqSp(-2.4f), lineHeight = cqSp(38f)),
modifier = Modifier.offset(x = -cq(1.5f)).pop(entrance(key, 250, 900)),
)
Text(
"alpha · the big one", color = c.onTertiaryC,
style = TextStyle(fontSize = cqSp(4f), fontWeight = FontWeight.Bold),
modifier = Modifier.padding(top = cq(4f)).up(entrance(key, 700), cq(7f))
.clip(RoundedCornerShape(50)).background(c.tertiaryC).padding(horizontal = cq(4.4f), vertical = cq(2.2f)),
)
}
Ticker(c.primary, c.onPrimary, "WIRELESS ✦ KNOB SUPPORT ✦ OFFLINE VOICE ✦ FASTER VIDEO ✦ AUTO UPDATES ✦ ", cq, cqSp, reverse = false, tilt = -4f,
key = key, modifier = Modifier.align(Alignment.BottomCenter).padding(bottom = height * 0.15f))
Confetti(c, key, Modifier.fillMaxSize())
}
}
/** A tilted band of words scrolling past, slid in from the side when the step opens. */
@Composable
private fun Ticker(
bg: Color, fg: Color, words: String, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit,
reverse: Boolean, tilt: Float, key: Any, modifier: Modifier,
) {
val enter = entrance(key, 400, 800)
val scroll by rememberInfiniteTransition(label = "ticker").animateFloat(
0f, 1f, infiniteRepeatable(tween(if (reverse) 20_000 else 16_000, easing = LinearEasing)), label = "ticker",
)
Box(
modifier
.fillMaxWidth()
.graphicsLayer {
rotationZ = tilt
scaleX = 1.2f
val e = Rise.transform(enter)
alpha = e
translationX = (1 - e) * -cq(30f).toPx()
}
.background(bg)
.padding(vertical = cq(2.6f)),
) {
Text(
words.repeat(4),
color = fg, maxLines = 1, softWrap = false,
style = TextStyle(fontSize = cqSp(5f), fontWeight = FontWeight.ExtraBold),
modifier = Modifier
.wrapContentWidth(Alignment.Start, unbounded = true)
.graphicsLayer {
// The text holds the words four times; sliding by half loops cleanly.
val shift = size.width / 2f * (if (reverse) 1 - scroll else scroll)
translationX = -shift
},
)
}
}
/** Material shapes thrown once from behind the version number. */
@Composable
private fun Confetti(c: IntroColors, key: Any, modifier: Modifier) {
val p = entrance(key, 450, 1300)
val bits = remember {
val r = Random(7)
List(22) { i ->
val a = i / 22f * 2 * PI.toFloat() + r.nextFloat() * 0.4f
val d = 30f + r.nextFloat() * 45f
Bit(cos(a) * d, sin(a) * d * 1.2f, r.nextInt(-270, 270).toFloat(), i % 4, i % 3)
}
}
if (p <= 0f || p >= 1f) return
val tones = listOf(c.primary, c.primaryC, c.tertiaryC, c.secondaryC)
Canvas(modifier) {
val unit = size.width / 100f
val e = CubicBezierEasing(0.15f, 0.7f, 0.3f, 1f).transform(p)
val alpha = if (p < 0.7f) 1f else 1f - (p - 0.7f) / 0.3f
val origin = Offset(size.width * 0.3f, size.height * 0.42f)
bits.forEach { b ->
val at = origin + Offset(b.dx * unit * e, b.dy * unit * e)
val s = 3.6f * unit * (0.3f + 0.7f * e)
rotate(b.spin * e, at) {
val color = tones[b.tone].copy(alpha = alpha)
when (b.shape) {
0 -> drawCircle(color, s / 2, at)
1 -> drawRoundRect(color, at - Offset(s / 2, s / 2), Size(s, s), androidx.compose.ui.geometry.CornerRadius(unit))
else -> drawRoundRect(color, at - Offset(s / 2, s / 4), Size(s, s / 2), androidx.compose.ui.geometry.CornerRadius(s / 4))
}
}
}
}
}
private class Bit(val dx: Float, val dy: Float, val spin: Float, val tone: Int, val shape: Int)
/** One idea per screen: a spinning cookie with an icon, a headline, a line of text, maybe a button. */
@Composable
private fun BadgeStep(
c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, badge: Color,
title: String, body: String, key: Any,
action: Pair<String, () -> Unit>? = null, outlined: Boolean = false,
extra: (@Composable () -> Unit)? = null,
icon: @Composable () -> Unit,
) {
val spin by rememberInfiniteTransition(label = "spin").animateFloat(
0f, 360f, infiniteRepeatable(tween(18_000, easing = LinearEasing)), label = "spin",
)
Column(
Modifier.fillMaxSize().padding(horizontal = cq(8f)),
verticalArrangement = Arrangement.Center,
) {
Box(Modifier.size(cq(34f)).pop(entrance(key, 50, 800)), contentAlignment = Alignment.Center) {
Cookie(badge, Modifier.fillMaxSize().graphicsLayer { rotationZ = spin })
Box(Modifier.size(cq(15f))) { icon() }
}
Spacer(Modifier.height(cq(8f)))
Text(
title, color = c.onSurface,
style = TextStyle(fontSize = cqSp(13.5f), fontWeight = FontWeight.Black, letterSpacing = cqSp(-0.6f), lineHeight = cqSp(12.5f)),
modifier = Modifier.up(entrance(key, 200, 700), cq(7f)),
)
Spacer(Modifier.height(cq(4f)))
Text(
body, color = c.onVariant,
style = TextStyle(fontSize = cqSp(4.5f), lineHeight = cqSp(6.5f)),
modifier = Modifier.up(entrance(key, 350, 700), cq(7f)),
)
extra?.invoke()
if (action != null) {
val (label, onClick) = action
Spacer(Modifier.height(if (outlined) cq(7f) else cq(9f)))
Text(
label,
color = if (outlined) c.primary else c.onPrimary,
style = TextStyle(fontSize = cqSp(if (outlined) 4.4f else 4.6f), fontWeight = FontWeight.Bold),
modifier = Modifier
.up(entrance(key, 500, 700), cq(7f))
.clip(RoundedCornerShape(50))
.then(
if (outlined) Modifier.border(BorderStroke(2.dp, c.outline), RoundedCornerShape(50))
else Modifier.background(c.primary),
)
.clickable(onClick = onClick)
.padding(horizontal = if (outlined) cq(7f) else cq(9f), vertical = if (outlined) cq(3.6f) else cq(4.2f)),
)
}
}
}
/**
* Voice: a few things to say, drifting past, and a button that fetches both speech models so the
* assistant works the first time someone tries it in the car.
*/
@Composable
private fun VoiceStep(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) {
val context = LocalContext.current
remember { SpeechModels.init(context) }
val needed = listOf(SpeechModels.Model.KROKO, SpeechModels.Model.PARAKEET_LARGE)
val states = needed.map { SpeechModels.state(it).collectAsState().value }
val ready = states.all { it == SpeechModels.State.Ready }
val label = when {
states.any { it is SpeechModels.State.Downloading } ->
"Downloading, ${needed.zip(states).sumOf { (m, st) -> m.downloadMb * percentOf(st) } / needed.sumOf { it.downloadMb }}%"
states.any { it is SpeechModels.State.Failed } -> "Download failed. Try again"
else -> "Download voice, ${needed.sumOf { it.downloadMb }} MB"
}
BadgeStep(
c, cq, cqSp, c.primaryC, "Just say it.",
"Tap the mic, or press the talk button on your wheel, and speak. Your phone turns speech " +
"into text by itself, and nothing you say leaves it. Say \"help\" in the car for the full list.",
key,
action = if (ready) null else label to { needed.forEach { SpeechModels.download(context, it) } },
outlined = true,
extra = { CommandDrift(c, cq, cqSp, key) },
) { MicIcon(c.onPrimaryC) }
}
private fun percentOf(state: SpeechModels.State) = when (state) {
SpeechModels.State.Ready -> 100
is SpeechModels.State.Downloading -> state.percent
else -> 0
}
private val SAMPLE_COMMANDS = listOf(
"Play Daft Punk", "What's playing?", "Next song",
"Call Mom", "Open Organic Maps", "Pause", "Help",
)
/** Example commands rising slowly through a window that fades at both edges. */
@Composable
private fun CommandDrift(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: Any) {
val scroll by rememberInfiniteTransition(label = "drift").animateFloat(
0f, 1f, infiniteRepeatable(tween(SAMPLE_COMMANDS.size * 2_200, easing = LinearEasing)), label = "drift",
)
Box(
Modifier
.padding(top = cq(6f))
.fillMaxWidth()
.height(cq(34f))
.up(entrance(key, 500, 700), cq(7f))
.graphicsLayer { compositingStrategy = CompositingStrategy.Offscreen }
.drawWithContent {
drawContent()
drawRect(
Brush.verticalGradient(0f to Color.Transparent, 0.25f to Color.Black, 0.75f to Color.Black, 1f to Color.Transparent),
blendMode = BlendMode.DstIn,
)
}
.clipToBounds(),
) {
Column(
Modifier
.wrapContentHeight(Alignment.Top, unbounded = true)
// The list is there twice; rising by half loops cleanly.
.graphicsLayer { translationY = -size.height / 2f * scroll },
verticalArrangement = Arrangement.spacedBy(cq(2.4f)),
) {
repeat(2) {
SAMPLE_COMMANDS.forEach { say ->
Text(
"\u201c$say\u201d", color = c.onSecondaryC,
style = TextStyle(fontSize = cqSp(4.4f), fontWeight = FontWeight.SemiBold),
modifier = Modifier
.clip(RoundedCornerShape(50))
.background(c.secondaryC)
.padding(horizontal = cq(4.4f), vertical = cq(2f)),
)
}
}
}
}
}
private val APP_PICKS = listOf(
Triple("Music", "Metrolist", "https://github.com/MetrolistGroup/Metrolist"),
Triple("Maps", "Vela", "https://github.com/PimpinPumpkin/Vela"),
Triple("Podcasts", "AntennaPod", "https://github.com/AntennaPod/AntennaPod"),
)
/** One row per pick, each opening the app's GitHub page; they arrive one after another. */
@Composable
private fun AppPicks(c: IntroColors, cq: (Float) -> Dp, cqSp: (Float) -> TextUnit, key: IntroStep) {
val context = LocalContext.current
Column(Modifier.padding(top = cq(6f)), verticalArrangement = Arrangement.spacedBy(cq(2.6f))) {
APP_PICKS.forEachIndexed { i, (kind, name, url) ->
Row(
Modifier
.fillMaxWidth()
.up(entrance(key, 500 + i * 120, 700), cq(7f))
.clip(RoundedCornerShape(cq(5f)))
.background(c.secondaryC)
.clickable {
runCatching {
context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(url)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK))
}
}
.padding(horizontal = cq(5f), vertical = cq(3.4f)),
verticalAlignment = Alignment.CenterVertically,
) {
Column(Modifier.weight(1f)) {
Text(kind.uppercase(), color = c.onSecondaryC.copy(alpha = 0.7f),
style = TextStyle(fontSize = cqSp(3.2f), fontWeight = FontWeight.Bold, letterSpacing = cqSp(0.3f)))
Text(name, color = c.onSecondaryC, style = TextStyle(fontSize = cqSp(5.2f), fontWeight = FontWeight.ExtraBold))
}
Text("GitHub ↗", color = c.onSecondaryC, style = TextStyle(fontSize = cqSp(3.8f), fontWeight = FontWeight.SemiBold))
}
}
}
}
/** Material's scalloped "cookie": a circle whose radius waves around its edge. */
@Composable
private fun Cookie(color: Color, modifier: Modifier) {
Canvas(modifier) {
val unit = size.minDimension / 100f
val path = Path()
var deg = 0
while (deg <= 360) {
val a = deg * PI / 180
val r = (44 + 4.5 * cos(9 * a)) * unit
val x = center.x + (r * cos(a)).toFloat()
val y = center.y + (r * sin(a)).toFloat()
if (deg == 0) path.moveTo(x, y) else path.lineTo(x, y)
deg += 3
}
path.close()
drawPath(path, color)
}
}
/** Draws in the mockup's 48-unit icon box. */
private inline fun DrawScope.inIconBox(block: DrawScope.() -> Unit) {
val s = size.minDimension / 48f
scale(s, s, Offset.Zero) { block() }
}
@Composable
private fun WifiIcon(color: Color) {
val t = rememberInfiniteTransition(label = "wifi")
val waves = (0..2).map { i ->
t.animateFloat(
0.35f, 1f,
infiniteRepeatable(tween(800, delayMillis = 0, easing = FastOutSlowInEasing), RepeatMode.Reverse,
initialStartOffset = androidx.compose.animation.core.StartOffset(i * 200)),
label = "wave$i",
)
}
Canvas(Modifier.fillMaxSize()) {
inIconBox {
val stroke = Stroke(5f, cap = StrokeCap.Round)
// The mockup's three arcs, inner to outer: radius r through (24 ± half, y).
listOf(Triple(8f, 6f, 34f), Triple(18f, 13f, 26f), Triple(28f, 20f, 18f)).forEachIndexed { i, (r, half, y) ->
val cy = y + sqrt(r * r - half * half)
val start = Math.toDegrees(atan2((y - cy).toDouble(), (-half).toDouble())).toFloat()
val end = Math.toDegrees(atan2((y - cy).toDouble(), half.toDouble())).toFloat()
drawArc(color.copy(alpha = waves[i].value), start, (end - start + 360f) % 360f, false,
Offset(24f - r, cy - r), Size(2 * r, 2 * r), style = stroke)
}
drawCircle(color, 3.6f, Offset(24f, 41f))
}
}
}
@Composable
private fun DialIcon(color: Color) {
val angle by rememberInfiniteTransition(label = "dial").animateFloat(
-70f, -70f,
infiniteRepeatable(keyframes {
durationMillis = 2600
-70f at 0 using CubicBezierEasing(0.5f, 0f, 0.3f, 1f)
70f at 1040
70f at 1430 using CubicBezierEasing(0.5f, 0f, 0.3f, 1f)
-70f at 2470
}),
label = "dial",
)
Canvas(Modifier.fillMaxSize()) {
inIconBox {
drawCircle(color, 19f, Offset(24f, 24f), style = Stroke(5f))
rotate(angle, Offset(24f, 24f)) {
drawRoundRect(color, Offset(21.5f, 7f), Size(5f, 13f), androidx.compose.ui.geometry.CornerRadius(2.5f))
}
}
}
}
@Composable
private fun MicIcon(color: Color) {
val t = rememberInfiniteTransition(label = "mic")
val waves = (0..1).map { i ->
t.animateFloat(
0.25f, 1f,
infiniteRepeatable(tween(700, easing = FastOutSlowInEasing), RepeatMode.Reverse,
initialStartOffset = androidx.compose.animation.core.StartOffset(i * 250)),
label = "mic$i",
)
}
Canvas(Modifier.fillMaxSize()) {
inIconBox {
drawRoundRect(color, Offset(18f, 5f), Size(12f, 22f), androidx.compose.ui.geometry.CornerRadius(6f))
val stroke = Stroke(4f, cap = StrokeCap.Round)
drawArc(color, 0f, 180f, false, Offset(12f, 14f), Size(24f, 22f), style = stroke)
drawLine(color, Offset(24f, 36f), Offset(24f, 42f), 4f, StrokeCap.Round)
// Sound waves either side, pulsing as if someone's talking.
listOf(1f to 0, -1f to 0, 1f to 1, -1f to 1).forEach { (side, i) ->
val r = if (i == 0) 15f else 21f
drawArc(color.copy(alpha = waves[i].value), if (side > 0) -35f else 145f, 70f, false,
Offset(24f - r, 16f - r), Size(2 * r, 2 * r), style = stroke)
}
}
}
}
@Composable
private fun StarIcon(color: Color) {
val turn by rememberInfiniteTransition(label = "star").animateFloat(
-8f, 8f, infiniteRepeatable(tween(1600, easing = FastOutSlowInEasing), RepeatMode.Reverse), label = "star",
)
Canvas(Modifier.fillMaxSize()) {
inIconBox {
val star = Path()
for (i in 0 until 10) {
val r = if (i % 2 == 0) 20f else 8.5f
val a = (-90 + i * 36) * PI / 180
val x = 24f + (r * cos(a)).toFloat()
val y = 25f + (r * sin(a)).toFloat()
if (i == 0) star.moveTo(x, y) else star.lineTo(x, y)
}
star.close()
rotate(turn, Offset(24f, 25f)) { drawPath(star, color) }
}
}
}
@Composable
private fun ShieldIcon(color: Color, tick: Color) {
val p = entrance(IntroStep.GRAPHENE, 800, 500)
Canvas(Modifier.fillMaxSize()) {
inIconBox {
val shield = Path().apply {
moveTo(24f, 4f); lineTo(40f, 10f); lineTo(40f, 22f)
cubicTo(40f, 32f, 33f, 40f, 24f, 44f)
cubicTo(15f, 40f, 8f, 32f, 8f, 22f)
lineTo(8f, 10f); close()
}
drawPath(shield, color)
drawPartialStroke(listOf(Offset(16f, 24f), Offset(22f, 30f), Offset(33f, 18f)), p, tick, 4.5f)
}
}
}
@Composable
private fun HeartIcon(color: Color) {
val beat by rememberInfiniteTransition(label = "beat").animateFloat(
1f, 1f,
infiniteRepeatable(keyframes {
durationMillis = 1400
1f at 0; 1.16f at 210; 1f at 420; 1.1f at 630; 1f at 840
}),
label = "beat",
)
Canvas(Modifier.fillMaxSize()) {
inIconBox {
scale(beat, beat, Offset(24f, 24f)) {
val heart = Path().apply {
moveTo(24f, 41f)
cubicTo(10f, 31f, 5f, 23f, 5f, 16f)
cubicTo(5f, 10f, 10f, 6f, 15f, 6f)
cubicTo(19f, 6f, 22f, 8f, 24f, 12f)
cubicTo(26f, 8f, 29f, 6f, 33f, 6f)
cubicTo(38f, 6f, 43f, 10f, 43f, 16f)
cubicTo(43f, 23f, 38f, 31f, 24f, 41f)
close()
}
drawPath(heart, color)
}
}
}
}
@Composable
private fun CheckIcon(color: Color) {
val p = entrance(IntroStep.DRIVE, 300, 500)
Canvas(Modifier.fillMaxSize()) {
inIconBox { drawPartialStroke(listOf(Offset(10f, 26f), Offset(20f, 35f), Offset(38f, 14f)), p, color, 5.5f) }
}
}
/** A polyline drawn up to [progress] of its length, so a tick can sweep in. */
private fun DrawScope.drawPartialStroke(points: List<Offset>, progress: Float, color: Color, width: Float) {
if (progress <= 0f) return
val lengths = points.zipWithNext { a, b -> (b - a).getDistance() }
var left = lengths.sum() * FastOutSlowInEasing.transform(progress.coerceAtMost(1f))
val path = Path().apply { moveTo(points[0].x, points[0].y) }
for (i in lengths.indices) {
if (left <= 0f) break
val f = (left / lengths[i]).coerceAtMost(1f)
val to = points[i] + (points[i + 1] - points[i]) * f
path.lineTo(to.x, to.y)
left -= lengths[i]
}
drawPath(path, color, style = Stroke(width, cap = StrokeCap.Round, join = StrokeJoin.Round))
}
@Composable
private fun Dots(c: IntroColors, count: Int, current: Int, cq: (Float) -> Dp, modifier: Modifier) {
Row(modifier, horizontalArrangement = Arrangement.spacedBy(cq(2f))) {
repeat(count) { i ->
val on = i == current
val width by animateDpAsState(if (on) cq(7f) else cq(2f), tween(300), label = "dot")
Box(
Modifier
.size(width, cq(2f))
.clip(RoundedCornerShape(50))
.background(if (on) c.primary else c.outline.copy(alpha = 0.45f)),
)
}
}
}
/** "Tap to continue", once the step has finished arriving, breathing softly. */
@Composable
private fun Hint(c: IntroColors, cqSp: (Float) -> TextUnit, key: Any, modifier: Modifier) {
val shown = entrance(key, 1600, 600)
val breathe by rememberInfiniteTransition(label = "hint").animateFloat(
0.75f, 0.35f, infiniteRepeatable(tween(1200), RepeatMode.Reverse), label = "hint",
)
Text(
"Tap to continue", color = c.onVariant,
style = TextStyle(fontSize = cqSp(3.6f), fontWeight = FontWeight.SemiBold),
modifier = modifier.graphicsLayer { alpha = if (shown < 1f) shown * 0.75f else breathe },
)
}
@@ -54,6 +54,9 @@ private val recommended = listOf(
RecommendedApp("ViVi Music", "com.vivi.vivimusic", "https://github.com/vivizzz007/vivi-music", "A YouTube Music player."),
RecommendedApp("Metrolist", "com.metrolist.music", "https://github.com/MetrolistGroup/Metrolist", "A YouTube Music player."),
),
"Podcasts" to listOf(
RecommendedApp("AntennaPod", "de.danoeh.antennapod", "https://github.com/AntennaPod/AntennaPod", "Podcasts, chapters and all."),
),
)
/** Opened from Settings: the same list the setup guide ends with. */
@@ -73,6 +73,7 @@ fun SettingsScreen(
onOpenRecommendedApps: () -> Unit,
onOpenWhatsNew: () -> Unit,
onReplayTutorial: () -> Unit,
onReplayIntro: () -> Unit,
atUsageNotes: Boolean = false,
) {
val scroll = rememberScrollState()
@@ -105,9 +106,10 @@ fun SettingsScreen(
verticalArrangement = Arrangement.spacedBy(24.dp),
) {
Spacer(Modifier.height(0.dp))
AboutSection(onOpenWhatsNew, onReplayTutorial)
AboutSection(onOpenWhatsNew, onReplayTutorial, onReplayIntro)
ConnectionSection()
CarAudioSection()
VoiceModelSection()
if (SHOW_RECOMMENDED_APPS) AppsSection(onOpenRecommendedApps)
CertificateSection()
SettingsSection("Usage notes", header = USAGE_STATS_INTRO) { UsageStatsRows() }
@@ -125,7 +127,7 @@ private fun Arrow() = Icon(
/** What's new, updates and the setup guide, one tap from the top. */
@Composable
private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit) {
private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Unit, onReplayIntro: () -> Unit) {
SettingsSection {
SettingsRow(
"What's new",
@@ -148,6 +150,16 @@ private fun AboutSection(onOpenWhatsNew: () -> Unit, onReplayTutorial: () -> Uni
icon = Icons.Filled.Refresh,
trailing = { Arrow() },
)
SettingsDivider()
SettingsRow(
"Replay the ${Intro.release} intro",
Modifier.clickable(onClick = onReplayIntro),
subtitle = "What's in this release, in a few taps",
icon = Icons.Filled.Star,
trailing = { Arrow() },
)
SettingsDivider()
AboutLinkRows()
}
}
@@ -0,0 +1,161 @@
package app.seb3thehacker.gearslip.ui
import android.content.Context
import android.content.Intent
import android.net.Uri
import androidx.compose.foundation.clickable
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.PaddingValues
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.automirrored.filled.KeyboardArrowRight
import androidx.compose.material.icons.automirrored.filled.Send
import androidx.compose.material.icons.filled.Build
import androidx.compose.material.icons.filled.Favorite
import androidx.compose.material3.Button
import androidx.compose.material3.Icon
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Surface
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.remember
import androidx.compose.runtime.setValue
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.unit.dp
/** Where Gearslip lives outside the app: donations, the chat room, and the code. */
internal object GearslipLinks {
const val SPONSORS = "https://github.com/sponsors/Seb3thehacker"
const val LIBERAPAY = "https://liberapay.com/Seb3thehacker"
const val MATRIX = "https://matrix.to/#/!mf0_mgBd5KqKpc9FfX_wHp1jFCcwAtPrndIn_oGIyMk?via=matrix.org"
const val SOURCE = "https://github.com/Seb3thehacker/gearslip"
fun open(context: Context, url: String) {
runCatching {
context.startActivity(Intent(Intent.ACTION_VIEW, Uri.parse(url)).addFlags(Intent.FLAG_ACTIVITY_NEW_TASK))
}
}
}
/** "Hide for now" on the home card puts it away for this long; the About page keeps the links. */
private const val HIDE_MS = 30L * 24 * 60 * 60 * 1000
private const val PREFS = "support"
private const val KEY_HIDDEN_UNTIL = "hidden_until"
private fun hiddenUntil(context: Context) =
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).getLong(KEY_HIDDEN_UNTIL, 0L)
/**
* The donation ask on Home: always in view, never in the way. It sits below the status and update
* cards, and "Hide for now" puts it away for a month.
*/
@Composable
internal fun SupportCard(modifier: Modifier = Modifier) {
val context = LocalContext.current
var hidden by remember { mutableStateOf(System.currentTimeMillis() < hiddenUntil(context)) }
if (hidden) return
val scheme = MaterialTheme.colorScheme
Surface(
modifier,
color = scheme.tertiaryContainer,
contentColor = scheme.onTertiaryContainer,
shape = MaterialTheme.shapes.extraLarge,
) {
Column(Modifier.fillMaxWidth().padding(20.dp)) {
Row(verticalAlignment = Alignment.CenterVertically) {
Icon(Icons.Filled.Favorite, contentDescription = null)
Text(
"Support Gearslip",
style = MaterialTheme.typography.titleLarge,
fontWeight = FontWeight.SemiBold,
modifier = Modifier.padding(start = 12.dp),
)
}
Text(
"Gearslip is free, with no ads and no tracking. If it gets you on the road, a " +
"donation keeps it going.",
style = MaterialTheme.typography.bodyMedium,
modifier = Modifier.padding(top = 8.dp),
)
Row(
Modifier.fillMaxWidth().padding(top = 12.dp),
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
TextButton(
onClick = {
context.getSharedPreferences(PREFS, Context.MODE_PRIVATE).edit()
.putLong(KEY_HIDDEN_UNTIL, System.currentTimeMillis() + HIDE_MS).apply()
hidden = true
},
contentPadding = PaddingValues(horizontal = 0.dp),
) { Text("Hide for now") }
Spacer(Modifier.weight(1f))
OutlinedButton(onClick = { GearslipLinks.open(context, GearslipLinks.LIBERAPAY) }) { Text("Liberapay") }
Button(onClick = { GearslipLinks.open(context, GearslipLinks.SPONSORS) }) { Text("GitHub") }
}
}
}
}
/** The About section's links: donating, the chat room, and the source code. */
@Composable
internal fun AboutLinkRows() {
val context = LocalContext.current
val arrow = @Composable {
Icon(
Icons.AutoMirrored.Filled.KeyboardArrowRight,
contentDescription = null,
tint = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
SettingsRow(
"Donate on GitHub Sponsors",
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.SPONSORS) },
subtitle = "Give once or monthly, through GitHub",
icon = Icons.Filled.Favorite,
trailing = arrow,
)
SettingsDivider()
SettingsRow(
"Donate on Liberapay",
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.LIBERAPAY) },
subtitle = "Give weekly or monthly, through a nonprofit",
icon = Icons.Filled.Favorite,
trailing = arrow,
)
SettingsDivider()
SettingsRow(
"Join the discussion on Matrix",
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.MATRIX) },
subtitle = "Questions, ideas, or a car to report",
icon = Icons.AutoMirrored.Filled.Send,
trailing = arrow,
)
SettingsDivider()
SettingsRow(
"Source code",
Modifier.clickable { GearslipLinks.open(context, GearslipLinks.SOURCE) },
subtitle = "Read or change Gearslip's code under the AGPL",
icon = Icons.Filled.Build,
trailing = arrow,
)
SettingsDivider()
SettingsRow(
"Voice credits",
subtitle = "Speech recognition by sherpa-onnx (Apache 2.0), with Kroko ASR by Banafo " +
"(CC BY-SA 4.0), and Parakeet and Canary by NVIDIA (CC BY 4.0)",
icon = Icons.Filled.Favorite,
)
}
@@ -0,0 +1,125 @@
package app.seb3thehacker.gearslip.ui
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxWidth
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.selection.selectable
import androidx.compose.material3.LinearProgressIndicator
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.RadioButton
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.collectAsState
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.platform.LocalContext
import androidx.compose.ui.semantics.Role
import androidx.compose.ui.unit.dp
import app.seb3thehacker.gearslip.speech.SpeechModels
import app.seb3thehacker.gearslip.speech.SpeechModels.Model
import app.seb3thehacker.gearslip.speech.SpeechModels.State
/**
* Which speech model the voice assistant listens with, and downloading it. Each is fetched once
* from sherpa-onnx's releases on GitHub; nothing is sent anywhere while listening.
*/
@Composable
internal fun VoiceModelSection() {
val context = LocalContext.current
remember { SpeechModels.init(context) }
val selected by SpeechModels.selected.collectAsState()
val final by SpeechModels.final.collectAsState()
SettingsSection(title = "Voice: live words") {
Text(
"The voice assistant turns speech into text on this phone, and nothing you say leaves it. " +
"It needs two models, about 515 MB together, downloaded once. Use Wi-Fi if you can. " +
"This one writes your words as you speak.",
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
)
Model.entries.filter { it.role == SpeechModels.Role.LIVE }.forEachIndexed { i, model ->
if (i > 0) SettingsDivider()
ModelRow(model, model == selected)
}
}
SettingsSection(title = "Voice: final words") {
Text(
"When you stop talking, this model reads the whole phrase again. It catches names and " +
"places the live model misses.",
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
)
val context = LocalContext.current
Row(
Modifier
.fillMaxWidth()
.selectable(final == null, role = Role.RadioButton) { SpeechModels.selectFinal(context, null) }
.padding(horizontal = 8.dp, vertical = 8.dp),
verticalAlignment = Alignment.CenterVertically,
) {
RadioButton(final == null, onClick = null, modifier = Modifier.padding(8.dp))
Column {
Text("None", style = MaterialTheme.typography.titleMedium)
Text(
"Keep the live words. Fastest, least accurate",
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
}
Model.entries.filter { it.role == SpeechModels.Role.FINAL }.forEach { model ->
SettingsDivider()
ModelRow(model, model == final)
}
}
}
@Composable
private fun ModelRow(model: Model, chosen: Boolean) {
val context = LocalContext.current
val state by SpeechModels.state(model).collectAsState()
Column(
Modifier
.fillMaxWidth()
.selectable(chosen, role = Role.RadioButton) { SpeechModels.select(context, model) }
.padding(horizontal = 8.dp, vertical = 8.dp),
) {
Row(verticalAlignment = Alignment.CenterVertically) {
RadioButton(chosen, onClick = null, modifier = Modifier.padding(8.dp))
Column(Modifier.weight(1f)) {
Text("${model.label} · ${model.downloadMb} MB", style = MaterialTheme.typography.titleMedium)
Text(
when (val s = state) {
State.Ready -> "Downloaded. ${model.detail}"
is State.Downloading -> "Downloading, ${s.percent}%"
is State.Failed -> "Download failed: ${s.reason}"
State.Missing -> model.detail
},
style = MaterialTheme.typography.bodySmall,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
Row(horizontalArrangement = Arrangement.spacedBy(4.dp)) {
when (state) {
State.Ready -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Delete") }
is State.Downloading -> TextButton(onClick = { SpeechModels.delete(context, model) }) { Text("Cancel") }
else -> OutlinedButton(onClick = { SpeechModels.download(context, model) }) { Text("Download") }
}
}
}
(state as? State.Downloading)?.let {
LinearProgressIndicator(
progress = { it.percent / 100f },
modifier = Modifier.fillMaxWidth().padding(start = 48.dp, end = 8.dp, top = 4.dp),
)
}
}
}
@@ -0,0 +1,12 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
package com.k2fsa.sherpa.onnx
data class FeatureConfig(
var sampleRate: Int = 16000,
var featureDim: Int = 80,
var dither: Float = 0.0f
)
fun getFeatureConfig(sampleRate: Int, featureDim: Int): FeatureConfig {
return FeatureConfig(sampleRate = sampleRate, featureDim = featureDim)
}
@@ -0,0 +1,8 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
package com.k2fsa.sherpa.onnx
data class HomophoneReplacerConfig(
var dictDir: String = "", // unused
var lexicon: String = "",
var ruleFsts: String = "",
)
@@ -0,0 +1,255 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
package com.k2fsa.sherpa.onnx
import android.content.res.AssetManager
data class OfflineRecognizerResult(
val text: String,
val tokens: Array<String>,
val timestamps: FloatArray,
val lang: String,
val emotion: String,
val event: String,
// valid only for TDT models
val durations: FloatArray,
)
data class OfflineTransducerModelConfig(
var encoder: String = "",
var decoder: String = "",
var joiner: String = "",
var qnnConfig: QnnConfig = QnnConfig(),
)
data class OfflineParaformerModelConfig(
var model: String = "",
var qnnConfig: QnnConfig = QnnConfig(),
)
data class OfflineNemoEncDecCtcModelConfig(
var model: String = "",
)
data class OfflineDolphinModelConfig(
var model: String = "",
)
data class OfflineZipformerCtcModelConfig(
var model: String = "",
var qnnConfig: QnnConfig = QnnConfig(),
)
data class OfflineWenetCtcModelConfig(
var model: String = "",
)
data class OfflineOmnilingualAsrCtcModelConfig(
var model: String = "",
)
data class OfflineMedAsrCtcModelConfig(
var model: String = "",
)
data class OfflineFireRedAsrCtcModelConfig(
var model: String = "",
)
data class OfflineFunAsrNanoModelConfig(
var encoderAdaptor: String = "",
var llm: String = "",
var embedding: String = "",
var tokenizer: String = "",
var systemPrompt: String = "You are a helpful assistant.",
var userPrompt: String = "语音转写:",
var maxNewTokens: Int = 512,
var temperature: Float = 1e-6f,
var topP: Float = 0.8f,
var seed: Int = 42,
var language: String = "",
var itn: Boolean = true,
var hotwords: String = "",
)
data class OfflineQwen3AsrModelConfig(
var convFrontend: String = "",
var encoder: String = "",
var decoder: String = "",
var tokenizer: String = "",
var maxTotalLen: Int = 512,
var maxNewTokens: Int = 128,
var temperature: Float = 1e-6f,
var topP: Float = 0.8f,
var seed: Int = 42,
var hotwords: String = "",
)
data class OfflineWhisperModelConfig(
var encoder: String = "",
var decoder: String = "",
var language: String = "en", // Used with multilingual model
var task: String = "transcribe", // transcribe or translate
var tailPaddings: Int = 1000, // Padding added at the end of the samples
var enableTokenTimestamps: Boolean = false,
var enableSegmentTimestamps: Boolean = false,
)
data class OfflineCanaryModelConfig(
var encoder: String = "",
var decoder: String = "",
var srcLang: String = "en",
var tgtLang: String = "en",
var usePnc: Boolean = true,
)
data class OfflineCohereTranscribeModelConfig(
var encoder: String = "",
var decoder: String = "",
var language: String = "",
var usePunct: Boolean = true,
var useItn: Boolean = true,
)
data class OfflineFireRedAsrModelConfig(
var encoder: String = "",
var decoder: String = "",
)
// For moonshine v1, you need four models.
// For moonshine v2, you need two models.
// - v1: preprocessor, encoder, uncachedDecoder, cachedDecoder
// - v2: encoder, mergedDecoder
data class OfflineMoonshineModelConfig(
var preprocessor: String = "",
var encoder: String = "",
var uncachedDecoder: String = "",
var cachedDecoder: String = "",
var mergedDecoder: String = "",
)
data class OfflineSenseVoiceModelConfig(
var model: String = "",
var language: String = "",
var useInverseTextNormalization: Boolean = true,
var qnnConfig: QnnConfig = QnnConfig(),
)
data class OfflineModelConfig(
var transducer: OfflineTransducerModelConfig = OfflineTransducerModelConfig(),
var paraformer: OfflineParaformerModelConfig = OfflineParaformerModelConfig(),
var whisper: OfflineWhisperModelConfig = OfflineWhisperModelConfig(),
var fireRedAsr: OfflineFireRedAsrModelConfig = OfflineFireRedAsrModelConfig(),
var moonshine: OfflineMoonshineModelConfig = OfflineMoonshineModelConfig(),
var nemo: OfflineNemoEncDecCtcModelConfig = OfflineNemoEncDecCtcModelConfig(),
var senseVoice: OfflineSenseVoiceModelConfig = OfflineSenseVoiceModelConfig(),
var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(),
var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(),
var medasr: OfflineMedAsrCtcModelConfig = OfflineMedAsrCtcModelConfig(),
var funasrNano: OfflineFunAsrNanoModelConfig = OfflineFunAsrNanoModelConfig(),
var qwen3Asr: OfflineQwen3AsrModelConfig = OfflineQwen3AsrModelConfig(),
var fireRedAsrCtc: OfflineFireRedAsrCtcModelConfig = OfflineFireRedAsrCtcModelConfig(),
var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
var cohereTranscribe: OfflineCohereTranscribeModelConfig =
OfflineCohereTranscribeModelConfig(),
var teleSpeech: String = "",
var numThreads: Int = 1,
var debug: Boolean = false,
var provider: String = "cpu",
var modelType: String = "",
var tokens: String = "",
var modelingUnit: String = "",
var bpeVocab: String = "",
)
data class OfflineRecognizerConfig(
var featConfig: FeatureConfig = FeatureConfig(),
var modelConfig: OfflineModelConfig = OfflineModelConfig(),
// var lmConfig: OfflineLMConfig(), // TODO(fangjun): enable it
var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
var decodingMethod: String = "greedy_search",
var maxActivePaths: Int = 4,
var hotwordsFile: String = "",
var hotwordsScore: Float = 1.5f,
var ruleFsts: String = "",
var ruleFars: String = "",
var blankPenalty: Float = 0.0f,
)
class OfflineRecognizer(
assetManager: AssetManager? = null,
val config: OfflineRecognizerConfig,
) {
private var ptr: Long
init {
ptr = if (assetManager != null) {
newFromAsset(assetManager, config)
} else {
newFromFile(config)
}
require(ptr != 0L) {
"Invalid OfflineRecognizerConfig: failed to create native OfflineRecognizer"
}
}
protected fun finalize() {
if (ptr != 0L) {
delete(ptr)
ptr = 0
}
}
fun release() = finalize()
fun createStream(): OfflineStream {
val p = createStream(ptr)
return OfflineStream(p)
}
fun createStream(hotwords: String): OfflineStream {
val p = createStreamWithHotwords(ptr, hotwords)
return OfflineStream(p)
}
fun getResult(stream: OfflineStream): OfflineRecognizerResult {
return getResult(stream.ptr)
}
fun decode(stream: OfflineStream) = decode(ptr, stream.ptr)
fun setConfig(config: OfflineRecognizerConfig) = setConfig(ptr, config)
private external fun delete(ptr: Long)
private external fun createStream(ptr: Long): Long
private external fun createStreamWithHotwords(ptr: Long, hotwords: String): Long
private external fun setConfig(ptr: Long, config: OfflineRecognizerConfig)
private external fun newFromAsset(
assetManager: AssetManager,
config: OfflineRecognizerConfig,
): Long
private external fun newFromFile(
config: OfflineRecognizerConfig,
): Long
private external fun decode(ptr: Long, streamPtr: Long)
private external fun getResult(streamPtr: Long): OfflineRecognizerResult
companion object {
init {
System.loadLibrary("sherpa-onnx-jni")
}
@JvmStatic
external fun prependAdspLibraryPath(newPath: String) // for qnn
}
}
@@ -0,0 +1,43 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
package com.k2fsa.sherpa.onnx
class OfflineStream(var ptr: Long) {
init {
require(ptr != 0L) { "Failed to create native OfflineStream" }
}
fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
acceptWaveform(ptr, samples, sampleRate)
fun setOption(key: String, value: String) = setOption(ptr, key, value)
fun getOption(key: String): String = getOption(ptr, key)
protected fun finalize() {
if (ptr != 0L) {
delete(ptr)
ptr = 0
}
}
fun release() = finalize()
fun use(block: (OfflineStream) -> Unit) {
try {
block(this)
} finally {
release()
}
}
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
private external fun setOption(ptr: Long, key: String, value: String)
private external fun getOption(ptr: Long, key: String): String
private external fun delete(ptr: Long)
companion object {
init {
System.loadLibrary("sherpa-onnx-jni")
}
}
}
@@ -0,0 +1,158 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
// The JNI names are fixed to this package; the upstream catalog of pretrained models is left out.
package com.k2fsa.sherpa.onnx
import android.content.res.AssetManager
data class EndpointRule(
var mustContainNonSilence: Boolean,
var minTrailingSilence: Float,
var minUtteranceLength: Float,
)
data class EndpointConfig(
var rule1: EndpointRule = EndpointRule(false, 2.4f, 0.0f),
var rule2: EndpointRule = EndpointRule(true, 1.4f, 0.0f),
var rule3: EndpointRule = EndpointRule(false, 0.0f, 20.0f)
)
data class OnlineTransducerModelConfig(
var encoder: String = "",
var decoder: String = "",
var joiner: String = "",
var qnnConfig: QnnConfig = QnnConfig(),
)
data class OnlineParaformerModelConfig(
var encoder: String = "",
var decoder: String = "",
)
data class OnlineZipformer2CtcModelConfig(
var model: String = "",
)
data class OnlineNeMoCtcModelConfig(
var model: String = "",
)
data class OnlineToneCtcModelConfig(
var model: String = "",
)
data class OnlineModelConfig(
var transducer: OnlineTransducerModelConfig = OnlineTransducerModelConfig(),
var paraformer: OnlineParaformerModelConfig = OnlineParaformerModelConfig(),
var zipformer2Ctc: OnlineZipformer2CtcModelConfig = OnlineZipformer2CtcModelConfig(),
var neMoCtc: OnlineNeMoCtcModelConfig = OnlineNeMoCtcModelConfig(),
var toneCtc: OnlineToneCtcModelConfig = OnlineToneCtcModelConfig(),
var tokens: String = "",
var numThreads: Int = 1,
var debug: Boolean = false,
var provider: String = "cpu",
var modelType: String = "",
var modelingUnit: String = "",
var bpeVocab: String = "",
)
data class OnlineLMConfig(
var model: String = "",
var scale: Float = 0.5f,
)
data class OnlineCtcFstDecoderConfig(
var graph: String = "",
var maxActive: Int = 3000,
)
data class OnlineRecognizerConfig(
var featConfig: FeatureConfig = FeatureConfig(),
var modelConfig: OnlineModelConfig = OnlineModelConfig(),
var lmConfig: OnlineLMConfig = OnlineLMConfig(),
var ctcFstDecoderConfig: OnlineCtcFstDecoderConfig = OnlineCtcFstDecoderConfig(),
var hr: HomophoneReplacerConfig = HomophoneReplacerConfig(),
var endpointConfig: EndpointConfig = EndpointConfig(),
var enableEndpoint: Boolean = true,
var decodingMethod: String = "greedy_search",
var maxActivePaths: Int = 4,
var hotwordsFile: String = "",
var hotwordsScore: Float = 1.5f,
var ruleFsts: String = "",
var ruleFars: String = "",
var blankPenalty: Float = 0.0f,
)
data class OnlineRecognizerResult(
val text: String,
val tokens: Array<String>,
val timestamps: FloatArray,
val ysProbs: FloatArray,
// TODO(fangjun): Add more fields
)
class OnlineRecognizer(
assetManager: AssetManager? = null,
val config: OnlineRecognizerConfig,
) {
private var ptr: Long
init {
ptr = if (assetManager != null) {
newFromAsset(assetManager, config)
} else {
newFromFile(config)
}
require(ptr != 0L) {
"Invalid OnlineRecognizerConfig: failed to create native OnlineRecognizer"
}
}
protected fun finalize() {
if (ptr != 0L) {
delete(ptr)
ptr = 0
}
}
fun release() = finalize()
fun createStream(hotwords: String = ""): OnlineStream {
val p = createStream(ptr, hotwords)
return OnlineStream(p)
}
fun reset(stream: OnlineStream) = reset(ptr, stream.ptr)
fun decode(stream: OnlineStream) = decode(ptr, stream.ptr)
fun isEndpoint(stream: OnlineStream) = isEndpoint(ptr, stream.ptr)
fun isReady(stream: OnlineStream) = isReady(ptr, stream.ptr)
fun getResult(stream: OnlineStream): OnlineRecognizerResult {
return getResult(ptr, stream.ptr)
}
private external fun delete(ptr: Long)
private external fun newFromAsset(
assetManager: AssetManager,
config: OnlineRecognizerConfig,
): Long
private external fun newFromFile(
config: OnlineRecognizerConfig,
): Long
private external fun createStream(ptr: Long, hotwords: String): Long
private external fun reset(ptr: Long, streamPtr: Long)
private external fun decode(ptr: Long, streamPtr: Long)
private external fun isEndpoint(ptr: Long, streamPtr: Long): Boolean
private external fun isReady(ptr: Long, streamPtr: Long): Boolean
private external fun getResult(ptr: Long, streamPtr: Long): OnlineRecognizerResult
companion object {
init {
System.loadLibrary("sherpa-onnx-jni")
}
@JvmStatic
external fun prependAdspLibraryPath(newPath: String) // for qnn
}
}
@@ -0,0 +1,47 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
package com.k2fsa.sherpa.onnx
class OnlineStream(var ptr: Long = 0) {
init {
require(ptr != 0L) { "Failed to create native OnlineStream" }
}
fun acceptWaveform(samples: FloatArray, sampleRate: Int) =
acceptWaveform(ptr, samples, sampleRate)
fun inputFinished() = inputFinished(ptr)
fun setOption(key: String, value: String) = setOption(ptr, key, value)
fun getOption(key: String): String = getOption(ptr, key)
protected fun finalize() {
if (ptr != 0L) {
delete(ptr)
ptr = 0
}
}
fun release() = finalize()
fun use(block: (OnlineStream) -> Unit) {
try {
block(this)
} finally {
release()
}
}
private external fun acceptWaveform(ptr: Long, samples: FloatArray, sampleRate: Int)
private external fun inputFinished(ptr: Long)
private external fun setOption(ptr: Long, key: String, value: String)
private external fun getOption(ptr: Long, key: String): String
private external fun delete(ptr: Long)
companion object {
init {
System.loadLibrary("sherpa-onnx-jni")
}
}
}
@@ -0,0 +1,8 @@
// From sherpa-onnx v1.13.8 (Apache-2.0, https://github.com/k2-fsa/sherpa-onnx), kotlin-api.
package com.k2fsa.sherpa.onnx
data class QnnConfig(
var backendLib: String = "",
var contextBinary: String = "",
var systemLib: String = "",
)
@@ -9,12 +9,10 @@
android:fillColor="#FF000000"
android:fillAlpha="0.35"/>
<path
android:pathData="M 54.00,36.18 L 58.64,30.70 L 67.20,34.24 L 66.60,41.40 L 71.82,54.00 L 77.30,58.64 L 73.76,67.20 L 66.60,66.60 L 66.60,66.60 L 67.20,73.76 L 58.64,77.30 L 54.00,71.82 L 54.00,71.82 L 49.36,77.30 L 40.80,73.76 L 41.40,66.60 L 41.40,66.60 L 34.24,67.20 L 30.70,58.64 L 36.18,54.00 L 36.18,54.00 L 30.70,49.36 L 34.24,40.80 L 41.40,41.40 L 41.40,41.40 L 40.80,34.24 L 49.36,30.70 L 54.00,36.18 Z"
android:pathData="M 54.00,36.18 L 58.64,30.70 L 67.20,34.24 L 66.60,41.40 L 71.82,54.00 L 77.30,58.64 L 73.76,67.20 L 66.60,66.60 L 66.60,66.60 L 67.20,73.76 L 58.64,77.30 L 54.00,71.82 L 54.00,71.82 L 49.36,77.30 L 40.80,73.76 L 41.40,66.60 L 41.40,66.60 L 34.24,67.20 L 30.70,58.64 L 36.18,54.00 L 36.18,54.00 L 30.70,49.36 L 34.24,40.80 L 41.40,41.40 L 41.40,41.40 L 40.80,34.24 L 49.36,30.70 L 54.00,36.18 Z M54,46.08 A7.92,7.92 0 1,1 53.99,46.08 Z"
android:fillType="evenOdd"
android:fillColor="#FF000000"/>
<path
android:pathData="M 78.61,40.70 L 85.31,43.29 L 84.74,52.55 L 77.78,54.31 Z"
android:fillColor="#FF000000"/>
<path
android:pathData="M54,46.08 A7.92,7.92 0 1,1 53.99,46.08 Z"
android:fillColor="#00000000"/>
</vector>
+2
View File
@@ -14,6 +14,7 @@ material = "1.14.0"
constraintlayout = "2.2.1"
zxing = "3.5.4"
okhttp = "5.4.0"
commonsCompress = "1.28.0"
kotlinxSerializationJson = "1.11.0"
ossLicenses = "17.5.1"
webkit = "1.16.0"
@@ -49,6 +50,7 @@ androidx-constraintlayout = { group = "androidx.constraintlayout", name = "const
google-material = { group = "com.google.android.material", name = "material", version.ref = "material" }
zxing-core = { group = "com.google.zxing", name = "core", version.ref = "zxing" }
okhttp = { group = "com.squareup.okhttp3", name = "okhttp", version.ref = "okhttp" }
commons-compress = { group = "org.apache.commons", name = "commons-compress", version.ref = "commonsCompress" }
kotlinx-serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerializationJson" }
google-oss-licenses = { group = "com.google.android.gms", name = "play-services-oss-licenses", version.ref = "ossLicenses" }
androidx-webkit = { group = "androidx.webkit", name = "webkit", version.ref = "webkit" }