diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5043aaf..0636864 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,6 +8,8 @@ on: jobs: build: runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 @@ -16,6 +18,8 @@ jobs: distribution: temurin java-version: "21" + - uses: gradle/actions/setup-gradle@v4 + - name: Write signing key and properties run: | echo "${{ secrets.KEYSTORE_BASE64 }}" | base64 -d > gearslip-release.jks @@ -32,7 +36,9 @@ jobs: - name: Locate APK id: apk run: | - apk=$(find gearslip/build/outputs/apk/release -name "*.apk" | head -n1) + # The build moves its APKs to /build/Gearslip-.apk (see renameApk). + apk=build/Gearslip-release.apk + test -f "$apk" || { echo "no APK at $apk"; ls -R build | head -50; exit 1; } echo "path=$apk" >> "$GITHUB_OUTPUT" - name: Create release diff --git a/.gitignore b/.gitignore index 1e4e04f..552ad99 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,6 @@ CLAUDE.md extraction/ .claude/ todo -logs/ \ No newline at end of file +logs/ +# Native build cache. +.cxx/ diff --git a/gearslip/build.gradle.kts b/gearslip/build.gradle.kts index 40a5506..cbc8d70 100644 --- a/gearslip/build.gradle.kts +++ b/gearslip/build.gradle.kts @@ -32,8 +32,8 @@ android { applicationId = "app.seb3thehacker.gearslip" minSdk = 31 targetSdk = 37 - versionCode = 5 - versionName = "0.0.05-alpha" + versionCode = 6 + versionName = "0.1.0-alpha" manifestPlaceholders["appLabel"] = "Gearslip" } @@ -78,6 +78,23 @@ android { targetCompatibility = JavaVersion.VERSION_21 } + defaultConfig { + // Speech-to-text ships for 64-bit ARM only: every phone Gearslip runs on, and one copy + // of the native library instead of four. + ndk { abiFilters += "arm64-v8a" } + } + ndkVersion = "27.1.12297006" + externalNativeBuild { + cmake { + path = file("src/main/cpp/CMakeLists.txt") + version = "3.22.1" + } + } + androidResources { + // Model weights barely compress, and stored plainly they stream straight out of the APK. + noCompress += "bin" + } + testOptions { // GearslipLog calls android.util.Log, which is a stub on the local JVM. unitTests.isReturnDefaultValues = true @@ -126,3 +143,4 @@ tasks.withType().configureEach { jvmTarget.set(JvmTarget.JVM_21) } } + diff --git a/gearslip/src/main/AndroidManifest.xml b/gearslip/src/main/AndroidManifest.xml index 077e32c..14f160e 100644 --- a/gearslip/src/main/AndroidManifest.xml +++ b/gearslip/src/main/AndroidManifest.xml @@ -85,6 +85,14 @@ + + + + + + + + + + + + @@ -128,7 +148,12 @@ - + + + + +#include +#include +#include +#include +#include +#include +#include "whisper.h" + +#define TAG "GearslipSpeech" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__) + +static whisper_context_params context_params() { + whisper_context_params p = whisper_context_default_params(); + p.use_gpu = false; + return p; +} + +// Streams a model straight out of the APK, so the bundled one is never copied to storage. +static size_t asset_read(void *ctx, void *output, size_t size) { + int n = AAsset_read(static_cast(ctx), output, size); + return n > 0 ? static_cast(n) : 0; +} +static bool asset_eof(void *ctx) { return AAsset_getRemainingLength64(static_cast(ctx)) <= 0; } +static void asset_close(void *ctx) { AAsset_close(static_cast(ctx)); } + +extern "C" JNIEXPORT jlong JNICALL +Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeInitAsset(JNIEnv *env, jobject, jobject assets, jstring jpath) { + AAssetManager *manager = AAssetManager_fromJava(env, assets); + const char *path = env->GetStringUTFChars(jpath, nullptr); + AAsset *asset = AAssetManager_open(manager, path, AASSET_MODE_STREAMING); + env->ReleaseStringUTFChars(jpath, path); + if (!asset) return 0; + whisper_model_loader loader = {asset, asset_read, asset_eof, asset_close}; + return reinterpret_cast(whisper_init_with_params(&loader, context_params())); +} + +// Bytes, not a jstring: NewStringUTF wants modified UTF-8 and aborts on some of what Whisper can +// write (four-byte characters), so Kotlin decodes the plain UTF-8 instead. +extern "C" JNIEXPORT jbyteArray JNICALL +Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeTranscribe( + JNIEnv *env, jobject, jlong handle, jfloatArray jsamples, jstring jlang, jint threads) { + auto *ctx = reinterpret_cast(handle); + if (!ctx) return env->NewByteArray(0); + jsize n = env->GetArrayLength(jsamples); + jfloat *samples = env->GetFloatArrayElements(jsamples, nullptr); + const char *lang = env->GetStringUTFChars(jlang, nullptr); + + whisper_full_params p = whisper_full_default_params(WHISPER_SAMPLING_GREEDY); + p.n_threads = threads; + p.language = lang; + p.translate = false; + p.no_context = true; + p.no_timestamps = true; + p.single_segment = true; + p.print_progress = false; + p.print_realtime = false; + p.print_special = false; + p.print_timestamps = false; + p.suppress_blank = true; + // The ACFT models are trained for a context cut to the audio's own length (1500 frames is + // 30 s, so 50 per second), which is what makes a three-second phrase fast. A little extra + // keeps the last word from being clipped. + p.audio_ctx = std::min(1500, static_cast(std::ceil(n / 320.0)) + 32); + + int rc = whisper_full(ctx, p, samples, n); + env->ReleaseStringUTFChars(jlang, lang); + env->ReleaseFloatArrayElements(jsamples, samples, JNI_ABORT); + if (rc != 0) { + LOGI("whisper_full failed: %d", rc); + return env->NewByteArray(0); + } + std::string text; + for (int i = 0; i < whisper_full_n_segments(ctx); i++) text += whisper_full_get_segment_text(ctx, i); + jbyteArray out = env->NewByteArray(static_cast(text.size())); + env->SetByteArrayRegion(out, 0, static_cast(text.size()), reinterpret_cast(text.data())); + return out; +} + +extern "C" JNIEXPORT void JNICALL +Java_app_seb3thehacker_gearslip_speech_SpeechEngine_nativeFree(JNIEnv *, jobject, jlong handle) { + if (handle) whisper_free(reinterpret_cast(handle)); +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt index 2c9dfbb..7c9cf73 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/AppSettings.kt @@ -70,14 +70,28 @@ object AppSettings { fun setGearslipEnabled(context: Context, value: Boolean) { prefs(context).edit().putBoolean(KEY_GEARSLIP_ENABLED, value).apply() + applyGearslipEnabled(context) + } + + /** + * Makes the USB alias match the saved switch. Called at every launch as well, so a switch + * saved without the alias following it (an older build crashed right there) heals itself. + * Only the alias changes: the driver's "always open with Gearslip" choice for the car is kept. + */ + fun applyGearslipEnabled(context: Context) { + val value = gearslipEnabled(context) val appContext = context.applicationContext - val component = ComponentName(appContext.packageName, "${appContext.packageName}.GearslipUsbAccessory") + // The class name comes from the code's namespace, not the package: a debug build installs + // as ".dev", but the alias is still app.seb3thehacker.gearslip.GearslipUsbAccessory. + val component = ComponentName(appContext.packageName, "${GearslipActivity::class.java.`package`!!.name}.GearslipUsbAccessory") val state = if (value) { PackageManager.COMPONENT_ENABLED_STATE_DEFAULT } else { PackageManager.COMPONENT_ENABLED_STATE_DISABLED } - context.applicationContext.packageManager - .setComponentEnabledSetting(component, state, PackageManager.DONT_KILL_APP) + val pm = appContext.packageManager + if (pm.getComponentEnabledSetting(component) == state) return + runCatching { pm.setComponentEnabledSetting(component, state, PackageManager.DONT_KILL_APP) } + .onFailure { GearslipLog.e("could not switch the USB connection ${if (value) "on" else "off"}", it) } } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/Changelog.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/Changelog.kt new file mode 100644 index 0000000..c6c3fa4 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/Changelog.kt @@ -0,0 +1,35 @@ +package app.seb3thehacker.gearslip + +/** + * What changed in each release, newest first, shown on the car screen after an update (and on + * the phone). Only the changes a driver would notice. Plain words, short lines: this is read in + * a parked car. + */ +object Changelog { + + class Release(val versionCode: Int, val versionName: String, val changes: List) + + val releases = listOf( + Release( + 6, "0.1.0-alpha", + listOf( + "Messaging apps such as Signal and Molly open to your recent chats.", + "Hear a message read aloud and reply by voice. Say \"cancel\" or \"change it\" before it sends, or tap a quick reply such as your arrival time.", + "Voice works on any phone, GrapheneOS included. Gearslip now turns speech into text itself, with no Google services.", + "New messages show as a small card at the top right. A bar shows how long until it closes or sends.", + "Only messages, missed calls, alarms and reminders pop up. Everything else waits on the dashboard.", + "Steering wheel buttons, control knobs and D-pads now work. Tell us if yours don't.", + "The map stays loaded when you switch screens, so it no longer flashes black.", + "Map apps draw more reliably and keep their connection when the phone is locked.", + "The Home button shows your map app, so it no longer appears twice.", + "The next turn shows in the nav bar when you leave the map.", + "Choose miles or kilometres under Settings > Units.", + "Try the voice assistant and the vehicle data screen under Settings > Experimental features.", + "Logs now say why a car ended the connection.", + ), + ), + ) + + /** The release that matches this build, if it has notes. */ + val current: Release? get() = releases.firstOrNull { it.versionCode == BuildConfig.VERSION_CODE } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt index 29207aa..8922ceb 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt @@ -113,6 +113,7 @@ class GearslipActivity : ComponentActivity(), Projection { GearslipLog.installCrashHandler() GearslipLog.i("Gearslip ready") CarSettings.init(this) + AppSettings.applyGearslipEnabled(this) // A capture that died without tidying up would leave notifications exposed to every // later screen share. Nothing is capturing if the service is not running, so undo it. if (!app.seb3thehacker.gearslip.audio.AudioCaptureService.running) { @@ -245,6 +246,12 @@ class GearslipActivity : ComponentActivity(), Projection { private fun handleIntent(intent: Intent?) { if (intent?.action != UsbManager.ACTION_USB_ACCESSORY_ATTACHED) return + // Switched off in Settings: the system shouldn't route the car here at all, but if it + // does, leave the accessory alone. + if (!AppSettings.gearslipEnabled(this)) { + GearslipLog.i("accessory attached, but Gearslip is switched off - not connecting") + return + } val accessory = intent.getParcelableExtra(UsbManager.EXTRA_ACCESSORY, UsbAccessory::class.java) if (accessory == null) { GearslipLog.e("attach intent carried no accessory") @@ -309,6 +316,9 @@ class GearslipActivity : ComponentActivity(), Projection { CarEnvironment.setDisplay(width, height, densityDpi) CarEnvironment.start(this) showCarUi() + app.seb3thehacker.gearslip.car.CarKeys.focusSink = { code, down -> + runOnUiThread { screenProjector?.dispatchKey(code, down) } + } } } @@ -323,6 +333,7 @@ class GearslipActivity : ComponentActivity(), Projection { override fun onProjectionStopped() { runOnUiThread { PhoneMirror.requestStop() + app.seb3thehacker.gearslip.car.CarKeys.focusSink = null screenProjector?.stop() screenProjector = null surface = null diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt index 1681a40..f3d7f8e 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipRunner.kt @@ -4,6 +4,7 @@ import android.os.Build import app.seb3thehacker.gearslip.audio.AudioLink import app.seb3thehacker.gearslip.car.CarAssistant import app.seb3thehacker.gearslip.car.CarEnvironment +import app.seb3thehacker.gearslip.car.CarKeys import app.seb3thehacker.gearslip.car.CarSensors import app.seb3thehacker.gearslip.car.CarServices import app.seb3thehacker.gearslip.car.CarSettings @@ -54,6 +55,7 @@ class GearslipRunner( @Volatile private var framesSent = 0 @Volatile private var dropped = 0 @Volatile private var inputChannelId: Int = -1 + @Volatile private var inputKeycodes: List = emptyList() @Volatile private var sensorChannelId: Int = -1 private var sensorTypes: List = emptyList() @Volatile private var audioLink: AudioLink? = null @@ -389,6 +391,7 @@ class GearslipRunner( log.w("no input service advertised - projection will be output-only") } else { inputChannelId = input.serviceId + inputKeycodes = input.keycodesSupported touchWidth = input.touchWidth touchHeight = input.touchHeight log.i( @@ -504,12 +507,19 @@ class GearslipRunner( */ private fun onInputMessage(messageId: Int, body: ByteArray) { when (messageId) { - MSG_CHANNEL_OPEN_RESPONSE -> - log.i("<- ChannelOpenResponse (input): status=${Protobuf.readInt32Field(body, 1)}") + MSG_CHANNEL_OPEN_RESPONSE -> { + val status = Protobuf.readInt32Field(body, 1) + log.i("<- ChannelOpenResponse (input): status=$status") + if (status == 0) sendKeyBinding() + } + + MSG_KEY_BINDING_RESPONSE -> + log.i("<- KeyBindingResponse: status=${Protobuf.readInt32Field(body, 1)}") MSG_INPUT_REPORT -> { val report = Wire.fields(body) Wire.bytes(report, 4)?.let(::onKeyEvent) + Wire.bytes(report, 6)?.let(::onRelativeEvent) val touch = Wire.bytes(report, 3) ?: Wire.bytes(report, 7) if (touch == null) { return @@ -544,36 +554,51 @@ class GearslipRunner( } } + /** + * Asks the head unit to forward its buttons. Many units send nothing from the steering wheel + * until the phone names the keys it wants. What the unit offered is asked for; a unit that + * offered nothing gets [CarKeys.WANTED], since some leave the list empty and send keys anyway. + * + * KeyBindingRequest { repeated int32 keycodes = 1 [packed = true]; } + */ + private fun sendKeyBinding() { + val keycodes = inputKeycodes.ifEmpty { CarKeys.WANTED } + val packed = keycodes.fold(ByteArray(0)) { acc, code -> acc + Protobuf.varint(code.toLong()) } + val body = Protobuf.varint(((1 shl 3) or 2).toLong()) + Protobuf.varint(packed.size.toLong()) + packed + send(MSG_KEY_BINDING_REQUEST, body, encrypted = true, channel = inputChannelId) + log.i("-> KeyBindingRequest(keycodes=$keycodes)") + } + /** * Steering-wheel and head-unit buttons: InputReport.key_event (field 4), shaped as * KeyEvent { repeated Key keys = 1; } * Key { keycode = 1; down = 2; metastate = 3; longpress = 4; } * - * Acted on the down edge only, so one press is one action. These are the same numeric - * codes as android.view.KeyEvent (the head unit reuses Android's own keycode space), which - * is why they line up with the constants below without needing a translation table. + * The codes are android.view.KeyEvent's; [CarKeys] decides what each one does. */ private fun onKeyEvent(keyEvent: ByteArray) { for (key in Wire.allBytes(Wire.fields(keyEvent), 1)) { val fields = Wire.fields(key) val keycode = Wire.varint(fields, 1)?.toInt() ?: continue val down = Wire.varint(fields, 2) == 1L - if (!down) continue - log.i("<- key event: keycode=$keycode") - when (keycode) { - KEYCODE_MEDIA_PLAY_PAUSE -> CarServices.media.togglePlay() - KEYCODE_MEDIA_NEXT -> CarServices.media.next() - KEYCODE_MEDIA_PREVIOUS -> CarServices.media.previous() - KEYCODE_VOICE_ASSIST, KEYCODE_SEARCH -> { - if (CarSettings.experimentalFeaturesEnabled.value && CarSettings.voiceAssistantEnabled.value) { - log.i("voice/PTT button pressed - starting the assistant") - CarAssistant.start() - } else { - log.i("voice/PTT button pressed - assistant is off in settings, ignoring") - } - } - else -> log.i("unhandled keycode=$keycode") - } + val longPress = Wire.varint(fields, 4) == 1L + log.i("<- key event: keycode=$keycode ${if (down) "down" else "up"}${if (longPress) " long" else ""}") + if (!CarKeys.onKey(keycode, down, longPress) && down) log.i("unhandled keycode=$keycode") + } + } + + /** + * The rotary knob: InputReport.relative_event (field 6), shaped as + * RelativeEvent { repeated Rel data = 1; } Rel { keycode = 1; delta = 2 (int32); } + */ + private fun onRelativeEvent(event: ByteArray) { + for (rel in Wire.allBytes(Wire.fields(event), 1)) { + val fields = Wire.fields(rel) + val keycode = Wire.varint(fields, 1)?.toInt() ?: continue + // int32, so a turn the other way arrives as a 64-bit two's complement varint. + val delta = Wire.varint(fields, 2)?.toInt() ?: continue + log.i("<- relative event: keycode=$keycode delta=$delta") + CarKeys.onRotate(delta) } } @@ -953,16 +978,11 @@ class GearslipRunner( const val VIDEO_FOCUS_PROJECTED = 1 const val SESSION_ID = 1 const val MSG_INPUT_REPORT = 32769 + const val MSG_KEY_BINDING_REQUEST = 32770 + const val MSG_KEY_BINDING_RESPONSE = 32771 const val ACTION_DOWN = 0 const val ACTION_UP = 1 - // android.view.KeyEvent codes the head unit re-sends as-is (aap_protobuf's KeyCode.proto - // mirrors Android's own keycode space). - const val KEYCODE_SEARCH = 84 - const val KEYCODE_MEDIA_PLAY_PAUSE = 85 - const val KEYCODE_MEDIA_NEXT = 87 - const val KEYCODE_MEDIA_PREVIOUS = 88 - const val KEYCODE_VOICE_ASSIST = 231 const val ACK_STALL_MS = 2_000L const val AUDIO_TRACE_MESSAGES = 6 const val PERIODIC_KEYFRAME_MS = 5_000L diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ScreenProjector.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ScreenProjector.kt index b909d45..3b9202c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ScreenProjector.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ScreenProjector.kt @@ -179,6 +179,28 @@ class ScreenProjector(private val context: Context) { if (action == MotionEvent.ACTION_UP || action == MotionEvent.ACTION_CANCEL) downTime = 0L } + /** + * A key from the car's D-pad or knob, as a hardware key event on the car display. Compose + * moves focus between buttons on arrows and Tab, and clicks the focused one on OK/Enter. + * [keyCode] of [CarKeys.KEYCODE_SHIFT_TAB] is Tab with Shift held. + */ + fun dispatchKey(keyCode: Int, down: Boolean) { + val target = root ?: return + // Nothing on the display takes keys until something in it holds focus. + if (target.findFocus() == null) target.requestFocus() + val shift = keyCode == app.seb3thehacker.gearslip.car.CarKeys.KEYCODE_SHIFT_TAB + val code = if (shift) android.view.KeyEvent.KEYCODE_TAB else keyCode + val now = SystemClock.uptimeMillis() + val event = android.view.KeyEvent( + now, now, + if (down) android.view.KeyEvent.ACTION_DOWN else android.view.KeyEvent.ACTION_UP, + code, 0, + if (shift) android.view.KeyEvent.META_SHIFT_ON or android.view.KeyEvent.META_SHIFT_LEFT_ON else 0, + android.view.KeyCharacterMap.VIRTUAL_KEYBOARD, 0, 0, android.view.InputDevice.SOURCE_DPAD, + ) + target.dispatchKeyEvent(event) + } + fun stop() { runCatching { presentation?.dismiss() } runCatching { owner?.destroy() } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt index 0ef8e53..84c61b3 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ServiceDiscovery.kt @@ -130,11 +130,14 @@ object ServiceDiscovery { val id = Wire.varint(service, 1)?.toInt() ?: continue val input = Wire.bytes(service, 4) ?: continue val inputFields = Wire.fields(input) - val touch = Wire.bytes(inputFields, 2) ?: continue - val touchFields = Wire.fields(touch) - val width = Wire.varint(touchFields, 1)?.toInt() ?: continue - val height = Wire.varint(touchFields, 2)?.toInt() ?: continue - val keycodes = Wire.bytes(inputFields, 1)?.let(::packedVarints).orEmpty() + // A unit with only a knob and buttons has no touchscreen entry; it still has keys. + val touchFields = Wire.bytes(inputFields, 2)?.let { Wire.fields(it) }.orEmpty() + val width = Wire.varint(touchFields, 1)?.toInt() ?: 0 + val height = Wire.varint(touchFields, 2)?.toInt() ?: 0 + // Declared packed, but some units send each keycode as its own field. + val keycodes = inputFields.filter { it.number == 1 }.flatMap { f -> + f.bytes?.let(::packedVarints) ?: listOf(f.varint.toInt()) + } return InputService(id, width, height, keycodes) } return null diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/VideoSource.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/VideoSource.kt index 0536281..b85398c 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/VideoSource.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/VideoSource.kt @@ -45,7 +45,8 @@ class VideoSource( private var frameIndex = 0L fun start() { - val format = MediaFormat.createVideoFormat(MIME, width, height).apply { + val bitRate = (width.toLong() * height * frameRate * BITS_PER_PIXEL).toInt().coerceIn(MIN_BIT_RATE, MAX_BIT_RATE) + fun format(tuned: Boolean) = MediaFormat.createVideoFormat(MIME, width, height).apply { setInteger( MediaFormat.KEY_COLOR_FORMAT, if (mode == Mode.SURFACE) { @@ -54,21 +55,37 @@ class VideoSource( MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible }, ) - setInteger(MediaFormat.KEY_BIT_RATE, 2_000_000) + setInteger(MediaFormat.KEY_BIT_RATE, bitRate) setInteger(MediaFormat.KEY_FRAME_RATE, frameRate) - setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) + // Drops already ask for a fresh keyframe, so a long interval costs nothing in recovery + // and spares the link a big frame every second. + setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, I_FRAME_INTERVAL_S) setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline) setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31) + if (tuned) { + // Hints only: a car screen wants each frame now, not the best frame a moment later. + setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR) + setInteger(MediaFormat.KEY_PRIORITY, 0) + setInteger(MediaFormat.KEY_LATENCY, 1) + } } - val encoder = MediaCodec.createEncoderByType(MIME) - encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + var encoder = MediaCodec.createEncoderByType(MIME) + try { + encoder.configure(format(tuned = true), null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + } catch (e: Exception) { + // Some encoders refuse a hint they don't know. The plain format is what always worked. + log.w("encoder refused the tuned format (${e.message}) - using the plain one") + encoder.release() + encoder = MediaCodec.createEncoderByType(MIME) + encoder.configure(format(tuned = false), null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) + } if (mode == Mode.SURFACE) inputSurface = encoder.createInputSurface() encoder.start() codec = encoder running = true - log.i("encoder started: ${width}x$height @ ${frameRate}fps, H.264 Baseline, mode=$mode") + log.i("encoder started: ${width}x$height @ ${frameRate}fps, ${bitRate / 1000} kbps, H.264 Baseline, mode=$mode") worker = Thread { runCatching { pump(encoder) } @@ -123,7 +140,10 @@ class VideoSource( frameIndex++ } - var outIndex = encoder.dequeueOutputBuffer(info, TIMEOUT_US) + // A surface-fed encoder has nothing to do between frames, so block until one is ready + // instead of sleeping a fixed time: less delay on screen, fewer wakeups for the CPU. + val wait = if (mode == Mode.SURFACE) SURFACE_TIMEOUT_US else TIMEOUT_US + var outIndex = encoder.dequeueOutputBuffer(info, wait) while (outIndex >= 0) { val buffer = encoder.getOutputBuffer(outIndex) if (buffer != null && info.size > 0) { @@ -144,7 +164,7 @@ class VideoSource( outIndex = encoder.dequeueOutputBuffer(info, 0) } - Thread.sleep(maxOf(1L, frameIntervalUs / 2000)) + if (mode != Mode.SURFACE) Thread.sleep(maxOf(1L, frameIntervalUs / 2000)) } } @@ -188,6 +208,13 @@ class VideoSource( private companion object { const val MIME = MediaFormat.MIMETYPE_VIDEO_AVC const val TIMEOUT_US = 10_000L + const val SURFACE_TIMEOUT_US = 100_000L + + /** Enough for sharp text and map lines; 800x480 at 30fps comes out at 4 Mbps. */ + const val BITS_PER_PIXEL = 0.3 + const val MIN_BIT_RATE = 4_000_000 + const val MAX_BIT_RATE = 12_000_000 + const val I_FRAME_INTERVAL_S = 5 /** (Y, Cb, Cr) - white, yellow, cyan, green, magenta, red, blue. */ val BAR_COLOURS = listOf( diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt index aec0b0d..5a37224 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioCaptureService.kt @@ -129,7 +129,7 @@ class AudioCaptureService : Service() { AudioRecord.Builder() .setAudioPlaybackCaptureConfig(config) .setAudioFormat(format) - .setBufferSizeInBytes(maxOf(minimum, CHUNK_BYTES * 4)) + .setBufferSizeInBytes(maxOf(minimum, CHUNK_BYTES * BUFFER_CHUNKS)) .build() } catch (t: Throwable) { GearslipLog.e("audio: could not set up playback capture", t) @@ -233,6 +233,8 @@ class AudioCaptureService : Service() { /** About 21 ms of 48 kHz stereo: small enough for lip-sync, big enough not to flood the link. */ private const val CHUNK_BYTES = 4096 + /** About half a second: room to wait out a slow ack from the car without losing sound. */ + private const val BUFFER_CHUNKS = 24 private const val LEVEL_LOG_MS = 3_000L private const val SILENCE_MS = 5_000L diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt index de826ca..46fdeb9 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/audio/AudioLink.kt @@ -37,6 +37,7 @@ class AudioLink( private var bytes = 0L private var dropped = 0 @Volatile private var configTimer: Thread? = null + private val ackLock = Object() private val config get() = service.configs.getOrNull(configIndex) ?: service.configs.firstOrNull() override val sampleRate get() = config?.sampleRate ?: 48_000 @@ -115,8 +116,11 @@ class AudioLink( MSG_ACK -> { val acked = Wire.varint(Wire.fields(body), 2)?.toInt() ?: 1 - inFlight = maxOf(0, inFlight - maxOf(1, acked)) - lastAckAt = System.currentTimeMillis() + synchronized(ackLock) { + inFlight = maxOf(0, inFlight - maxOf(1, acked)) + lastAckAt = System.currentTimeMillis() + ackLock.notifyAll() + } } // Whatever the head unit answers Setup with, we need its bytes to identify it. @@ -166,6 +170,7 @@ class AudioLink( override fun stop() { if (!started) return started = false + synchronized(ackLock) { ackLock.notifyAll() } runCatching { sendOnChannel(MSG_STOP, Protobuf.varintField(1, SESSION_ID.toLong())) if (focusRequested) sendControl(MSG_FOCUS_REQUEST, Protobuf.varintField(1, FOCUS_RELEASE.toLong())) @@ -179,16 +184,24 @@ class AudioLink( override fun write(pcm: ByteArray, size: Int) { if (!started) return - if (inFlight >= maxUnacked) { - // A lost ack must not silence the stream for good; after a stall assume the window cleared. - if (System.currentTimeMillis() - lastAckAt > STALL_MS) { - GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window") - inFlight = 0 - } else { - dropped++ - return + // Wait for the car to catch up rather than throw the chunk away: a dropped chunk is a gap + // the driver hears, while a short wait is soaked up by the capture buffer behind us. + synchronized(ackLock) { + while (started && inFlight >= maxUnacked) { + val waited = System.currentTimeMillis() - lastAckAt + if (waited > STALL_MS) { + // A lost ack must not silence the stream for good; after a stall assume the window cleared. + GearslipLog.w("audio: no ack for ${STALL_MS}ms with $inFlight in flight - reopening the window") + inFlight = 0 + break + } + ackLock.wait(STALL_MS - waited + 1) } } + if (!started) { + dropped++ + return + } val timestamp = ByteArray(8) val micros = System.nanoTime() / 1_000 for (i in 0 until 8) timestamp[i] = ((micros shr ((7 - i) * 8)) and 0xFF).toByte() @@ -199,7 +212,7 @@ class AudioLink( started = false return } - inFlight++ + synchronized(ackLock) { inFlight++ } chunks++ bytes += size // The first chunk proves the whole path; after that, an occasional line is enough to show diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAppStage.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAppStage.kt index c4ef3b4..b16475d 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAppStage.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAppStage.kt @@ -18,6 +18,7 @@ import androidx.compose.runtime.Composable import androidx.compose.runtime.DisposableEffect import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue @@ -96,7 +97,8 @@ private fun AppSurface( // connection with it and the map freezes for good. The nav bar hiding for the search keyboard // and the lyrics pane opening both resize this area. androidx.compose.foundation.layout.BoxWithConstraints(modifier) { - androidx.compose.runtime.key(constraints.maxWidth, constraints.maxHeight) { + val epoch by connection.surfaceEpoch.collectAsState() + androidx.compose.runtime.key(constraints.maxWidth, constraints.maxHeight, epoch) { SurfaceHost(connection, frame, onSize = { size = it }, Modifier.fillMaxSize()) } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt index a4e1fe9..4577b41 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarAssistant.kt @@ -12,17 +12,17 @@ import android.speech.SpeechRecognizer import android.speech.tts.TextToSpeech import android.speech.tts.UtteranceProgressListener import androidx.core.content.ContextCompat +import app.seb3thehacker.gearslip.call.CallUiState +import app.seb3thehacker.gearslip.call.CarCalls +import app.seb3thehacker.gearslip.speech.SpeechEngine import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow /** - * A voice assistant built entirely on Android's own speech APIs - [SpeechRecognizer] to listen, - * [TextToSpeech] to answer - rather than a bundled recognition engine (see the conversation this - * was scoped from: a project like Dicio ships its own STT model and isn't built to be embedded - * as a library, which cuts against how the rest of Gearslip is built). Both APIs just talk to - * whatever the system has configured, so this works unchanged on GrapheneOS against GrapheneOS - * Speech Services, or on stock Android against whatever the phone shipped with - Gearslip never - * names an engine. + * A voice assistant: Gearslip's own [SpeechEngine] (whisper.cpp) to listen, the system's + * [TextToSpeech] to answer. Listening used to go through the system's [SpeechRecognizer] alone, + * but many phones have no recognizer at all (GrapheneOS ships only text-to-speech), so the + * system one is now just the fallback for when the bundled engine can't run. * * Listening and speaking live here; deciding what a command *means* is [AssistantCommands], kept * separate because it needs a [CarNavigator] and a frame to act on, neither of which this object @@ -80,55 +80,21 @@ object CarAssistant { _state.value = State(Phase.ERROR, reply = "Needs the microphone permission.", missingPermission = true) return } - if (!SpeechRecognizer.isRecognitionAvailable(context)) { - _state.value = State(Phase.ERROR, reply = "No speech recognizer is set up on this phone.") + // Android hands every app but the dialer a silent microphone during a call. + if (CarCalls.state.value is CallUiState.Active) { + _state.value = State(Phase.ERROR, reply = "Voice commands work once the call ends.") return } - tts?.stop() _state.value = State(Phase.LISTENING) - - val r = SpeechRecognizer.createSpeechRecognizer(context) - recognizer = r - r.setRecognitionListener(object : RecognitionListener { - override fun onReadyForSpeech(params: Bundle?) {} - override fun onBeginningOfSpeech() {} - override fun onRmsChanged(rmsdB: Float) {} - override fun onBufferReceived(buffer: ByteArray?) {} - - override fun onEndOfSpeech() { - _state.value = _state.value.copy(phase = Phase.THINKING) - } - - override fun onError(error: Int) { - releaseRecognizer() - _state.value = State(Phase.ERROR, reply = errorMessage(error)) - } - - override fun onResults(results: Bundle?) { - releaseRecognizer() - val text = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull().orEmpty() - if (text.isBlank()) { - _state.value = State(Phase.ERROR, reply = "Didn't catch that.") - } else { - _state.value = _state.value.copy(phase = Phase.THINKING, heard = text) - _pendingCommand.value = text - } - } - - override fun onPartialResults(partialResults: Bundle?) { - val text = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull() - if (text != null) _state.value = _state.value.copy(heard = text) - } - - override fun onEvent(eventType: Int, params: Bundle?) {} - }) - r.startListening( - Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { - putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) - putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) - putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + listenOnce( + onPartial = { text -> _state.value = _state.value.copy(heard = text) }, + onHeard = { _state.value = _state.value.copy(phase = Phase.THINKING) }, + onResult = { text -> + _state.value = _state.value.copy(phase = Phase.THINKING, heard = text) + _pendingCommand.value = text }, + onError = { message -> _state.value = State(Phase.ERROR, reply = message) }, ) } @@ -166,7 +132,118 @@ object CarAssistant { engine.speak(text, TextToSpeech.QUEUE_FLUSH, null, "gearslip-assistant") } + /** + * Speaks [text] for someone other than the assistant (a message read aloud), then runs + * [onDone] on the main thread - straight away if speech isn't available. + */ + fun say(text: String, onDone: () -> Unit) { + val engine = tts + if (engine == null || !ttsReady || text.isBlank()) { + mainHandler.post(onDone) + return + } + engine.setOnUtteranceProgressListener(object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) {} + override fun onDone(utteranceId: String?) { mainHandler.post(onDone) } + @Deprecated("Deprecated in Java, no replacement on this API level") + override fun onError(utteranceId: String?) { mainHandler.post(onDone) } + // Stopped part way: whoever stopped it has already moved on. + override fun onStop(utteranceId: String?, interrupted: Boolean) {} + }) + engine.speak(text, TextToSpeech.QUEUE_FLUSH, null, "gearslip-say") + } + + fun stopSpeaking() { tts?.stop() } + + /** + * Listens once, the same way the assistant does, and hands back what was said without acting + * on it. Callbacks arrive on the main thread; [onError] gets a sentence fit to show. + */ + fun dictate( + onPartial: (String) -> Unit, + onResult: (String) -> Unit, + onError: (String) -> Unit, + cue: Boolean = true, + onSpeaking: () -> Unit = {}, + ) { + cancel() + listenOnce(onPartial, onHeard = {}, onResult, onError, cue, onSpeaking) + } + + /** + * One phrase, through Gearslip's own [SpeechEngine]. The phone's recognizer is only a fallback + * for when the engine itself can't run, since plenty of phones (GrapheneOS among them) have none. + */ + private fun listenOnce( + onPartial: (String) -> Unit, + onHeard: () -> Unit, + onResult: (String) -> Unit, + onError: (String) -> Unit, + cue: Boolean = true, + onSpeaking: () -> Unit = {}, + ) { + val context = appContext ?: return onError("Speech isn't ready yet.") + SpeechEngine.listen( + context, + cue = cue, + onSpeaking = onSpeaking, + onHeard = onHeard, + onResult = onResult, + onError = { message -> + if (message == SpeechEngine.NOT_LOADED && SpeechRecognizer.isRecognitionAvailable(context)) { + systemListen(context, onPartial, onHeard, onResult, onError) + } else { + onError(message) + } + }, + ) + } + + private fun systemListen( + context: Context, + onPartial: (String) -> Unit, + onHeard: () -> Unit, + onResult: (String) -> Unit, + onError: (String) -> Unit, + ) { + val r = SpeechRecognizer.createSpeechRecognizer(context) + recognizer = r + r.setRecognitionListener(object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) {} + override fun onBeginningOfSpeech() {} + override fun onRmsChanged(rmsdB: Float) {} + override fun onBufferReceived(buffer: ByteArray?) {} + override fun onEndOfSpeech() = onHeard() + override fun onEvent(eventType: Int, params: Bundle?) {} + + override fun onError(error: Int) { + releaseRecognizer() + onError(errorMessage(error)) + } + + override fun onResults(results: Bundle?) { + releaseRecognizer() + val text = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull().orEmpty() + if (text.isBlank()) onError("Didn't catch that.") else onResult(text) + } + + override fun onPartialResults(partialResults: Bundle?) { + partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)?.firstOrNull()?.let(onPartial) + } + }) + r.startListening( + Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true) + putExtra(RecognizerIntent.EXTRA_CALLING_PACKAGE, context.packageName) + }, + ) + } + + fun stopListening() = releaseRecognizer() + private fun releaseRecognizer() { + SpeechEngine.cancel() recognizer?.let { runCatching { it.cancel() }; it.destroy() } recognizer = null } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarHomeScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarHomeScreen.kt index 7191589..0354f22 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarHomeScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarHomeScreen.kt @@ -82,6 +82,15 @@ fun CarHome() { val navigator = LocalCarNavigator.current val panel = navigator.sidePanel + + // Coming back to the map from another screen: make sure the nav app is drawing into a live + // surface. The first composition is skipped, since the surface is only just being lent. + val onHome = navigator.current == CarScreen.Home + val seenHome = remember { booleanArrayOf(false) } + LaunchedEffect(onHome) { + if (onHome && seenHome[0]) CarServices.nav.refreshSurface() + if (onHome) seenHome[0] = true + } val splitScreen = phase == CarMedia.Phase.READY && now.isActive && panel != SidePanel.NONE // No outer padding on any side, split screen or not - only the gap between the two panes diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt new file mode 100644 index 0000000..3eeae41 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarKeys.kt @@ -0,0 +1,137 @@ +package app.seb3thehacker.gearslip.car + +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.view.KeyEvent +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.call.CallUiState +import app.seb3thehacker.gearslip.call.CarCalls +import kotlinx.coroutines.flow.MutableSharedFlow +import kotlinx.coroutines.flow.SharedFlow + +/** + * The car's own buttons: steering wheel, rotary knob, D-pad and the hard keys around the screen. + * + * The head unit sends Android keycodes as they are, so the numbers here are android.view.KeyEvent's. + * Media, voice and call keys act straight away. Keys that move around the car UI (back, home, the + * knob) need a [CarNavigator], which only [CarUi] has, so they go out on [screenKeys]. Focus keys + * (D-pad, knob turns, OK) become real key events on the car's display through [focusSink], where + * Compose moves focus between buttons the same way it would with a keyboard. + */ +object CarKeys { + + /** A key that changes which screen is showing. */ + enum class ScreenKey { BACK, HOME, MAP, MEDIA, PHONE } + + private val _screenKeys = MutableSharedFlow(extraBufferCapacity = 8) + val screenKeys: SharedFlow = _screenKeys + + /** Delivers an Android key event to the car display; set while the car UI is showing. */ + @Volatile var focusSink: ((keyCode: Int, down: Boolean) -> Unit)? = null + + private var appContext: Context? = null + private val main = Handler(Looper.getMainLooper()) + + fun init(context: Context) { appContext = context.applicationContext } + + /** + * Every key worth asking the head unit for. The binding request lists what the head unit + * offered; when it offers nothing (some units leave the list empty and send keys anyway), + * this is what's asked for instead. + */ + val WANTED = listOf( + KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE, KeyEvent.KEYCODE_MEDIA_PLAY, KeyEvent.KEYCODE_MEDIA_PAUSE, + KeyEvent.KEYCODE_MEDIA_NEXT, KeyEvent.KEYCODE_MEDIA_PREVIOUS, KeyEvent.KEYCODE_MEDIA_STOP, + KeyEvent.KEYCODE_SEARCH, KeyEvent.KEYCODE_VOICE_ASSIST, + KeyEvent.KEYCODE_CALL, KeyEvent.KEYCODE_ENDCALL, + KeyEvent.KEYCODE_BACK, KeyEvent.KEYCODE_HOME, + KeyEvent.KEYCODE_DPAD_UP, KeyEvent.KEYCODE_DPAD_DOWN, KeyEvent.KEYCODE_DPAD_LEFT, + KeyEvent.KEYCODE_DPAD_RIGHT, KeyEvent.KEYCODE_DPAD_CENTER, KeyEvent.KEYCODE_ENTER, + KEYCODE_ROTARY_CONTROLLER, KEYCODE_MEDIA, KEYCODE_NAVIGATION, KEYCODE_TEL, + ) + + /** + * One key edge from the head unit. Most keys act on the press; focus keys pass both edges on, + * so a held D-pad key repeats the way it would anywhere else. Returns false for a key nothing + * here knows, for the caller to log. + */ + fun onKey(keyCode: Int, down: Boolean, longPress: Boolean): Boolean { + if (keyCode in FOCUS_KEYS) { + focusSink?.invoke(keyCode, down) + return true + } + if (!down) return keyCode in WANTED + val media = CarServices.media + when (keyCode) { + KeyEvent.KEYCODE_MEDIA_PLAY_PAUSE -> media.togglePlay() + KeyEvent.KEYCODE_MEDIA_PLAY -> if (!media.now.value.playing) media.togglePlay() + KeyEvent.KEYCODE_MEDIA_PAUSE, KeyEvent.KEYCODE_MEDIA_STOP -> if (media.now.value.playing) media.togglePlay() + KeyEvent.KEYCODE_MEDIA_NEXT -> media.next() + KeyEvent.KEYCODE_MEDIA_PREVIOUS -> media.previous() + KeyEvent.KEYCODE_SEARCH, KeyEvent.KEYCODE_VOICE_ASSIST -> voice() + KeyEvent.KEYCODE_CALL -> call() + KeyEvent.KEYCODE_ENDCALL -> endCall() + KeyEvent.KEYCODE_BACK -> screen(ScreenKey.BACK) + KeyEvent.KEYCODE_HOME -> screen(ScreenKey.HOME) + KEYCODE_NAVIGATION -> screen(ScreenKey.MAP) + KEYCODE_MEDIA -> screen(ScreenKey.MEDIA) + KEYCODE_TEL -> screen(ScreenKey.PHONE) + else -> return false + } + if (longPress) GearslipLog.i("keys: long press on $keyCode") + return true + } + + /** A turn of the rotary knob: each click moves focus one step, like Tab or Shift+Tab. */ + fun onRotate(delta: Int) { + val sink = focusSink ?: return + val code = if (delta > 0) KeyEvent.KEYCODE_TAB else KEYCODE_SHIFT_TAB + repeat(kotlin.math.abs(delta).coerceAtMost(MAX_ROTARY_STEPS)) { + sink(code, true) + sink(code, false) + } + } + + /** + * The voice key: answers the message on screen if there is one, otherwise starts the + * assistant when it's turned on. + */ + private fun voice() = main.post { + if (VoiceReply.onVoiceKey()) return@post + if (CarSettings.experimentalFeaturesEnabled.value && CarSettings.voiceAssistantEnabled.value) { + CarAssistant.start() + } else { + GearslipLog.i("keys: voice pressed, but the assistant is off in settings") + } + } + + /** Answers a ringing call; with nothing ringing, opens the phone screen. */ + private fun call() = main.post { + val context = appContext ?: return@post + if (CarCalls.state.value is CallUiState.Ringing) CarCalls.answer(context) else screen(ScreenKey.PHONE) + } + + private fun endCall() = main.post { + if (CarCalls.state.value is CallUiState.Ringing) CarCalls.decline() + else GearslipLog.i("keys: end call pressed with no ringing call; hanging up needs the dialer") + } + + private fun screen(key: ScreenKey) { _screenKeys.tryEmit(key) } + + // aap_protobuf's KeyCode.proto: car-only keys past the end of Android's own range. + const val KEYCODE_ROTARY_CONTROLLER = 65536 + const val KEYCODE_MEDIA = 65537 + const val KEYCODE_NAVIGATION = 65538 + const val KEYCODE_TEL = 65540 + + /** Stands in for Shift+Tab: the sink sends Tab with the shift meta state. */ + const val KEYCODE_SHIFT_TAB = -KeyEvent.KEYCODE_TAB + + private const val MAX_ROTARY_STEPS = 5 + + private val FOCUS_KEYS = setOf( + KeyEvent.KEYCODE_DPAD_UP, KeyEvent.KEYCODE_DPAD_DOWN, KeyEvent.KEYCODE_DPAD_LEFT, + KeyEvent.KEYCODE_DPAD_RIGHT, KeyEvent.KEYCODE_DPAD_CENTER, KeyEvent.KEYCODE_ENTER, + ) +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarLauncher.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarLauncher.kt index 04125e4..66a410f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarLauncher.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarLauncher.kt @@ -128,13 +128,18 @@ internal object LauncherCache { // Playing - so only the label needs disambiguating, not the tile itself. val mediaPackages = mediaApps.map { it.component.packageName }.toSet() - val nav = navApps.map { + val messagingApps = MessagingCatalog.installed(context) + // Google Messages ships its own car app, which won't start under Gearslip. Gearslip's + // own Messages screen does the same job, so a messenger gets only that tile. + val messagingPackages = messagingApps.map { it.packageName }.toSet() + + val nav = navApps.filter { it.component.packageName !in messagingPackages }.map { val label = if (it.component.packageName in mediaPackages) "${it.label} · Browse" else it.label Entry(label, iconOf(it.component.packageName), template = it) } val media = mediaApps.map { Entry(it.label, iconOf(it.component.packageName), media = it) } // A messenger that is also a car media app keeps only its player tile. - val messaging = MessagingCatalog.installed(context) + val messaging = messagingApps .filter { it.packageName !in mediaPackages } .map { Entry(it.label, iconOf(it.packageName), messaging = it) } return (nav + media + messaging).sortedBy { it.label.lowercase() }.also { flow.value = it } @@ -192,12 +197,14 @@ fun CarLauncher() { var showBadgeKey by remember { mutableStateOf(false) } fun tileOf(entry: Entry): Tile { - val pkg = entry.packageName + // The badges rate an app's own car screens; a messenger shown on Gearslip's own Messages + // screen has none, so it gets no badge. + val pkg = entry.packageName?.takeIf { entry.messaging == null } val id = entry.componentId return Tile( entry.label, entry.icon, entry.builtIn?.glyph, - verified = pkg != null && KnownApps.works(pkg), - broken = pkg != null && KnownApps.isBroken(pkg), + verified = pkg != null && (KnownApps.works(pkg) || (entry.media != null && KnownApps.playerWorks(pkg))), + broken = pkg != null && (KnownApps.isBroken(pkg) || (entry.template != null && KnownApps.screenBroken(pkg))), partial = pkg != null && KnownApps.isPartial(pkg), onLongClick = id?.let { { navigator.showAppMenu(it) } }, ) { openApp(entry, id?.let { running[it] }, navigator, frame) } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt index ad1086e..cb8e114 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarServices.kt @@ -59,6 +59,10 @@ object CarServices { app.seb3thehacker.gearslip.notify.CarMotion.start(appContext) app.seb3thehacker.gearslip.call.CarCalls.start(appContext) CarAssistant.init(appContext) + CarKeys.init(appContext) + VoiceReply.init(appContext) + // Loaded now, while nothing's waiting on it, so the first reply isn't slower than the rest. + app.seb3thehacker.gearslip.speech.SpeechEngine.warm(appContext) } // --- map ----------------------------------------------------------------------------- diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt index 0371290..3804a5a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettings.kt @@ -12,6 +12,8 @@ enum class NightMode { AUTO, DAY, NIGHT } /** Distances and temperatures. [AUTO] goes by the phone's region. */ enum class Units { AUTO, IMPERIAL, METRIC } +/** Where a new message shows: [NAV_BAR] turns the clock button into it, [CARD] floats a card top right. */ + /** The car UI's own colours. Separate from [NightMode], which is about the light outside (the map). */ enum class AppTheme { PHONE, LIGHT, DARK } @@ -36,6 +38,8 @@ object CarSettings { private const val KEY_CAR_SENSORS = "car_sensors_enabled" private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning" private const val KEY_UNITS = "car_units" + private const val KEY_WHATS_NEW = "car_whats_new_after_updates" + private const val KEY_WHATS_NEW_SEEN = "car_whats_new_seen_code" /** How many shortcuts the nav bar makes room for; a driver reaching for one needs it to * still be on the bar, not off the edge of a long list. */ @@ -68,6 +72,36 @@ object CarSettings { val units: StateFlow = unitsFlow + + private val whatsNewFlow = MutableStateFlow(true) + private val whatsNewSeenFlow = MutableStateFlow(0) + private val whatsNewOpenFlow = MutableStateFlow(false) + + /** Whether the car screen shows what's new once after each update. */ + val whatsNewAfterUpdates: StateFlow = whatsNewFlow + + /** Version code whose notes the car screen last showed. */ + val whatsNewSeen: StateFlow = whatsNewSeenFlow + + /** Opened by hand from Settings, whatever the two above say. Not saved. */ + val whatsNewOpen: StateFlow = whatsNewOpenFlow + + fun setWhatsNewAfterUpdates(on: Boolean) { + whatsNewFlow.value = on + AppSettings.putString(app, KEY_WHATS_NEW, on.toString()) + } + + fun openWhatsNew() { whatsNewOpenFlow.value = true } + + /** Closes the notes and marks [versionCode]'s as read. */ + fun closeWhatsNew(versionCode: Int) { + whatsNewOpenFlow.value = false + whatsNewSeenFlow.value = versionCode + AppSettings.putString(app, KEY_WHATS_NEW_SEEN, versionCode.toString()) + } + + + /** Whether to show miles, feet and °F; [Units.AUTO] follows the countries that still use them. */ fun imperial(): Boolean = when (unitsFlow.value) { Units.IMPERIAL -> true @@ -161,6 +195,8 @@ object CarSettings { unitsFlow.value = runCatching { Units.valueOf(AppSettings.getString(app, KEY_UNITS, Units.AUTO.name)) }.getOrDefault(Units.AUTO) + whatsNewFlow.value = AppSettings.getString(app, KEY_WHATS_NEW, "true") == "true" + whatsNewSeenFlow.value = AppSettings.getString(app, KEY_WHATS_NEW_SEEN, "0").toIntOrNull() ?: 0 } fun setScale(value: Float) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt index d56b940..3eec102 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarSettingsScreen.kt @@ -1,5 +1,6 @@ package app.seb3thehacker.gearslip.car + import app.seb3thehacker.gearslip.car.theme.* import android.os.Build import androidx.compose.foundation.layout.Arrangement @@ -50,6 +51,7 @@ fun CarSettingsScreen() { val open by CarSettings.openOnConnect.collectAsState() val night by CarSettings.nightMode.collectAsState() val units by CarSettings.units.collectAsState() + val whatsNew by CarSettings.whatsNewAfterUpdates.collectAsState() val pipeAudio by CarSettings.pipeAudio.collectAsState() val autoplay by CarSettings.autoplay.collectAsState() val theme by CarSettings.appTheme.collectAsState() @@ -149,6 +151,20 @@ fun CarSettingsScreen() { Info("Video", display.ifEmpty { "-" }) Info("Certificate", cert?.headline ?: "Checking…") } + + Column(verticalArrangement = Arrangement.spacedBy(8.dp)) { + Text("About", style = MaterialTheme.typography.titleMedium, color = MaterialTheme.colorScheme.primary) + Info("Version", app.seb3thehacker.gearslip.BuildConfig.VERSION_NAME) + Info("Build", app.seb3thehacker.gearslip.BuildConfig.VERSION_CODE.toString()) + ExperimentalFeatureRow( + "What's new after updates", + "Show the list of changes once, the first time the car connects after an update.", + whatsNew, + ) { CarSettings.setWhatsNewAfterUpdates(it) } + GsButton(onClick = CarSettings::openWhatsNew, tone = GsTone.Tonal, modifier = Modifier.height(56.dp)) { + Text("What's new", style = MaterialTheme.typography.titleMedium) + } + } } } @@ -237,7 +253,7 @@ private fun ExperimentalFeatureRow( ) } Spacer(Modifier.width(12.dp)) - Switch(checked = checked, onCheckedChange = onCheckedChange) + GsSwitch(checked, onCheckedChange) } } } @@ -358,3 +374,4 @@ private fun StepButton(text: String, modifier: Modifier, onClick: () -> Unit) { Text(text, style = MaterialTheme.typography.titleLarge) } } + diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt index 95177a6..72bbb72 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarUi.kt @@ -137,6 +137,22 @@ fun CarUi() { } } + // The car's own back, home, map, media and phone keys. + LaunchedEffect(navigator) { + CarKeys.screenKeys.collect { key -> + when (key) { + CarKeys.ScreenKey.BACK -> when { + navigator.appMenu != null -> navigator.dismissAppMenu() + VoiceReply.state.value.phase != VoiceReply.Phase.IDLE -> VoiceReply.cancel() + else -> navigator.back() + } + CarKeys.ScreenKey.HOME, CarKeys.ScreenKey.MAP -> navigator.home() + CarKeys.ScreenKey.MEDIA -> if (CarServices.mediaApp.value != null) navigator.media() else navigator.apps() + CarKeys.ScreenKey.PHONE -> BuiltInApps.Phone.open(navigator) + } + } + } + CompositionLocalProvider( LocalDensity provides Density(base.density * scale, base.fontScale), LocalCarNavigator provides navigator, @@ -204,10 +220,12 @@ fun CarUi() { } } } + PopupTimer() // The dashboard is already listing them, so it needs no popup. - if (screen !is CarScreen.Notifications && screen != CarScreen.Dashboard) { - NotificationPopup(Modifier.align(Alignment.TopCenter)) - } + NotificationPopup( + showNew = screen !is CarScreen.Notifications && screen != CarScreen.Dashboard, + modifier = Modifier.align(Alignment.TopEnd), + ) // Above the popup: a call is more urgent than any notification. CallOverlay(Modifier.align(Alignment.TopCenter)) AssistantOverlay(Modifier.align(Alignment.TopCenter)) @@ -227,6 +245,7 @@ fun CarUi() { } // Over everything, nav bar included: a long press on an app, anywhere. AppMenu(navigator, rememberRunningApps()) + if (safetyWarningSeen) CarWhatsNew() } } } @@ -282,10 +301,46 @@ private fun CarSafetyWizard(onDone: () -> Unit) { private fun BrowseAppScreen(navigator: CarNavigator) { val status by CarServices.browse.status.collectAsState() val frame by CarEnvironment.frame.collectAsState() - CarAppStage(CarServices.browse, status, frame) { + val template by CarServices.browse.template.collectAsState() + val leave = { CarServices.stopBrowse() navigator.back() } + // An app that never answers is as stuck as one that says no; don't leave the driver on "Loading". + var timedOut by remember { mutableStateOf(false) } + LaunchedEffect(template == null) { + timedOut = false + if (template == null) { + delay(APP_START_TIMEOUT_MS) + timedOut = true + } + } + val failed = status.phase == CarAppConnection.Phase.REJECTED || status.phase == CarAppConnection.Phase.FAILED + if (failed || timedOut) { + AppWontStart(status.app ?: "This app", leave) + return + } + CarAppStage(CarServices.browse, status, frame, onDisconnect = leave) +} + +private const val APP_START_TIMEOUT_MS = 15_000L + +/** What a car app that won't run under Gearslip gets instead of a loading screen that never ends. */ +@Composable +private fun AppWontStart(app: String, onBack: () -> Unit) { + Column( + Modifier.fillMaxSize().padding(24.dp), + verticalArrangement = Arrangement.spacedBy(12.dp, Alignment.CenterVertically), + horizontalAlignment = Alignment.CenterHorizontally, + ) { + Text("$app doesn't work with Gearslip yet.", style = MaterialTheme.typography.titleLarge, fontWeight = FontWeight.SemiBold) + Text( + "It didn't start. Some apps only run on Google's Android Auto.", + style = MaterialTheme.typography.bodyLarge, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + GsButton(onClick = onBack, tone = GsTone.Tonal) { Text("Back") } + } } /** @@ -379,7 +434,7 @@ private fun CarNavBar(navigator: CarNavigator) { // breadcrumb trail is how you retrace your steps, but reaching for either of // these two from three screens deep in a media app shouldn't mean reading it // first. - NavItem(ImageVector.vectorResource(R.drawable.navigation_24), navigator.current == CarScreen.Home) { navigator.home() } + NavHomeButton(navigator, entries, running) Spacer(Modifier.width(6.dp)) NavItem(MediaIcons.Apps, navigator.current == CarScreen.Apps) { navigator.apps() } val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState() @@ -391,12 +446,14 @@ private fun CarNavBar(navigator: CarNavigator) { // Pinned shortcuts, then whatever else is open right now. A pinned app that's also // open stays in its pinned spot, marked with a dot, instead of showing up twice. + // The map app lives on the Home button, so it never gets a second icon here. val runningEntries = running.keys - .filter { it !in pinnedIds } + .filter { it !in pinnedIds && running[it] != RunningSlot.NAV } .mapNotNull { id -> entries.find { it.componentId == id } } - if (pinnedEntries.isNotEmpty() || runningEntries.isNotEmpty()) NavDivider() - pinnedEntries.forEach { entry -> NavEntryIcon(entry, running, navigator, frame) } - if (pinnedEntries.isNotEmpty() && runningEntries.isNotEmpty()) NavDivider() + val shownPins = pinnedEntries.filter { running[it.componentId] != RunningSlot.NAV } + if (shownPins.isNotEmpty() || runningEntries.isNotEmpty()) NavDivider() + shownPins.forEach { entry -> NavEntryIcon(entry, running, navigator, frame) } + if (shownPins.isNotEmpty() && runningEntries.isNotEmpty()) NavDivider() runningEntries.forEach { entry -> NavEntryIcon(entry, running, navigator, frame) } // Next turn first, off the map only; the player shares the space when it fits, @@ -464,6 +521,35 @@ private fun CarNavBar(navigator: CarNavigator) { } } +/** + * Home: the map, with the player or lyrics beside it. While a map app runs, the button wears that + * app's icon and a long press offers to close it, so the app needs no icon of its own on the bar. + */ +@Composable +private fun NavHomeButton(navigator: CarNavigator, entries: List, running: Map) { + val selected = navigator.current == CarScreen.Home + val navId = running.entries.firstOrNull { it.value == RunningSlot.NAV }?.key + val navEntry = navId?.let { id -> entries.find { it.componentId == id } } + GsIconBox( + { navigator.home() }, + colors = navColors(selected), + latched = selected, + onLongClick = navId?.let { id -> { navigator.showAppMenu(id) } }, + ) { + val icon = navEntry?.icon + if (icon != null) { + Box(Modifier.padding(NAV_ITEM_PADDING).size(28.dp), contentAlignment = Alignment.Center) { + Image(icon, navEntry.label, Modifier.size(28.dp).clip(RoundedCornerShape(7.dp))) + } + } else { + Icon( + ImageVector.vectorResource(R.drawable.navigation_24), contentDescription = "Home", + modifier = Modifier.padding(NAV_ITEM_PADDING).size(28.dp), + ) + } + } +} + /** Splits the nav bar's sections: Home and Apps, pinned apps, open apps. */ @Composable private fun NavDivider() { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarWhatsNew.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarWhatsNew.kt new file mode 100644 index 0000000..66cf8bb --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/CarWhatsNew.kt @@ -0,0 +1,113 @@ +package app.seb3thehacker.gearslip.car + +import androidx.compose.foundation.background +import androidx.compose.foundation.clickable +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.foundation.verticalScroll +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.produceState +import kotlinx.coroutines.delay +import androidx.compose.runtime.remember +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.Changelog +import app.seb3thehacker.gearslip.car.theme.GsButton +import app.seb3thehacker.gearslip.car.theme.GsSwitch +import app.seb3thehacker.gearslip.notify.CarMotion + +/** + * What's new in this build, once, the first time the car screen shows after an update - or any + * time from Settings > About. Held back while the car is moving: a list to read is for when + * parked. The switch turns the automatic showing off (or back on); it doesn't close the card. + */ +@Composable +fun CarWhatsNew() { + val release = Changelog.current ?: return + val afterUpdates by CarSettings.whatsNewAfterUpdates.collectAsState() + val seen by CarSettings.whatsNewSeen.collectAsState() + val open by CarSettings.whatsNewOpen.collectAsState() + // Polled: CarMotion is a plain value, and the card should appear once the car stops. + val moving by produceState(CarMotion.moving) { + while (true) { + value = CarMotion.moving + delay(5_000) + } + } + val due = afterUpdates && seen < release.versionCode && !moving + if (!open && !due) return + val close = { CarSettings.closeWhatsNew(release.versionCode) } + + Box( + Modifier + .fillMaxSize() + .background(Color.Black.copy(alpha = 0.5f)) + .clickable(remember { MutableInteractionSource() }, indication = null, onClick = close), + contentAlignment = Alignment.Center, + ) { + Surface( + shape = RoundedCornerShape(24.dp), + color = MaterialTheme.colorScheme.surfaceContainerHigh, + // Swallows taps on the card itself, so only the scrim around it closes. + modifier = Modifier + .padding(16.dp) + .widthIn(max = 640.dp) + .clickable(remember { MutableInteractionSource() }, indication = null) {}, + ) { + Column(Modifier.padding(20.dp), verticalArrangement = Arrangement.spacedBy(14.dp)) { + Text( + "What's new in ${release.versionName}", + style = MaterialTheme.typography.headlineSmall, + fontWeight = FontWeight.Bold, + ) + Column( + Modifier.weight(1f, fill = false).verticalScroll(rememberScrollState()), + verticalArrangement = Arrangement.spacedBy(10.dp), + ) { + release.changes.forEach { change -> + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Box( + Modifier + .padding(top = 9.dp) + .size(7.dp) + .background(MaterialTheme.colorScheme.primary, CircleShape), + ) + Text(change, style = MaterialTheme.typography.bodyLarge) + } + } + } + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(14.dp)) { + GsSwitch(afterUpdates, CarSettings::setWhatsNewAfterUpdates) + Text( + "Show after updates", + style = MaterialTheme.typography.titleMedium, + modifier = Modifier.weight(1f), + ) + GsButton(onClick = close, modifier = Modifier.height(56.dp)) { + Text("Got it", style = MaterialTheme.typography.titleMedium) + } + } + } + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagePopup.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagePopup.kt new file mode 100644 index 0000000..605a38d --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagePopup.kt @@ -0,0 +1,271 @@ +package app.seb3thehacker.gearslip.car + +import android.os.SystemClock +import androidx.compose.foundation.background +import androidx.compose.foundation.horizontalScroll +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.widthIn +import androidx.compose.foundation.rememberScrollState +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Check +import androidx.compose.material.icons.filled.Close +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableLongStateOf +import androidx.compose.runtime.produceState +import androidx.compose.runtime.withFrameMillis +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.vector.ImageVector +import androidx.compose.ui.res.vectorResource +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.style.TextOverflow +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.R +import app.seb3thehacker.gearslip.car.theme.GsButton +import app.seb3thehacker.gearslip.car.theme.GsIconButton +import app.seb3thehacker.gearslip.car.theme.GsProgressBar +import app.seb3thehacker.gearslip.car.theme.GsTone +import app.seb3thehacker.gearslip.notify.CarNotification +import app.seb3thehacker.gearslip.notify.CarNotifications +import kotlinx.coroutines.delay + +/* + * A new message on the car screen: a card at the top right, over the player beside the map on + * Home, so the road stays in view. It offers two things - hear it, or answer it by voice - and + * shows [VoiceReply]'s progress once either starts. A bar along its bottom empties as the card + * runs out of time, whatever it's waiting to do: go away, or send. + */ + +/** How long a new message stays up on its own. */ +private const val POPUP_MS = 5_000L + +/** When the plain popup goes away, for its bar; set by [PopupTimer]. */ +private val popupUntil = mutableLongStateOf(0L) + +/** + * Times the popup out. A voice reply in progress holds it, and each new message in the same chat + * starts the clock again. + */ +@Composable +internal fun PopupTimer() { + val popup by CarNotifications.popup.collectAsState() + val voice by VoiceReply.state.collectAsState() + val n = popup ?: return + val busy = voice.phase != VoiceReply.Phase.IDLE + LaunchedEffect(n.id, busy) { + if (busy) return@LaunchedEffect + popupUntil.longValue = SystemClock.uptimeMillis() + POPUP_MS + delay(POPUP_MS) + CarNotifications.dismissPopup(n.id) + } +} + +@Composable +internal fun ReadGlyph(modifier: Modifier = Modifier) = + Icon(ImageVector.vectorResource(R.drawable.volume_up_24), null, modifier) + +@Composable +internal fun ReplyGlyph(modifier: Modifier = Modifier) = + Icon(ImageVector.vectorResource(R.drawable.voice_selection_24), null, modifier) + +/** "Alex", or "Alex · 3 new" when a burst of messages from one chat has been merged. */ +private fun CarNotification.heading(): String { + val who = title.ifBlank { appLabel } + return if (burst > 1) "$who · $burst new" else who +} + +/** The card. It shows the voice reply too, wherever that was started from. */ +@Composable +fun NotificationPopup(showNew: Boolean, modifier: Modifier = Modifier) { + val popup by CarNotifications.popup.collectAsState() + val voice by VoiceReply.state.collectAsState() + val target = voice.target + if (voice.phase != VoiceReply.Phase.IDLE && target != null) { + PopupCard(voice.until, voice.span, modifier) { VoiceReplyBody(voice, target) } + return + } + val n = popup?.takeIf { showNew } ?: return + PopupCard(popupUntil.longValue, POPUP_MS, modifier) { + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp)) { + NotificationIcon(n, size = 44.dp) + Column(Modifier.weight(1f)) { + Text(n.heading(), style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold, maxLines = 1, overflow = TextOverflow.Ellipsis) + Text(n.text, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant, maxLines = 1, overflow = TextOverflow.Ellipsis) + } + GsIconButton(Icons.Filled.Close, "Dismiss", { CarNotifications.dismissPopup(n.id) }, size = 40.dp, iconSize = 20.dp) + } + Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { + GsButton(onClick = { VoiceReply.read(n) }, tone = GsTone.Neutral, modifier = Modifier.weight(1f)) { + ReadGlyph(Modifier.size(20.dp)) + Text("Read", Modifier.padding(start = 8.dp)) + } + if (n.reply != null) { + GsButton(onClick = { VoiceReply.reply(n) }, modifier = Modifier.weight(1f)) { + ReplyGlyph(Modifier.size(20.dp)) + Text("Reply", Modifier.padding(start = 8.dp)) + } + } + } + } +} + +/** + * The card's frame. When [until] is set, a bar along the bottom empties over [span] ms, so the + * driver sees at a glance how long is left before the card acts. + */ +@Composable +private fun PopupCard(until: Long, span: Long, modifier: Modifier, content: @Composable () -> Unit) { + Surface( + shape = RoundedCornerShape(20.dp), + color = MaterialTheme.colorScheme.surfaceContainerHigh, + contentColor = MaterialTheme.colorScheme.onSurface, + shadowElevation = 8.dp, + modifier = modifier.padding(12.dp).widthIn(min = 300.dp, max = 400.dp).fillMaxWidth(0.34f), + ) { + Column(Modifier.padding(14.dp), verticalArrangement = Arrangement.spacedBy(12.dp)) { + content() + if (until > 0L && span > 0L) { + val left by rememberTimeLeft(until, span) + GsProgressBar(left, Modifier.fillMaxWidth(), height = 14.dp) + } + } + } +} + +/** The share of [span] still to go before [until], updated every frame so the bar moves smoothly. */ +@Composable +private fun rememberTimeLeft(until: Long, span: Long) = produceState(fractionLeft(until, span), until, span) { + while (value > 0f) { + withFrameMillis { value = fractionLeft(until, span) } + } +} + +private fun fractionLeft(until: Long, span: Long): Float = + ((until - SystemClock.uptimeMillis()).toFloat() / span).coerceIn(0f, 1f) + +/** Each step of a voice reply, with the buttons that fit it. */ +@Composable +private fun VoiceReplyBody(s: VoiceReply.State, n: CarNotification) { + val navigator = LocalCarNavigator.current + val who = n.title.ifBlank { n.appLabel } + Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(12.dp)) { + when (s.phase) { + VoiceReply.Phase.OFFERING, VoiceReply.Phase.ASKING, VoiceReply.Phase.LISTENING -> ListeningDot() + VoiceReply.Phase.SENT -> Icon(Icons.Filled.Check, null, Modifier.size(32.dp), tint = MaterialTheme.colorScheme.primary) + else -> NotificationIcon(n, size = 44.dp) + } + Column(Modifier.weight(1f)) { + Text(voiceHeading(s, who), style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.Bold, maxLines = 1, overflow = TextOverflow.Ellipsis) + voiceDetail(s, n)?.let { + Text(it, style = MaterialTheme.typography.bodyMedium, color = MaterialTheme.colorScheme.onSurfaceVariant, maxLines = 3, overflow = TextOverflow.Ellipsis) + } + } + } + if (s.phase == VoiceReply.Phase.CONFIRMING) { + Text("Say “cancel” or “change it”", style = MaterialTheme.typography.labelLarge, color = MaterialTheme.colorScheme.primary) + } + if ((s.phase == VoiceReply.Phase.READ || s.phase == VoiceReply.Phase.OFFERING) && n.reply != null) { + QuickReplies(n) + } + Row(horizontalArrangement = Arrangement.spacedBy(10.dp)) { + val wide = Modifier.weight(1f) + when (s.phase) { + VoiceReply.Phase.READING, VoiceReply.Phase.READ -> { + GsButton(onClick = VoiceReply::cancel, tone = GsTone.Neutral, modifier = wide) { + Text(if (s.phase == VoiceReply.Phase.READING) "Stop" else "Close") + } + if (n.reply != null) { + GsButton(onClick = { VoiceReply.reply(n, readFirst = false) }, modifier = wide) { + ReplyGlyph(Modifier.size(20.dp)) + Text("Reply", Modifier.padding(start = 8.dp)) + } + } + } + VoiceReply.Phase.OFFERING -> { + GsButton(onClick = VoiceReply::cancel, tone = GsTone.Neutral, modifier = wide) { Text("No") } + GsButton(onClick = { VoiceReply.reply(n, readFirst = false) }, modifier = wide) { + ReplyGlyph(Modifier.size(20.dp)) + Text("Reply", Modifier.padding(start = 8.dp)) + } + } + VoiceReply.Phase.ASKING, VoiceReply.Phase.LISTENING -> + GsButton(onClick = VoiceReply::cancel, tone = GsTone.Neutral, modifier = wide) { Text("Cancel") } + VoiceReply.Phase.CONFIRMING -> { + GsButton(onClick = VoiceReply::cancel, tone = GsTone.Neutral, modifier = wide) { Text("Cancel") } + GsButton(onClick = VoiceReply::sendNow, modifier = wide) { Text("Send now") } + } + VoiceReply.Phase.ERROR -> { + GsButton(onClick = VoiceReply::cancel, tone = GsTone.Neutral, modifier = wide) { Text("Close") } + if (n.reply != null) { + GsButton(onClick = { VoiceReply.reply(n, readFirst = false) }, tone = GsTone.Tonal, modifier = wide) { Text("Try again") } + GsButton( + onClick = { VoiceReply.cancel(); navigator.notifications(replyTo = n.key) }, + tone = GsTone.Neutral, modifier = wide, + ) { Text("Type") } + } + } + VoiceReply.Phase.SENT, VoiceReply.Phase.IDLE -> Unit + } + } +} + +private fun voiceHeading(s: VoiceReply.State, who: String): String = when (s.phase) { + VoiceReply.Phase.READING, VoiceReply.Phase.READ -> who + VoiceReply.Phase.OFFERING, VoiceReply.Phase.ASKING -> "Reply to $who" + VoiceReply.Phase.LISTENING -> "Listening" + VoiceReply.Phase.CONFIRMING -> "Sending to $who" + VoiceReply.Phase.SENT -> "Sent to $who" + VoiceReply.Phase.ERROR -> "Couldn't reply" + VoiceReply.Phase.IDLE -> "" +} + +private fun voiceDetail(s: VoiceReply.State, n: CarNotification): String? = when (s.phase) { + VoiceReply.Phase.READING, VoiceReply.Phase.READ -> n.text + VoiceReply.Phase.OFFERING -> s.heard.ifBlank { "Do you want to reply? Say yes or no." } + VoiceReply.Phase.ASKING -> "Get ready to say your reply." + VoiceReply.Phase.LISTENING -> s.heard.ifBlank { "Say your reply" } + VoiceReply.Phase.CONFIRMING, VoiceReply.Phase.SENT -> "“${s.heard}”" + VoiceReply.Phase.ERROR -> s.error + VoiceReply.Phase.IDLE -> null +} + +/** Ready-made answers, one tap each: sent at once, with no voice. */ +@Composable +internal fun QuickReplies(n: CarNotification, modifier: Modifier = Modifier) { + Row( + modifier.fillMaxWidth().horizontalScroll(rememberScrollState()), + horizontalArrangement = Arrangement.spacedBy(8.dp), + ) { + VoiceReply.quickReplies().forEach { text -> + GsButton(onClick = { VoiceReply.quickReply(n, text) }, tone = GsTone.Neutral) { + Text(text, style = MaterialTheme.typography.labelLarge, maxLines = 1) + } + } + } +} + +@Composable +private fun ListeningDot(modifier: Modifier = Modifier) { + Box( + modifier.size(44.dp).clip(CircleShape).background(MaterialTheme.colorScheme.primary), + contentAlignment = Alignment.Center, + ) { + MicGlyph(Modifier.size(24.dp)) + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagesScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagesScreen.kt index 5d8dee5..218ff2f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagesScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/MessagesScreen.kt @@ -94,7 +94,8 @@ fun MessagesScreen(packageName: String) { items(threads, key = { it.key }) { n -> ConversationCard( n, - onReply = { navigator.notifications(replyTo = n.key) }, + onRead = { VoiceReply.read(n) }, + onReply = { VoiceReply.reply(n) }, onMarkRead = { CarNotifications.markThreadRead(context, n) }, ) } @@ -104,12 +105,13 @@ fun MessagesScreen(packageName: String) { } @Composable -private fun ConversationCard(n: CarNotification, onReply: () -> Unit, onMarkRead: () -> Unit) { +private fun ConversationCard(n: CarNotification, onRead: () -> Unit, onReply: () -> Unit, onMarkRead: () -> Unit) { var expanded by rememberSaveable(n.key) { mutableStateOf(false) } // An app that posts plain notifications instead of a conversation still gets one line. val lines = n.lines.ifEmpty { listOf(ChatLine(n.title.ifBlank { n.appLabel }, n.text.ifBlank { n.title }, n.postedAt)) } val shown = if (expanded) lines else lines.takeLast(COLLAPSED_LINES) val dim = n.read || n.replied + var quick by rememberSaveable(n.key) { mutableStateOf(false) } Surface( shape = RoundedCornerShape(16.dp), @@ -150,11 +152,26 @@ private fun ConversationCard(n: CarNotification, onReply: () -> Unit, onMarkRead ) } shown.forEach { MessageLine(it, showSender = n.isGroup) } + if (quick) QuickReplies(n, Modifier.padding(top = 6.dp)) } Column(horizontalAlignment = Alignment.End, verticalArrangement = Arrangement.spacedBy(8.dp)) { - when { - n.replied -> Icon(Icons.Filled.Check, contentDescription = "Replied", tint = MaterialTheme.colorScheme.primary) - n.reply != null -> GsButton(onClick = onReply, tone = GsTone.Tonal) { Text("Reply") } + if (n.replied) { + Icon(Icons.Filled.Check, contentDescription = "Replied", tint = MaterialTheme.colorScheme.primary) + } + GsButton(onClick = onRead, tone = GsTone.Neutral) { + ReadGlyph(Modifier.size(20.dp)) + Text("Read", Modifier.padding(start = 8.dp)) + } + if (n.reply != null) { + GsButton(onClick = onReply, tone = GsTone.Tonal) { + ReplyGlyph(Modifier.size(20.dp)) + Text("Reply", Modifier.padding(start = 8.dp)) + } + } + if (n.reply != null) { + GsButton(onClick = { quick = !quick }, tone = GsTone.Neutral) { + Text(if (quick) "Hide quick replies" else "Quick reply") + } } if (n.markRead != null) { GsButton(onClick = onMarkRead, tone = GsTone.Neutral) { Text("Mark read") } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NotificationsScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NotificationsScreen.kt index a1b4996..31ef3a1 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NotificationsScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/NotificationsScreen.kt @@ -161,7 +161,10 @@ private fun NotificationRow(n: CarNotification, onReply: (CarNotification) -> Un } when { n.replied -> Icon(Icons.Filled.Check, contentDescription = "Replied", tint = MaterialTheme.colorScheme.primary) - n.reply != null -> GsButton(onClick = { onReply(n) }, tone = GsTone.Tonal) { Text("Reply") } + n.reply != null -> GsButton(onClick = { VoiceReply.reply(n) }, tone = GsTone.Tonal) { + ReplyGlyph(Modifier.size(20.dp)) + Text("Reply", Modifier.padding(start = 8.dp)) + } } } } @@ -227,53 +230,3 @@ private fun ReplyPane(n: CarNotification, onDone: () -> Unit) { ) } } - -/** - * The popup shown over whatever the car is displaying. It times itself out, and tapping it opens - * the history - or straight into a reply, when the app allows one. - */ -@Composable -fun NotificationPopup(modifier: Modifier = Modifier) { - val popup by CarNotifications.popup.collectAsState() - val navigator = LocalCarNavigator.current - val n = popup ?: return - - LaunchedEffect(n.id) { - delay(POPUP_MS) - CarNotifications.dismissPopup(n.id) - } - - Surface( - shape = RoundedCornerShape(20.dp), - color = MaterialTheme.colorScheme.surfaceContainerHigh, - contentColor = MaterialTheme.colorScheme.onSurface, - shadowElevation = 8.dp, - modifier = modifier - .padding(top = 8.dp) - .fillMaxWidth(0.7f) - .clickable { navigator.dashboard() }, - ) { - Row( - Modifier.padding(12.dp), - verticalAlignment = Alignment.CenterVertically, - horizontalArrangement = Arrangement.spacedBy(12.dp), - ) { - NotificationIcon(n) - Column(Modifier.weight(1f)) { - Text(n.appLabel, style = MaterialTheme.typography.labelMedium, maxLines = 1) - if (n.title.isNotBlank()) { - Text(n.title, style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold, maxLines = 1, overflow = TextOverflow.Ellipsis) - } - if (n.text.isNotBlank()) { - Text(n.text, style = MaterialTheme.typography.bodyMedium, maxLines = 2, overflow = TextOverflow.Ellipsis) - } - } - if (n.reply != null) { - GsButton(onClick = { navigator.notifications(replyTo = n.key) }, tone = GsTone.Tonal) { Text("Reply") } - } - GsIconButton(Icons.Filled.Close, "Dismiss", { CarNotifications.dismissPopup(n.id) }) - } - } -} - -private const val POPUP_MS = 8_000L diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceReply.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceReply.kt new file mode 100644 index 0000000..f364d93 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/VoiceReply.kt @@ -0,0 +1,353 @@ +package app.seb3thehacker.gearslip.car + +import android.content.Context +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import androidx.car.app.navigation.model.NavigationTemplate +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.call.CallUiState +import app.seb3thehacker.gearslip.call.CarCalls +import app.seb3thehacker.gearslip.host.clockTime +import app.seb3thehacker.gearslip.host.remainingTime +import app.seb3thehacker.gearslip.notify.CarNotification +import app.seb3thehacker.gearslip.notify.CarNotifications +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow + +/** + * Messages by voice, the way Android Auto does it: a message is read aloud, "Do you want to + * reply?", a yes gets "Go ahead", the reply is read back, and it sends after a few seconds unless + * cancelled. Reply does the same; only once the message has just been read does it skip straight + * to "What's your reply?". While the reply counts down, saying "cancel" or "change it" stops it or + * starts it again. A quick reply sends a ready-made answer with no voice at all. Listening and speaking + * go through [CarAssistant], so this hears through the same microphone and recognizer. + * + * Every step runs on the main thread. Each run gets a new [run] number, and callbacks from an + * older run are dropped, so a cancel can never be undone by a late recognizer result. + */ +object VoiceReply { + + enum class Phase { + IDLE, + /** Reading the message aloud. */ + READING, + /** Read; the Reply button stays up for a moment in case the driver wants it. */ + READ, + /** Asking "Do you want to reply?" and listening for a yes. */ + OFFERING, + /** Saying "What's your reply?". */ + ASKING, + LISTENING, + /** + * Reading the reply back, then counting down to [State.until] while listening for + * "cancel" or "change it". [State.until] is 0 while the countdown is held. + */ + CONFIRMING, + SENT, + ERROR, + } + + data class State( + val phase: Phase = Phase.IDLE, + val target: CarNotification? = null, + val heard: String = "", + val error: String = "", + /** + * Uptime at which this step ends on its own (the card goes, or the reply sends), and how + * long the step was given, for the card's bar. 0 when nothing is timed. + */ + val until: Long = 0L, + val span: Long = 0L, + ) + + const val SEND_DELAY_MS = 5_000L + private const val LINGER_MS = 5_000L + private const val SENT_MS = 2_000L + + private val _state = MutableStateFlow(State()) + val state: StateFlow = _state + + private val main = Handler(Looper.getMainLooper()) + private var appContext: Context? = null + private var run = 0 + + fun init(context: Context) { appContext = context.applicationContext } + + val active: Boolean get() = _state.value.phase != Phase.IDLE + + /** Reads [n]'s newest messages aloud. */ + fun read(n: CarNotification) { + val id = begin(State(Phase.READING, n)) + CarAssistant.say(spoken(n)) { + if (id != run) return@say + when { + n.reply == null -> finish() + // The microphone is silent during a call, so there's no point asking. + CarCalls.state.value is CallUiState.Active -> idleAfterRead(id) + else -> offer(id) + } + } + } + + /** "Do you want to reply?" A yes goes on to the reply; anything else ends quietly. */ + private fun offer(id: Int) { + _state.value = _state.value.copy(phase = Phase.OFFERING, heard = "") + CarAssistant.say("Do you want to reply?") { + if (id != run) return@say + CarAssistant.dictate( + onPartial = { text -> if (id == run) _state.value = _state.value.copy(heard = text) }, + onResult = { text -> + if (id != run) return@dictate + if (isYes(text)) { + _state.value = _state.value.copy(phase = Phase.ASKING, heard = "") + CarAssistant.say("Go ahead.") { if (id == run) listen(id) } + } else { + finish() + } + }, + // Heard nothing: leave the Reply button up for a moment, the same as after reading. + onError = { if (id == run) idleAfterRead(id) }, + ) + } + } + + private fun idleAfterRead(id: Int) { + _state.value = _state.value.copy(phase = Phase.READ, heard = "") + lingerThenIdle(id) + } + + private fun isYes(text: String): Boolean { + val words = words(text) + if (words.any { it in NO_WORDS }) return false + return words.any { it in YES_WORDS } + } + + /** + * Reads [n] aloud and asks whether to reply, or, when the driver has just heard it, asks for + * the reply straight away. Then listens, reads the reply back and sends it. + */ + fun reply(n: CarNotification, readFirst: Boolean = true) { + if (n.reply == null) { + val id = begin(State(Phase.ERROR, n, error = "${n.appLabel} doesn't allow replies from the car.")) + lingerThenIdle(id) + return + } + // Android hands every app but the dialer a silent microphone during a call. + if (CarCalls.state.value is CallUiState.Active) { + val id = begin(State(Phase.ERROR, n, error = "Voice reply works once the call ends.")) + lingerThenIdle(id) + return + } + if (!readFirst) { + val id = begin(State(Phase.ASKING, n)) + ask(id) + return + } + // Read it, then "Do you want to reply?", the same as Read. + read(n) + } + + private fun ask(id: Int) { + CarAssistant.say("What's your reply?") { + if (id == run) listen(id) + } + } + + private fun listen(id: Int) { + _state.value = _state.value.copy(phase = Phase.LISTENING, heard = "") + CarAssistant.dictate( + onPartial = { text -> if (id == run) _state.value = _state.value.copy(heard = text) }, + onResult = { text -> if (id == run) confirm(id, text) }, + onError = { message -> + if (id != run) return@dictate + _state.value = _state.value.copy(phase = Phase.ERROR, error = message) + lingerThenIdle(id) + }, + ) + } + + private fun confirm(id: Int, text: String) { + _state.value = _state.value.copy(phase = Phase.CONFIRMING, heard = text, until = 0L, span = 0L) + // The countdown starts once the driver has heard the whole reply, not while it's read out. + CarAssistant.say("Sending: $text") { + if (id != run) return@say + startCountdown(id, SEND_DELAY_MS) + // A moment for the read-back to finish playing in the car before the microphone opens. + main.postDelayed({ if (id == run) listenForCommand(id) }, ECHO_MS) + } + } + + private fun startCountdown(id: Int, ms: Long) { + val sendAt = SystemClock.uptimeMillis() + ms + _state.value = _state.value.copy(until = sendAt, span = ms) + main.removeCallbacksAndMessages(SEND_TOKEN) + main.postAtTime({ if (id == run) send() }, SEND_TOKEN, sendAt) + } + + /** + * Listens through the countdown for "cancel", "change it" or "send". The countdown holds from + * the first word, so a reply can't go out while the driver is still saying "cancel". + */ + private fun listenForCommand(id: Int) { + CarAssistant.dictate( + onPartial = {}, + cue = false, + onSpeaking = { + if (id != run) return@dictate + main.removeCallbacksAndMessages(SEND_TOKEN) + _state.value = _state.value.copy(until = 0L, span = 0L) + }, + onResult = { text -> + if (id != run) return@dictate + when (command(text)) { + Command.CANCEL -> { + finish() + CarAssistant.say("Not sent.") {} + } + Command.CHANGE -> { + main.removeCallbacksAndMessages(SEND_TOKEN) + _state.value = _state.value.copy(phase = Phase.ASKING, heard = "", until = 0L, span = 0L) + ask(id) + } + Command.SEND -> send() + // Talk that wasn't meant for us, a passenger or the radio: carry on counting. + null -> startCountdown(id, RESUME_MS) + } + }, + onError = { if (id == run && _state.value.until == 0L) startCountdown(id, RESUME_MS) }, + ) + } + + private enum class Command { CANCEL, CHANGE, SEND } + + private fun command(text: String): Command? { + val words = words(text) + return when { + words.any { it in CANCEL_WORDS } -> Command.CANCEL + words.any { it in CHANGE_WORDS } -> Command.CHANGE + words.any { it in SEND_WORDS } -> Command.SEND + else -> null + } + } + + private fun words(text: String) = text.lowercase().split(Regex("[^a-z']+")).filter { it.isNotEmpty() } + + /** + * Ready-made replies for a tap instead of a voice reply. The first gives the arrival time when + * the map is following a route. + */ + fun quickReplies(): List = listOfNotNull(etaReply(), "Driving, I'll reply soon.", "On my way.") + + private fun etaReply(): String? { + val estimate = (CarServices.nav.template.value as? NavigationTemplate)?.destinationTravelEstimate ?: return null + val at = estimate.arrivalTimeAtDestination.clockTime() + val left = remainingTime(estimate.remainingTimeSeconds) + return when { + at.isNotEmpty() -> "Driving, I'll be there around $at." + left.isNotEmpty() -> "Driving, I'll be there in about $left." + else -> null + } + } + + /** Sends [text] to [n] straight away, as if it had been said and confirmed. */ + fun quickReply(n: CarNotification, text: String) { + if (n.reply == null) return + begin(State(Phase.CONFIRMING, n, heard = text)) + send() + } + + /** Sends a reply that's counting down, without waiting for the rest of it. */ + fun sendNow() { + if (_state.value.phase == Phase.CONFIRMING) send() + } + + private fun send() { + val s = _state.value + val n = s.target ?: return finish() + val context = appContext ?: return finish() + val id = ++run + CarAssistant.stopListening() + CarAssistant.stopSpeaking() + if (CarNotifications.reply(context, n, s.heard.trim())) { + GearslipLog.i("voice reply: sent to ${n.appLabel}") + CarAssistant.say("Message sent.") {} + CarNotifications.dismissPopup(n.id) + _state.value = s.copy(phase = Phase.SENT, until = SystemClock.uptimeMillis() + SENT_MS, span = SENT_MS) + main.postDelayed({ if (id == run) finish() }, SENT_MS) + } else { + _state.value = s.copy(phase = Phase.ERROR, error = "${n.appLabel} withdrew the reply. Open the app on the phone to answer.") + lingerThenIdle(id) + } + } + + /** Stops wherever it is. Nothing is sent. */ + fun cancel() { + if (!active) return + CarAssistant.stopListening() + CarAssistant.stopSpeaking() + finish() + } + + /** + * The steering wheel's voice key. With a message on screen it answers that message, and + * during a countdown it sends straight away. Returns false when there's nothing to answer, so + * the key can start the assistant instead. + */ + fun onVoiceKey(): Boolean { + val s = _state.value + when (s.phase) { + // Already heard, so straight to the reply. + Phase.READ, Phase.READING, Phase.OFFERING, Phase.ERROR -> s.target?.let { reply(it, readFirst = false); return true } + Phase.CONFIRMING -> { sendNow(); return true } + Phase.ASKING, Phase.LISTENING, Phase.SENT -> return true + Phase.IDLE -> Unit + } + val popup = CarNotifications.popup.value ?: return false + if (popup.reply == null) return false + reply(popup) + return true + } + + private fun begin(state: State): Int { + CarAssistant.stopListening() + CarAssistant.stopSpeaking() + main.removeCallbacksAndMessages(null) + _state.value = state + return ++run + } + + private fun lingerThenIdle(id: Int) { + _state.value = _state.value.copy(until = SystemClock.uptimeMillis() + LINGER_MS, span = LINGER_MS) + main.postDelayed({ if (id == run) finish() }, LINGER_MS) + } + + private fun finish() { + run++ + main.removeCallbacksAndMessages(null) + _state.value = State() + } + + /** "Alex says: ..." or, in a group, "Sam in Climbing says: ...", for the newest messages. */ + private fun spoken(n: CarNotification): String { + val from = n.title.ifBlank { n.appLabel } + val lines = n.lines.filter { it.sender != null }.takeLast(n.burst.coerceIn(1, MAX_READ)) + if (lines.isEmpty()) return "$from says: ${n.text}" + return lines.joinToString(". ") { line -> + val who = if (n.isGroup) "${line.sender} in $from" else from + "$who says: ${line.text}" + } + } + + private const val MAX_READ = 3 + private val YES_WORDS = setOf("yes", "yeah", "yep", "yup", "sure", "ok", "okay", "reply", "please") + private val NO_WORDS = setOf("no", "nope", "nah", "cancel", "don't", "stop") + private val CANCEL_WORDS = setOf("cancel", "stop", "don't", "no", "nope") + private val CHANGE_WORDS = setOf("change", "redo", "again", "edit", "wrong", "fix") + private val SEND_WORDS = setOf("send", "yes", "yeah", "ok", "okay", "go") + + private val SEND_TOKEN = Any() + private const val ECHO_MS = 300L + /** After talk that wasn't a command, the reply waits this long again before it goes. */ + private const val RESUME_MS = 3_000L +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsControls.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsControls.kt index eca8a87..a1bd95b 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsControls.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsControls.kt @@ -22,6 +22,7 @@ import androidx.compose.foundation.border import androidx.compose.foundation.combinedClickable import androidx.compose.foundation.interaction.MutableInteractionSource import androidx.compose.foundation.interaction.PressInteraction +import androidx.compose.foundation.interaction.collectIsFocusedAsState import androidx.compose.foundation.layout.Arrangement import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.BoxScope @@ -228,10 +229,15 @@ private fun Modifier.gsPressable( onClick: () -> Unit, onLongClick: (() -> Unit)? = null, role: Role = Role.Button, + shape: Shape = gsShape(), ): Modifier { val look = gsLook() + // Reached with the car's knob or D-pad rather than a finger: a ring shows where focus is. + val focused by source.collectIsFocusedAsState() + val ring = MaterialTheme.colorScheme.primary return this .scale(if (pressed) look.pressedScale else 1f) + .then(if (focused) Modifier.border(3.dp, ring, shape) else Modifier) .combinedClickable( interactionSource = source, indication = null, @@ -268,7 +274,7 @@ fun GsButton( Row( modifier .defaultMinSize(minWidth = 64.dp, minHeight = 48.dp) - .gsPressable(source, pressed, enabled, onClick) + .gsPressable(source, pressed, enabled, onClick, shape = shape) .gsSurface(c.container, shape, c.border, pressed) .padding(contentPadding), horizontalArrangement = Arrangement.Center, @@ -317,7 +323,7 @@ fun GsIconBox( val pressed = rememberPressed(source) Box( modifier - .gsPressable(source, pressed, enabled, onClick, onLongClick) + .gsPressable(source, pressed, enabled, onClick, onLongClick, shape = shape) .gsSurface(c.container, shape, c.border, pressed, latched), contentAlignment = Alignment.Center, ) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSeekBar.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSeekBar.kt index 0f5737b..b1293f4 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSeekBar.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSeekBar.kt @@ -1,13 +1,14 @@ package app.seb3thehacker.gearslip.car.theme +import androidx.compose.foundation.background import androidx.compose.foundation.gestures.detectHorizontalDragGestures import androidx.compose.foundation.gestures.detectTapGestures import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.BoxWithConstraints +import androidx.compose.foundation.layout.fillMaxHeight import androidx.compose.foundation.layout.fillMaxWidth import androidx.compose.foundation.layout.height -import androidx.compose.foundation.layout.offset -import androidx.compose.foundation.layout.size +import androidx.compose.foundation.layout.padding import androidx.compose.foundation.layout.width import androidx.compose.foundation.shape.CircleShape import androidx.compose.material3.MaterialTheme @@ -18,14 +19,75 @@ import androidx.compose.runtime.remember import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.clip +import androidx.compose.ui.draw.drawWithContent +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.lerp import androidx.compose.ui.input.pointer.pointerInput +import androidx.compose.ui.unit.Dp +import androidx.compose.ui.unit.coerceAtLeast import androidx.compose.ui.unit.dp /** - * A seek bar in the current theme's finish: the track is set into the surface (drawn latched, so - * themes with a pressed-in look sink it), the played part and the knob are raised like a button. - * Tapping or dragging seeks; the position is committed on release rather than on every pixel of a - * drag, because each commit is a request to the media app. + * A capsule progress bar: a tinted track with a glossy fill set inside it, lighter along the + * top and darker along the bottom. The fill never gets narrower than it is tall, so its ends stay + * round at any value above zero. + */ +@Composable +fun GsProgressBar( + fraction: Float, + modifier: Modifier = Modifier, + height: Dp = 22.dp, + color: Color = MaterialTheme.colorScheme.primary, + /** Brightens the fill, for a bar that's being dragged. */ + active: Boolean = false, +) { + val scheme = MaterialTheme.colorScheme + val shown = fraction.coerceIn(0f, 1f) + val inset = 3.dp + val fill = if (active) lerp(color, Color.White, 0.12f) else color + BoxWithConstraints( + modifier + .fillMaxWidth() + .height(height) + .clip(CircleShape) + .background(lerp(scheme.surfaceContainerHighest, color, 0.12f)) + .padding(inset), + contentAlignment = Alignment.CenterStart, + ) { + if (shown <= 0f) return@BoxWithConstraints + val inner = maxHeight + Box( + Modifier + .width((maxWidth * shown).coerceAtLeast(inner)) + .fillMaxHeight() + .clip(CircleShape) + .background( + Brush.verticalGradient( + 0f to lerp(fill, Color.White, 0.30f), + 0.55f to fill, + 1f to lerp(fill, Color.Black, 0.22f), + ), + ) + // The gloss: a soft white band across the top half. + .drawWithContent { + drawContent() + drawRect( + Brush.verticalGradient( + 0f to Color.White.copy(alpha = 0.35f), + 0.5f to Color.White.copy(alpha = 0.05f), + 0.5f to Color.Transparent, + ), + ) + }, + ) + } +} + +/** + * [GsProgressBar] that seeks: tapping or dragging moves it. The position is committed on release + * rather than on every pixel of a drag, because each commit is a request to the media app. */ @Composable fun GsSeekBar( @@ -34,16 +96,12 @@ fun GsSeekBar( modifier: Modifier = Modifier, enabled: Boolean = true, ) { - val scheme = MaterialTheme.colorScheme var dragging by remember { mutableStateOf(null) } - val shown = (dragging ?: fraction).coerceIn(0f, 1f) - val track = 8.dp - val knob = 20.dp - - BoxWithConstraints( + // A taller touch area than the bar itself, so it's easy to hit on a car screen. + Box( modifier .fillMaxWidth() - .height(32.dp) + .height(36.dp) .then( if (!enabled) Modifier else Modifier .pointerInput(Unit) { detectTapGestures { onSeek((it.x / size.width).coerceIn(0f, 1f)) } } @@ -55,29 +113,8 @@ fun GsSeekBar( ) { change, _ -> dragging = (change.position.x / size.width).coerceIn(0f, 1f) } }, ), - contentAlignment = Alignment.CenterStart, + contentAlignment = Alignment.Center, ) { - // The knob's centre travels from half a knob in to half a knob from the end, so it never - // hangs off either edge; the fill runs to its centre. - val travel = maxWidth - knob - val centre = knob / 2 + travel * shown - Box( - Modifier - .fillMaxWidth() - .height(track) - .gsSurface(scheme.onSurface.copy(alpha = 0.14f), CircleShape, latched = true), - ) - Box( - Modifier - .width(centre) - .height(track) - .gsSurface(scheme.primary, CircleShape), - ) - Box( - Modifier - .offset(x = centre - knob / 2) - .size(knob) - .gsSurface(scheme.primary, CircleShape, pressed = dragging != null), - ) + GsProgressBar(dragging ?: fraction, active = dragging != null) } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSwitch.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSwitch.kt new file mode 100644 index 0000000..6eed6a3 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/car/theme/GsSwitch.kt @@ -0,0 +1,43 @@ +package app.seb3thehacker.gearslip.car.theme + +import androidx.compose.animation.core.animateDpAsState +import androidx.compose.animation.core.tween +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.offset +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.shape.CircleShape +import androidx.compose.material3.MaterialTheme +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.unit.dp + +/** + * An on/off switch in the current theme's finish: the track is pressed in while on, the knob is + * raised like a button. The whole switch is one tap target, as big as a nav bar button. + */ +@Composable +fun GsSwitch(checked: Boolean, onCheckedChange: (Boolean) -> Unit, modifier: Modifier = Modifier) { + val scheme = MaterialTheme.colorScheme + val knob = 26.dp + val travel = 64.dp - knob - 8.dp + val x by animateDpAsState(if (checked) travel else 0.dp, tween(140), label = "switch") + GsIconBox( + onClick = { onCheckedChange(!checked) }, + modifier = modifier.size(width = 64.dp, height = 36.dp), + colors = if (checked) GsColors(scheme.primary, scheme.onPrimary) else GsColors(scheme.surfaceContainerHighest, scheme.onSurfaceVariant), + shape = CircleShape, + latched = checked, + ) { + Box(Modifier.matchParentSize().padding(4.dp), contentAlignment = Alignment.CenterStart) { + Box( + Modifier + .offset(x = x) + .size(knob) + .gsSurface(if (checked) scheme.onPrimary else scheme.outline, CircleShape), + ) + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt index 7672ef2..e015781 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/host/CarAppConnection.kt @@ -453,6 +453,33 @@ class CarAppConnection(private val context: Context) { } } + private val _surfaceEpoch = MutableStateFlow(0) + + /** Bumped when the lent surface turns out to be dead, so the stage builds a fresh one. */ + val surfaceEpoch: StateFlow = _surfaceEpoch.asStateFlow() + + /** + * Lends the app its surface again: called when the map comes back into view. An app that + * draws through its own VirtualDisplay (Organic Maps does) can be left pointing at a surface + * that has since died, and draws black until it's told about a live one. Taking the surface + * back and lending it again repoints it; a surface that has itself died is replaced. + */ + fun refreshSurface() { + main.post { + val current = lent ?: return@post + if (!current.surface.isValid) { + GearslipLog.w("host: the lent surface died; building a new one") + lent = null + sendSurfaceDestroyed(current) + _surfaceEpoch.value = _surfaceEpoch.value + 1 + return@post + } + GearslipLog.i("host: lending the surface again as the map comes back") + sendSurfaceDestroyed(current) + sendSurfaceAvailable() + } + } + private fun sendSurfaceAvailable() { val callback = surfaceCallback ?: return val surface = lent ?: return diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/host/KnownApps.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/host/KnownApps.kt index c03c079..19f585f 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/host/KnownApps.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/host/KnownApps.kt @@ -20,6 +20,26 @@ object KnownApps { fun works(packageName: String): Boolean = packageName in working + /** + * Apps whose media player works through Gearslip's own player even though their car screen + * doesn't. Only the player tile gets the check, not the "· Browse" one. + */ + private val workingPlayer = setOf( + "com.spotify.music", // Spotify: play, browse, seek through Gearslip's player + ) + + fun playerWorks(packageName: String): Boolean = packageName in workingPlayer + + /** + * Apps whose car screen refuses Gearslip but whose player works. Only the "· Browse" tile + * gets the red X. Car library 1.9 and later accepts only Google's host on a phone. + */ + private val brokenScreen = setOf( + "com.spotify.music", // Spotify 9.1.86 and later: car library 1.9 rejects Gearslip + ) + + fun screenBroken(packageName: String): Boolean = packageName in brokenScreen + /** * Registers a car template service but never gives Gearslip a usable screen - left off the * launcher entirely rather than shown as a tile that goes nowhere. @@ -43,7 +63,6 @@ object KnownApps { /** Shown on the launcher with a yellow mark: starts and can be used, but not cleanly enough to call working. */ private val partial = setOf( "app.vela", // Vela Maps: connects and draws, but rough enough not to call it working yet - "com.spotify.music", // Spotify: Now Playing tile works, browse tile rejects Gearslip as a host ) fun isPartial(packageName: String): Boolean = packageName in partial diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarNotifications.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarNotifications.kt index 66b18a3..171607e 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarNotifications.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/CarNotifications.kt @@ -42,6 +42,8 @@ data class CarNotification( /** The app's own "mark as read" action, fired once the driver has opened the thread. */ val markRead: Notification.Action? = null, val read: Boolean = false, + /** How many posts this popup stands for: new messages in one chat update it, not queue. */ + val burst: Int = 1, ) /** @@ -78,7 +80,10 @@ object CarNotifications { fun nextId(): Long = nextId++ - /** [quiet] adds to the history without a popup or unread badge (used to seed on connect). */ + /** + * [quiet] adds to the history without a popup or unread badge: used to seed on connect, and + * for anything that isn't worth interrupting a drive for (see the listener's filter). + */ fun post(notification: CarNotification, quiet: Boolean) { _history.update { list -> (listOf(notification) + list.filterNot { it.key == notification.key }).take(MAX_HISTORY) @@ -88,7 +93,9 @@ object CarNotifications { if (quiet || echo) return // History still keeps it; only the popup is held back while driving. if (CarMotion.moving) return - _popup.value = notification + _popup.update { shown -> + if (shown?.key == notification.key) notification.copy(burst = shown.burst + 1) else notification + } } /** The phone dismissed it. It stays in history, but its reply intent is no longer trustworthy. */ diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/GearslipNotificationListener.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/GearslipNotificationListener.kt index 99b21ed..3906878 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/GearslipNotificationListener.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/GearslipNotificationListener.kt @@ -30,7 +30,10 @@ class GearslipNotificationListener : NotificationListenerService() { override fun onNotificationRemoved(sbn: StatusBarNotification) = CarNotifications.gone(sbn.key) private fun ingest(sbn: StatusBarNotification, quiet: Boolean) { - if (sbn.packageName == packageName) return + // Gearslip's own notifications are skipped, except a dev build's test message. + if (sbn.packageName == packageName && + !(app.seb3thehacker.gearslip.BuildConfig.DEBUG && sbn.notification.channelId == TestMessage.CHANNEL) + ) return val n = sbn.notification // Ongoing entries (media, navigation, downloads) and group headers are status, not news. if (n.flags and (Notification.FLAG_ONGOING_EVENT or Notification.FLAG_GROUP_SUMMARY) != 0) return @@ -41,6 +44,8 @@ class GearslipNotificationListener : NotificationListenerService() { val text = (extras.getCharSequence(Notification.EXTRA_BIG_TEXT) ?: extras.getCharSequence(Notification.EXTRA_TEXT))?.toString().orEmpty() if (title.isBlank() && text.isBlank()) return + // The player and the map already show these. + if (extras.containsKey(Notification.EXTRA_MEDIA_SESSION) || n.category == Notification.CATEGORY_NAVIGATION) return val style = runCatching { NotificationCompat.MessagingStyle.extractMessagingStyleFromNotification(n) }.getOrNull() // In MessagingStyle a message with no sender is the phone owner's own. @@ -66,10 +71,25 @@ class GearslipNotificationListener : NotificationListenerService() { isGroup = style?.isGroupConversation == true, markRead = markReadOf(n), ), - quiet, + quiet || !alerts(sbn, style != null), ) } + /** + * Whether a notification is worth a popup in the car: messages, missed calls, alarms and + * reminders. Everything else still lands in the dashboard's list, just without interrupting. + * An incoming call is left out too, since the call card already covers it. + */ + private fun alerts(sbn: StatusBarNotification, conversation: Boolean): Boolean { + if (sbn.packageName in SYSTEM_PACKAGES) return false + val ranking = Ranking() + if (currentRanking?.getRanking(sbn.key, ranking) == true && + ranking.importance < android.app.NotificationManager.IMPORTANCE_DEFAULT + ) return false + val n = sbn.notification + return conversation || replyOf(n) != null || n.category in ALERT_CATEGORIES + } + private fun markReadOf(n: Notification): Notification.Action? = (n.actions.orEmpty().toList() + Notification.WearableExtender(n).actions) .firstOrNull { it.actionIntent != null && it.semanticAction == Notification.Action.SEMANTIC_ACTION_MARK_AS_READ } @@ -98,6 +118,14 @@ class GearslipNotificationListener : NotificationListenerService() { private companion object { const val ICON_PX = 96 const val MAX_LINES = 8 + val ALERT_CATEGORIES = setOf( + Notification.CATEGORY_MESSAGE, + Notification.CATEGORY_MISSED_CALL, + Notification.CATEGORY_ALARM, + Notification.CATEGORY_REMINDER, + Notification.CATEGORY_EVENT, + ) + val SYSTEM_PACKAGES = setOf("android", "com.android.systemui", "com.android.vending", "com.google.android.gms") val QUIET_CATEGORIES = setOf( Notification.CATEGORY_TRANSPORT, Notification.CATEGORY_PROGRESS, diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/TestMessage.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/TestMessage.kt new file mode 100644 index 0000000..1376e11 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/notify/TestMessage.kt @@ -0,0 +1,112 @@ +package app.seb3thehacker.gearslip.notify + +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.BroadcastReceiver +import android.content.Context +import android.content.Intent +import androidx.core.app.NotificationCompat +import androidx.core.app.NotificationManagerCompat +import androidx.core.app.Person +import androidx.core.app.RemoteInput +import app.seb3thehacker.gearslip.GearslipLog + +/** + * Dev builds only: posts a chat message from Gearslip to itself, shaped like Signal's or + * WhatsApp's (MessagingStyle, a Reply action, a Mark as read action), so popups and voice reply + * can be tried without asking someone to text you. The listener lets these through only in a + * debug build; everything else Gearslip posts is still ignored. + */ +object TestMessage { + + const val CHANNEL = "gearslip_test_message" + private const val NOTIFICATION_ID = 77 + private const val KEY_REPLY = "reply" + private const val ACTION_REPLY = "app.seb3thehacker.gearslip.TEST_REPLY" + private const val ACTION_READ = "app.seb3thehacker.gearslip.TEST_READ" + + private val SAMPLES = listOf( + "Alex" to "Running ten minutes late, save me a seat?", + "Sam" to "Are you still coming over tonight?", + "Jordan" to "Can you grab milk on the way home?", + "Riley" to "Call me when you're parked.", + ) + private var next = 0 + private val me = Person.Builder().setName("You").build() + + fun post(context: Context) { + val (name, text) = SAMPLES[next++ % SAMPLES.size] + val sender = Person.Builder().setName(name).build() + val style = NotificationCompat.MessagingStyle(me) + .addMessage(text, System.currentTimeMillis(), sender) + show(context, style) + GearslipLog.i("test message: posted \"$text\" from $name") + } + + private fun show(context: Context, style: NotificationCompat.MessagingStyle) { + val manager = context.getSystemService(NotificationManager::class.java) + // High importance, like a real messaging app's chats: lower ones never pop up in the car. + manager.createNotificationChannel( + NotificationChannel(CHANNEL, "Test messages", NotificationManager.IMPORTANCE_HIGH), + ) + val replyAction = NotificationCompat.Action.Builder( + android.R.drawable.ic_menu_send, "Reply", + broadcast(context, ACTION_REPLY, mutable = true), + ) + .addRemoteInput(RemoteInput.Builder(KEY_REPLY).setLabel("Reply").build()) + .setSemanticAction(NotificationCompat.Action.SEMANTIC_ACTION_REPLY) + .setShowsUserInterface(false) + .build() + val readAction = NotificationCompat.Action.Builder( + android.R.drawable.ic_menu_view, "Mark as read", + broadcast(context, ACTION_READ, mutable = false), + ) + .setSemanticAction(NotificationCompat.Action.SEMANTIC_ACTION_MARK_AS_READ) + .setShowsUserInterface(false) + .build() + val notification = NotificationCompat.Builder(context, CHANNEL) + .setSmallIcon(android.R.drawable.sym_action_chat) + .setStyle(style) + .setCategory(NotificationCompat.CATEGORY_MESSAGE) + .setPriority(NotificationCompat.PRIORITY_HIGH) + .addAction(replyAction) + .addAction(readAction) + .setAutoCancel(true) + .build() + runCatching { NotificationManagerCompat.from(context).notify(NOTIFICATION_ID, notification) } + .onFailure { GearslipLog.w("test message: could not post (${it.message})") } + } + + private fun broadcast(context: Context, action: String, mutable: Boolean): PendingIntent = + PendingIntent.getBroadcast( + context, action.hashCode(), + Intent(context, TestMessageReceiver::class.java).setAction(action), + PendingIntent.FLAG_UPDATE_CURRENT or + if (mutable) PendingIntent.FLAG_MUTABLE else PendingIntent.FLAG_IMMUTABLE, + ) + + internal fun onAction(context: Context, intent: Intent) { + when (intent.action) { + ACTION_REPLY -> { + val text = RemoteInput.getResultsFromIntent(intent)?.getCharSequence(KEY_REPLY)?.toString() + GearslipLog.i("test message: reply received: \"$text\"") + // Show the reply in the conversation, the way a chat app confirms it was sent. + val active = context.getSystemService(NotificationManager::class.java) + .activeNotifications.firstOrNull { it.id == NOTIFICATION_ID }?.notification + val style = active?.let { NotificationCompat.MessagingStyle.extractMessagingStyleFromNotification(it) } + ?: NotificationCompat.MessagingStyle(me) + style.addMessage(text.orEmpty(), System.currentTimeMillis(), null as Person?) + show(context, style) + } + ACTION_READ -> { + GearslipLog.i("test message: marked as read") + NotificationManagerCompat.from(context).cancel(NOTIFICATION_ID) + } + } + } +} + +class TestMessageReceiver : BroadcastReceiver() { + override fun onReceive(context: Context, intent: Intent) = TestMessage.onAction(context, intent) +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt new file mode 100644 index 0000000..3626c15 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/speech/SpeechEngine.kt @@ -0,0 +1,228 @@ +package app.seb3thehacker.gearslip.speech + +import android.Manifest +import android.annotation.SuppressLint +import android.content.Context +import android.content.pm.PackageManager +import android.content.res.AssetManager +import android.media.AudioFormat +import android.media.AudioManager +import android.media.AudioRecord +import android.media.MediaRecorder +import android.media.ToneGenerator +import android.os.Handler +import android.os.Looper +import android.os.SystemClock +import app.seb3thehacker.gearslip.GearslipLog +import java.util.Locale +import java.util.concurrent.Executors +import kotlin.math.log10 +import kotlin.math.max +import kotlin.math.sqrt + +/** + * Gearslip's own speech-to-text, so voice works on phones with no speech service (GrapheneOS ships + * none). whisper.cpp runs FUTO's ACFT Whisper model, which are tuned for short phrases: a reply + * of a few seconds comes back in well under a second on a recent phone. + * + * Whisper needs the whole phrase before it starts, so this records from the phone's microphone, + * decides when the driver has stopped talking (a stretch of quiet after speech), then transcribes + * once. Everything runs on one worker thread; callbacks arrive on the main thread. + */ +object SpeechEngine { + + /** FUTO's ACFT Whisper tiny (Apache-2.0), re-quantized to 5 bits, inside the APK. */ + private const val MODEL = "speech/tiny_acft_q5_1.bin" + + /** The error for an engine that can't run at all, which callers may answer with a fallback. */ + const val NOT_LOADED = "The speech model didn't load." + + private const val RATE = 16_000 + private const val FRAME = 480 // 30 ms + private const val MAX_MS = 15_000L + private const val NO_SPEECH_MS = 6_000L + private const val END_SILENCE_MS = 1_100L + private const val CALIBRATE_MS = 300L + private const val PRE_ROLL = RATE * 3 / 10 // keep 300 ms before the first word + private const val CUE_VOLUME = 80 + private const val CUE_MS = 150 + /** The car plays what the phone sends a moment late; waiting this out keeps the beep off the recording. */ + private const val CUE_TAIL_MS = 250L + + private val worker = Executors.newSingleThreadExecutor { Thread(it, "gearslip-speech") } + private val main = Handler(Looper.getMainLooper()) + + private var libraryLoaded = false + private var handle = 0L + @Volatile private var run = 0 + + /** Loads the model ahead of time, so the first reply doesn't wait for it. */ + fun warm(context: Context) { + val app = context.applicationContext + worker.execute { ensureLoaded(app) } + } + + /** + * Listens for one phrase. [cue] plays a short tone first, so the driver knows when to talk. + * [onSpeaking] fires at the first word, [onHeard] when the driver stops and the words are being + * worked out; [onResult] gets the text, [onError] a sentence fit to show on the car screen. + */ + fun listen( + context: Context, + cue: Boolean = true, + onSpeaking: () -> Unit = {}, + onHeard: () -> Unit, + onResult: (String) -> Unit, + onError: (String) -> Unit, + ) { + val app = context.applicationContext + if (app.checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + onError("Needs the microphone permission.") + return + } + val id = ++run + fun post(block: () -> Unit) { main.post { if (id == run) block() } } + worker.execute { + if (id != run) return@execute + if (!ensureLoaded(app)) return@execute post { onError(NOT_LOADED) } + if (cue) cue() + if (id != run) return@execute + val audio = try { + record(id) { post(onSpeaking) } + } catch (t: Throwable) { + GearslipLog.e("speech: recording failed", t) + return@execute post { onError("The microphone isn't available right now.") } + } + if (id != run) return@execute + if (audio == null) return@execute post { onError("Didn't catch that.") } + post(onHeard) + val started = SystemClock.uptimeMillis() + val text = clean(transcribe(audio)) + GearslipLog.i( + "speech: ${audio.size * 1000L / RATE} ms of audio took ${SystemClock.uptimeMillis() - started} ms", + ) + post { if (text.isEmpty()) onError("Didn't catch that.") else onResult(text) } + } + } + + /** Stops listening; a phrase already being worked out is dropped. */ + fun cancel() { run++ } + + /** + * A short beep on the media stream, so it plays through the car like everything else, then a + * pause long enough for it to finish there before the microphone opens and hears it. + */ + private fun cue() { + val tone = runCatching { ToneGenerator(AudioManager.STREAM_MUSIC, CUE_VOLUME) }.getOrNull() ?: return + try { + tone.startTone(ToneGenerator.TONE_PROP_BEEP, CUE_MS) + Thread.sleep(CUE_MS + CUE_TAIL_MS) + } finally { + tone.release() + } + } + + // --- recording -------------------------------------------------------------------------- + + /** + * Records until the driver stops talking. Returns the phrase from just before the first word, + * or null when nobody spoke. The speech threshold follows the noise in the first 300 ms, so a + * loud cabin needs a louder voice than a quiet one to count as talking. + */ + @SuppressLint("MissingPermission") // checked in listen() + private fun record(id: Int, onSpeaking: () -> Unit): FloatArray? { + val min = AudioRecord.getMinBufferSize(RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT) + val recorder = AudioRecord( + MediaRecorder.AudioSource.VOICE_RECOGNITION, RATE, + AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, max(min, FRAME * 8), + ) + check(recorder.state == AudioRecord.STATE_INITIALIZED) { "AudioRecord didn't initialize" } + val samples = FloatArray((RATE * MAX_MS / 1000).toInt()) + var count = 0 + val frame = ShortArray(FRAME) + var noise = 0.0 + var noiseFrames = 0 + var speechStart = -1 + var lastSpeech = 0 + recorder.startRecording() + try { + while (id == run && count + FRAME <= samples.size) { + val n = recorder.read(frame, 0, FRAME) + if (n <= 0) continue + var sum = 0.0 + for (i in 0 until n) { + val v = frame[i] / 32768f + samples[count + i] = v + sum += v * v + } + count += n + val db = 20 * log10(sqrt(sum / n) + 1e-9) + val ms = count * 1000L / RATE + if (ms <= CALIBRATE_MS) { + noise += db; noiseFrames++ + continue + } + val floor = if (noiseFrames > 0) noise / noiseFrames else -60.0 + val talking = db > max(floor + 12, -45.0) + if (talking) { + if (speechStart < 0) { + speechStart = max(0, count - n - PRE_ROLL) + onSpeaking() + } + lastSpeech = count + } + if (speechStart < 0 && ms > NO_SPEECH_MS) return null + if (speechStart >= 0 && (count - lastSpeech) * 1000L / RATE > END_SILENCE_MS) break + } + } finally { + runCatching { recorder.stop() } + recorder.release() + } + if (speechStart < 0) return null + return samples.copyOfRange(speechStart, count) + } + + // --- the model -------------------------------------------------------------------------- + + private fun ensureLoaded(context: Context): Boolean { + if (handle != 0L) return true + if (!libraryLoaded) { + runCatching { System.loadLibrary("gearslip_speech") } + .onFailure { GearslipLog.e("speech: native library missing", it); return false } + libraryLoaded = true + } + val started = SystemClock.uptimeMillis() + handle = nativeInitAsset(context.assets, MODEL) + if (handle == 0L) { + GearslipLog.w("speech: couldn't load the model") + return false + } + GearslipLog.i("speech: loaded the model in ${SystemClock.uptimeMillis() - started} ms") + return true + } + + private fun transcribe(audio: FloatArray): String { + val bytes = nativeTranscribe(handle, audio, language(), threads()) + return String(bytes, Charsets.UTF_8) + } + + /** The phone's language, as Whisper names it. Older Java codes differ for a few. */ + private fun language(): String = when (val lang = Locale.getDefault().language) { + "iw" -> "he" + "in" -> "id" + "ji" -> "yi" + "" -> "en" + else -> lang + } + + /** The big cores only: more threads than that slow Whisper down on a phone. */ + private fun threads() = Runtime.getRuntime().availableProcessors().coerceIn(1, 8).let { if (it >= 8) 4 else max(1, it / 2) } + + /** Whisper marks silence and noise as "[BLANK_AUDIO]" or "(wind blowing)"; none of it is words. */ + private fun clean(text: String): String = + text.replace(Regex("""\[[^\]]*\]|\([^)]*\)|\*[^*]*\*"""), " ").replace(Regex("\\s+"), " ").trim() + + private external fun nativeInitAsset(assets: AssetManager, path: String): Long + private external fun nativeTranscribe(handle: Long, samples: FloatArray, language: String, threads: Int): ByteArray + private external fun nativeFree(handle: Long) +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt index 1d44605..4d11e69 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/GearslipApp.kt @@ -1,6 +1,12 @@ package app.seb3thehacker.gearslip.ui +import android.Manifest +import android.app.Activity +import android.content.pm.ActivityInfo +import android.content.pm.PackageManager import androidx.activity.compose.BackHandler +import androidx.activity.compose.rememberLauncherForActivityResult +import androidx.activity.result.contract.ActivityResultContracts import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.WindowInsets @@ -16,6 +22,7 @@ import androidx.compose.material3.Surface import androidx.compose.material3.Text import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember @@ -31,7 +38,7 @@ import app.seb3thehacker.gearslip.AppSettings import app.seb3thehacker.gearslip.BuildConfig import app.seb3thehacker.gearslip.CertProvider -private enum class Screen { HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, HELP } +private enum class Screen { HOME, LOGS, SETTINGS, CAR_PREVIEW, CERT_SETUP, SETUP_GUIDE, HELP, WHATS_NEW } /** Three screens and a back stack of depth one: no navigation library needed. */ @Composable @@ -52,6 +59,25 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) { } BackHandler(enabled = screen != Screen.HOME) { screen = Screen.HOME } + // Home stays upright; every other screen turns with the phone. + LaunchedEffect(screen) { + (context as? Activity)?.requestedOrientation = if (screen == Screen.HOME) { + ActivityInfo.SCREEN_ORIENTATION_PORTRAIT + } else { + ActivityInfo.SCREEN_ORIENTATION_UNSPECIFIED + } + } + + // The guide asks for this now, but anyone who finished it before that never saw the page. + val notificationPermission = rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) {} + LaunchedEffect(Unit) { + if (!needsSetupGuide && + context.checkSelfPermission(Manifest.permission.POST_NOTIFICATIONS) != PackageManager.PERMISSION_GRANTED + ) { + notificationPermission.launch(Manifest.permission.POST_NOTIFICATIONS) + } + } + Column( Modifier .fillMaxSize() @@ -67,8 +93,10 @@ fun GearslipApp(onDisconnect: () -> Unit, onRequestCallScreening: () -> Unit) { onOpenSettings = { screen = Screen.SETTINGS }, onOpenCarPreview = { screen = Screen.CAR_PREVIEW }, onOpenHelp = { screen = Screen.HELP }, + onOpenWhatsNew = { screen = Screen.WHATS_NEW }, onDisconnect = onDisconnect, ) + Screen.WHATS_NEW -> WhatsNewScreen(onBack = { screen = Screen.HOME }) Screen.LOGS -> LogsScreen(onBack = { screen = Screen.HOME }) Screen.SETTINGS -> SettingsScreen(onBack = { screen = Screen.HOME }) Screen.CAR_PREVIEW -> CarPreviewScreen(onBack = { screen = Screen.HOME }) diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HelpScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HelpScreen.kt index 097ae82..7cb8595 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HelpScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HelpScreen.kt @@ -53,7 +53,8 @@ private fun setupSteps(onRequestCallScreening: () -> Unit) = listOf( Step( "Allow the microphone, so music can reach the car.", "Android asks for the microphone before it will let any app pass music along. " + - "Gearslip never listens to you: it copies what a music app is playing, and nothing else.", + "Gearslip listens only when you tap Reply or press the voice button. The rest of the " + + "time it copies what a music app is playing, and nothing else.", ), Step( "Optional: stop Android asking about audio every drive.", diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt index 3b54467..7082c1a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/HomeScreen.kt @@ -16,6 +16,8 @@ import androidx.compose.foundation.verticalScroll import androidx.compose.material.icons.Icons import androidx.compose.material.icons.automirrored.filled.KeyboardArrowRight import androidx.compose.material.icons.filled.Info +import androidx.compose.material.icons.filled.Send +import androidx.compose.material.icons.filled.Star import androidx.compose.material.icons.filled.List import androidx.compose.material.icons.filled.PlayArrow import androidx.compose.material.icons.filled.Settings @@ -37,7 +39,9 @@ import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.unit.dp import androidx.lifecycle.compose.collectAsStateWithLifecycle +import app.seb3thehacker.gearslip.BuildConfig import app.seb3thehacker.gearslip.SessionStatus +import app.seb3thehacker.gearslip.notify.TestMessage @Composable fun HomeScreen( @@ -45,6 +49,7 @@ fun HomeScreen( onOpenSettings: () -> Unit, onOpenCarPreview: () -> Unit, onOpenHelp: () -> Unit, + onOpenWhatsNew: () -> Unit, onDisconnect: () -> Unit, ) { val status by SessionStatus.state.collectAsStateWithLifecycle() @@ -110,6 +115,13 @@ fun HomeScreen( ActionRow(Icons.Filled.List, "Live logs", onOpenLogs) RowDivider() ActionRow(Icons.Filled.Info, "Connection help", onOpenHelp) + RowDivider() + ActionRow(Icons.Filled.Star, "What's new", onOpenWhatsNew) + if (BuildConfig.DEBUG) { + val context = LocalContext.current + RowDivider() + ActionRow(Icons.Filled.Send, "Send a test message") { TestMessage.post(context) } + } } } } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt index a39cf98..711a2a1 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/SetupGuideScreen.kt @@ -153,6 +153,17 @@ private fun pages(context: Context, onRequestCallScreening: () -> Unit): List Unit) { + Scaffold( + containerColor = MaterialTheme.colorScheme.background, + topBar = { + TopAppBar( + title = { Text("What's new") }, + navigationIcon = { + IconButton(onClick = onBack) { + Icon(Icons.AutoMirrored.Filled.ArrowBack, contentDescription = "Back") + } + }, + ) + }, + ) { padding -> + Column( + Modifier + .fillMaxSize() + .padding(padding) + .padding(horizontal = 24.dp) + .verticalScroll(rememberScrollState()), + verticalArrangement = Arrangement.spacedBy(28.dp), + ) { + Spacer(Modifier.height(0.dp)) + Changelog.releases.forEach { release -> + Column(verticalArrangement = Arrangement.spacedBy(12.dp)) { + Text( + if (release.versionCode == BuildConfig.VERSION_CODE) "${release.versionName} (this version)" else release.versionName, + style = MaterialTheme.typography.titleLarge, + fontWeight = FontWeight.Bold, + color = MaterialTheme.colorScheme.primary, + ) + release.changes.forEach { change -> + Row(horizontalArrangement = Arrangement.spacedBy(12.dp)) { + Box( + Modifier + .padding(top = 8.dp) + .size(6.dp) + .background(MaterialTheme.colorScheme.primary, CircleShape), + ) + Text(change, style = MaterialTheme.typography.bodyLarge) + } + } + } + } + Spacer(Modifier.height(16.dp)) + } + } +} diff --git a/gearslip/src/main/res/drawable/voice_selection_24.xml b/gearslip/src/main/res/drawable/voice_selection_24.xml new file mode 100644 index 0000000..bd77da3 --- /dev/null +++ b/gearslip/src/main/res/drawable/voice_selection_24.xml @@ -0,0 +1,9 @@ + + + diff --git a/gearslip/src/main/res/drawable/volume_up_24.xml b/gearslip/src/main/res/drawable/volume_up_24.xml new file mode 100644 index 0000000..3057051 --- /dev/null +++ b/gearslip/src/main/res/drawable/volume_up_24.xml @@ -0,0 +1,9 @@ + + +