car: an experimental turn card on the instrument cluster, for cars with a cluster screen

This commit is contained in:
Seb3thehacker committed 2026-10-26 16:50:36 -07:00
1 parent b168eb7c04
commit 83c33eeea3
10 files changed
+400 -7

No files matched your search

@@ -0,0 +1,164 @@
package app.seb3thehacker.gearslip
import android.content.Context
import android.os.Handler
import android.os.Looper
import app.seb3thehacker.gearslip.car.ClusterScreen
/**
* Gearslip's picture on the car's instrument-cluster screen: a second video sink, for a car that
* offers one, showing the turn card ([ClusterScreen]). The same exchange as the main screen's,
* on its own channel, with its own encoder and its own private display:
*
* -> CHANNEL_OPEN_REQUEST <- CHANNEL_OPEN_RESPONSE
* -> Setup { H.264 } <- Config { status, max_unacked, configuration_indices }
* -> VideoFocusRequest <- VideoFocusNotification { projected }
* -> Start, CodecConfig, Data... <- Ack
*
* No touch: a cluster has none. Experimental, behind [car.CarSettings.clusterVideoEnabled], and
* untested in a car: none in the notes has a cluster sink.
*/
internal class ClusterVideo(
private val context: Context,
private val service: ServiceDiscovery.VideoService,
private val send: (messageId: Int, body: ByteArray) -> Unit,
) {
private val log = GearslipLog.tagged("CLUSTER")
private val main = Handler(Looper.getMainLooper())
val channelId get() = service.serviceId
private var configIndex = 0
@Volatile private var started = false
@Volatile private var running = true
@Volatile private var source: VideoSource? = null
private var projector: ScreenProjector? = null
private val ackLock = Object()
@Volatile private var inFlight = 0
@Volatile private var maxUnacked = 1
@Volatile private var lastAckAt = 0L
private var frames = 0
fun onMessage(messageId: Int, body: ByteArray) {
when (messageId) {
GearslipRunner.MSG_CHANNEL_OPEN_RESPONSE -> {
val status = Protobuf.readInt32Field(body, 1) ?: 0
log.i("<- ChannelOpenResponse: status=$status")
if (status != 0) return
send(GearslipRunner.MSG_MEDIA_SETUP, Protobuf.varintField(1, GearslipRunner.CODEC_H264_BP.toLong()))
log.i("-> Setup(codec=VIDEO_H264_BP)")
}
GearslipRunner.MSG_MEDIA_CONFIG -> {
val fields = Wire.fields(body)
Wire.varint(fields, 2)?.toInt()?.let { maxUnacked = maxOf(1, it) }
val offered = fields.filter { it.number == 3 && it.wireType == 0 }.map { it.varint.toInt() }
// The cluster gets the smallest picture it offers: a turn card needs no more.
configIndex = offered.minByOrNull { service.configs.getOrNull(it)?.pixelCount() ?: Int.MAX_VALUE } ?: 0
log.i("<- Config status=${Wire.varint(fields, 1)} max_unacked=$maxUnacked offered=$offered - using $configIndex")
val focus = Protobuf.varintField(2, GearslipRunner.VIDEO_FOCUS_PROJECTED.toLong()) + Protobuf.varintField(3, 1L)
send(GearslipRunner.MSG_VIDEO_FOCUS_REQUEST, focus)
log.i("-> VideoFocusRequest(mode=PROJECTED)")
}
GearslipRunner.MSG_VIDEO_FOCUS_NOTIFICATION -> {
val mode = Wire.varint(Wire.fields(body), 1)?.toInt() ?: GearslipRunner.VIDEO_FOCUS_PROJECTED
log.i("<- VideoFocusNotification: mode=$mode")
val projected = mode == GearslipRunner.VIDEO_FOCUS_PROJECTED || mode == GearslipRunner.VIDEO_FOCUS_PROJECTED_NO_INPUT
if (projected && !started) startStream()
}
GearslipRunner.MSG_MEDIA_ACK -> {
val acked = Wire.varint(Wire.fields(body), 2)?.toInt() ?: 1
synchronized(ackLock) {
inFlight = maxOf(0, inFlight - maxOf(1, acked))
lastAckAt = System.currentTimeMillis()
ackLock.notifyAll()
}
}
else -> log.i("<- unhandled message id=$messageId (${body.size} bytes)")
}
}
private fun startStream() {
val config = service.configs.getOrNull(configIndex) ?: service.configs.firstOrNull() ?: return
val (width, height) = config.pixelSize() ?: run {
log.w("unsupported cluster resolution ${config.resolutionName}")
return
}
started = true
send(
GearslipRunner.MSG_MEDIA_START,
Protobuf.varintField(1, GearslipRunner.SESSION_ID.toLong()) + Protobuf.varintField(2, configIndex.toLong()),
)
log.i("-> Start(config=$configIndex -> $config)")
lastAckAt = System.currentTimeMillis()
val video = VideoSource(
width = width,
height = height,
frameRate = FRAME_RATE,
mode = VideoSource.Mode.SURFACE,
wireless = true, // a small share of the link: the main screen comes first
onCodecConfig = { csd ->
send(GearslipRunner.MSG_MEDIA_CODEC_CONFIG, csd)
log.i("-> CodecConfig (${csd.size} bytes)")
},
onFrame = { data, pts, _ -> sendFrame(data, pts) },
)
source = video
runCatching { video.start() }.onFailure {
log.e("cluster encoder failed to start", it)
return
}
val surface = video.inputSurface ?: return
val density = config.density.takeIf { it > 0 } ?: 160
main.post {
if (!running) return@post
projector = ScreenProjector(context).also {
it.start(surface, width, height, density, name = "gearslip-cluster") {
// The margins the car crops off are kept clear, as on the main screen.
ClusterScreen(config.widthMargin, config.heightMargin)
}
}
}
}
/** As on the main screen: wait for the car's acks rather than drop a frame. */
private fun sendFrame(data: ByteArray, presentationTimeUs: Long) {
synchronized(ackLock) {
while (running && inFlight >= maxUnacked) {
val since = System.currentTimeMillis() - lastAckAt
if (since > GearslipRunner.ACK_STALL_MS) {
inFlight = 0
lastAckAt = System.currentTimeMillis()
break
}
ackLock.wait(GearslipRunner.ACK_STALL_MS - since + 1)
}
}
if (!running) return
val timestamp = ByteArray(8)
for (i in 0 until 8) timestamp[i] = ((presentationTimeUs shr ((7 - i) * 8)) and 0xFF).toByte()
synchronized(ackLock) { inFlight++ }
runCatching { send(GearslipRunner.MSG_MEDIA_DATA, timestamp + data) }.onFailure { running = false }
if (++frames <= 2) log.i("-> Data frame #$frames (${data.size} bytes)")
}
fun stop() {
running = false
synchronized(ackLock) { ackLock.notifyAll() }
main.post {
projector?.stop()
projector = null
}
source?.stop()
source = null
}
private companion object {
// A turn card changes a few times a minute; the encoder repeats the last frame between.
const val FRAME_RATE = 15
}
}
@@ -97,6 +97,7 @@ class GearslipRunner(
@Volatile private var sensorChannelId: Int = -1
@Volatile private var mediaStatus: CarMediaStatus? = null
@Volatile private var phoneStatus: app.seb3thehacker.gearslip.call.CarPhoneStatus? = null
@Volatile private var clusterVideo: ClusterVideo? = null
@Volatile private var clusterNav: ClusterNav? = null
private var sensorTypes: List<Int> = emptyList()
@Volatile private var audioLink: AudioLink? = null
@@ -194,6 +195,8 @@ class GearslipRunner(
mediaStatus = null
phoneStatus?.stop()
phoneStatus = null
clusterVideo?.stop()
clusterVideo = null
clusterNav?.stop()
clusterNav = null
lastSensorNight = null
@@ -243,6 +246,8 @@ class GearslipRunner(
mediaStatus?.onMessage(messageId, body)
} else if (frame.channel == phoneStatus?.channelId) {
phoneStatus?.onMessage(messageId, body)
} else if (frame.channel == clusterVideo?.channelId) {
clusterVideo?.onMessage(messageId, body)
} else if (frame.channel == guidanceLink?.channelId) {
guidanceLink?.onMessage(messageId, body)
} else if (frame.channel == CarBluetooth.channelId) {
@@ -520,6 +525,7 @@ class GearslipRunner(
startMicChannel(serviceDiscoveryResponse)
startMediaStatusChannel(serviceDiscoveryResponse)
startPhoneStatusChannel(serviceDiscoveryResponse)
startClusterVideo(serviceDiscoveryResponse)
startClusterNavChannel(serviceDiscoveryResponse)
startBluetooth(serviceDiscoveryResponse)
startVendorChannel(serviceDiscoveryResponse)
@@ -636,6 +642,19 @@ class GearslipRunner(
openChannel(channel)
}
/** The turn card on the instrument cluster, for a car with a cluster screen: experimental, off by default. */
private fun startClusterVideo(serviceDiscoveryResponse: ByteArray) {
val sink = ServiceDiscovery.findClusterVideoService(serviceDiscoveryResponse) ?: return
log.i("cluster video sink: channel=${sink.serviceId} display=${sink.displayId} type=${sink.displayType}")
if (!CarSettings.experimentalFeaturesEnabled.value || !CarSettings.clusterVideoEnabled.value) {
log.i("cluster screen is off in the car's Settings (experimental) - leaving that sink alone")
return
}
val context = app.seb3thehacker.gearslip.car.CarEnvironment.appContext ?: return
clusterVideo = ClusterVideo(context, sink) { id, body -> send(id, body, encrypted = true, channel = sink.serviceId) }
openChannel(sink.serviceId)
}
/** Calls, for the car's own screens: the cluster, the steering wheel display. */
private fun startPhoneStatusChannel(serviceDiscoveryResponse: ByteArray) {
val channel = ServiceDiscovery.findPhoneStatusChannel(serviceDiscoveryResponse) ?: run {
@@ -1480,7 +1499,7 @@ class GearslipRunner(
}
}
private companion object {
internal companion object {
/** The stats window adaptive bitrate reads, matching the watchdog's sleep. */
const val ADAPT_WINDOW_MS = 5_000L
@@ -51,8 +51,17 @@ object ServiceDiscovery {
"$resolutionName @ $frameRateName, density=$density, margin=${widthMargin}x$heightMargin"
}
/** [displayId] is MediaSinkService field 6: 0, the default, is the main screen. */
class VideoService(val serviceId: Int, val codecType: Int, val configs: List<VideoConfig>, val displayId: Int = 0) {
/**
* [displayId] is MediaSinkService field 6: 0, the default, is the main screen. [displayType]
* is field 7 (0 main, 1 instrument cluster, 2 auxiliary), or null when the car leaves it out.
*/
class VideoService(
val serviceId: Int,
val codecType: Int,
val configs: List<VideoConfig>,
val displayId: Int = 0,
val displayType: Int? = null,
) {
val codecName: String get() = CODECS[codecType] ?: "unknown($codecType)"
}
@@ -283,6 +292,19 @@ object ServiceDiscovery {
return sinks.firstOrNull { it.displayId == 0 } ?: sinks.firstOrNull()
}
/**
* The instrument cluster's video sink: one marked as a cluster, or, from a car that marks
* none, a second sink beside the main screen's. Null when the car has no cluster screen.
*/
fun findClusterVideoService(response: ByteArray): VideoService? {
val sinks = videoServices(response)
val main = findVideoService(response) ?: return null
return sinks.firstOrNull { it.displayType == DISPLAY_TYPE_CLUSTER && it.serviceId != main.serviceId }
?: sinks.firstOrNull { it.displayType == null && it.serviceId != main.serviceId && it.displayId != main.displayId }
}
private const val DISPLAY_TYPE_CLUSTER = 1
private fun videoServices(response: ByteArray): List<VideoService> = buildList {
for (channel in Wire.allBytes(Wire.fields(response), 1)) {
val service = Wire.fields(channel)
@@ -304,7 +326,7 @@ object ServiceDiscovery {
heightMargin = Wire.varint(f, 4)?.toInt() ?: 0,
)
}
add(VideoService(id, codecType, configs, Wire.varint(sinkFields, 6)?.toInt() ?: 0))
add(VideoService(id, codecType, configs, Wire.varint(sinkFields, 6)?.toInt() ?: 0, Wire.varint(sinkFields, 7)?.toInt()))
}
}
@@ -64,7 +64,9 @@ object CarEnvironment {
}
@Volatile private var sensorNight: Boolean? = null
private var appContext: Context? = null
/** Set once the car UI starts; the cluster screen draws with it too. */
var appContext: Context? = null
private set
private var running = false
private val handler = Handler(Looper.getMainLooper())
@@ -75,6 +75,7 @@ private const val KEY_LOCATION_WARNING = "car_location_warning"
private const val KEY_SYSTEM_SPEECH = "car_system_speech"
private const val KEY_CAR_SENSORS = "car_sensors_enabled"
private const val KEY_ADAPTIVE_VIDEO = "adaptive_video_enabled"
private const val KEY_CLUSTER_VIDEO = "cluster_video_enabled"
private const val KEY_DEBUG_BADGE = "debug_badge"
private const val KEY_SEEN_SAFETY_WARNING = "car_seen_safety_warning"
private const val KEY_UNITS = "car_units"
@@ -124,6 +125,7 @@ private const val KEY_LOCATION_WARNING = "car_location_warning"
private val systemSpeechFlow = MutableStateFlow(false)
private val carSensorsFlow = MutableStateFlow(false)
private val adaptiveVideoFlow = MutableStateFlow(false)
private val clusterVideoFlow = MutableStateFlow(false)
private val debugBadgeFlow = MutableStateFlow(true)
private val seenSafetyWarningFlow = MutableStateFlow(false)
private val unitsFlow = MutableStateFlow(Units.AUTO)
@@ -293,6 +295,13 @@ private const val KEY_LOCATION_WARNING = "car_location_warning"
*/
val adaptiveVideoEnabled: StateFlow<Boolean> = adaptiveVideoFlow
/**
* Gearslip's turn card on a car's instrument-cluster screen, for a car that offers one as a
* second video sink. Applies only while [experimentalFeaturesEnabled] is on too, and from the
* next connection; untested in a car so far.
*/
val clusterVideoEnabled: StateFlow<Boolean> = clusterVideoFlow
/** Debug builds only: the car screen's DEBUG BUILD badge, which screenshots for the README leave out. */
val debugBadge: StateFlow<Boolean> = debugBadgeFlow
@@ -348,6 +357,7 @@ private const val KEY_LOCATION_WARNING = "car_location_warning"
systemSpeechFlow.value = AppSettings.getString(app, KEY_SYSTEM_SPEECH, "false") == "true"
carSensorsFlow.value = AppSettings.getString(app, KEY_CAR_SENSORS, "false") == "true"
adaptiveVideoFlow.value = AppSettings.getString(app, KEY_ADAPTIVE_VIDEO, "false") == "true"
clusterVideoFlow.value = AppSettings.getString(app, KEY_CLUSTER_VIDEO, "false") == "true"
debugBadgeFlow.value = AppSettings.getString(app, KEY_DEBUG_BADGE, "true") == "true"
seenSafetyWarningFlow.value = AppSettings.getString(app, KEY_SEEN_SAFETY_WARNING, "false") == "true"
unitsFlow.value = runCatching {
@@ -466,6 +476,11 @@ private const val KEY_LOCATION_WARNING = "car_location_warning"
AppSettings.putString(app, KEY_ADAPTIVE_VIDEO, on.toString())
}
fun setClusterVideoEnabled(on: Boolean) {
clusterVideoFlow.value = on
AppSettings.putString(app, KEY_CLUSTER_VIDEO, on.toString())
}
fun setCarSensorsEnabled(on: Boolean) {
carSensorsFlow.value = on
AppSettings.putString(app, KEY_CAR_SENSORS, on.toString())
@@ -78,6 +78,7 @@ fun CarSettingsScreen() {
val experimentalFeatures by CarSettings.experimentalFeaturesEnabled.collectAsState()
val carSensors by CarSettings.carSensorsEnabled.collectAsState()
val adaptiveVideo by CarSettings.adaptiveVideoEnabled.collectAsState()
val clusterVideo by CarSettings.clusterVideoEnabled.collectAsState()
val systemSpeech by CarSettings.systemSpeech.collectAsState()
val display by CarEnvironment.display.collectAsState()
val vehicle by CarEnvironment.vehicle.collectAsState()
@@ -313,7 +314,7 @@ fun CarSettingsScreen() {
}
if (section == chosen && experimentalFeatures) item {
ExperimentalFeaturesList(carSensors, adaptiveVideo)
ExperimentalFeaturesList(carSensors, adaptiveVideo, clusterVideo)
}
if (section == chosen) item {
@@ -445,7 +446,7 @@ private fun <T> ChoiceButtons(options: List<Pair<T, String>>, selected: T, onSel
* so the list reads fine whether there's one entry or several.
*/
@Composable
private fun ExperimentalFeaturesList(carSensors: Boolean, adaptiveVideo: Boolean) {
private fun ExperimentalFeaturesList(carSensors: Boolean, adaptiveVideo: Boolean, clusterVideo: Boolean) {
Column(
Modifier.fillMaxWidth(),
verticalArrangement = Arrangement.spacedBy(10.dp),
@@ -460,6 +461,11 @@ private fun ExperimentalFeaturesList(carSensors: Boolean, adaptiveVideo: Boolean
description = stringResource(R.string.carset_adaptive_note),
checked = adaptiveVideo,
) { CarSettings.setAdaptiveVideoEnabled(it) }
ExperimentalFeatureRow(
title = stringResource(R.string.car_cluster_video),
description = stringResource(R.string.car_cluster_video_note),
checked = clusterVideo,
) { CarSettings.setClusterVideoEnabled(it) }
}
}
@@ -0,0 +1,140 @@
package app.seb3thehacker.gearslip.car
import androidx.compose.foundation.background
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.BoxWithConstraints
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.Spacer
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.width
import androidx.compose.material3.LocalContentColor
import androidx.compose.material3.MaterialTheme
import androidx.compose.material3.Text
import androidx.compose.material3.darkColorScheme
import androidx.compose.runtime.Composable
import androidx.compose.runtime.CompositionLocalProvider
import androidx.compose.runtime.getValue
import androidx.compose.runtime.produceState
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.platform.LocalDensity
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
import androidx.compose.ui.unit.dp
import app.seb3thehacker.gearslip.host.clockTime
import app.seb3thehacker.gearslip.host.display
import app.seb3thehacker.gearslip.host.remainingTime
import kotlinx.coroutines.delay
/**
* The instrument-cluster screen: the next turn, big enough to read in a glance past the wheel.
* The arrow and the distance to it, what to do there, and the time, distance and arrival left
* along the bottom. With no route it shows what's playing, quietly, or nothing.
*
* Always dark, like the dials around it. [marginX] and [marginY] are the pixels the car crops off
* the frame, kept clear.
*/
@Composable
internal fun ClusterScreen(marginX: Int, marginY: Int) {
// The map app's turn changes a few times a minute: read it once a second.
val turn by produceState<ClusterTurn?>(null) {
while (true) {
value = ClusterTurn.now()
delay(1_000)
}
}
val density = LocalDensity.current
MaterialTheme(colorScheme = darkColorScheme()) {
CompositionLocalProvider(LocalContentColor provides Color.White) {
BoxWithConstraints(
Modifier.fillMaxSize().background(Color.Black).padding(
horizontal = with(density) { (marginX / 2).toDp() } + 24.dp,
vertical = with(density) { (marginY / 2).toDp() } + 16.dp,
),
contentAlignment = Alignment.CenterStart,
) {
// Sized off the height: clusters run from short strips to near squares.
val unit = maxHeight / 12
val t = turn
if (t == null) {
NowPlayingLine(Modifier.align(Alignment.Center))
return@BoxWithConstraints
}
Column(verticalArrangement = Arrangement.spacedBy(unit * 0.4f)) {
Row(verticalAlignment = Alignment.CenterVertically) {
CarGlyph(t.icon, Modifier.size(unit * 3.4f))
Spacer(Modifier.width(unit))
Text(
t.distance,
fontSize = with(density) { (unit * 2.2f).toSp() },
fontWeight = FontWeight.Bold,
maxLines = 1,
)
}
if (t.what.isNotEmpty()) {
Text(
t.what,
fontSize = with(density) { (unit * 1.1f).toSp() },
fontWeight = FontWeight.SemiBold,
maxLines = 2,
overflow = TextOverflow.Ellipsis,
)
}
if (t.trip.isNotEmpty()) {
Text(
t.trip,
fontSize = with(density) { (unit * 0.85f).toSp() },
color = Color.White.copy(alpha = 0.7f),
maxLines = 1,
)
}
}
}
}
}
}
@Composable
private fun NowPlayingLine(modifier: Modifier) {
val now = CarServices.mediaOrNull()?.now?.value
if (now == null || now.title.isEmpty()) return
Text(
listOf(now.title, now.artist).filter { it.isNotEmpty() }.joinToString(" · "),
modifier,
style = MaterialTheme.typography.titleLarge,
color = Color.White.copy(alpha = 0.7f),
maxLines = 1,
overflow = TextOverflow.Ellipsis,
)
}
/** What the cluster shows of a route, read from [NavTurn] and the trip's estimate to the destination. */
private class ClusterTurn(
val icon: androidx.car.app.model.CarIcon?,
val distance: String,
val what: String,
val trip: String,
) {
companion object {
fun now(): ClusterTurn? {
val turn = NavTurn.current() ?: return null
if (turn.rerouting) return null
val step = turn.step
val cue = step.cue?.toString().orEmpty()
val what = cue.ifEmpty { step.road?.toString().orEmpty() }
val estimate = CarServices.navOrNull()?.trip?.value?.destinationTravelEstimates?.firstOrNull()
val trip = estimate?.let { e ->
listOf(
e.remainingTimeSeconds.takeIf { it >= 0 }?.let { remainingTime(it) }.orEmpty(),
e.remainingDistance.display(),
e.arrivalTimeAtDestination.clockTime(),
).filter { it.isNotEmpty() }.joinToString(" · ")
}.orEmpty()
return ClusterTurn(step.maneuver?.icon, turn.distance.display(), what, trip)
}
}
}
@@ -202,6 +202,7 @@ private fun GroupRows(group: CarGroup) {
val experimental by CarSettings.experimentalFeaturesEnabled.collectAsState()
val carSensors by CarSettings.carSensorsEnabled.collectAsState()
val adaptiveVideo by CarSettings.adaptiveVideoEnabled.collectAsState()
val clusterVideo by CarSettings.clusterVideoEnabled.collectAsState()
when (group) {
CarGroup.SCREEN -> {
@@ -278,6 +279,13 @@ private fun GroupRows(group: CarGroup) {
CarSettings::setAdaptiveVideoEnabled,
note = stringResource(R.string.car_adaptive_video_note),
)
SettingsDivider()
Toggle(
stringResource(R.string.car_cluster_video),
clusterVideo,
CarSettings::setClusterVideoEnabled,
note = stringResource(R.string.car_cluster_video_note),
)
}
}
}