diff --git a/.gitignore b/.gitignore index ba9ca43..fdd2771 100644 --- a/.gitignore +++ b/.gitignore @@ -15,8 +15,6 @@ keystore.properties .cxx/ # Testing -/docs/cert-sweep.sh -extraction/ /cloned-repos docs-internal/ @@ -31,6 +29,4 @@ CLAUDE.md # Other archive/ - -# Side project notes, kept out of the repo -backseat-jury.md +.wrangler/ \ No newline at end of file diff --git a/README.md b/README.md index e3e6938..704c7c1 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,27 @@ mode. adb commands: one removes the audio dialog for good, and the other keeps notifications visible while audio plays. +## Usage notes + +Gearslip can send short notes that show which cars it works in. They stay off unless you +turn them on, in setup or in Settings. + +- **When they go out.** When you open Gearslip, at most once a day, and after each drive. + Nothing runs in the background. +- **What they carry.** The Gearslip version, plus whatever you tick: the Android version + and whether the phone runs GrapheneOS, the phone model, whether a car connected that + day, and each car's head unit, model, year, screen size, and connection result. + Settings shows the exact note. +- **What they leave out.** Your name, accounts, location, contacts, messages, and + anything you type or play. +- **How phones are counted.** A random number counts each phone once. The server keeps + only a salted hash of it. +- **Where they go.** To a Cloudflare Worker, whose code is in + [stats-worker/](stats-worker/). Cloudflare sees your IP address in delivering a note, + as it would for any website; the Worker doesn't store it. + +Turn the notes off, and they stop at once. + ## Building ``` diff --git a/gearslip/build.gradle.kts b/gearslip/build.gradle.kts index 58659fa..9a919c5 100644 --- a/gearslip/build.gradle.kts +++ b/gearslip/build.gradle.kts @@ -35,6 +35,9 @@ android { versionCode = 7 versionName = "0.1.10-alpha" manifestPlaceholders["appLabel"] = "Gearslip" + // Where opted-in phones send their daily usage note: the stats-worker/ Worker's address, + // set as gearslipStatsUrl in gradle.properties. Left empty, nothing is ever sent. + buildConfigField("String", "STATS_URL", "\"${project.findProperty("gearslipStatsUrl") ?: ""}\"") } signingConfigs { diff --git a/gearslip/src/main/AndroidManifest.xml b/gearslip/src/main/AndroidManifest.xml index 248e512..03c99d9 100644 --- a/gearslip/src/main/AndroidManifest.xml +++ b/gearslip/src/main/AndroidManifest.xml @@ -74,6 +74,8 @@ LAUNCHER query below to find, so the setup guide's Android Auto check needs this to see whether the package is installed at all. --> + + diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt index ec403c1..8fe7da3 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/GearslipActivity.kt @@ -27,6 +27,7 @@ import app.seb3thehacker.gearslip.car.CarEnvironment import app.seb3thehacker.gearslip.car.CarSettings import app.seb3thehacker.gearslip.car.CarUi import app.seb3thehacker.gearslip.car.Prefetch +import app.seb3thehacker.gearslip.stats.UsageStats import app.seb3thehacker.gearslip.ui.GearslipApp import app.seb3thehacker.gearslip.ui.GearslipTheme import java.io.FileInputStream @@ -150,6 +151,7 @@ class GearslipActivity : ComponentActivity(), Projection { override fun onStart() { super.onStart() Prefetch.warm(this) + UsageStats.onAppOpened(this) } override fun onNewIntent(intent: Intent) { diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/SessionReport.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/SessionReport.kt index 53ff774..fe2a01a 100644 --- a/gearslip/src/main/java/app/seb3thehacker/gearslip/SessionReport.kt +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/SessionReport.kt @@ -1,6 +1,7 @@ package app.seb3thehacker.gearslip import android.content.Context +import app.seb3thehacker.gearslip.stats.UsageStats import android.os.Build /** @@ -48,7 +49,10 @@ object SessionReport { @Volatile private var lastState = "" private var printed = false + private var appContext: Context? = null + fun init(context: Context) { + appContext = context.applicationContext val info = runCatching { context.packageManager.getPackageInfo(context.packageName, 0) }.getOrNull() appVersion = info?.versionName ?: "?" appCode = info?.longVersionCode ?: 0 @@ -59,6 +63,8 @@ object SessionReport { category = Category.NONE failure = ""; protocolVersion = ""; certSource = ""; certIssuer = "" tlsProtocol = ""; tlsCipher = ""; authStatus = null + headUnitInfo = null + carScreen = ""; carDpi = 0 headUnit = ""; profile = ""; video = ""; lastStream = ""; lastState = "" printed = false } @@ -68,6 +74,14 @@ object SessionReport { fun tls(protocol: String, cipher: String) { tlsProtocol = protocol; tlsCipher = cipher } fun auth(status: Int) { authStatus = status } fun headUnit(info: String) { headUnit = info } + + /** Who the car said it is, for the opt-in usage note; null until service discovery. */ + @Volatile var headUnitInfo: ServiceDiscovery.HeadUnitInfo? = null + + /** The car screen's usable size once its margins are cropped, and its density; blank before video starts. */ + @Volatile var carScreen = ""; private set + @Volatile var carDpi = 0; private set + fun carScreen(size: String, dpi: Int) { carScreen = size; carDpi = dpi } fun profile(name: String?) { profile = name ?: "none matched" } fun video(summary: String) { video = summary } fun audio(summary: String) { audio = summary } @@ -121,6 +135,18 @@ object SessionReport { if (printed || startedAt == 0L) return printed = true render().lines().forEach { GearslipLog.i(it) } + appContext?.let { context -> + // Video started means the car showed Gearslip: that's a working car, and unplugging or + // switching the car off at the end shows up as a USB or ByeBye "failure" that isn't one. + val reached = carScreen.isNotEmpty() + val outcome = when { + reached && category in setOf(Category.NONE, Category.USB, Category.BYEBYE) -> "connected" + reached -> "connected, then ${category.label}" + category == Category.NONE -> "ended before video" + else -> category.label + } + runCatching { UsageStats.recordSession(context, headUnitInfo, protocolVersion, outcome, carScreen, carDpi) } + } GearslipLog.flush() } diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/stats/UsageStats.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/stats/UsageStats.kt new file mode 100644 index 0000000..aa82a01 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/stats/UsageStats.kt @@ -0,0 +1,235 @@ +package app.seb3thehacker.gearslip.stats + +import android.app.job.JobScheduler +import android.content.Context +import android.os.Build +import app.seb3thehacker.gearslip.BuildConfig +import app.seb3thehacker.gearslip.GearslipLog +import app.seb3thehacker.gearslip.ServiceDiscovery +import org.json.JSONArray +import org.json.JSONObject +import java.net.HttpURLConnection +import java.net.URL +import java.time.LocalDate +import java.util.UUID +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Opt-in usage counts: a short note to the stats Worker (stats-worker/ in the repo). Off until the + * driver turns it on in setup or Settings. Only the app version is always sent; everything else + * is one of the driver's [Choices]. A random install ID counts a phone once; nothing ties it to a + * person, and the Worker keeps no IP addresses. + * + * Notes go out only while Gearslip is in use: the first time the app is opened each day, and + * after every car session. Nothing runs in the background, so a phone that stops using Gearslip + * goes quiet. A note that fails waits for the next one. With no [BuildConfig.STATS_URL] set at + * build time, nothing is sent. + */ +object UsageStats { + + /** What the driver agreed to share beyond the app version. */ + data class Choices( + val android: Boolean = true, + val phone: Boolean = true, + val cars: Boolean = true, + ) + + private const val PREFS = "usage_stats" + private const val KEY_ENABLED = "enabled" + private const val KEY_ASKED = "asked" + private const val KEY_ID = "install_id" + /** The phone details the server last got, and when, so they go again only on a change. */ + private const val KEY_SENT_DETAILS = "sent_details" + private const val KEY_DETAILS_DAY = "details_day" + private const val DETAILS_REFRESH_DAYS = 30L + private const val KEY_CARS = "cars" + private const val KEY_SENT_DAY = "sent_day" + /** The daily background job of earlier builds, cancelled wherever it's still scheduled. */ + private const val OLD_JOB_ID = 4_207 + private const val MAX_CARS = 10 + + private fun prefs(context: Context) = context.getSharedPreferences(PREFS, Context.MODE_PRIVATE) + + /** + * Dev builds are the developer's own, so they always send everything and never ask: no switch + * in Settings, no page in setup. Only release builds offer the choice. + */ + val choosable: Boolean get() = !BuildConfig.DEBUG + + fun enabled(context: Context): Boolean = !choosable || prefs(context).getBoolean(KEY_ENABLED, false) + + /** Whether the driver has answered the question in setup, either way. */ + fun asked(context: Context): Boolean = !choosable || prefs(context).getBoolean(KEY_ASKED, false) + + fun choices(context: Context): Choices { + if (!choosable) return Choices() + val p = prefs(context) + return Choices( + android = p.getBoolean("share_android", true), + phone = p.getBoolean("share_phone", true), + cars = p.getBoolean("share_cars", true), + ) + } + + fun set(context: Context, enabled: Boolean, choices: Choices = choices(context)) { + prefs(context).edit() + .putBoolean(KEY_ENABLED, enabled) + .putBoolean(KEY_ASKED, true) + .putBoolean("share_android", choices.android) + .putBoolean("share_phone", choices.phone) + .putBoolean("share_cars", choices.cars) + .apply() + if (enabled) sendSoon(context) + else prefs(context).edit().remove(KEY_CARS).remove(KEY_SENT_DAY).apply() + } + + /** Gearslip was opened: send today's note unless one already went out. */ + fun onAppOpened(context: Context) { + context.getSystemService(JobScheduler::class.java)?.cancel(OLD_JOB_ID) + if (prefs(context).getString(KEY_SENT_DAY, null) != LocalDate.now().toString()) sendSoon(context) + } + + private val sending = AtomicBoolean(false) + @Volatile private var sendAgain = false + /** Guards the saved car list, which a session's end and a finished send both rewrite. */ + private val carsLock = Any() + + /** + * Sends off the calling thread. A call while a note is in flight sends once more after it, so + * a car recorded mid-send still goes out. + */ + private fun sendSoon(context: Context) { + if (!enabled(context)) return + if (!sending.compareAndSet(false, true)) { sendAgain = true; return } + val app = context.applicationContext + Thread({ + try { + do { sendAgain = false } while (send(app) && sendAgain) + } finally { + sending.set(false) + } + }, "gearslip-stats").start() + } + + private fun installId(context: Context): String { + val p = prefs(context) + return p.getString(KEY_ID, null) ?: UUID.randomUUID().toString().also { p.edit().putString(KEY_ID, it).apply() } + } + + /** + * A car session ended. [info] is null when the car hung up before it said who it is, for + * instance after rejecting the certificate; the protocol version and [outcome] still say a lot. + */ + fun recordSession( + context: Context, + info: ServiceDiscovery.HeadUnitInfo?, + protocol: String, + outcome: String, + screen: String = "", + dpi: Int = 0, + ) { + if (!enabled(context)) return + val p = prefs(context) + synchronized(carsLock) { addCar(p, info, protocol, outcome, screen, dpi) } + sendSoon(context) + } + + private fun addCar( + p: android.content.SharedPreferences, + info: ServiceDiscovery.HeadUnitInfo?, + protocol: String, + outcome: String, + screen: String, + dpi: Int, + ) { + val cars = runCatching { JSONArray(p.getString(KEY_CARS, "[]")) }.getOrDefault(JSONArray()) + val car = JSONObject() + .put("name", info?.headUnitName.orEmpty()) + .put("car", info?.carModel.orEmpty()) + .put("year", info?.carYear.orEmpty()) + .put("make", info?.headUnitMake.orEmpty()) + .put("model", info?.headUnitModel.orEmpty()) + .put("protocol", protocol) + .put("result", outcome) + if (screen.isNotEmpty()) car.put("screen", screen).put("dpi", dpi) + // The same car and outcome twice in a day is one fact, not two. + val seen = (0 until cars.length()).any { cars.optJSONObject(it)?.toString() == car.toString() } + if (!seen && cars.length() < MAX_CARS) cars.put(car) + p.edit().putString(KEY_CARS, cars.toString()).apply() + } + + // GrapheneOS reports the same build fields as stock Pixel firmware; its own apps give it away. + private fun isGrapheneOs(context: Context) = + runCatching { context.packageManager.getPackageInfo("app.grapheneos.setupwizard", 0) }.isSuccess + + private fun details(context: Context, c: Choices) = JSONObject().apply { + if (c.android) put("android", Build.VERSION.RELEASE).put("os", if (isGrapheneOs(context)) "GrapheneOS" else "other") + if (c.phone) put("phone", "${Build.MANUFACTURER} ${Build.MODEL}") + } + + /** + * What a note carries. With [everything], every field the driver shares, as Settings shows it; + * otherwise exactly what the next note would send, which leaves out unchanged phone details. + */ + fun preview(context: Context, everything: Boolean = false): JSONObject { + val c = choices(context) + val p = prefs(context) + val note = JSONObject() + .put("id", installId(context)) + .put("version", BuildConfig.VERSION_NAME) + .put("code", BuildConfig.VERSION_CODE) + .put("build", if (BuildConfig.DEBUG) "dev" else "release") + // Phone details change rarely, so they ride along only when they differ from what the + // server last got, or once a month in case it lost them. Unticked ones are left out, and + // the server keeps the last value it had. + val details = details(context, c) + val stale = p.getString(KEY_DETAILS_DAY, null)?.let { LocalDate.parse(it).plusDays(DETAILS_REFRESH_DAYS) <= LocalDate.now() } ?: true + if (details.length() > 0 && (everything || stale || details.toString() != p.getString(KEY_SENT_DETAILS, null))) { + details.keys().forEach { note.put(it, details.get(it)) } + } + if (c.cars) note.put("cars", runCatching { JSONArray(p.getString(KEY_CARS, "[]")) }.getOrDefault(JSONArray())) + return note + } + + /** Sends the note now; false when it didn't get through. */ + private fun send(context: Context): Boolean { + if (!enabled(context)) return true + val endpoint = BuildConfig.STATS_URL + if (endpoint.isBlank()) return true + val p = prefs(context) + val note = synchronized(carsLock) { preview(context) } + val sentCars = note.optJSONArray("cars")?.toString() + val sentDetails = if (note.has("android") || note.has("phone")) details(context, choices(context)).toString() else null + val body = note.toString().toByteArray() + return runCatching { + val conn = (URL("$endpoint/ping").openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + doOutput = true + connectTimeout = 15_000 + readTimeout = 15_000 + setRequestProperty("Content-Type", "application/json") + } + conn.outputStream.use { it.write(body) } + val code = conn.responseCode + conn.disconnect() + if (code in 200..299) { + // Sent: the cars start over, unless a session ended meanwhile and added one, which + // stays for the next note. + synchronized(carsLock) { + val edit = p.edit().putString(KEY_SENT_DAY, LocalDate.now().toString()) + if (sentDetails != null) edit.putString(KEY_SENT_DETAILS, sentDetails).putString(KEY_DETAILS_DAY, LocalDate.now().toString()) + if (sentCars == null || p.getString(KEY_CARS, "[]") == sentCars) edit.remove(KEY_CARS) + edit.apply() + } + GearslipLog.i("stats: note sent") + true + } else { + GearslipLog.w("stats: the server answered $code") + false + } + }.getOrElse { + GearslipLog.w("stats: could not send: ${it.message}") + false + } + } +} diff --git a/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt new file mode 100644 index 0000000..baeb2b9 --- /dev/null +++ b/gearslip/src/main/java/app/seb3thehacker/gearslip/ui/UsageStatsChoices.kt @@ -0,0 +1,148 @@ +package app.seb3thehacker.gearslip.ui + +import androidx.compose.animation.AnimatedVisibility +import androidx.compose.foundation.clickable +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.ColumnScope +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.selection.toggleable +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Info +import androidx.compose.material.icons.filled.KeyboardArrowDown +import androidx.compose.material.icons.filled.KeyboardArrowUp +import androidx.compose.material.icons.filled.Send +import androidx.compose.material3.Checkbox +import androidx.compose.material3.Icon +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.Surface +import androidx.compose.material3.Switch +import androidx.compose.material3.Text +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.semantics.Role +import androidx.compose.ui.text.AnnotatedString +import androidx.compose.ui.text.SpanStyle +import androidx.compose.ui.text.buildAnnotatedString +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.withStyle +import androidx.compose.ui.unit.dp +import app.seb3thehacker.gearslip.BuildConfig +import app.seb3thehacker.gearslip.stats.UsageStats + +internal const val USAGE_STATS_TITLE = "Help show which cars work" + +private const val INTRO_START = + "Gearslip can't tell which cars it works in unless drivers say so. Turn this on, and Gearslip " + + "sends a short note when you open it and after each drive, never in the background. The " + + "note carries the Gearslip version and whatever you tick below. " +private const val INTRO_PRIVACY = + "It holds no name, account, or location, only a random number so your phone counts once. " + + "Cloudflare carries the note, and the server keeps no IP address." +private const val INTRO_END = + " Nothing helps the project more: these notes tell the next driver whether their car " + + "will work before they install." + +/** The pitch for the usage notes, with the privacy promise in bold so it isn't missed. */ +internal val USAGE_STATS_INTRO: AnnotatedString = buildAnnotatedString { + append(INTRO_START) + withStyle(SpanStyle(fontWeight = FontWeight.Bold)) { append(INTRO_PRIVACY) } + append(INTRO_END) +} + +/** The usage notes card for the setup guide: the same rows Settings shows, minus the note. */ +@Composable +internal fun UsageStatsChoices(modifier: Modifier = Modifier) { + SettingsCard(modifier) { UsageStatsRows(showNote = false) } +} + +/** + * The on switch, what to share, and (with [showNote]) the note itself, saved as they change. + * Rows only: the caller puts them in a card. + */ +@Composable +internal fun ColumnScope.UsageStatsRows(showNote: Boolean) { + val context = LocalContext.current + var on by remember { mutableStateOf(UsageStats.enabled(context)) } + var choices by remember { mutableStateOf(UsageStats.choices(context)) } + var noteOpen by remember { mutableStateOf(false) } + fun save() = UsageStats.set(context, on, choices) + + SettingsRow( + "Send usage notes", + Modifier.toggleable(on, role = Role.Switch) { on = it; save() }, + subtitle = if (BuildConfig.STATS_URL.isBlank()) "This build has no stats server, so it sends nothing" else null, + icon = Icons.Filled.Send, + trailing = { Switch(checked = on, onCheckedChange = null) }, + ) + AnimatedVisibility(on) { + Column { + SettingsDivider() + Text( + "Also share", + style = MaterialTheme.typography.labelLarge, + color = MaterialTheme.colorScheme.primary, + modifier = Modifier.padding(start = 56.dp, end = 16.dp, top = 12.dp, bottom = 4.dp), + ) + Choice("Android version, and whether it's GrapheneOS", choices.android) { choices = choices.copy(android = it); save() } + Choice("Phone model", choices.phone) { choices = choices.copy(phone = it); save() } + Choice("Each car's head unit, model, year, and screen size, and whether it connected", choices.cars) { + choices = choices.copy(cars = it) + save() + } + if (showNote) { + SettingsDivider() + SettingsRow( + "What gets sent", + Modifier.clickable { noteOpen = !noteOpen }, + subtitle = "The Android version and phone model go only when they change, and once a month", + icon = Icons.Filled.Info, + trailing = { + Icon( + if (noteOpen) Icons.Filled.KeyboardArrowUp else Icons.Filled.KeyboardArrowDown, + contentDescription = if (noteOpen) "Hide the note" else "Show the note", + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + }, + ) + AnimatedVisibility(noteOpen) { + // Rebuilt from the saved choices each time one changes, so it always matches. + val note = remember(choices) { UsageStats.preview(context, everything = true).toString(2) } + Surface( + color = MaterialTheme.colorScheme.surfaceContainerLowest, + shape = MaterialTheme.shapes.medium, + modifier = Modifier.fillMaxWidth().padding(start = 16.dp, end = 16.dp, bottom = 16.dp), + ) { + Text( + note, + style = MaterialTheme.typography.bodySmall, + fontFamily = FontFamily.Monospace, + modifier = Modifier.padding(12.dp), + ) + } + } + } + } + } +} + +/** A share choice: the label lines up with the rows' titles, the checkbox sits at the end. */ +@Composable +private fun Choice(label: String, checked: Boolean, onChange: (Boolean) -> Unit) { + Row( + Modifier.fillMaxWidth().toggleable(checked, role = Role.Checkbox, onValueChange = onChange) + .padding(start = 56.dp, end = 16.dp, top = 4.dp, bottom = 4.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + Text(label, style = MaterialTheme.typography.bodyLarge, modifier = Modifier.weight(1f).padding(end = 12.dp)) + Checkbox(checked = checked, onCheckedChange = null, modifier = Modifier.padding(vertical = 8.dp)) + } +} diff --git a/gradle.properties b/gradle.properties index ad5fff4..6d8348a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,4 +21,6 @@ kotlin.code.style=official # resources declared in the library itself and none from the library's dependencies, # thereby reducing the size of the R class for that library android.nonTransitiveRClass=true -android.dependency.useConstraints=false \ No newline at end of file +android.dependency.useConstraints=false +# Where opted-in phones send their daily usage note (stats-worker/). Blank sends nothing. +gearslipStatsUrl=https://gearslip-stats.gearslip.workers.dev diff --git a/stats-worker/.gitignore b/stats-worker/.gitignore new file mode 100644 index 0000000..18433e3 --- /dev/null +++ b/stats-worker/.gitignore @@ -0,0 +1,3 @@ +.dev.vars +.wrangler/ +node_modules/ diff --git a/stats-worker/schema.sql b/stats-worker/schema.sql new file mode 100644 index 0000000..b04466d --- /dev/null +++ b/stats-worker/schema.sql @@ -0,0 +1,52 @@ +-- One row per phone per day: the first note that day counts, later ones update it. +CREATE TABLE IF NOT EXISTS pings ( + day TEXT NOT NULL, -- YYYY-MM-DD, UTC + uid TEXT NOT NULL, -- salted hash of the phone's random install ID + version TEXT NOT NULL, + code INTEGER, + build TEXT, + android TEXT, -- these four are from older builds; details now live in phones + os TEXT, + phone TEXT, + car_today INTEGER, + PRIMARY KEY (day, uid) +); + +-- Each car a phone saw that day, and how the session ended. +CREATE TABLE IF NOT EXISTS cars ( + day TEXT NOT NULL, + uid TEXT NOT NULL, + name TEXT, car TEXT, year TEXT, make TEXT, model TEXT, + protocol TEXT, + result TEXT, + screen TEXT, -- usable pixels, e.g. 1280x720 + dpi INTEGER, + PRIMARY KEY (day, uid, name, car, year, protocol, result) +); + +CREATE INDEX IF NOT EXISTS pings_uid ON pings (uid); + +-- GitHub's running download count for each release's APK, saved once a day. +CREATE TABLE IF NOT EXISTS downloads ( + day TEXT NOT NULL, + tag TEXT NOT NULL, + count INTEGER NOT NULL, + PRIMARY KEY (day, tag) +); + +-- Which releases are pre-releases, so the dashboard can pick out the latest full release. +CREATE TABLE IF NOT EXISTS releases ( + tag TEXT PRIMARY KEY, + prerelease INTEGER NOT NULL, + published TEXT +); + +-- Each phone's latest details. Phones send them only when they change (and once a month), so +-- a field a note leaves out keeps its last value. +CREATE TABLE IF NOT EXISTS phones ( + uid TEXT PRIMARY KEY, + android TEXT, + os TEXT, + phone TEXT, + updated TEXT +); diff --git a/stats-worker/src/index.js b/stats-worker/src/index.js new file mode 100644 index 0000000..fd0b3b1 --- /dev/null +++ b/stats-worker/src/index.js @@ -0,0 +1,293 @@ +// Gearslip usage notes: opt-in phones POST one short note a day, and a private dashboard counts +// them. No IP address is stored, and the phone's random ID is kept only as a salted hash. + +const MAX_BODY = 8 * 1024 +const MAX_CARS = 10 +const short = (v, n = 80) => (typeof v === 'string' ? v.slice(0, n) : null) + +// The tables as schema.sql has them, plus the columns added since, so a database made by an +// older version catches up on its own. Each runs once per Worker instance; an ALTER whose +// column already exists fails, and that's fine. +const SCHEMA = [ + `CREATE TABLE IF NOT EXISTS pings (day TEXT NOT NULL, uid TEXT NOT NULL, version TEXT NOT NULL, + code INTEGER, build TEXT, android TEXT, os TEXT, phone TEXT, car_today INTEGER, PRIMARY KEY (day, uid))`, + `CREATE TABLE IF NOT EXISTS cars (day TEXT NOT NULL, uid TEXT NOT NULL, name TEXT, car TEXT, year TEXT, + make TEXT, model TEXT, protocol TEXT, result TEXT, screen TEXT, dpi INTEGER, + PRIMARY KEY (day, uid, name, car, year, protocol, result))`, + `CREATE INDEX IF NOT EXISTS pings_uid ON pings (uid)`, + `CREATE TABLE IF NOT EXISTS downloads (day TEXT NOT NULL, tag TEXT NOT NULL, count INTEGER NOT NULL, PRIMARY KEY (day, tag))`, + `CREATE TABLE IF NOT EXISTS releases (tag TEXT PRIMARY KEY, prerelease INTEGER NOT NULL, published TEXT)`, + // Each phone's latest details. Phones send them only when they change, so a field a note + // leaves out keeps its last value. + `CREATE TABLE IF NOT EXISTS phones (uid TEXT PRIMARY KEY, android TEXT, os TEXT, phone TEXT, updated TEXT)`, +] +const ADDED_COLUMNS = [ + `ALTER TABLE pings ADD COLUMN os TEXT`, + `ALTER TABLE cars ADD COLUMN screen TEXT`, + `ALTER TABLE cars ADD COLUMN dpi INTEGER`, + // Older builds sent details in every note: carry the latest of those over once. + `INSERT OR IGNORE INTO phones (uid, android, os, phone, updated) + SELECT uid, android, os, phone, day FROM pings p + WHERE day = (SELECT MAX(day) FROM pings q WHERE q.uid = p.uid) + AND (android IS NOT NULL OR phone IS NOT NULL)`, +] +let schemaReady = null +function ensureSchema(env) { + schemaReady ??= (async () => { + for (const sql of SCHEMA) await env.DB.prepare(sql).run() + for (const sql of ADDED_COLUMNS) await env.DB.prepare(sql).run().catch(() => {}) + })().catch((e) => { schemaReady = null; throw e }) + return schemaReady +} + +export default { + async fetch(request, env) { + await ensureSchema(env) + const url = new URL(request.url) + if (request.method === 'POST' && url.pathname === '/ping') return ping(request, env) + if (request.method === 'GET' && (url.pathname === '/' || url.pathname === '/api/summary')) { + if (!env.DASHBOARD_KEY || url.searchParams.get('key') !== env.DASHBOARD_KEY) { + return new Response('Not found', { status: 404 }) + } + // The daily trigger normally saves today's counts; this covers a missed or late one. + await snapshotDownloads(env, false).catch((e) => console.log('github:', e.message)) + const summary = await summarize(env) + if (url.pathname === '/api/summary') return Response.json(summary) + return new Response(dashboard(summary), { headers: { 'content-type': 'text/html; charset=utf-8' } }) + } + return new Response('Not found', { status: 404 }) + }, + + async scheduled(event, env) { + await ensureSchema(env) + await snapshotDownloads(env, true) + }, +} + +// Saves each release's APK download count for today. GitHub only keeps a running total, so +// downloads per day come from the difference between days. +async function snapshotDownloads(env, force) { + if (!env.GITHUB_REPO) return + const day = today() + // Today's counts and the release list both have to be there; either missing means fetch again. + if (!force && (await env.DB.prepare(`SELECT 1 WHERE EXISTS (SELECT 1 FROM downloads WHERE day = ?) + AND EXISTS (SELECT 1 FROM releases)`).bind(day).first())) return + const headers = { 'user-agent': 'gearslip-stats', accept: 'application/vnd.github+json' } + if (env.GITHUB_TOKEN) headers.authorization = `Bearer ${env.GITHUB_TOKEN}` + const res = await fetch(`https://api.github.com/repos/${env.GITHUB_REPO}/releases?per_page=100`, { headers }) + if (!res.ok) throw new Error(`GitHub answered ${res.status}`) + const statements = (await res.json()).filter((release) => !release.draft).flatMap((release) => { + const count = release.assets.filter((a) => a.name.endsWith('.apk')).reduce((n, a) => n + a.download_count, 0) + return [ + env.DB.prepare('INSERT OR REPLACE INTO downloads (day, tag, count) VALUES (?, ?, ?)').bind(day, release.tag_name, count), + env.DB.prepare('INSERT OR REPLACE INTO releases (tag, prerelease, published) VALUES (?, ?, ?)') + .bind(release.tag_name, release.prerelease ? 1 : 0, release.published_at), + ] + }) + if (statements.length) await env.DB.batch(statements) +} + +const today = () => new Date().toISOString().slice(0, 10) +// Tags are "v0.1.10-alpha" or "0.0.05-alpha"; the app reports "0.1.10-alpha". +const versionOf = (tag) => tag.replace(/^v/, '') + +async function ping(request, env) { + const text = await request.text() + if (text.length > MAX_BODY) return new Response('Too large', { status: 413 }) + let note + try { + note = JSON.parse(text) + } catch { + return new Response('Bad JSON', { status: 400 }) + } + const id = short(note.id, 64) + const version = short(note.version, 32) + if (!id || !version) return new Response('Missing id or version', { status: 400 }) + + const uid = await hash(`${env.SALT}:${id}`) + const day = today() + const android = short(note.android, 16), os = short(note.os, 16), phone = short(note.phone) + const statements = [ + env.DB.prepare( + `INSERT INTO pings (day, uid, version, code, build) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (day, uid) DO UPDATE SET version = excluded.version, code = excluded.code, build = excluded.build`, + ).bind(day, uid, version, Number.isInteger(note.code) ? note.code : null, short(note.build, 16)), + ] + if (android || os || phone) { + statements.push( + env.DB.prepare( + `INSERT INTO phones (uid, android, os, phone, updated) VALUES (?, ?, ?, ?, ?) + ON CONFLICT (uid) DO UPDATE SET android = COALESCE(excluded.android, android), + os = COALESCE(excluded.os, os), phone = COALESCE(excluded.phone, phone), updated = excluded.updated`, + ).bind(uid, android, os, phone, day), + ) + } + for (const car of Array.isArray(note.cars) ? note.cars.slice(0, MAX_CARS) : []) { + statements.push( + env.DB.prepare( + `INSERT OR IGNORE INTO cars (day, uid, name, car, year, make, model, protocol, result, screen, dpi) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).bind(day, uid, short(car.name) ?? '', short(car.car) ?? '', short(car.year, 8) ?? '', + short(car.make) ?? '', short(car.model) ?? '', short(car.protocol, 8) ?? '', short(car.result) ?? '', + short(car.screen, 16), Number.isInteger(car.dpi) ? car.dpi : null), + ) + } + await env.DB.batch(statements) + return new Response(null, { status: 204 }) +} + +async function hash(text) { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)) + return [...new Uint8Array(digest)].slice(0, 16).map((b) => b.toString(16).padStart(2, '0')).join('') +} + +async function summarize(env) { + const all = async (sql) => (await env.DB.prepare(sql).all()).results + const one = async (sql) => (await env.DB.prepare(sql).first()) ?? {} + const latest = await all(`SELECT d.tag, d.count, r.prerelease, r.published FROM downloads d + LEFT JOIN releases r ON r.tag = d.tag + WHERE d.day = (SELECT MAX(day) FROM downloads)`) + const seen = Object.fromEntries((await all(`SELECT version, COUNT(DISTINCT uid) AS n FROM pings + WHERE build = 'release' GROUP BY version`)).map((r) => [r.version, r.n])) + // The newest full release, the one most people are sent to; pre-releases don't count. + const newest = await one(`SELECT tag FROM releases WHERE prerelease = 0 ORDER BY published DESC LIMIT 1`) + const latestRelease = newest.tag ? { tag: newest.tag, downloads: latest.find((r) => r.tag === newest.tag)?.count ?? 0 } : null + const releases = latest.map((r) => ({ + tag: r.tag, + kind: r.prerelease ? 'Pre-release' : r.prerelease === 0 ? 'Release' : '', + published: (r.published ?? '').slice(0, 10), + downloads: r.count, + phones: seen[versionOf(r.tag)] ?? 0, + })) + .map((r) => ({ ...r, share: r.phones && r.downloads ? `${Math.round((100 * r.phones) / r.downloads)}%` : '' })) + .sort((a, b) => b.published.localeCompare(a.published)) + // Only releases that have sent anything can say what share of people send; older ones predate the notes. + const counted = releases.filter((r) => r.phones > 0) + const countedDownloads = counted.reduce((n, r) => n + r.downloads, 0) + const countedPhones = counted.reduce((n, r) => n + r.phones, 0) + const totals = await all(`SELECT day, SUM(count) AS total FROM downloads GROUP BY day ORDER BY day`) + const newPhones = Object.fromEntries((await all(`SELECT first_day AS day, COUNT(*) AS n FROM + (SELECT uid, MIN(day) AS first_day FROM pings WHERE build = 'release' GROUP BY uid) + GROUP BY first_day`)).map((r) => [r.day, r.n])) + const perDay = totals.slice(1).map((r, i) => ({ day: r.day, downloads: r.total - totals[i].total, new_phones: newPhones[r.day] ?? 0 })) + return { + latestRelease, + releases, + totalDownloads: releases.reduce((n, r) => n + r.downloads, 0), + share: countedDownloads ? Math.round((100 * countedPhones) / countedDownloads) : null, + perDay: perDay.slice(-60).reverse(), + daily: await all(`SELECT p.day, COUNT(*) AS phones, + SUM(EXISTS (SELECT 1 FROM cars c WHERE c.uid = p.uid AND c.day = p.day + AND c.result LIKE 'connected%')) AS in_a_car, + SUM(p.day = f.first_day) AS new_phones + FROM pings p JOIN (SELECT uid, MIN(day) AS first_day FROM pings GROUP BY uid) f + ON f.uid = p.uid + WHERE p.day >= date('now', '-60 days') GROUP BY p.day ORDER BY p.day`), + // Android runs no code at uninstall, so a phone that stopped sending is the nearest sign. + new30: (await one(`SELECT COUNT(*) AS n FROM (SELECT uid, MIN(day) AS first_day FROM pings GROUP BY uid) + WHERE first_day >= date('now', '-30 days')`)).n ?? 0, + quiet: (await one(`SELECT COUNT(*) AS n FROM (SELECT uid, MAX(day) AS last_day FROM pings GROUP BY uid) + WHERE last_day < date('now', '-14 days')`)).n ?? 0, + everyone: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM pings`)).n ?? 0, + active7: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM pings WHERE day >= date('now', '-7 days')`)).n ?? 0, + active30: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM pings WHERE day >= date('now', '-30 days')`)).n ?? 0, + driving30: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM cars + WHERE result LIKE 'connected%' AND day >= date('now', '-30 days')`)).n ?? 0, + versions: await all(`SELECT version, COUNT(DISTINCT uid) AS phones FROM pings + WHERE day >= date('now', '-7 days') GROUP BY version ORDER BY phones DESC`), + // Details come from each phone's latest, counted for phones that sent anything in 30 days. + android: await all(`SELECT android, COUNT(*) AS phones FROM phones + WHERE android IS NOT NULL AND uid IN (SELECT uid FROM pings WHERE day >= date('now', '-30 days')) + GROUP BY android ORDER BY phones DESC`), + screens: await all(`SELECT screen, COUNT(DISTINCT uid) AS phones FROM cars + WHERE screen IS NOT NULL GROUP BY screen ORDER BY phones DESC`), + os: await all(`SELECT os, COUNT(*) AS phones FROM phones + WHERE os IS NOT NULL AND uid IN (SELECT uid FROM pings WHERE day >= date('now', '-30 days')) + GROUP BY os ORDER BY phones DESC`), + phones: await all(`SELECT phone, COUNT(*) AS phones FROM phones + WHERE phone IS NOT NULL AND uid IN (SELECT uid FROM pings WHERE day >= date('now', '-30 days')) + GROUP BY phone ORDER BY phones DESC LIMIT 30`), + cars: await all(`SELECT name, car, year, make, model, protocol, + GROUP_CONCAT(DISTINCT screen || CASE WHEN dpi THEN ' @ ' || dpi || ' dpi' ELSE '' END) AS screen, + SUM(result = 'connected') AS connected, + SUM(result != 'connected') AS failed, + GROUP_CONCAT(DISTINCT CASE WHEN result != 'connected' THEN result END) AS failures, + COUNT(DISTINCT uid) AS phones, MAX(day) AS last_seen + FROM cars GROUP BY name, car, year, make, model, protocol + ORDER BY phones DESC, connected DESC LIMIT 200`), + } +} + +const esc = (v) => String(v ?? '').replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c]) + +function table(rows, columns) { + if (!rows.length) return '

Nothing yet.

' + const head = columns.map(([, label]) => `${esc(label)}`).join('') + const body = rows.map((r) => `${columns.map(([key]) => `${esc(r[key])}`).join('')}`).join('') + return `
${head}${body}
` +} + +function chart(daily) { + if (!daily.length) return '

No notes yet.

' + const w = 600, h = 160, max = Math.max(...daily.map((d) => d.phones), 1), bw = w / daily.length + const bars = daily.map((d, i) => { + const ph = (d.phones / max) * (h - 20), ch = ((d.in_a_car ?? 0) / max) * (h - 20) + return `${esc(d.day)}: ${d.phones} phones` + + `${esc(d.day)}: ${d.in_a_car ?? 0} in a car` + }).join('') + return `${bars} +

phones that day of those, in a car

` +} + +function dashboard(s) { + return ` +Gearslip usage + +

Gearslip usage

From phones that opted in to daily notes.

+${s.latestRelease ? `
${s.latestRelease.downloads}downloads of ${esc(s.latestRelease.tag)}, the latest release
` : ''} +
+
${s.active7}phones, last 7 days
+
${s.active30}phones, last 30 days
+
${s.driving30}used in a car, last 30 days
+
${s.new30}new phones, last 30 days
+
${s.quiet}gone quiet (nothing in 14 days)
+
${s.everyone}phones ever
+
+

Android tells an app nothing when it's uninstalled, so "gone quiet" is the nearest +sign: phones that sent notes before and stopped. Turning the notes off looks the same.

+

Downloads and phones

+
+
${s.totalDownloads}APK downloads on GitHub, all releases
+
${s.share == null ? '–' : s.share + '%'}of downloads send notes
+
+

The share counts only releases that have sent at least one note, and only release builds. +A download isn't always a new person: updating means downloading again, and some people download +twice. So read it as a rough guide, and multiply the phones above by about ${s.share ? Math.round(100 / s.share) : '–'} to estimate everyone.

+${table(s.releases, [['tag', 'Release'], ['kind', 'Type'], ['published', 'Published'], ['downloads', 'Downloads'], ['phones', 'Phones sending'], ['share', 'Share']])} +

Downloads per day

${s.perDay.length ? table(s.perDay, [['day', 'Day'], ['downloads', 'Downloads'], ['new_phones', 'New phones']]) + : '

Counts are saved once a day, so this fills in from tomorrow.

'} +

Phones per day

${chart(s.daily)} +

New phones per day

${table(s.daily.filter((d) => d.new_phones > 0).slice().reverse(), [['day', 'Day'], ['new_phones', 'New phones']])} +

Cars

${table(s.cars, [['name', 'Head unit'], ['car', 'Car'], ['year', 'Year'], ['make', 'Maker'], ['model', 'Unit model'], + ['protocol', 'Protocol'], ['screen', 'Screen'], ['connected', 'Connected'], ['failed', 'Failed'], ['failures', 'Why it failed'], ['phones', 'Phones'], ['last_seen', 'Last seen']])} +

Car screens

${table(s.screens, [['screen', 'Usable size'], ['phones', 'Phones']])} +

Versions, last 7 days

${table(s.versions, [['version', 'Version'], ['phones', 'Phones']])} +

Phones

${table(s.phones, [['phone', 'Phone'], ['phones', 'Count']])} +

GrapheneOS, last 30 days

${table(s.os, [['os', 'System'], ['phones', 'Phones']])} +

Android

${table(s.android, [['android', 'Android'], ['phones', 'Count']])} +` +} diff --git a/stats-worker/wrangler.toml b/stats-worker/wrangler.toml new file mode 100644 index 0000000..e80c185 --- /dev/null +++ b/stats-worker/wrangler.toml @@ -0,0 +1,23 @@ +# Gearslip's opt-in usage notes. Deploy with `npx wrangler deploy` from this folder. +# Setup, once: +# npx wrangler d1 create gearslip-stats (paste the id it prints below) +# npx wrangler d1 execute gearslip-stats --remote --file schema.sql +# npx wrangler secret put SALT (any long random string; keep it) +# npx wrangler secret put DASHBOARD_KEY (the key for the dashboard link) +name = "gearslip-stats" +main = "src/index.js" +compatibility_date = "2026-10-01" + +[[d1_databases]] +binding = "DB" +database_name = "gearslip-stats" +database_id = "201521a7-0b4a-4fff-ab11-d575b821528a" + +# The public repo whose release downloads the dashboard compares against. If GitHub starts +# refusing (60 requests an hour without a token), add one: npx wrangler secret put GITHUB_TOKEN +[vars] +GITHUB_REPO = "Seb3thehacker/gearslip" + +# Save the download counts once a day, so the dashboard can show downloads per day. +[triggers] +crons = ["17 6 * * *"]