From 25b8bbb1a54231784bea7c85cdbaa01a69942191 Mon Sep 17 00:00:00 2001 From: Seb3thehacker Date: Tue, 6 Oct 2026 23:33:18 -0700 Subject: [PATCH] keep the stats server out of the app repo --- .gitignore | 3 +- stats-worker/.gitignore | 3 - stats-worker/schema.sql | 52 ------- stats-worker/src/index.js | 299 ------------------------------------- stats-worker/wrangler.toml | 23 --- 5 files changed, 2 insertions(+), 378 deletions(-) delete mode 100644 stats-worker/.gitignore delete mode 100644 stats-worker/schema.sql delete mode 100644 stats-worker/src/index.js delete mode 100644 stats-worker/wrangler.toml diff --git a/.gitignore b/.gitignore index fdd2771..d72c470 100644 --- a/.gitignore +++ b/.gitignore @@ -29,4 +29,5 @@ CLAUDE.md # Other archive/ -.wrangler/ \ No newline at end of file +.wrangler/ +stats-worker/ diff --git a/stats-worker/.gitignore b/stats-worker/.gitignore deleted file mode 100644 index 18433e3..0000000 --- a/stats-worker/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -.dev.vars -.wrangler/ -node_modules/ diff --git a/stats-worker/schema.sql b/stats-worker/schema.sql deleted file mode 100644 index b04466d..0000000 --- a/stats-worker/schema.sql +++ /dev/null @@ -1,52 +0,0 @@ --- One row per phone per day: the first note that day counts, later ones update it. -CREATE TABLE IF NOT EXISTS pings ( - day TEXT NOT NULL, -- YYYY-MM-DD, UTC - uid TEXT NOT NULL, -- salted hash of the phone's random install ID - version TEXT NOT NULL, - code INTEGER, - build TEXT, - android TEXT, -- these four are from older builds; details now live in phones - os TEXT, - phone TEXT, - car_today INTEGER, - PRIMARY KEY (day, uid) -); - --- Each car a phone saw that day, and how the session ended. -CREATE TABLE IF NOT EXISTS cars ( - day TEXT NOT NULL, - uid TEXT NOT NULL, - name TEXT, car TEXT, year TEXT, make TEXT, model TEXT, - protocol TEXT, - result TEXT, - screen TEXT, -- usable pixels, e.g. 1280x720 - dpi INTEGER, - PRIMARY KEY (day, uid, name, car, year, protocol, result) -); - -CREATE INDEX IF NOT EXISTS pings_uid ON pings (uid); - --- GitHub's running download count for each release's APK, saved once a day. -CREATE TABLE IF NOT EXISTS downloads ( - day TEXT NOT NULL, - tag TEXT NOT NULL, - count INTEGER NOT NULL, - PRIMARY KEY (day, tag) -); - --- Which releases are pre-releases, so the dashboard can pick out the latest full release. -CREATE TABLE IF NOT EXISTS releases ( - tag TEXT PRIMARY KEY, - prerelease INTEGER NOT NULL, - published TEXT -); - --- Each phone's latest details. Phones send them only when they change (and once a month), so --- a field a note leaves out keeps its last value. -CREATE TABLE IF NOT EXISTS phones ( - uid TEXT PRIMARY KEY, - android TEXT, - os TEXT, - phone TEXT, - updated TEXT -); diff --git a/stats-worker/src/index.js b/stats-worker/src/index.js deleted file mode 100644 index 6d34d12..0000000 --- a/stats-worker/src/index.js +++ /dev/null @@ -1,299 +0,0 @@ -// Gearslip usage notes: opt-in phones POST one short note a day, and a private dashboard counts -// them. No IP address is stored, and the phone's random ID is kept only as a salted hash. - -const MAX_BODY = 8 * 1024 -const MAX_CARS = 10 -const short = (v, n = 80) => (typeof v === 'string' ? v.slice(0, n) : null) - -// The tables as schema.sql has them, plus the columns added since, so a database made by an -// older version catches up on its own. Each runs once per Worker instance; an ALTER whose -// column already exists fails, and that's fine. -const SCHEMA = [ - `CREATE TABLE IF NOT EXISTS pings (day TEXT NOT NULL, uid TEXT NOT NULL, version TEXT NOT NULL, - code INTEGER, build TEXT, android TEXT, os TEXT, phone TEXT, car_today INTEGER, PRIMARY KEY (day, uid))`, - `CREATE TABLE IF NOT EXISTS cars (day TEXT NOT NULL, uid TEXT NOT NULL, name TEXT, car TEXT, year TEXT, - make TEXT, model TEXT, protocol TEXT, result TEXT, screen TEXT, dpi INTEGER, - PRIMARY KEY (day, uid, name, car, year, protocol, result))`, - `CREATE INDEX IF NOT EXISTS pings_uid ON pings (uid)`, - `CREATE TABLE IF NOT EXISTS downloads (day TEXT NOT NULL, tag TEXT NOT NULL, count INTEGER NOT NULL, PRIMARY KEY (day, tag))`, - `CREATE TABLE IF NOT EXISTS releases (tag TEXT PRIMARY KEY, prerelease INTEGER NOT NULL, published TEXT)`, - // Each phone's latest details. Phones send them only when they change, so a field a note - // leaves out keeps its last value. - `CREATE TABLE IF NOT EXISTS phones (uid TEXT PRIMARY KEY, android TEXT, os TEXT, phone TEXT, updated TEXT)`, -] -const ADDED_COLUMNS = [ - `ALTER TABLE pings ADD COLUMN os TEXT`, - `ALTER TABLE cars ADD COLUMN screen TEXT`, - `ALTER TABLE cars ADD COLUMN dpi INTEGER`, - // Older builds sent details in every note: carry the latest of those over once. - `INSERT OR IGNORE INTO phones (uid, android, os, phone, updated) - SELECT uid, android, os, phone, day FROM pings p - WHERE day = (SELECT MAX(day) FROM pings q WHERE q.uid = p.uid) - AND (android IS NOT NULL OR phone IS NOT NULL)`, -] -let schemaReady = null -function ensureSchema(env) { - schemaReady ??= (async () => { - for (const sql of SCHEMA) await env.DB.prepare(sql).run() - for (const sql of ADDED_COLUMNS) await env.DB.prepare(sql).run().catch(() => {}) - })().catch((e) => { schemaReady = null; throw e }) - return schemaReady -} - -export default { - async fetch(request, env) { - await ensureSchema(env) - const url = new URL(request.url) - if (request.method === 'POST' && url.pathname === '/ping') return ping(request, env) - if (request.method === 'GET' && (url.pathname === '/' || url.pathname === '/api/summary')) { - if (!env.DASHBOARD_KEY || url.searchParams.get('key') !== env.DASHBOARD_KEY) { - return new Response('Not found', { status: 404 }) - } - // Opening the dashboard refreshes today's counts, at most every few minutes, so it matches - // GitHub instead of showing the morning's numbers. GitHub allows 60 requests an hour. - const stale = Date.now() - lastFetched > REFRESH_MS - await snapshotDownloads(env, stale).catch((e) => console.log('github:', e.message)) - const summary = await summarize(env) - if (url.pathname === '/api/summary') return Response.json(summary) - return new Response(dashboard(summary), { headers: { 'content-type': 'text/html; charset=utf-8' } }) - } - return new Response('Not found', { status: 404 }) - }, - - async scheduled(event, env) { - await ensureSchema(env) - await snapshotDownloads(env, true) - }, -} - -// Saves each release's APK download count for today. GitHub only keeps a running total, so -// downloads per day come from the difference between days. -let lastFetched = 0 -const REFRESH_MS = 5 * 60 * 1000 - -async function snapshotDownloads(env, force) { - if (!env.GITHUB_REPO) return - const day = today() - // Today's counts and the release list both have to be there; either missing means fetch again. - if (!force && (await env.DB.prepare(`SELECT 1 WHERE EXISTS (SELECT 1 FROM downloads WHERE day = ?) - AND EXISTS (SELECT 1 FROM releases)`).bind(day).first())) return - const headers = { 'user-agent': 'gearslip-stats', accept: 'application/vnd.github+json' } - if (env.GITHUB_TOKEN) headers.authorization = `Bearer ${env.GITHUB_TOKEN}` - const res = await fetch(`https://api.github.com/repos/${env.GITHUB_REPO}/releases?per_page=100`, { headers }) - if (!res.ok) throw new Error(`GitHub answered ${res.status}`) - const statements = (await res.json()).filter((release) => !release.draft).flatMap((release) => { - const count = release.assets.filter((a) => a.name.endsWith('.apk')).reduce((n, a) => n + a.download_count, 0) - return [ - env.DB.prepare('INSERT OR REPLACE INTO downloads (day, tag, count) VALUES (?, ?, ?)').bind(day, release.tag_name, count), - env.DB.prepare('INSERT OR REPLACE INTO releases (tag, prerelease, published) VALUES (?, ?, ?)') - .bind(release.tag_name, release.prerelease ? 1 : 0, release.published_at), - ] - }) - if (statements.length) await env.DB.batch(statements) - lastFetched = Date.now() -} - -const today = () => new Date().toISOString().slice(0, 10) -// Tags are "v0.1.10-alpha" or "0.0.05-alpha"; the app reports "0.1.10-alpha". -const versionOf = (tag) => tag.replace(/^v/, '') - -async function ping(request, env) { - const text = await request.text() - if (text.length > MAX_BODY) return new Response('Too large', { status: 413 }) - let note - try { - note = JSON.parse(text) - } catch { - return new Response('Bad JSON', { status: 400 }) - } - const id = short(note.id, 64) - const version = short(note.version, 32) - if (!id || !version) return new Response('Missing id or version', { status: 400 }) - - const uid = await hash(`${env.SALT}:${id}`) - const day = today() - const android = short(note.android, 16), os = short(note.os, 16), phone = short(note.phone) - const statements = [ - env.DB.prepare( - `INSERT INTO pings (day, uid, version, code, build) VALUES (?, ?, ?, ?, ?) - ON CONFLICT (day, uid) DO UPDATE SET version = excluded.version, code = excluded.code, build = excluded.build`, - ).bind(day, uid, version, Number.isInteger(note.code) ? note.code : null, short(note.build, 16)), - ] - if (android || os || phone) { - statements.push( - env.DB.prepare( - `INSERT INTO phones (uid, android, os, phone, updated) VALUES (?, ?, ?, ?, ?) - ON CONFLICT (uid) DO UPDATE SET android = COALESCE(excluded.android, android), - os = COALESCE(excluded.os, os), phone = COALESCE(excluded.phone, phone), updated = excluded.updated`, - ).bind(uid, android, os, phone, day), - ) - } - for (const car of Array.isArray(note.cars) ? note.cars.slice(0, MAX_CARS) : []) { - statements.push( - env.DB.prepare( - `INSERT OR IGNORE INTO cars (day, uid, name, car, year, make, model, protocol, result, screen, dpi) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - ).bind(day, uid, short(car.name) ?? '', short(car.car) ?? '', short(car.year, 8) ?? '', - short(car.make) ?? '', short(car.model) ?? '', short(car.protocol, 8) ?? '', short(car.result) ?? '', - short(car.screen, 16), Number.isInteger(car.dpi) ? car.dpi : null), - ) - } - await env.DB.batch(statements) - return new Response(null, { status: 204 }) -} - -async function hash(text) { - const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text)) - return [...new Uint8Array(digest)].slice(0, 16).map((b) => b.toString(16).padStart(2, '0')).join('') -} - -async function summarize(env) { - const all = async (sql) => (await env.DB.prepare(sql).all()).results - const one = async (sql) => (await env.DB.prepare(sql).first()) ?? {} - const latest = await all(`SELECT d.tag, d.count, r.prerelease, r.published FROM downloads d - LEFT JOIN releases r ON r.tag = d.tag - WHERE d.day = (SELECT MAX(day) FROM downloads)`) - const seen = Object.fromEntries((await all(`SELECT version, COUNT(DISTINCT uid) AS n FROM pings - WHERE build = 'release' GROUP BY version`)).map((r) => [r.version, r.n])) - // The newest full release, the one most people are sent to; pre-releases don't count. - const newest = await one(`SELECT tag FROM releases WHERE prerelease = 0 ORDER BY published DESC LIMIT 1`) - const latestRelease = newest.tag ? { tag: newest.tag, downloads: latest.find((r) => r.tag === newest.tag)?.count ?? 0 } : null - const releases = latest.map((r) => ({ - tag: r.tag, - kind: r.prerelease ? 'Pre-release' : r.prerelease === 0 ? 'Release' : '', - published: (r.published ?? '').slice(0, 10), - downloads: r.count, - phones: seen[versionOf(r.tag)] ?? 0, - })) - .map((r) => ({ ...r, share: r.phones && r.downloads ? `${Math.round((100 * r.phones) / r.downloads)}%` : '' })) - .sort((a, b) => b.published.localeCompare(a.published)) - // Only releases that have sent anything can say what share of people send; older ones predate the notes. - const counted = releases.filter((r) => r.phones > 0) - const countedDownloads = counted.reduce((n, r) => n + r.downloads, 0) - const countedPhones = counted.reduce((n, r) => n + r.phones, 0) - const totals = await all(`SELECT day, SUM(count) AS total FROM downloads GROUP BY day ORDER BY day`) - const newPhones = Object.fromEntries((await all(`SELECT first_day AS day, COUNT(*) AS n FROM - (SELECT uid, MIN(day) AS first_day FROM pings WHERE build = 'release' GROUP BY uid) - GROUP BY first_day`)).map((r) => [r.day, r.n])) - const perDay = totals.slice(1).map((r, i) => ({ day: r.day, downloads: r.total - totals[i].total, new_phones: newPhones[r.day] ?? 0 })) - return { - latestRelease, - releases, - totalDownloads: releases.reduce((n, r) => n + r.downloads, 0), - share: countedDownloads ? Math.round((100 * countedPhones) / countedDownloads) : null, - perDay: perDay.slice(-60).reverse(), - daily: await all(`SELECT p.day, COUNT(*) AS phones, - SUM(EXISTS (SELECT 1 FROM cars c WHERE c.uid = p.uid AND c.day = p.day - AND c.result LIKE 'connected%')) AS in_a_car, - SUM(p.day = f.first_day) AS new_phones - FROM pings p JOIN (SELECT uid, MIN(day) AS first_day FROM pings GROUP BY uid) f - ON f.uid = p.uid - WHERE p.day >= date('now', '-60 days') GROUP BY p.day ORDER BY p.day`), - // Android runs no code at uninstall, so a phone that stopped sending is the nearest sign. - new30: (await one(`SELECT COUNT(*) AS n FROM (SELECT uid, MIN(day) AS first_day FROM pings GROUP BY uid) - WHERE first_day >= date('now', '-30 days')`)).n ?? 0, - quiet: (await one(`SELECT COUNT(*) AS n FROM (SELECT uid, MAX(day) AS last_day FROM pings GROUP BY uid) - WHERE last_day < date('now', '-14 days')`)).n ?? 0, - everyone: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM pings`)).n ?? 0, - active7: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM pings WHERE day >= date('now', '-7 days')`)).n ?? 0, - active30: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM pings WHERE day >= date('now', '-30 days')`)).n ?? 0, - driving30: (await one(`SELECT COUNT(DISTINCT uid) AS n FROM cars - WHERE result LIKE 'connected%' AND day >= date('now', '-30 days')`)).n ?? 0, - versions: await all(`SELECT version, COUNT(DISTINCT uid) AS phones FROM pings - WHERE day >= date('now', '-7 days') GROUP BY version ORDER BY phones DESC`), - // Details come from each phone's latest, counted for phones that sent anything in 30 days. - android: await all(`SELECT android, COUNT(*) AS phones FROM phones - WHERE android IS NOT NULL AND uid IN (SELECT uid FROM pings WHERE day >= date('now', '-30 days')) - GROUP BY android ORDER BY phones DESC`), - screens: await all(`SELECT screen, COUNT(DISTINCT uid) AS phones FROM cars - WHERE screen IS NOT NULL GROUP BY screen ORDER BY phones DESC`), - os: await all(`SELECT os, COUNT(*) AS phones FROM phones - WHERE os IS NOT NULL AND uid IN (SELECT uid FROM pings WHERE day >= date('now', '-30 days')) - GROUP BY os ORDER BY phones DESC`), - phones: await all(`SELECT phone, COUNT(*) AS phones FROM phones - WHERE phone IS NOT NULL AND uid IN (SELECT uid FROM pings WHERE day >= date('now', '-30 days')) - GROUP BY phone ORDER BY phones DESC LIMIT 30`), - cars: await all(`SELECT name, car, year, make, model, protocol, - GROUP_CONCAT(DISTINCT screen || CASE WHEN dpi THEN ' @ ' || dpi || ' dpi' ELSE '' END) AS screen, - SUM(result = 'connected') AS connected, - SUM(result != 'connected') AS failed, - GROUP_CONCAT(DISTINCT CASE WHEN result != 'connected' THEN result END) AS failures, - COUNT(DISTINCT uid) AS phones, MAX(day) AS last_seen - FROM cars GROUP BY name, car, year, make, model, protocol - ORDER BY phones DESC, connected DESC LIMIT 200`), - } -} - -const esc = (v) => String(v ?? '').replace(/[&<>"]/g, (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"' })[c]) - -function table(rows, columns) { - if (!rows.length) return '

Nothing yet.

' - const head = columns.map(([, label]) => `${esc(label)}`).join('') - const body = rows.map((r) => `${columns.map(([key]) => `${esc(r[key])}`).join('')}`).join('') - return `
${head}${body}
` -} - -function chart(daily) { - if (!daily.length) return '

No notes yet.

' - const w = 600, h = 160, max = Math.max(...daily.map((d) => d.phones), 1), bw = w / daily.length - const bars = daily.map((d, i) => { - const ph = (d.phones / max) * (h - 20), ch = ((d.in_a_car ?? 0) / max) * (h - 20) - return `${esc(d.day)}: ${d.phones} phones` + - `${esc(d.day)}: ${d.in_a_car ?? 0} in a car` - }).join('') - return `${bars} -

phones that day of those, in a car

` -} - -function dashboard(s) { - return ` -Gearslip usage - -

Gearslip usage

From phones that opted in to daily notes.

-${s.latestRelease ? `
${s.latestRelease.downloads}downloads of ${esc(s.latestRelease.tag)}, the latest release
` : ''} -
-
${s.active7}phones, last 7 days
-
${s.active30}phones, last 30 days
-
${s.driving30}used in a car, last 30 days
-
${s.new30}new phones, last 30 days
-
${s.quiet}gone quiet (nothing in 14 days)
-
${s.everyone}phones ever
-
-

Android tells an app nothing when it's uninstalled, so "gone quiet" is the nearest -sign: phones that sent notes before and stopped. Turning the notes off looks the same.

-

Downloads and phones

-
-
${s.totalDownloads}APK downloads on GitHub, all releases
-
${s.share == null ? '–' : s.share + '%'}of downloads send notes
-
-

The share counts only releases that have sent at least one note, and only release builds. -A download isn't always a new person: updating means downloading again, and some people download -twice. So read it as a rough guide, and multiply the phones above by about ${s.share ? Math.round(100 / s.share) : '–'} to estimate everyone.

-${table(s.releases, [['tag', 'Release'], ['kind', 'Type'], ['published', 'Published'], ['downloads', 'Downloads'], ['phones', 'Phones sending'], ['share', 'Share']])} -

Downloads per day

${s.perDay.length ? table(s.perDay, [['day', 'Day'], ['downloads', 'Downloads'], ['new_phones', 'New phones']]) - : '

Counts are saved once a day, so this fills in from tomorrow.

'} -

Phones per day

${chart(s.daily)} -

New phones per day

${table(s.daily.filter((d) => d.new_phones > 0).slice().reverse(), [['day', 'Day'], ['new_phones', 'New phones']])} -

Cars

${table(s.cars, [['name', 'Head unit'], ['car', 'Car'], ['year', 'Year'], ['make', 'Maker'], ['model', 'Unit model'], - ['protocol', 'Protocol'], ['screen', 'Screen'], ['connected', 'Connected'], ['failed', 'Failed'], ['failures', 'Why it failed'], ['phones', 'Phones'], ['last_seen', 'Last seen']])} -

Car screens

${table(s.screens, [['screen', 'Usable size'], ['phones', 'Phones']])} -

Versions, last 7 days

${table(s.versions, [['version', 'Version'], ['phones', 'Phones']])} -

Phones

${table(s.phones, [['phone', 'Phone'], ['phones', 'Count']])} -

GrapheneOS, last 30 days

${table(s.os, [['os', 'System'], ['phones', 'Phones']])} -

Android

${table(s.android, [['android', 'Android'], ['phones', 'Count']])} -` -} diff --git a/stats-worker/wrangler.toml b/stats-worker/wrangler.toml deleted file mode 100644 index e80c185..0000000 --- a/stats-worker/wrangler.toml +++ /dev/null @@ -1,23 +0,0 @@ -# Gearslip's opt-in usage notes. Deploy with `npx wrangler deploy` from this folder. -# Setup, once: -# npx wrangler d1 create gearslip-stats (paste the id it prints below) -# npx wrangler d1 execute gearslip-stats --remote --file schema.sql -# npx wrangler secret put SALT (any long random string; keep it) -# npx wrangler secret put DASHBOARD_KEY (the key for the dashboard link) -name = "gearslip-stats" -main = "src/index.js" -compatibility_date = "2026-10-01" - -[[d1_databases]] -binding = "DB" -database_name = "gearslip-stats" -database_id = "201521a7-0b4a-4fff-ab11-d575b821528a" - -# The public repo whose release downloads the dashboard compares against. If GitHub starts -# refusing (60 requests an hour without a token), add one: npx wrangler secret put GITHUB_TOKEN -[vars] -GITHUB_REPO = "Seb3thehacker/gearslip" - -# Save the download counts once a day, so the dashboard can show downloads per day. -[triggers] -crons = ["17 6 * * *"]