diff --git a/android/app/src/androidTest/java/com/github/quarck/calnotify/calendar/Uid2445PopulationProbeTest.kt b/android/app/src/androidTest/java/com/github/quarck/calnotify/calendar/Uid2445PopulationProbeTest.kt new file mode 100644 index 000000000..0b085d883 --- /dev/null +++ b/android/app/src/androidTest/java/com/github/quarck/calnotify/calendar/Uid2445PopulationProbeTest.kt @@ -0,0 +1,215 @@ +// +// Copyright (C) 2025 William Harris (wharris+cnplus@upscalews.com) +// + +package com.github.quarck.calnotify.calendar + +import android.Manifest +import android.content.Context +import android.provider.CalendarContract +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.rule.GrantPermissionRule +import com.github.quarck.calnotify.logs.DevLog +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Diagnostic probe for the portable event identity work. + * + * Answers one question before any of that plan gets built: **is + * [CalendarContract.Events.UID_2445] actually populated on real calendars?** + * + * The exact-match path (Phases 0-5 of + * `docs/dev_todo/portable_event_identity.md`) depends entirely on that column + * having values, but there is a long-standing unresolved Android issue titled + * "CalendarContract.Events.UID_2445 column is always null" + * (https://issuetracker.google.com/issues/37053160). If the column turns out to + * be sparse, the exact path is not viable and Phase 6's content heuristic + * becomes the primary mechanism instead - a materially different plan. + * + * This is a **diagnostic, not an assertion**. It reads whatever calendars are on + * the device and reports; it never fails on low coverage, because "this device + * has no synced calendars" is a property of the device, not a bug. Read the + * verdict in logcat: + * + * ``` + * adb logcat -s CNPlus:* | grep UID2445_PROBE + * ``` + * + * Run against a device with real (ideally Google-synced) calendars - an empty + * emulator tells you nothing: + * + * ``` + * .\gradlew.bat :app:connectedX8664DebugAndroidTest \ + * -Pandroid.testInstrumentationRunnerArguments.class=com.github.quarck.calnotify.calendar.Uid2445PopulationProbeTest + * ``` + */ +@RunWith(AndroidJUnit4::class) +class Uid2445PopulationProbeTest { + + private lateinit var context: Context + + @get:Rule + val permissionRule: GrantPermissionRule = GrantPermissionRule.grant( + Manifest.permission.READ_CALENDAR, + Manifest.permission.WRITE_CALENDAR + ) + + @Before + fun setup() { + context = InstrumentationRegistry.getInstrumentation().targetContext + } + + /** Per-account-type tally of how many events carry a usable identifier. */ + private data class Tally( + var total: Int = 0, + var uidNonNull: Int = 0, + var syncIdNonNull: Int = 0, + /** Events with neither identifier - unrecoverable by the exact path. */ + var neither: Int = 0 + ) + + @Test + fun probeUid2445Population() { + val calendars = readCalendars() + + if (calendars.isEmpty()) { + report(emptyMap(), 0) + DevLog.warn(LOG_TAG, "$TAG no calendars on this device - probe is inconclusive") + return + } + + val byAccountType = mutableMapOf() + var scanned = 0 + + for ((calendarId, accountType) in calendars) { + val tally = byAccountType.getOrPut(accountType) { Tally() } + scanned += tallyEventsForCalendar(calendarId, tally) + } + + report(byAccountType, scanned) + } + + /** @return calendar id -> account type for every calendar on the device. */ + private fun readCalendars(): List> { + val result = mutableListOf>() + val projection = arrayOf( + CalendarContract.Calendars._ID, + CalendarContract.Calendars.ACCOUNT_TYPE + ) + + context.contentResolver.query( + CalendarContract.Calendars.CONTENT_URI, projection, null, null, null + )?.use { cursor -> + while (cursor.moveToNext()) { + result.add(cursor.getLong(0) to (cursor.getString(1) ?: "(null)")) + } + } + return result + } + + /** Adds this calendar's events into [tally]. @return number of events scanned. */ + private fun tallyEventsForCalendar(calendarId: Long, tally: Tally): Int { + val projection = arrayOf( + CalendarContract.Events.UID_2445, + CalendarContract.Events._SYNC_ID + ) + // Skip tombstones - deleted rows would understate real coverage. + val selection = + "${CalendarContract.Events.CALENDAR_ID} = ? AND " + + "(${CalendarContract.Events.DELETED} IS NULL OR ${CalendarContract.Events.DELETED} = 0)" + + var scanned = 0 + context.contentResolver.query( + CalendarContract.Events.CONTENT_URI, + projection, + selection, + arrayOf(calendarId.toString()), + null + )?.use { cursor -> + while (cursor.moveToNext()) { + val hasUid = !cursor.getString(0).isNullOrBlank() + val hasSyncId = !cursor.getString(1).isNullOrBlank() + + tally.total++ + if (hasUid) tally.uidNonNull++ + if (hasSyncId) tally.syncIdNonNull++ + if (!hasUid && !hasSyncId) tally.neither++ + scanned++ + } + } + return scanned + } + + private fun report(byAccountType: Map, scanned: Int) { + val overall = Tally() + byAccountType.values.forEach { + overall.total += it.total + overall.uidNonNull += it.uidNonNull + overall.syncIdNonNull += it.syncIdNonNull + overall.neither += it.neither + } + + DevLog.info(LOG_TAG, "$TAG ================ UID_2445 POPULATION ================") + DevLog.info(LOG_TAG, "$TAG calendars=${byAccountType.size} accountTypes, events scanned=$scanned") + DevLog.info(LOG_TAG, "$TAG ${"accountType".padEnd(28)} ${"total".padStart(6)} ${"uid".padStart(6)} ${"pct".padStart(5)} ${"syncId".padStart(7)} ${"neither".padStart(8)}") + + for ((accountType, t) in byAccountType.entries.sortedByDescending { it.value.total }) { + DevLog.info( + LOG_TAG, + "$TAG ${accountType.take(28).padEnd(28)} ${t.total.toString().padStart(6)} " + + "${t.uidNonNull.toString().padStart(6)} ${pct(t.uidNonNull, t.total).padStart(5)} " + + "${t.syncIdNonNull.toString().padStart(7)} ${t.neither.toString().padStart(8)}" + ) + } + + DevLog.info(LOG_TAG, "$TAG ${"-".repeat(66)}") + DevLog.info( + LOG_TAG, + "$TAG ${"OVERALL".padEnd(28)} ${overall.total.toString().padStart(6)} " + + "${overall.uidNonNull.toString().padStart(6)} ${pct(overall.uidNonNull, overall.total).padStart(5)} " + + "${overall.syncIdNonNull.toString().padStart(7)} ${overall.neither.toString().padStart(8)}" + ) + DevLog.info(LOG_TAG, "$TAG VERDICT: ${verdict(overall)}") + DevLog.info(LOG_TAG, "$TAG ====================================================") + } + + private fun pct(part: Int, whole: Int): String = + if (whole == 0) "n/a" else "${part * 100 / whole}%" + + /** + * Maps coverage onto the decision the plan actually needs to make. + * Thresholds are judgement calls, deliberately stated so the reasoning is + * visible rather than buried in a number. + */ + private fun verdict(overall: Tally): String { + if (overall.total == 0) + return "INCONCLUSIVE - no events found; re-run on a device with real calendars" + + val uidPct = overall.uidNonNull * 100 / overall.total + val eitherPct = (overall.total - overall.neither) * 100 / overall.total + + return when { + uidPct >= 90 -> + "VIABLE - UID_2445 populated for $uidPct% of events; exact path (Phases 0-5) stands" + eitherPct >= 90 -> + "VIABLE WITH FALLBACK - UID_2445 only $uidPct%, but $eitherPct% have UID or _SYNC_ID; " + + "exact path stands and the _SYNC_ID fallback carries real weight" + uidPct >= 50 -> + "MIXED - UID_2445 only $uidPct% and $eitherPct% have either; exact path works for " + + "some events, Phase 6 heuristic needed to cover the rest" + else -> + "NOT VIABLE - UID_2445 only $uidPct%; the exact path cannot carry this plan. " + + "Phase 6 heuristic becomes primary - revisit the plan before building Phases 0-5" + } + } + + companion object { + private const val LOG_TAG = "Uid2445Probe" + /** Greppable marker so the report is easy to pull out of logcat. */ + private const val TAG = "UID2445_PROBE" + } +} diff --git a/docs/dev_todo/portable_event_identity.md b/docs/dev_todo/portable_event_identity.md index cea3b7bed..25fc297f3 100644 --- a/docs/dev_todo/portable_event_identity.md +++ b/docs/dev_todo/portable_event_identity.md @@ -42,7 +42,7 @@ Store durable, provider-independent identity alongside every stored event so tha This is the single most important thing to understand about the design, and it is easy to misread. -`UID_2445` is **not** something that can be looked up from an orphaned row. The app has no way to ask the new device "what is the UID of event `91427`?", because `91427` is a row number that means nothing there. The UID has to be read from the provider on the **old** device, while the IDs are still valid, and stored. +The event UID (in practice `_SYNC_ID` — see the probe results below) is **not** something that can be looked up from an orphaned row. The app has no way to ask the new device "what is the sync id of event `91427`?", because `91427` is a row number that means nothing there. It has to be read from the provider on the **old** device, while the IDs are still valid, and stored. So the mechanism is a snapshot, not a lookup: @@ -96,7 +96,7 @@ Phase 6 exists precisely because the precondition above will not always hold — |---|---|---| | Where to store identity | A **new, dedicated Room database** with typed columns | Keeps the scarce reserved text columns free for data that must live in the event row. Identity is off the hot path, joinable by key, and stays out of the Supabase sync payload by construction. | | Calendar identity | Reuse `CalendarBackupInfo` (account name/type, owner, displayName, name) | Already exists, already has a tested 3-tier fallback matcher, already the proven format in settings backup. | -| Event identity | `Events.UID_2445`, falling back to `Events._SYNC_ID` | `UID_2445` is the iCalendar UID — globally stable and identical across devices for the same Google/CalDAV event. Available since API 17; minSdk is 24. | +| Event identity | **`Events._SYNC_ID`**, with `Events.UID_2445` read opportunistically | **Measured, not assumed:** on a real device `UID_2445` was null for all 4761 events while `_SYNC_ID` was populated and unique for 100% of them. See the probe results below. | | When identity is captured | On every event write (add/update), best-effort | Cheap, keeps identity fresh, and means any future backup is restorable without a migration pass. | | When re-resolution runs | Lazily, on a detected restore, **retrying** until resolved — plus a manual trigger | Calendars often sync onto the phone *after* our first launch, so a one-shot pass would match nothing. | | Manual trigger UX | Mirror the existing pull-to-refresh + overflow "Refresh" in `prefs/CalendarsActivity.kt` | That screen already requests a calendar sync then reloads, which is exactly the shape needed here. Reusing a familiar interaction beats inventing a new one. | @@ -143,7 +143,8 @@ One table, keyed to match `eventsV9`'s primary key so rows join cleanly. One col | `calendarOwnerAccount` | String | `Calendars.OWNER_ACCOUNT` | | `calendarDisplayName` | String | `Calendars.CALENDAR_DISPLAY_NAME` — e.g. "Work" | | `calendarName` | String | `Calendars.NAME` | -| `eventUid` | String? | `Events.UID_2445`, fallback `_SYNC_ID`. Nullable — not every event has one | +| `eventSyncId` | String? | `Events._SYNC_ID` — the primary identifier, populated for 100% of events measured | +| `eventUid` | String? | `Events.UID_2445` — opportunistic; null on Google, may be populated by other providers | | `originalCalendarId` | Long | The `cid` in force when this row was captured | | `originalEventId` | Long | The `id` in force when this row was captured | | `capturedAtTime` | Long | Via `CNPlusClockInterface`, never `System.currentTimeMillis()` | @@ -173,53 +174,112 @@ flowchart TD B -->|"account tuple
(the email)"| C["cid = 12 ✅"] C --> D{"Which event
inside it?"} D -->|"the email again"| E["800 candidates ❌
they all share it"] - D -->|"UID_2445"| F["id = 55310 ✅
exactly one"] + D -->|"_SYNC_ID"| F["id = 55310 ✅
exactly one"] ``` So the plan uses **both**: the email tuple to find the calendar, then the UID to find the event within it. -### What `UID_2445` actually is +### What these identifiers actually are -It's the [RFC 5545](https://datatracker.ietf.org/doc/html/rfc5545#section-3.8.4.7) iCalendar `UID` property — the identifier the calendar *format* uses, as opposed to `_ID`, which is just a row number in the local SQLite database. The `2445` is a fossil: RFC 2445 was the original iCalendar spec, obsoleted by 5545, but Android kept the constant name. It's `CalendarContract.Events.UID_2445`, present since API 17 (your minSdk is 24), and holds a string like: +Two columns carry a server-assigned identifier, and the distinction turned out to matter — see the measured result below. -``` -040000008200E00074C5B7101A82E00800000000B0F2C8B5A1D9DA01000000000000000 -``` +**`UID_2445`** is the [RFC 5545](https://datatracker.ietf.org/doc/html/rfc5545#section-3.8.4.7) iCalendar `UID` property — the identifier the calendar *format* uses, as opposed to `_ID`, which is just a row number in the local SQLite database. The `2445` is a fossil: RFC 2445 was the original iCalendar spec, obsoleted by 5545, but Android kept the constant name. It is the *correct* identifier in principle, and **empty in practice on Google calendars.** -or, on Google Calendar, typically something closer to `abc123def456@google.com`. +**`_SYNC_ID`** is the sync adapter's own key for the event. On Google it holds values like `nekuken5tb3bvpoj8ncg4er7f0` or `20261031_jg75kt4ps505q4u01pbpictp2o` — Google's event-id encoding. Less standard than the iCalendar UID, but it is the one that is actually populated. -The key property: **the server assigns it, so it's the same string on every device that syncs that event.** `_ID` is assigned locally by whichever device happened to insert the row first, which is exactly why it doesn't survive a restore. +The key property, which both share: **the server assigns it, so it is the same string on every device that syncs that event.** `_ID` is assigned locally by whichever device happened to insert the row first, which is exactly why it does not survive a restore. -To be explicit, since this is the easy misreading: the UID is useful only because we **stored it on the old device**. It is a value we carry with us, not one the new device can derive from an orphaned row — see the precondition in the Goal. +To be explicit, since this is the easy misreading: the identifier is useful only because we **stored it on the old device**. It is a value we carry with us, not one the new device can derive from an orphaned row — see the precondition in the Goal. ```mermaid flowchart LR - G["Google Calendar
UID abc123@google.com"] --> P1["Old phone
_ID 91427"] + G["Google Calendar
_SYNC_ID nekuken5tb3bv..."] --> P1["Old phone
_ID 91427"] G --> P2["New phone
_ID 55310"] P1 -.->|"backup restores
_ID 91427"| P2 - P2 --> X["91427 doesn't exist here ❌
UID abc123 does ✅"] + P2 --> X["91427 doesn't exist here ❌
sync id nekuken5tb3bv... does ✅"] ``` ### Why not title + time instead -Title+time heuristics produce false positives on recurring and duplicated events (a weekly standup is many rows with identical titles), and break the moment the user edits a title. The UID survives renames, reschedules, and recurrence. +Title+time heuristics produce false positives on recurring and duplicated events (a weekly standup is many rows with identical titles), and break the moment the user edits a title. A server-assigned id survives renames, reschedules, and recurrence. ### Caveats -**Locally-created events that never synced to an account may have a null/empty `UID_2445`.** Those events are also the ones least likely to exist on the new phone at all — a local-only calendar isn't restored by Google. They degrade to unresolved and keep their stale ID: no crash, no data loss. +**Locally-created events that never synced to an account have neither identifier.** AOSP's `CalendarProvider2` notes that *"if this event hasn't been sync'ed with the server yet, the `_sync_id` field will be null"*, and `UID_2445` is empty on Google regardless. Such events are also the least likely to exist on the new phone at all — a local-only calendar is not restored by Google. They degrade to unresolved and keep their stale ID: no crash, no data loss. + +#### RESULT: `UID_2445` is empty; `_SYNC_ID` is the real identifier + +**Measured 2026-09-20, Pixel 10 Pro Fold (API 37), 16 Google calendars across two accounts:** + +``` +accountType total uid pct syncId neither +com.google 4761 0 0% 4761 0 +---------------------------------------------------------------------- +OVERALL 4761 0 0% 4761 0 +``` + +The long-standing Android issue is real and current: **`UID_2445` is null for every single event.** The column exists and is queryable, but Google's sync adapter never populates it. + +`_SYNC_ID` is the opposite — populated for 100% of events, and **4761 unique values across 4761 events**, i.e. perfectly unique with no collisions. Values look like `nekuken5tb3bvpoj8ncg4er7f0` and `20261031_jg75kt4ps505q4u01pbpictp2o`, which is Google's own event-id encoding. + +**Consequences for this plan:** + +1. **`_SYNC_ID` becomes the primary event identifier, not the fallback.** Read `UID_2445` opportunistically — it costs one column and may be populated by non-Google providers (CalDAV, Exchange) — but nothing should depend on it. +2. **The exact-match path survives intact.** This was the real risk the probe existed to check, and it came back fine: there *is* a stable, server-assigned, unique per-event identifier. Only its name changes. +3. **Phase 6 stays optional.** It is still the fallback for missing identity, not the primary mechanism. + +**Recurring events behave well**, which matters for the Phase 1b re-key: + +- A recurring series has **one** `_SYNC_ID` for the parent event, not one per instance — so `(sync_id, instanceStartTime)` identifies a specific occurrence. +- Recurrence **exceptions** (1021 of them here) carry `original_sync_id` pointing at the parent series, so a modified single occurrence stays traceable. + +**Caveat carried forward:** this is one device with one provider type (`com.google`). The never-synced local-event case still degrades to unresolved, exactly as the Caveats section describes — this device simply has no local-only calendars to demonstrate it. Re-run the probe on a device with Exchange or CalDAV accounts before assuming the same holds there. + +**Validated against the live app database on the same device** (368 stored events, Room active): + +| Check | Result | +|---|---| +| Stored `cid` values | 2 calendars (6, 16), both still present | +| Stored `id` resolves in provider | 362 / 368 | +| **Phase 2 backfill would capture `_SYNC_ID`** | **368 / 368 (100%)** | +| Reserved `s2` column empty | 368 / 368 — the plan's premise holds | + +The 6 whose *instance* had vanished are snoozed occurrences of deleted recurring series; their parent event rows still exist, so backfill still reaches a `_SYNC_ID` for them. That is why backfill scores 100% while direct instance resolution scores 362. + +**The calendar matcher was also validated.** All 16 calendars on this device produce a **unique** tier-1 key (`account_name` + `account_type` + `ownerAccount`) — zero ambiguity, despite only two distinct account names across them. `ownerAccount` is what disambiguates, which confirms `findMatchingCalendarId`'s existing three-tier design is right for this setup rather than merely plausible. + +`./scripts/capture_calendar_snapshot.sh` saves all of this to `./tmp/` (gitignored — it is real calendar data) so the analysis can be re-run without a device. Verified equivalent: replaying the backfill simulation from a snapshot reproduces the same 368/368. + +Reproduce with `./scripts/probe_uid2445.sh`. The app-database checks used `adb exec-out run-as com.github.quarck.calnotify cat databases/RoomEvents` — note `exec-out`, not `shell`, or the binary is corrupted in transit, and pull the `-wal` file too or the DB reads as malformed. + +#### How that was measured + +The exact-match path assumes a stable per-event identifier exists. A long-standing Android issue titled ["CalendarContract.Events.UID_2445 column is always null"](https://issuetracker.google.com/issues/37053160) put that in doubt, so it was tested rather than trusted — and the issue turned out to be accurate. + +Implemented as `androidTest/.../calendar/Uid2445PopulationProbeTest.kt`. It is a **diagnostic, not an assertion**: it reports and never fails on low coverage, since "this device has no synced calendars" is a property of the device rather than a bug. It also tallies `_SYNC_ID` alongside `UID_2445` so the fallback's real value gets measured rather than assumed, and counts events carrying *neither* — those are the ones the exact path can never recover. + +``` +.\gradlew.bat :app:connectedX8664DebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.class=com.github.quarck.calnotify.calendar.Uid2445PopulationProbeTest +adb logcat -s Uid2445Probe:* | grep UID2445_PROBE +``` + +Run it against a device with **real, Google-synced calendars** — a clean emulator has no synced events and will report `INCONCLUSIVE`, which is not evidence either way. + +There is also a no-build version, `scripts/probe_uid2445.sh`, which reads the provider over `adb shell content query` and prints the same table and verdict. Prefer it for a quick answer; it needs no compile, install, or instrumentation run. + +Its tallying logic lives in `scripts/lib/uid2445_tally.awk` and is covered by `scripts/lib/test_uid2445_tally.sh`, which runs against captured provider output in `scripts/lib/testdata/` — so the parsing can be verified without a device. -`_SYNC_ID` is the fallback when `UID_2445` is empty, but it does **not** stack the way one might assume. AOSP's `CalendarProvider2` notes that *"if this event hasn't been sync'ed with the server yet, the `_sync_id` field will be null"* — so `_SYNC_ID` is null in exactly the never-synced case described above. The fallback covers only the narrower situation where an event *did* sync but the provider populated `_SYNC_ID` without `UID_2445`. Worth having, but the never-synced case degrades to unresolved regardless of it. +#### Can this run against a backup instead of a live device? -#### De-risk before building: confirm `UID_2445` is actually populated +**No, and the reason is worth recording because it constrains more than this probe.** -The entire exact-match path assumes this column has values. There is a long-standing Android issue titled ["CalendarContract.Events.UID_2445 column is always null"](https://issuetracker.google.com/issues/37053160) whose resolution is not publicly readable, so the assumption should be tested rather than trusted. +`UID_2445` lives in the **Calendar Provider's** database (`com.android.providers.calendar`) — a different app. This app's backup covers only its own data: per `res/xml/backup_rules.xml`, the `eventsV9` databases and two prefs files. It contains no calendar-provider rows at all, so there is nothing in a CNPlus backup to measure. -**Do this first, before any of Phase 0.** One throwaway query across a few real calendars — count non-null `UID_2445` against total events, broken down by account type — settles it in minutes. +Reading the provider's own database directly is also not available: `adb shell run-as` only works on your own debuggable package, so `com.android.providers.calendar` is out of reach without root. -The result changes the plan materially: +The wider consequence for this plan: **the provider is only ever readable live.** That is exactly why identity must be captured at write time (Phases 0/2) rather than reconstructed later, and why an already-restored device has nothing but row content to match on (Phase 6). -- **Well populated** ⇒ proceed as written; Phase 6 stays an optional safety net. -- **Sparse or null on common providers** ⇒ the exact path is not viable, and **Phase 6's heuristic becomes the primary mechanism** rather than the fallback. That is a different plan, and better to discover now than after Phases 0–5 are built on it. +Re-run it on any device with a non-Google provider (Exchange, CalDAV) before assuming these numbers generalize — the measurement so far covers `com.google` only. **The email tuple is not perfectly unique either** — this is why `findMatchingCalendarId()` already has three tiers. One account can expose several calendars (primary, birthdays, a shared team calendar), all with the same `ACCOUNT_NAME`. That's why the match uses account name + type + owner, and only falls back to display name. @@ -255,7 +315,7 @@ One identity row per stored event, holding **everything needed to find that even Resolution consumes the identity rows described above. A restored event needs two lookups, in order: 1. **Calendar**: `findMatchingCalendarId(context, storedBackupInfo)` → new `cid`. Reuses the existing 3-tier matcher untouched. -2. **Event**: query `Events.CONTENT_URI` for `UID_2445 = ? AND CALENDAR_ID = ?` (scoped to the just-matched calendar to avoid cross-calendar collisions) → new `id`. +2. **Event**: query `Events.CONTENT_URI` for `_sync_id = ? AND CALENDAR_ID = ?` (scoped to the just-matched calendar to avoid cross-calendar collisions) → new `id`. Fall back to `UID_2445` where the stored row has one. #### The `-1L` sentinel must be guarded explicitly @@ -324,7 +384,7 @@ flowchart TD D -->|"no / -1L"| Y["Unresolved — keep row,
cid untouched, retry"] D -->|yes| S["UPDATE cid — safe,
commit now ✅"] - S --> E["Query Events for
UID_2445 in that calendar"] + S --> E["Query Events for
_SYNC_ID in that calendar"] E --> F{"Event
matched?"} F -->|no| X["Stop — cid gain kept,
mark unresolved, retry"] F -->|yes| G{"id already
current?"} @@ -376,11 +436,11 @@ The key insight: the calendar half of this problem was already solved once for s ### Phase 0: Capture identity at write time -**Before anything else:** confirm `UID_2445` is actually populated on real calendars (see Design Decisions). If it is sparse, the exact-match path is not viable and Phase 6 becomes primary — that determination should happen before writing any of the code below. +**Prerequisite met.** The probe has run (see Design Decisions): `UID_2445` is null on Google calendars, but `_SYNC_ID` is populated and unique for 100% of events, so the exact-match path stands with `_SYNC_ID` as the identifier. **0a — Identity storage.** New `identitystorage/` package following the shape of `monitorstorage/`: `EventIdentityEntity` (the schema in Design Decisions), `EventIdentityDao`, and `EventIdentityDatabase` at `version = 1`, name `RoomEventIdentity`. No legacy predecessor and no copy-migration — this is a fresh database. Use `CrSqliteRoomFactory` for consistency with the existing three. -**0b — Read the UID from the provider.** Add `Events.UID_2445` (with `_SYNC_ID` fallback) to the projection in `CalendarProvider.getEvent()` (`calendar/CalendarProvider.kt:418-439`) and expose it on `EventRecord`. Keep it nullable — not every event has one. +**0b — Read the identifiers from the provider.** Add `Events._SYNC_ID` (primary) and `Events.UID_2445` (opportunistic) to the projection in `CalendarProvider.getEvent()` (`calendar/CalendarProvider.kt:418-439`) and expose both on `EventRecord`. Keep both nullable — never-synced events have neither. **0c — Persist it.** Write an identity row whenever an event is added or updated in `ApplicationController`, keyed `(eventId, instanceStartTime)`. Best-effort: a failure to capture identity must never fail the event write itself. @@ -487,7 +547,7 @@ Phases 0–5 all depend on the precondition in the Goal: identity was captured o **The matching pass.** For each unresolved row with no identity: -1. **Skip recurring events outright** (`isRepeating`). They are the known weak case — a weekly standup has many identically-titled instances, so the signals cannot separate them reliably. Not worth the risk for the fraction it would recover; decline and move on. +1. **Skip recurring events outright** (`isRepeating`). They are the known weak case — a weekly standup has many identically-titled instances, so the signals cannot separate them reliably. Not worth the risk for the fraction it would recover; decline and move on. (Note this limitation applies only to Phase 6's content heuristic — the exact path handles recurrence fine, since a series has a single `_SYNC_ID` and exceptions carry `original_sync_id`.) 2. Query `CalendarContract.Instances.query()` over a narrow window bracketing the stored `instanceStartTime` (the existing instance-scan code at `CalendarProvider.kt:1594` already uses this API, so the query shape is proven). 3. Filter candidates by exact `title` match, then `isAllDay`, then `location` where present. 4. Accept **only when exactly one candidate survives.** Two or more ⇒ ambiguous ⇒ leave unresolved. @@ -526,9 +586,9 @@ Those limits are acceptable *because the alternative is nothing*. The failure mo | File | Changes | |---|---| -| `calendar/CalendarProvider.kt` | Add `UID_2445`/`_SYNC_ID` to `getEvent()` projection; add a lookup-by-UID query | +| `calendar/CalendarProvider.kt` | Add `_SYNC_ID`/`UID_2445` to `getEvent()` projection; add a lookup-by-sync-id query | | `calendar/CalendarProviderInterface.kt` | Declare the new lookup | -| `calendar/EventRecord.kt` | Carry nullable `eventUid` | +| `calendar/EventRecord.kt` | Carry nullable `eventSyncId` and `eventUid` | | `app/ApplicationController.kt` | Write identity rows on event add/update; **fix `restoreToActive()` (line 1321) to use the stored identity instead of re-querying the stale ID** | | `backup/SettingsBackupManager.kt` | Extract calendar-remap logic for reuse in Phase 5 | | `prefs/MiscSettingsFragmentX.kt` | Manual re-link action | diff --git a/scripts/capture_calendar_snapshot.sh b/scripts/capture_calendar_snapshot.sh new file mode 100755 index 000000000..c415072f8 --- /dev/null +++ b/scripts/capture_calendar_snapshot.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# +# Capture a device's calendar state for offline analysis. +# +# Lets the portable-event-identity work continue without keeping a device +# attached: the snapshot holds everything the probes and simulations need. +# +# WHAT THIS CAN AND CANNOT CAPTURE +# +# The Calendar Provider's own database (com.android.providers.calendar) is NOT +# pullable -- `run-as` only works on your own debuggable package, and that is a +# different app. What we capture instead is its full *contents*, exported via +# `content query` as text. That is enough to re-run any read-only analysis +# offline; it is not a restorable database file. +# +# Our own app databases (Events/RoomEvents etc.) ARE pulled verbatim, because +# they belong to a debuggable package we control. +# +# PRIVACY: output contains real calendar data -- event titles, times, account +# names. It is written under ./tmp/, which is gitignored. Do not commit it. +# +# Usage: ./scripts/capture_calendar_snapshot.sh [output_dir] + +set -euo pipefail + +readonly APP_PACKAGE="com.github.quarck.calnotify" +readonly PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly OUT_DIR="${1:-${PROJECT_ROOT}/tmp/calendar_snapshot_$(date +%Y%m%d_%H%M%S)}" + +die() { echo "Error: $*" >&2; exit 1; } + +command -v adb >/dev/null 2>&1 || die "adb not found." +adb get-state >/dev/null 2>&1 || die "no device connected." + +mkdir -p "${OUT_DIR}/provider" "${OUT_DIR}/app_databases" + +echo "== calendar snapshot ==" +echo "destination: ${OUT_DIR}" +echo + +# ----------------------------------------------------------- device info --- +{ + echo "captured_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + for prop in ro.product.model ro.build.version.sdk ro.build.version.release ro.build.fingerprint; do + echo "${prop}=$(adb shell getprop "$prop" 2>/dev/null | tr -d '\r')" + done +} > "${OUT_DIR}/device_info.txt" +echo "device_info.txt" + +# ------------------------------------------------------- provider export --- +# Exported as text: the provider's own DB file is unreachable without root. +dump_provider() { + local name="$1" uri="$2" projection="$3" + adb shell "content query --uri '${uri}' --projection '${projection}' --where \"1=1\"" 2>/dev/null \ + | tr -d '\r' > "${OUT_DIR}/provider/${name}.txt" + echo "provider/${name}.txt ($(wc -l < "${OUT_DIR}/provider/${name}.txt") rows)" +} + +dump_provider calendars \ + "content://com.android.calendar/calendars" \ + "_id:account_name:account_type:ownerAccount:name:calendar_displayName:calendar_color:sync_events:visible:isPrimary:calendar_timezone:calendar_access_level" + +# Everything the identity and heuristic work needs, including deleted rows so +# tombstone handling can be exercised offline. +dump_provider events \ + "content://com.android.calendar/events" \ + "_id:calendar_id:_sync_id:uid2445:original_sync_id:original_id:title:eventLocation:dtstart:dtend:duration:allDay:rrule:rdate:eventStatus:deleted:lastSynced" + +dump_provider reminders \ + "content://com.android.calendar/reminders" \ + "_id:event_id:minutes:method" + +# Instances need an explicit time window in the URI. +/- 1 year, matching the +# provider's own typical sync horizon. +now_ms=$(( $(date +%s) * 1000 )) +year_ms=$(( 365 * 24 * 60 * 60 * 1000 )) +adb shell "content query --uri 'content://com.android.calendar/instances/when/$((now_ms - year_ms))/$((now_ms + year_ms))' --projection 'event_id:begin:end:startDay:endDay' --where \"1=1\"" 2>/dev/null \ + | tr -d '\r' > "${OUT_DIR}/provider/instances.txt" +echo "provider/instances.txt ($(wc -l < "${OUT_DIR}/provider/instances.txt") rows)" + +# --------------------------------------------------------- app databases --- +# exec-out, NOT shell: `adb shell` mangles binary streams and the database +# reads back as malformed. The -wal file matters just as much as the main file; +# without it recent writes are missing and integrity_check fails. +if adb shell "run-as ${APP_PACKAGE} true" >/dev/null 2>&1; then + echo + for db in $(adb shell "run-as ${APP_PACKAGE} ls databases" 2>/dev/null | tr -d '\r'); do + case "$db" in + *-journal|*-shm) continue ;; # -shm is rebuildable; journals are noise + esac + adb exec-out run-as "${APP_PACKAGE}" cat "databases/${db}" \ + > "${OUT_DIR}/app_databases/${db}" 2>/dev/null || true + echo "app_databases/${db} ($(wc -c < "${OUT_DIR}/app_databases/${db}") bytes)" + done + + adb exec-out run-as "${APP_PACKAGE}" tar c shared_prefs 2>/dev/null \ + | tar x -C "${OUT_DIR}" 2>/dev/null \ + && echo "shared_prefs/" || echo "shared_prefs/ (skipped)" +else + echo + echo "NOTE: ${APP_PACKAGE} not installed or not debuggable - skipping app databases." >&2 +fi + +# ------------------------------------------------------------- provenance --- +cat > "${OUT_DIR}/README.md" < +run_tally() { + awk -f "$AWK_PROGRAM" "${DATA}/$1" "${DATA}/$2" +} + +# expect_contains +expect_contains() { + local description="$1" output="$2" needle="$3" + + if printf '%s' "$output" | grep -qF -- "$needle"; then + echo " PASS ${description}" + else + echo " FAIL ${description}" + echo " expected to find: ${needle}" + echo " in output:" + printf '%s\n' "$output" | sed 's/^/ /' + failures=$((failures + 1)) + fi +} + +echo "== uid2445_tally.awk ==" + +# --- mixed: one synced calendar, one local, one tombstone ------------------- +out="$(run_tally calendars_mixed.txt events_mixed.txt)" + +# google calendar has 4 rows but one is deleted=1, so the total must be 3. +expect_contains "tombstones excluded from totals" "$out" "com.google 3" +expect_contains "local calendar reports 0% uid" "$out" "LOCAL 2 0 0%" +expect_contains "events with no identifier counted as neither" "$out" "OVERALL 5 2 40% 3 2" +expect_contains "low coverage reports NOT VIABLE" "$out" "VERDICT: NOT VIABLE" + +# --- all events carry a UID ------------------------------------------------- +out="$(run_tally calendars_google.txt events_all_uid.txt)" +expect_contains "full coverage reports VIABLE" "$out" "VERDICT: VIABLE - UID_2445 populated for 100%" + +# --- UID absent but _SYNC_ID present --------------------------------------- +out="$(run_tally calendars_google.txt events_sync_id_only.txt)" +expect_contains "sync_id-only coverage reports VIABLE WITH FALLBACK" "$out" "VERDICT: VIABLE WITH FALLBACK" + +# --- no events at all ------------------------------------------------------- +out="$(run_tally calendars_google.txt events_empty.txt)" +expect_contains "empty provider reports INCONCLUSIVE" "$out" "VERDICT: INCONCLUSIVE" + +echo +if [ "$failures" -eq 0 ]; then + echo "All tests passed." +else + echo "${failures} test(s) failed." +fi +exit "$failures" diff --git a/scripts/lib/testdata/calendars_google.txt b/scripts/lib/testdata/calendars_google.txt new file mode 100644 index 000000000..4f1edf8e3 --- /dev/null +++ b/scripts/lib/testdata/calendars_google.txt @@ -0,0 +1 @@ +Row: 0 _id=7, account_type=com.google, account_name=will@example.com diff --git a/scripts/lib/testdata/calendars_mixed.txt b/scripts/lib/testdata/calendars_mixed.txt new file mode 100644 index 000000000..47fc224e9 --- /dev/null +++ b/scripts/lib/testdata/calendars_mixed.txt @@ -0,0 +1,2 @@ +Row: 0 _id=1, account_type=com.google, account_name=will@example.com +Row: 1 _id=2, account_type=LOCAL, account_name=local diff --git a/scripts/lib/testdata/events_all_uid.txt b/scripts/lib/testdata/events_all_uid.txt new file mode 100644 index 000000000..8328e1d33 --- /dev/null +++ b/scripts/lib/testdata/events_all_uid.txt @@ -0,0 +1,4 @@ +Row: 0 calendar_id=7, uid2445=a1@google.com, _sync_id=s1, deleted=0 +Row: 1 calendar_id=7, uid2445=a2@google.com, _sync_id=s2, deleted=0 +Row: 2 calendar_id=7, uid2445=a3@google.com, _sync_id=s3, deleted=0 +Row: 3 calendar_id=7, uid2445=a4@google.com, _sync_id=s4, deleted=0 diff --git a/scripts/lib/testdata/events_empty.txt b/scripts/lib/testdata/events_empty.txt new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/lib/testdata/events_mixed.txt b/scripts/lib/testdata/events_mixed.txt new file mode 100644 index 000000000..21afc9a69 --- /dev/null +++ b/scripts/lib/testdata/events_mixed.txt @@ -0,0 +1,6 @@ +Row: 0 calendar_id=1, uid2445=abc123@google.com, _sync_id=sync1, deleted=0 +Row: 1 calendar_id=1, uid2445=def456@google.com, _sync_id=sync2, deleted=0 +Row: 2 calendar_id=1, uid2445=NULL, _sync_id=sync3, deleted=0 +Row: 3 calendar_id=1, uid2445=ghi789@google.com, _sync_id=sync4, deleted=1 +Row: 4 calendar_id=2, uid2445=NULL, _sync_id=NULL, deleted=0 +Row: 5 calendar_id=2, uid2445=NULL, _sync_id=NULL, deleted=0 diff --git a/scripts/lib/testdata/events_sync_id_only.txt b/scripts/lib/testdata/events_sync_id_only.txt new file mode 100644 index 000000000..e6727eba3 --- /dev/null +++ b/scripts/lib/testdata/events_sync_id_only.txt @@ -0,0 +1,4 @@ +Row: 0 calendar_id=7, uid2445=NULL, _sync_id=s1, deleted=0 +Row: 1 calendar_id=7, uid2445=NULL, _sync_id=s2, deleted=0 +Row: 2 calendar_id=7, uid2445=NULL, _sync_id=s3, deleted=0 +Row: 3 calendar_id=7, uid2445=NULL, _sync_id=s4, deleted=0 diff --git a/scripts/lib/uid2445_tally.awk b/scripts/lib/uid2445_tally.awk new file mode 100644 index 000000000..1b1530259 --- /dev/null +++ b/scripts/lib/uid2445_tally.awk @@ -0,0 +1,141 @@ +# Tally UID_2445 / _SYNC_ID coverage from Calendar Provider rows. +# +# Used by scripts/probe_uid2445.sh. Kept as a separate file so it can be read +# and tested on its own: +# +# awk -f scripts/lib/uid2445_tally.awk calendars.txt events.txt +# +# Input is two `adb shell content query` outputs, as two separate files, +# calendars FIRST and events SECOND. Order matters: the account_type map has to +# be built before events can be attributed to it. +# +# A row looks like: +# Row: 0 _id=1, account_type=com.google, account_name=will@example.com +# Row: 0 calendar_id=1, uid2445=abc@google.com, _sync_id=s1, deleted=0 + +# ---------------------------------------------------------------- parsing --- + +# Return the value of `key` in a "Row: N k=v, k=v" line, or "" if absent. +# +# Keys are compared EXACTLY after splitting on ", ". Two bugs this avoids: +# - a substring match lets "_id" match inside "calendar_id" +# - regex trimming breaks on values containing "@" or "." -- i.e. every +# UID and every email address +function field(line, key, parts, count, i, eq) { + sub(/^Row: [0-9]+ /, "", line) + count = split(line, parts, ", ") + + for (i = 1; i <= count; i++) { + eq = index(parts[i], "=") + if (eq > 0 && substr(parts[i], 1, eq - 1) == key) + return substr(parts[i], eq + 1) + } + return "" +} + +# The provider prints absent values as the literal string "NULL". +function has_value(v) { + return (v != "" && v != "NULL") +} + +# ---------------------------------------------------------------- reading --- + +# Which file we are in: 1 = calendars, 2 = events. Using FNR/NR rather than +# matching on field names, because "_sync_id=" would otherwise be caught by any +# pattern looking for "_id=". +FNR == 1 { file_index++ } + +# Calendar row: remember which account type this calendar belongs to. +file_index == 1 { + id = field($0, "_id") + if (id != "") { + type = field($0, "account_type") + account_type_of[id] = (type == "" ? "(null)" : type) + } + next +} + +# Event row: count it against its calendar's account type. +{ + if (field($0, "deleted") == "1") + next # tombstone: would understate real coverage + + calendar_id = field($0, "calendar_id") + account = (calendar_id in account_type_of) \ + ? account_type_of[calendar_id] : "(unknown calendar)" + + got_uid = has_value(field($0, "uid2445")) + got_sync_id = has_value(field($0, "_sync_id")) + + events[account]++ + total_events++ + + if (got_uid) { + with_uid[account]++ + total_with_uid++ + } + if (got_sync_id) { + with_sync_id[account]++ + total_with_sync_id++ + } + if (!got_uid && !got_sync_id) { + with_neither[account]++ + total_with_neither++ + } +} + +# --------------------------------------------------------------- reporting --- + +function percent(part, whole) { + return (whole == 0) ? 0 : (part * 100 / whole) +} + +function print_row(label, total, uid, sync_id, neither) { + printf "%-26s %7d %7d %5d%% %8d %9d\n", + label, total, uid, percent(uid, total), sync_id, neither +} + +# Map coverage onto the decision the plan actually has to make. Thresholds are +# judgement calls, spelled out here rather than left implicit. +function verdict(uid_pct, either_pct) { + if (uid_pct >= 90) + return sprintf("VIABLE - UID_2445 populated for %d%% of events; " \ + "exact path (Phases 0-5) stands", uid_pct) + + if (either_pct >= 90) + return sprintf("VIABLE WITH FALLBACK - UID_2445 only %d%%, but %d%% " \ + "have UID or _SYNC_ID; the _SYNC_ID fallback carries real weight", + uid_pct, either_pct) + + if (uid_pct >= 50) + return sprintf("MIXED - UID_2445 %d%%, either %d%%; exact path works " \ + "for some events, Phase 6 heuristic needed for the rest", + uid_pct, either_pct) + + return sprintf("NOT VIABLE - UID_2445 only %d%%; the exact path cannot " \ + "carry this plan. Phase 6 heuristic becomes primary - revisit the " \ + "plan before building Phases 0-5", uid_pct) +} + +END { + if (total_events == 0) { + print "VERDICT: INCONCLUSIVE - no events found." + print "Re-run on a device with real, synced calendars." + exit + } + + printf "%-26s %7s %7s %6s %8s %9s\n", + "accountType", "total", "uid", "pct", "syncId", "neither" + + for (account in events) + print_row(account, events[account], with_uid[account] + 0, + with_sync_id[account] + 0, with_neither[account] + 0) + + print "----------------------------------------------------------------------" + print_row("OVERALL", total_events, total_with_uid + 0, + total_with_sync_id + 0, total_with_neither + 0) + + print "" + print "VERDICT: " verdict(percent(total_with_uid, total_events), + percent(total_events - total_with_neither, total_events)) +} diff --git a/scripts/probe_uid2445.sh b/scripts/probe_uid2445.sh new file mode 100755 index 000000000..22ca94a2f --- /dev/null +++ b/scripts/probe_uid2445.sh @@ -0,0 +1,93 @@ +#!/bin/bash +# +# Probe whether CalendarContract.Events.UID_2445 is populated on a device. +# +# Answers the question that gates Phase 0 of +# docs/dev_todo/portable_event_identity.md: does UID_2445 actually have values +# on real calendars? There is a long-standing unresolved Android issue claiming +# it is always null (https://issuetracker.google.com/issues/37053160), and the +# plan's exact-match path depends entirely on that column. +# +# Read-only. Needs no build, install, or instrumentation run -- it reads the +# Calendar Provider over `adb shell content query`. +# +# REQUIRES A LIVE DEVICE with real, synced calendars. It cannot read an app +# backup: UID_2445 lives in the Calendar Provider's database +# (com.android.providers.calendar), a different app that we do not back up. +# See "Can this run against a backup instead of a live device?" in the plan. +# +# Usage: +# ./scripts/probe_uid2445.sh [output_file] + +set -euo pipefail + +readonly OUTPUT_FILE="${1:-}" +readonly SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +readonly TALLY_AWK="${SCRIPT_DIR}/lib/uid2445_tally.awk" + +readonly CALENDARS_URI="content://com.android.calendar/calendars" +readonly EVENTS_URI="content://com.android.calendar/events" + +die() { + echo "Error: $*" >&2 + exit 1 +} + +# `content query` prints one "Row: N k=v, k=v" line per row. Strip CRs, which +# adb adds on some platforms and which would otherwise end up inside values. +query_provider() { + local uri="$1" projection="$2" + adb shell content query --uri "$uri" --projection "$projection" 2>/dev/null | tr -d '\r' +} + +check_prerequisites() { + command -v adb >/dev/null 2>&1 \ + || die "adb not found. Install Android SDK platform tools." + + adb get-state >/dev/null 2>&1 \ + || die "no device connected. This probe needs a live device with real synced calendars; it cannot read an app backup." + + [ -f "$TALLY_AWK" ] || die "missing $TALLY_AWK" +} + +print_device_header() { + local model api + model="$(adb shell getprop ro.product.model 2>/dev/null | tr -d '\r')" + api="$(adb shell getprop ro.build.version.sdk 2>/dev/null | tr -d '\r')" + + echo "== UID_2445 population probe ==" + echo "device: ${model} (API ${api})" + echo +} + +main() { + check_prerequisites + print_device_header + + local calendars events + calendars="$(query_provider "$CALENDARS_URI" "_id:account_type:account_name")" + [ -n "$calendars" ] \ + || die "no calendars readable. Grant calendar permission to the shell, or check the device." + + echo "calendars found:" + echo "$calendars" | sed 's/^/ /' + echo + + events="$(query_provider "$EVENTS_URI" "calendar_id:uid2445:_sync_id:deleted")" + + # Calendars first: the tally needs the id -> account_type map before it can + # attribute events. Order matters, which is why they are passed separately + # rather than concatenated. + local report + report="$(awk -f "$TALLY_AWK" <(echo "$calendars") <(echo "$events"))" + + echo "$report" + + if [ -n "$OUTPUT_FILE" ]; then + { print_device_header; echo "$report"; } > "$OUTPUT_FILE" + echo + echo "Saved to: $OUTPUT_FILE" + fi +} + +main "$@"