From b731d92ee6bba4689495ad7afa40dcdf6d968f43 Mon Sep 17 00:00:00 2001 From: Matthew Raymer Date: Mon, 6 Oct 2025 13:07:15 +0000 Subject: [PATCH] chore: planning document almsot ready --- ...STARRED_PROJECTS_POLLING_IMPLEMENTATION.md | 222 ++++++------------ 1 file changed, 67 insertions(+), 155 deletions(-) diff --git a/doc/STARRED_PROJECTS_POLLING_IMPLEMENTATION.md b/doc/STARRED_PROJECTS_POLLING_IMPLEMENTATION.md index c4c0215..e3e4f59 100644 --- a/doc/STARRED_PROJECTS_POLLING_IMPLEMENTATION.md +++ b/doc/STARRED_PROJECTS_POLLING_IMPLEMENTATION.md @@ -61,7 +61,7 @@ User-Agent: TimeSafari-DailyNotificationPlugin/1.0.0 "version": "1.0.0" }, "previousClaim": { - "jwtId": "1703980800_xyz789_ghi01234", + "jwtId": "1703980800_xyz789_0badf00d", "claimType": "project_update", "claimData": { "status": "in_progress", @@ -78,7 +78,7 @@ User-Agent: TimeSafari-DailyNotificationPlugin/1.0.0 "hitLimit": false, "pagination": { "hasMore": true, - "nextAfterId": "1704153600_mno345_pqr67890" + "nextAfterId": "1704153600_mno345_0badf00d" } } ``` @@ -219,7 +219,7 @@ function compareJwtIds(a: string, b: string): number { return a.localeCompare(b); } -// Example: "1704067200_abc123_def45678" < "1704153600_xyz789_ghi01234" +// Example: "1704067200_abc123_def45678" < "1704153600_xyz789_0badf00d" ``` **Eventual Consistency Bounds**: @@ -247,7 +247,7 @@ X-Idempotency-Key: {uuid} **Request Body**: ```json { - "acknowledgedJwtIds": ["1704067200_abc123_def45678", "1704153600_mno345_pqr67890"], + "acknowledgedJwtIds": ["1704067200_abc123_def45678", "1704153600_mno345_0badf00d"], "acknowledgedAt": "2025-01-01T12:00:00Z", "clientVersion": "TimeSafari-DailyNotificationPlugin/1.0.0" } @@ -822,7 +822,9 @@ document.addEventListener('visibilitychange', () => { const lastPoll = localStorage.getItem('lastPollTimestamp'); const now = Date.now(); if (now - parseInt(lastPoll) > 3600000) { // 1 hour - pollStarredProjects(); + pollStarredProjects().then(() => { + localStorage.setItem('lastPollTimestamp', now.toString()); + }); } } }); @@ -841,13 +843,15 @@ document.addEventListener('visibilitychange', () => { ↓ [Make API Call] ← [Valid Config] ← [Has Starred Projects] ← [Has Last Ack ID] ↓ ↓ ↓ ↓ -[Network Error] [Parse Response] [Process Results] [Update Watermark] +[Network Error] [Parse Response] [Process Results] [Generate Notifications] ↓ ↓ ↓ ↓ -[Retry Logic] [Generate Notifications] [Success] [Commit State] +[Retry Logic] [Schedule Delivery] [Success] [Acknowledge Delivery] ↓ ↓ ↓ ↓ -[Exponential Backoff] [Schedule Delivery] [End] [End] - ↓ ↓ -[Max Retries] [End] +[Exponential Backoff] [Acknowledge Delivery] [End] [Update Watermark] + ↓ ↓ ↓ ↓ +[Max Retries] [Update Watermark] [End] [Commit State] + ↓ ↓ ↓ ↓ +[End] [Commit State] [End] [End] ↓ [End] ``` @@ -858,12 +862,13 @@ document.addEventListener('visibilitychange', () => { 3. **Check Last Ack ID**: If `lastAckedStarredPlanChangesJwtId` is null, run Bootstrap Watermark; then continue 4. **Make API Call**: Execute authenticated POST request 5. **Process Results**: Parse response and extract change count -6. **Update Watermark**: Advance watermark only after successful delivery AND acknowledgment -7. **Generate Notifications**: Create user notifications for changes +6. **Generate Notifications**: Create user notifications for changes +7. **Update Watermark**: Advance watermark only after successful delivery AND acknowledgment #### Watermark Bootstrap Path **Bootstrap Implementation**: + ```typescript async function bootstrapWatermark(activeDid: string, starredPlanHandleIds: string[]): Promise { try { @@ -930,10 +935,10 @@ CREATE INDEX idx_outbox_undelivered ON notification_outbox(delivered_at) WHERE d **Atomic Transaction Pattern**: ```sql --- Phase 1: Atomic commit of watermark + outbox +-- Phase 1: Atomic commit of outbox only (watermark stays unchanged) BEGIN TRANSACTION; INSERT INTO notification_outbox (jwt_id, content) VALUES (?, ?); -UPDATE settings SET lastAckedStarredPlanChangesJwtId = ? WHERE accountDid = ?; +-- Do NOT update watermark here - it advances only after delivery + acknowledgment COMMIT; ``` @@ -1011,7 +1016,7 @@ async function processPollingResults(results: PollingResult[]): Promise { } // 3. Dispatcher delivers notifications - await notificationDispatcher.processOutbox(); + const { deliveredJwtIds, latestJwtId } = await notificationDispatcher.processOutbox(); // 4. After successful delivery, call acknowledgment endpoint await acknowledgeDeliveredNotifications(deliveredJwtIds); @@ -1047,7 +1052,7 @@ UPDATE notification_outbox SET delivered_at = datetime('now'), acknowledged_at = ``` **Recovery Rules**: -- **Crash After Watermark Update**: On restart, check `notification_pending` table for uncommitted notifications +- **Crash After Watermark Update**: On restart, check `notification_outbox` table for uncommitted notifications - **Crash Before Watermark Update**: Safe to retry - no state change occurred - **Partial Notification Failure**: Rollback watermark update, retry entire transaction - **Acknowledgment Endpoint**: Call `POST /api/v2/plans/acknowledge` after successful notification delivery @@ -1055,13 +1060,13 @@ UPDATE notification_outbox SET delivered_at = datetime('now'), acknowledged_at = **Recovery Implementation**: ```typescript async function recoverPendingNotifications(): Promise { - const pending = await db.query('SELECT * FROM notification_pending WHERE created_at < ?', + const pending = await db.query('SELECT * FROM notification_outbox WHERE created_at < ?', [Date.now() - 300000]); // 5 minutes ago for (const notification of pending) { try { await scheduleNotification(notification.content); - await db.query('DELETE FROM notification_pending WHERE id = ?', [notification.id]); + await db.query('DELETE FROM notification_outbox WHERE id = ?', [notification.id]); } catch (error) { // Log error, will retry on next recovery cycle console.error('Failed to recover notification:', error); @@ -1118,7 +1123,7 @@ Body: "You have {count} new updates in your starred projects" **Deep Link Routes**: ``` -timesafari://projects/updates?jwtIds=1704067200_abc123_def45678,1704153600_mno345_pqr67890 +timesafari://projects/updates?jwtIds=1704067200_abc123_def45678,1704153600_mno345_0badf00d timesafari://projects/{projectId}/details?jwtId=1704067200_abc123_def45678 timesafari://notifications/starred-projects timesafari://projects/updates?shortlink=abc123def456789 @@ -1153,7 +1158,7 @@ function validateDeepLinkParams(params: any): DeepLinkValidation { ```typescript // Server generates shortlink for large result sets const shortlink = await generateShortlink({ - jwtIds: ['1704067200_abc123_def45678', '1704153600_mno345_pqr67890', /* ... 50 more */], + jwtIds: ['1704067200_abc123_def45678', '1704153600_mno345_0badf00d', /* ... 50 more */], expiresAt: Date.now() + 86400000 // 24 hours }); @@ -1384,125 +1389,6 @@ const PII_REDACTION_PATTERNS = [ ### Testing Artifacts -#### Mock Fixtures - -**Empty Response**: -```json -{ - "data": [], - "hitLimit": false, - "pagination": { - "hasMore": false, - "nextAfterId": null - } -} -``` - -**Small Response (3 items)**: -```json -{ - "data": [ - { - "planSummary": { - "jwtId": "1704067200_abc123_def45678", - "handleId": "test_project_1", - "name": "Test Project 1", - "description": "First test project" - }, - "previousClaim": { - "jwtId": "1703980800_xyz789_ghi01234", - "claimType": "project_update" - } - }, - { - "planSummary": { - "jwtId": "1704153600_mno345_pqr67890", - "handleId": "test_project_2", - "name": "Test Project 2", - "description": "Second test project" - }, - "previousClaim": { - "jwtId": "1704067200_stu901_vwx23456", - "claimType": "project_update" - } - }, - { - "planSummary": { - "jwtId": "1704240000_new123_0badf00d", - "handleId": "test_project_3", - "name": "Test Project 3", - "description": "Third test project" - }, - "previousClaim": { - "jwtId": "1704153600_old456_1cafebad", - "claimType": "project_update" - } - } - ], - "hitLimit": false, - "pagination": { - "hasMore": false, - "nextAfterId": null - } -} -``` - -**Paginated Response**: -```json -{ - "data": [...], // 100 items - "hitLimit": true, - "pagination": { - "hasMore": true, - "nextAfterId": "1704153600_mno345_pqr67890" - } -} -``` - -**Rate Limited Response (canonical format)**: -```json -{ - "error": "Rate limit exceeded", - "code": "RATE_LIMIT_EXCEEDED", - "retryAfter": 60, - "details": { - "limit": 100, - "window": "1m", - "remaining": 0, - "resetAt": "2024-01-01T12:01:00Z" - }, - "requestId": "req_jkl012" -} -``` - -**Contract Tests**: -```typescript -// JWT ID comparison helper -function compareJwtIds(a: string, b: string): number { - return a.localeCompare(b); // Lexicographic comparison for fixed-width format -} - -describe('StarredProjectsPolling Contract Tests', () => { - test('should maintain JWT ID ordering', () => { - const response = mockPaginatedResponse(); - const jwtIds = response.data.map(item => item.planSummary.jwtId); - const sortedIds = [...jwtIds].sort(compareJwtIds); - expect(jwtIds).toEqual(sortedIds); - }); - - test('should handle watermark movement correctly', () => { - const initialWatermark = '1704067200_abc123_def45678'; - const response = mockSmallResponse(); - const newWatermark = getNewWatermark(response); - expect(compareJwtIds(newWatermark, initialWatermark)).toBeGreaterThan(0); - }); - - test('should respect pagination limits', () => { - const response = mockPaginatedResponse(); - expect(response.data.length).toBeLessThanOrEqual(100); - }); -}); -``` #### Testing Fixtures & SLAs @@ -1533,16 +1419,16 @@ describe('StarredProjectsPolling Contract Tests', () => { "issuerDid": "did:key:test_issuer_1", "agentDid": "did:key:test_agent_1", "locLat": 40.7128, - "locLon": -74.0060, + "locLon": -74.0060 }, "previousClaim": { - "jwtId": "1703980800_xyz789_ghi01234", + "jwtId": "1703980800_xyz789_0badf00d", "claimType": "project_update" } }, { "planSummary": { - "jwtId": "1704153600_mno345_pqr67890", + "jwtId": "1704153600_mno345_0badf00d", "handleId": "test_project_2", "name": "Test Project 2", "description": "Second test project", @@ -1552,7 +1438,7 @@ describe('StarredProjectsPolling Contract Tests', () => { "locLon": null }, "previousClaim": { - "jwtId": "1704067200_stu901_vwx23456", + "jwtId": "1704067200_stu901_1cafebad", "claimType": "project_update" } }, @@ -1568,7 +1454,7 @@ describe('StarredProjectsPolling Contract Tests', () => { "locLon": -122.4194 }, "previousClaim": { - "jwtId": "1704153600_old456_1cafebad", + "jwtId": "1704153600_old456_0badf00d", "claimType": "project_update" } } @@ -1585,12 +1471,12 @@ describe('StarredProjectsPolling Contract Tests', () => { ```json { "data": [ - // ... 100 items with jwtIds from "1704067200_abc123_def45678" to "1704153600_xyz789_ghi01234" + // ... 100 items with jwtIds from "1704067200_abc123_def45678" to "1704153600_xyz789_0badf00d" ], "hitLimit": true, "pagination": { "hasMore": true, - "nextAfterId": "1704240000_new123_0badf00d" + "nextAfterId": "1704240000_new123_1cafebad" } } ``` @@ -1625,7 +1511,7 @@ describe('StarredProjectsPolling Contract Tests', () => { "locLon": null }, "previousClaim": { - "jwtId": "1703980800_xyz789_ghi01234", + "jwtId": "1703980800_xyz789_0badf00d", "claimType": "project_update" } } @@ -1644,7 +1530,7 @@ describe('StarredProjectsPolling Contract Tests', () => { "data": [ { "planSummary": { - "jwtId": "1704153600_mno345_pqr67890", + "jwtId": "1704153600_mno345_0badf00d", "handleId": "test_project_2", "locLat": null, "locLon": null @@ -1655,7 +1541,7 @@ describe('StarredProjectsPolling Contract Tests', () => { "jwtId": "1704067200_abc123_def45678", "handleId": "test_project_1", "locLat": 40.7128, - "locLon": -74.0060, + "locLon": -74.0060 } } ], @@ -2166,11 +2052,24 @@ class StarredProjectsPollingManager { return StarredProjectsPollingResult(changeCount: 0, hitLimit: false, error: "No starred projects") } - // 3. Check if we have last acknowledged ID + // 3. Check if we have last acknowledged ID, bootstrap if missing guard let lastAckedId = config.lastAckedStarredPlanChangesJwtId, !lastAckedId.isEmpty else { - print("\(TAG): No last acknowledged ID, skipping poll") - return StarredProjectsPollingResult(changeCount: 0, hitLimit: false, error: "No last acknowledged ID") + print("\(TAG): No last acknowledged ID, running bootstrap watermark") + let bootstrapWatermark = try await bootstrapWatermark(activeDid: config.activeDid, starredPlanHandleIds: starredPlanHandleIds) + if let bootstrapWatermark = bootstrapWatermark { + // Update config with bootstrap watermark and reload + try await updateLastAckedId(bootstrapWatermark, for: config.activeDid) + let updatedConfig = try await getUserConfiguration() + if let updatedConfig = updatedConfig { + print("\(TAG): Bootstrap watermark set: \(bootstrapWatermark)") + // Continue with updated config + return try await pollStarredProjectChangesWithConfig(updatedConfig) + } + } else { + print("\(TAG): Bootstrap watermark failed, skipping poll") + return StarredProjectsPollingResult(changeCount: 0, hitLimit: false, error: "Bootstrap watermark failed") + } } // 4. Make API call @@ -2331,10 +2230,23 @@ export class StarredProjectsPollingManager { return { changeCount: 0, hitLimit: false, error: 'No starred projects' }; } - // 3. Check if we have last acknowledged ID + // 3. Check if we have last acknowledged ID, bootstrap if missing if (!config.lastAckedStarredPlanChangesJwtId) { - console.log('StarredProjectsPollingManager: No last acknowledged ID, skipping poll'); - return { changeCount: 0, hitLimit: false, error: 'No last acknowledged ID' }; + console.log('StarredProjectsPollingManager: No last acknowledged ID, running bootstrap watermark'); + const bootstrapWatermark = await this.bootstrapWatermark(config.activeDid, config.starredPlanHandleIds); + if (bootstrapWatermark) { + // Update config with bootstrap watermark and reload + await this.updateLastAckedId(bootstrapWatermark, config.activeDid); + const updatedConfig = await this.getUserConfiguration(); + if (updatedConfig) { + console.log(`StarredProjectsPollingManager: Bootstrap watermark set: ${bootstrapWatermark}`); + // Continue with updated config + return await this.pollStarredProjectChangesWithConfig(updatedConfig); + } + } else { + console.log('StarredProjectsPollingManager: Bootstrap watermark failed, skipping poll'); + return { changeCount: 0, hitLimit: false, error: 'Bootstrap watermark failed' }; + } } // 4. Make API call