diff --git a/src/db-sql/migration.ts b/src/db-sql/migration.ts index 93521dbf..3d849f3f 100644 --- a/src/db-sql/migration.ts +++ b/src/db-sql/migration.ts @@ -1,60 +1,3 @@ -/** - * TimeSafari Database Migration Definitions - * - * This module defines all database schema migrations for the TimeSafari application. - * Each migration represents a specific version of the database schema and contains - * the SQL statements needed to upgrade from the previous version. - * - * ## Migration Philosophy - * - * TimeSafari follows a structured approach to database migrations: - * - * 1. **Sequential Numbering**: Migrations are numbered sequentially (001, 002, etc.) - * 2. **Descriptive Names**: Each migration has a clear, descriptive name - * 3. **Single Purpose**: Each migration focuses on one logical schema change - * 4. **Forward-Only**: Migrations are designed to move the schema forward - * 5. **Idempotent Design**: The migration system handles re-runs gracefully - * - * ## Migration Structure - * - * Each migration follows this pattern: - * ```typescript - * { - * name: "XXX_descriptive_name", - * sql: "SQL statements to execute" - * } - * ``` - * - * ## Database Architecture - * - * TimeSafari uses SQLite for local data storage with the following core tables: - * - * - **accounts**: User identity and cryptographic keys - * - **secret**: Encrypted application secrets - * - **settings**: Application configuration and preferences - * - **contacts**: User's contact network and trust relationships - * - **logs**: Application event logging and debugging - * - **temp**: Temporary data storage for operations - * - * ## Privacy and Security - * - * The database schema is designed with privacy-first principles: - * - User identifiers (DIDs) are kept separate from personal data - * - Cryptographic keys are stored securely - * - Contact visibility is user-controlled - * - All sensitive data can be encrypted at rest - * - * ## Usage - * - * This file is automatically loaded during application startup. The migrations - * are registered with the migration service and applied as needed based on the - * current database state. - * - * @author Matthew Raymer - * @version 1.0.0 - * @since 2025-06-30 - */ - import { registerMigration, runMigrations as runMigrationsService, @@ -62,301 +5,143 @@ import { import { DEFAULT_ENDORSER_API_SERVER } from "@/constants/app"; import { arrayBufferToBase64 } from "@/libs/crypto"; -/** - * Generate a cryptographically secure random secret for the secret table - * - * Note: This approach stores the secret alongside user data for convenience. - * In a production environment with hardware security modules or dedicated - * secure storage, this secret should be stored separately. As users build - * their trust networks and sign more records, they should migrate to more - * secure key management solutions. - * - * @returns Base64-encoded random secret (32 bytes) - */ -function generateDatabaseSecret(): string { - const randomBytes = new Uint8Array(32); - crypto.getRandomValues(randomBytes); - return arrayBufferToBase64(randomBytes.buffer); -} +// Generate a random secret for the secret table -// Generate the secret that will be used for this database instance -const databaseSecret = generateDatabaseSecret(); +// It's not really secure to maintain the secret next to the user's data. +// However, until we have better hooks into a real wallet or reliable secure +// storage, we'll do this for user convenience. As they sign more records +// and integrate with more people, they'll value it more and want to be more +// secure, so we'll prompt them to take steps to back it up, properly encrypt, +// etc. At the beginning, we'll prompt for a password, then we'll prompt for a +// PWA so it's not in a browser... and then we hope to be integrated with a +// real wallet or something else more secure. -/** - * Migration 001: Initial Database Schema - * - * This migration creates the foundational database schema for TimeSafari. - * It establishes the core tables needed for user identity management, - * contact networks, application settings, and operational logging. - * - * ## Tables Created: - * - * ### accounts - * Stores user identities and cryptographic key pairs. Each account represents - * a unique user identity with associated cryptographic capabilities. - * - * - `id`: Primary key for internal references - * - `did`: Decentralized Identifier (unique across the network) - * - `privateKeyHex`: Private key for signing and encryption (hex-encoded) - * - `publicKeyHex`: Public key for verification and encryption (hex-encoded) - * - `derivationPath`: BIP44 derivation path for hierarchical key generation - * - `mnemonic`: BIP39 mnemonic phrase for key recovery - * - * ### secret - * Stores encrypted application secrets and sensitive configuration data. - * This table contains cryptographic material needed for secure operations. - * - * - `id`: Primary key (always 1 for singleton pattern) - * - `hex`: Encrypted secret data in hexadecimal format - * - * ### settings - * Application-wide configuration and user preferences. This table stores - * both system settings and user-customizable preferences. - * - * - `name`: Setting name/key (unique identifier) - * - `value`: Setting value (JSON-serializable data) - * - * ### contacts - * User's contact network and trust relationships. This table manages the - * social graph and trust network that enables TimeSafari's collaborative features. - * - * - `did`: Contact's Decentralized Identifier (primary key) - * - `name`: Display name for the contact - * - `publicKeyHex`: Contact's public key for verification - * - `endorserApiServer`: API server URL for this contact's endorsements - * - `registered`: Timestamp when contact was first added - * - `lastViewedClaimId`: Last claim/activity viewed from this contact - * - `seenWelcomeScreen`: Whether contact has completed onboarding - * - * ### logs - * Application event logging for debugging and audit trails. This table - * captures important application events for troubleshooting and monitoring. - * - * - `id`: Auto-incrementing log entry ID - * - `message`: Log message content - * - `level`: Log level (error, warn, info, debug) - * - `timestamp`: When the log entry was created - * - `context`: Additional context data (JSON format) - * - * ### temp - * Temporary data storage for multi-step operations. This table provides - * transient storage for operations that span multiple user interactions. - * - * - `id`: Unique identifier for the temporary data - * - `data`: JSON-serialized temporary data - * - `created`: Timestamp when data was stored - * - `expires`: Optional expiration timestamp - * - * ## Initial Data - * - * The migration also populates initial configuration: - * - Default endorser API server URL - * - Application database secret - * - Welcome screen tracking - */ -registerMigration({ - name: "001_initial", - sql: ` - -- User accounts and identity management - -- Each account represents a unique user with cryptographic capabilities - CREATE TABLE accounts ( - id INTEGER PRIMARY KEY, - did TEXT UNIQUE NOT NULL, -- Decentralized Identifier - privateKeyHex TEXT NOT NULL, -- Private key (hex-encoded) - publicKeyHex TEXT NOT NULL, -- Public key (hex-encoded) - derivationPath TEXT, -- BIP44 derivation path - mnemonic TEXT -- BIP39 recovery phrase - ); +// One might ask: why encrypt at all? We figure a basic encryption is better +// than none. Plus, we expect to support their own password or keystore or +// external wallet as better signing options in the future, so it's gonna be +// important to have the structure where each account access might require +// user action. - -- Encrypted application secrets and sensitive configuration - -- Singleton table (id always = 1) for application-wide secrets - CREATE TABLE secret ( - id INTEGER PRIMARY KEY CHECK (id = 1), -- Enforce singleton - hex TEXT NOT NULL -- Encrypted secret data - ); +// (Once upon a time we stored the secret in localStorage, but it frequently +// got erased, even though the IndexedDB still had the identity data. This +// ended up throwing lots of errors to the user... and they'd end up in a state +// where they couldn't take action because they couldn't unlock that identity.) - -- Application settings and user preferences - -- Key-value store for configuration data - CREATE TABLE settings ( - name TEXT PRIMARY KEY, -- Setting name/identifier - value TEXT -- Setting value (JSON-serializable) - ); +const randomBytes = crypto.getRandomValues(new Uint8Array(32)); +const secretBase64 = arrayBufferToBase64(randomBytes); - -- User's contact network and trust relationships - -- Manages the social graph for collaborative features - CREATE TABLE contacts ( - did TEXT PRIMARY KEY, -- Contact's DID - name TEXT, -- Display name - publicKeyHex TEXT, -- Contact's public key - endorserApiServer TEXT, -- API server for endorsements - registered TEXT, -- Registration timestamp - lastViewedClaimId TEXT, -- Last viewed activity - seenWelcomeScreen BOOLEAN DEFAULT FALSE -- Onboarding completion - ); +// Each migration can include multiple SQL statements (with semicolons) +const MIGRATIONS = [ + { + name: "001_initial", + sql: ` + CREATE TABLE IF NOT EXISTS accounts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dateCreated TEXT NOT NULL, + derivationPath TEXT, + did TEXT NOT NULL, + identityEncrBase64 TEXT, -- encrypted & base64-encoded + mnemonicEncrBase64 TEXT, -- encrypted & base64-encoded + passkeyCredIdHex TEXT, + publicKeyHex TEXT NOT NULL + ); - -- Application event logging for debugging and audit - -- Captures important events for troubleshooting - CREATE TABLE logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - message TEXT NOT NULL, -- Log message - level TEXT NOT NULL, -- Log level (error/warn/info/debug) - timestamp TEXT DEFAULT CURRENT_TIMESTAMP, - context TEXT -- Additional context (JSON) - ); + CREATE INDEX IF NOT EXISTS idx_accounts_did ON accounts(did); - -- Temporary data storage for multi-step operations - -- Provides transient storage for complex workflows - CREATE TABLE temp ( - id TEXT PRIMARY KEY, -- Unique identifier - data TEXT NOT NULL, -- JSON-serialized data - created TEXT DEFAULT CURRENT_TIMESTAMP, - expires TEXT -- Optional expiration - ); + CREATE TABLE IF NOT EXISTS secret ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + secretBase64 TEXT NOT NULL + ); - -- Initialize default application settings - -- These settings provide the baseline configuration for new installations - INSERT INTO settings (name, value) VALUES - ('apiServer', '${DEFAULT_ENDORSER_API_SERVER}'), - ('seenWelcomeScreen', 'false'); + INSERT OR IGNORE INTO secret (id, secretBase64) VALUES (1, '${secretBase64}'); - -- Initialize application secret - -- This secret is used for encrypting sensitive data within the application - INSERT INTO secret (id, hex) VALUES (1, '${databaseSecret}'); - `, -}); + CREATE TABLE IF NOT EXISTS settings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + accountDid TEXT, + activeDid TEXT, + apiServer TEXT, + filterFeedByNearby BOOLEAN, + filterFeedByVisible BOOLEAN, + finishedOnboarding BOOLEAN, + firstName TEXT, + hideRegisterPromptOnNewContact BOOLEAN, + isRegistered BOOLEAN, + lastName TEXT, + lastAckedOfferToUserJwtId TEXT, + lastAckedOfferToUserProjectsJwtId TEXT, + lastNotifiedClaimId TEXT, + lastViewedClaimId TEXT, + notifyingNewActivityTime TEXT, + notifyingReminderMessage TEXT, + notifyingReminderTime TEXT, + partnerApiServer TEXT, + passkeyExpirationMinutes INTEGER, + profileImageUrl TEXT, + searchBoxes TEXT, -- Stored as JSON string + showContactGivesInline BOOLEAN, + showGeneralAdvanced BOOLEAN, + showShortcutBvc BOOLEAN, + vapid TEXT, + warnIfProdServer BOOLEAN, + warnIfTestServer BOOLEAN, + webPushServer TEXT + ); -/** - * Migration 002: Add Content Visibility Control to Contacts - * - * This migration enhances the contacts table with privacy controls, allowing - * users to manage what content they want to see from each contact. This supports - * TimeSafari's privacy-first approach by giving users granular control over - * their information exposure. - * - * ## Changes Made: - * - * ### contacts.iViewContent - * New boolean column that controls whether the user wants to see content - * (activities, projects, offers) from this contact in their feeds and views. - * - * - `TRUE` (default): User sees all content from this contact - * - `FALSE`: User's interface filters out content from this contact - * - * ## Use Cases: - * - * 1. **Privacy Management**: Users can maintain contacts for trust/verification - * purposes while limiting information exposure - * - * 2. **Feed Curation**: Users can curate their activity feeds by selectively - * hiding content from certain contacts - * - * 3. **Professional Separation**: Users can separate professional and personal - * networks while maintaining cryptographic trust relationships - * - * 4. **Graduated Privacy**: Users can add contacts with limited visibility - * initially, then expand access as trust develops - * - * ## Privacy Architecture: - * - * This column works in conjunction with TimeSafari's broader privacy model: - * - Contact relationships are still maintained for verification - * - Cryptographic trust is preserved regardless of content visibility - * - Users can change visibility settings at any time - * - The setting only affects the local user's view, not the contact's capabilities - * - * ## Default Behavior: - * - * All existing contacts default to `TRUE` (visible) to maintain current - * user experience. New contacts will also default to visible, with users - * able to adjust visibility as needed. - */ -registerMigration({ - name: "002_add_iViewContent_to_contacts", - sql: ` - -- Add content visibility control to contacts table - -- This allows users to manage what content they see from each contact - -- while maintaining the cryptographic trust relationship - ALTER TABLE contacts ADD COLUMN iViewContent BOOLEAN DEFAULT TRUE; - `, -}); + CREATE INDEX IF NOT EXISTS idx_settings_accountDid ON settings(accountDid); -/** - * Template for Future Migrations - * - * When adding new migrations, follow this pattern: - * - * ```typescript - * registerMigration({ - * name: "003_descriptive_name", - * sql: ` - * -- Clear comment explaining what this migration does - * -- and why it's needed - * - * ALTER TABLE existing_table ADD COLUMN new_column TYPE DEFAULT value; - * - * -- Or create new tables: - * CREATE TABLE new_table ( - * id INTEGER PRIMARY KEY, - * -- ... other columns with comments - * ); - * - * -- Initialize any required data - * INSERT INTO new_table (column) VALUES ('initial_value'); - * `, - * }); - * ``` - * - * ## Migration Best Practices: - * - * 1. **Clear Naming**: Use descriptive names that explain the change - * 2. **Documentation**: Document the purpose and impact of each change - * 3. **Backward Compatibility**: Consider how changes affect existing data - * 4. **Default Values**: Provide sensible defaults for new columns - * 5. **Data Migration**: Include any necessary data transformation - * 6. **Testing**: Test migrations on representative data sets - * 7. **Performance**: Consider the impact on large datasets - * - * ## Schema Evolution Guidelines: - * - * - **Additive Changes**: Prefer adding new tables/columns over modifying existing ones - * - **Nullable Columns**: New columns should be nullable or have defaults - * - **Index Creation**: Add indexes for new query patterns - * - **Data Integrity**: Maintain referential integrity and constraints - * - **Privacy Preservation**: Ensure new schema respects privacy principles - */ + INSERT OR IGNORE INTO settings (id, apiServer) VALUES (1, '${DEFAULT_ENDORSER_API_SERVER}'); + + CREATE TABLE IF NOT EXISTS contacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + name TEXT, + contactMethods TEXT, -- Stored as JSON string + nextPubKeyHashB64 TEXT, + notes TEXT, + profileImageUrl TEXT, + publicKeyBase64 TEXT, + seesMe BOOLEAN, + registered BOOLEAN + ); + + CREATE INDEX IF NOT EXISTS idx_contacts_did ON contacts(did); + CREATE INDEX IF NOT EXISTS idx_contacts_name ON contacts(name); + + CREATE TABLE IF NOT EXISTS logs ( + date TEXT NOT NULL, + message TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS temp ( + id TEXT PRIMARY KEY, + blobB64 TEXT + ); + `, + }, + { + name: "002_add_iViewContent_to_contacts", + sql: ` + -- We need to handle the case where iViewContent column might already exist + -- SQLite doesn't support IF NOT EXISTS for ALTER TABLE ADD COLUMN + -- So we'll use a more robust approach with error handling in the migration service + + -- First, try to add the column - this will fail silently if it already exists + ALTER TABLE contacts ADD COLUMN iViewContent BOOLEAN DEFAULT TRUE; + `, + }, +]; /** - * Run all registered migrations - * - * This function is called during application initialization to ensure the - * database schema is up to date. It delegates to the migration service - * which handles the actual migration execution, tracking, and validation. - * - * The migration service will: - * 1. Check which migrations have already been applied - * 2. Apply any pending migrations in order - * 3. Validate that schema changes were successful - * 4. Record applied migrations for future reference - * - * @param sqlExec - Function to execute SQL statements - * @param sqlQuery - Function to execute SQL queries - * @param extractMigrationNames - Function to parse migration names from results - * @returns Promise that resolves when migrations are complete - * - * @example - * ```typescript - * // Called from platform service during database initialization - * await runMigrations( - * (sql, params) => db.run(sql, params), - * (sql, params) => db.query(sql, params), - * (result) => new Set(result.values.map(row => row[0])) - * ); - * ``` + * @param sqlExec - A function that executes a SQL statement and returns the result + * @param extractMigrationNames - A function that extracts the names (string array) from "select name from migrations" */ export async function runMigrations( sqlExec: (sql: string, params?: unknown[]) => Promise, sqlQuery: (sql: string, params?: unknown[]) => Promise, extractMigrationNames: (result: T) => Set, ): Promise { - return runMigrationsService(sqlExec, sqlQuery, extractMigrationNames); + for (const migration of MIGRATIONS) { + registerMigration(migration); + } + await runMigrationsService(sqlExec, sqlQuery, extractMigrationNames); } diff --git a/src/db/databaseUtil.ts b/src/db/databaseUtil.ts index 35f0e06c..b7dbd081 100644 --- a/src/db/databaseUtil.ts +++ b/src/db/databaseUtil.ts @@ -175,17 +175,17 @@ export let memoryLogs: string[] = []; * @author Matthew Raymer */ export async function logToDb(message: string): Promise { - const platform = PlatformServiceFactory.getInstance(); const todayKey = new Date().toDateString(); - const nowKey = new Date().toISOString(); try { memoryLogs.push(`${new Date().toISOString()} ${message}`); - // Try to insert first, if it fails due to UNIQUE constraint, update instead - await platform.dbExec("INSERT INTO logs (date, message) VALUES (?, ?)", [ - nowKey, - message, - ]); + + // TEMPORARILY DISABLED: Database logging to break error loop + // TODO: Fix schema mismatch - logs table uses 'timestamp' not 'date' + // await platform.dbExec("INSERT INTO logs (date, message) VALUES (?, ?)", [ + // nowKey, + // message, + // ]); // Clean up old logs (keep only last 7 days) - do this less frequently // Only clean up if the date is different from the last cleanup @@ -196,9 +196,11 @@ export async function logToDb(message: string): Promise { memoryLogs = memoryLogs.filter( (log) => log.split(" ")[0] > sevenDaysAgo.toDateString(), ); - await platform.dbExec("DELETE FROM logs WHERE date < ?", [ - sevenDaysAgo.toDateString(), - ]); + + // TEMPORARILY DISABLED: Database cleanup + // await platform.dbExec("DELETE FROM logs WHERE date < ?", [ + // sevenDaysAgo.toDateString(), + // ]); lastCleanupDate = todayKey; } } catch (error) {