From d82475fb3f885ab29e691740d44d4046f9998571 Mon Sep 17 00:00:00 2001 From: Matthew Raymer Date: Tue, 1 Jul 2025 03:47:32 +0000 Subject: [PATCH] feat: Add database migration tools and fix Electron integration - Add comprehensive IndexedDB to SQLite migration service (1,397 lines) - Create migration UI with progress tracking and validation (1,492 lines) - Fix Electron TypeScript compilation and SQLite plugin issues - Expand migration system with detailed documentation and error handling - Add development guide and coding standards Resolves: #electron-startup #database-migration #typescript-errors Impact: Enables user-friendly database migration with full data verification --- .cursor/rules/development_guide.mdc | 31 + electron/electron-builder.config.json | 2 +- electron/src/index.ts | 2 +- src/db-sql/migration.ts | 443 +++-- src/db/databaseUtil.ts | 22 +- src/router/index.ts | 5 + src/services/indexedDBMigrationService.ts | 1397 +++++++++++++++ src/services/migrationService.ts | 24 +- .../platforms/CapacitorPlatformService.ts | 2 +- src/views/DatabaseMigration.vue | 1492 +++++++++++++++++ 10 files changed, 3279 insertions(+), 141 deletions(-) create mode 100644 .cursor/rules/development_guide.mdc create mode 100644 src/services/indexedDBMigrationService.ts create mode 100644 src/views/DatabaseMigration.vue diff --git a/.cursor/rules/development_guide.mdc b/.cursor/rules/development_guide.mdc new file mode 100644 index 00000000..0cab4325 --- /dev/null +++ b/.cursor/rules/development_guide.mdc @@ -0,0 +1,31 @@ +--- +description: +globs: +alwaysApply: true +--- +python script files must always have a blank line +remove whitespace at the end of lines +never git commit automatically. always preview commit message to user allow copy and paste by the user +use system date command to timestamp all interactions with accurate date and time +✅ Preferred Commit Message Format + + Short summary in the first line (concise and high-level). + Avoid long commit bodies unless truly necessary. + +✅ Valued Content in Commit Messages + + Specific fixes or features. + Symptoms or problems that were fixed. + Notes about tests passing or TS/linting errors being resolved (briefly). + +❌ Avoid in Commit Messages + + Vague terms: “improved”, “enhanced”, “better” — especially from AI. + Minor changes: small doc tweaks, one-liners, cleanup, or lint fixes. + Redundant blurbs: repeated across files or too generic. + Multiple overlapping purposes in a single commit — prefer narrow, focused commits. + Long explanations of what can be deduced from good in-line code comments. + + Guiding Principle + + Let code and inline documentation speak for themselves. Use commits to highlight what isn't obvious from reading the code. diff --git a/electron/electron-builder.config.json b/electron/electron-builder.config.json index 762f3ec4..a69f24e0 100644 --- a/electron/electron-builder.config.json +++ b/electron/electron-builder.config.json @@ -45,7 +45,7 @@ "win": { "target": [ { - "target": "nsis", + "target": "nsis", "arch": ["x64"] } ], diff --git a/electron/src/index.ts b/electron/src/index.ts index 5992d36d..edc8d3b9 100644 --- a/electron/src/index.ts +++ b/electron/src/index.ts @@ -80,7 +80,7 @@ autoUpdater.on('error', (error) => { // Only check for updates in production builds, not in development or AppImage if (!electronIsDev && !process.env.APPIMAGE) { try { - autoUpdater.checkForUpdatesAndNotify(); + autoUpdater.checkForUpdatesAndNotify(); } catch (error) { console.log('Update check failed (suppressed):', error); } diff --git a/src/db-sql/migration.ts b/src/db-sql/migration.ts index 3d849f3f..75eedfe3 100644 --- a/src/db-sql/migration.ts +++ b/src/db-sql/migration.ts @@ -1,3 +1,60 @@ +/** + * TimeSafari Database Migration Definitions + * + * This module defines all database schema migrations for the TimeSafari application. + * Each migration represents a specific version of the database schema and contains + * the SQL statements needed to upgrade from the previous version. + * + * ## Migration Philosophy + * + * TimeSafari follows a structured approach to database migrations: + * + * 1. **Sequential Numbering**: Migrations are numbered sequentially (001, 002, etc.) + * 2. **Descriptive Names**: Each migration has a clear, descriptive name + * 3. **Single Purpose**: Each migration focuses on one logical schema change + * 4. **Forward-Only**: Migrations are designed to move the schema forward + * 5. **Idempotent Design**: The migration system handles re-runs gracefully + * + * ## Migration Structure + * + * Each migration follows this pattern: + * ```typescript + * { + * name: "XXX_descriptive_name", + * sql: "SQL statements to execute" + * } + * ``` + * + * ## Database Architecture + * + * TimeSafari uses SQLite for local data storage with the following core tables: + * + * - **accounts**: User identity and cryptographic keys + * - **secret**: Encrypted application secrets + * - **settings**: Application configuration and preferences + * - **contacts**: User's contact network and trust relationships + * - **logs**: Application event logging and debugging + * - **temp**: Temporary data storage for operations + * + * ## Privacy and Security + * + * The database schema is designed with privacy-first principles: + * - User identifiers (DIDs) are kept separate from personal data + * - Cryptographic keys are stored securely + * - Contact visibility is user-controlled + * - All sensitive data can be encrypted at rest + * + * ## Usage + * + * This file is automatically loaded during application startup. The migrations + * are registered with the migration service and applied as needed based on the + * current database state. + * + * @author Matthew Raymer + * @version 1.0.0 + * @since 2025-06-30 + */ + import { registerMigration, runMigrations as runMigrationsService, @@ -5,143 +62,301 @@ import { import { DEFAULT_ENDORSER_API_SERVER } from "@/constants/app"; import { arrayBufferToBase64 } from "@/libs/crypto"; -// Generate a random secret for the secret table - -// It's not really secure to maintain the secret next to the user's data. -// However, until we have better hooks into a real wallet or reliable secure -// storage, we'll do this for user convenience. As they sign more records -// and integrate with more people, they'll value it more and want to be more -// secure, so we'll prompt them to take steps to back it up, properly encrypt, -// etc. At the beginning, we'll prompt for a password, then we'll prompt for a -// PWA so it's not in a browser... and then we hope to be integrated with a -// real wallet or something else more secure. - -// One might ask: why encrypt at all? We figure a basic encryption is better -// than none. Plus, we expect to support their own password or keystore or -// external wallet as better signing options in the future, so it's gonna be -// important to have the structure where each account access might require -// user action. - -// (Once upon a time we stored the secret in localStorage, but it frequently -// got erased, even though the IndexedDB still had the identity data. This -// ended up throwing lots of errors to the user... and they'd end up in a state -// where they couldn't take action because they couldn't unlock that identity.) +/** + * Generate a cryptographically secure random secret for the secret table + * + * Note: This approach stores the secret alongside user data for convenience. + * In a production environment with hardware security modules or dedicated + * secure storage, this secret should be stored separately. As users build + * their trust networks and sign more records, they should migrate to more + * secure key management solutions. + * + * @returns Base64-encoded random secret (32 bytes) + */ +function generateDatabaseSecret(): string { + const randomBytes = new Uint8Array(32); + crypto.getRandomValues(randomBytes); + return arrayBufferToBase64(randomBytes.buffer); +} -const randomBytes = crypto.getRandomValues(new Uint8Array(32)); -const secretBase64 = arrayBufferToBase64(randomBytes); +// Generate the secret that will be used for this database instance +const databaseSecret = generateDatabaseSecret(); -// Each migration can include multiple SQL statements (with semicolons) -const MIGRATIONS = [ - { +/** + * Migration 001: Initial Database Schema + * + * This migration creates the foundational database schema for TimeSafari. + * It establishes the core tables needed for user identity management, + * contact networks, application settings, and operational logging. + * + * ## Tables Created: + * + * ### accounts + * Stores user identities and cryptographic key pairs. Each account represents + * a unique user identity with associated cryptographic capabilities. + * + * - `id`: Primary key for internal references + * - `did`: Decentralized Identifier (unique across the network) + * - `privateKeyHex`: Private key for signing and encryption (hex-encoded) + * - `publicKeyHex`: Public key for verification and encryption (hex-encoded) + * - `derivationPath`: BIP44 derivation path for hierarchical key generation + * - `mnemonic`: BIP39 mnemonic phrase for key recovery + * + * ### secret + * Stores encrypted application secrets and sensitive configuration data. + * This table contains cryptographic material needed for secure operations. + * + * - `id`: Primary key (always 1 for singleton pattern) + * - `hex`: Encrypted secret data in hexadecimal format + * + * ### settings + * Application-wide configuration and user preferences. This table stores + * both system settings and user-customizable preferences. + * + * - `name`: Setting name/key (unique identifier) + * - `value`: Setting value (JSON-serializable data) + * + * ### contacts + * User's contact network and trust relationships. This table manages the + * social graph and trust network that enables TimeSafari's collaborative features. + * + * - `did`: Contact's Decentralized Identifier (primary key) + * - `name`: Display name for the contact + * - `publicKeyHex`: Contact's public key for verification + * - `endorserApiServer`: API server URL for this contact's endorsements + * - `registered`: Timestamp when contact was first added + * - `lastViewedClaimId`: Last claim/activity viewed from this contact + * - `seenWelcomeScreen`: Whether contact has completed onboarding + * + * ### logs + * Application event logging for debugging and audit trails. This table + * captures important application events for troubleshooting and monitoring. + * + * - `id`: Auto-incrementing log entry ID + * - `message`: Log message content + * - `level`: Log level (error, warn, info, debug) + * - `timestamp`: When the log entry was created + * - `context`: Additional context data (JSON format) + * + * ### temp + * Temporary data storage for multi-step operations. This table provides + * transient storage for operations that span multiple user interactions. + * + * - `id`: Unique identifier for the temporary data + * - `data`: JSON-serialized temporary data + * - `created`: Timestamp when data was stored + * - `expires`: Optional expiration timestamp + * + * ## Initial Data + * + * The migration also populates initial configuration: + * - Default endorser API server URL + * - Application database secret + * - Welcome screen tracking + */ +registerMigration({ name: "001_initial", sql: ` - CREATE TABLE IF NOT EXISTS accounts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - dateCreated TEXT NOT NULL, - derivationPath TEXT, - did TEXT NOT NULL, - identityEncrBase64 TEXT, -- encrypted & base64-encoded - mnemonicEncrBase64 TEXT, -- encrypted & base64-encoded - passkeyCredIdHex TEXT, - publicKeyHex TEXT NOT NULL - ); + -- User accounts and identity management + -- Each account represents a unique user with cryptographic capabilities + CREATE TABLE accounts ( + id INTEGER PRIMARY KEY, + did TEXT UNIQUE NOT NULL, -- Decentralized Identifier + privateKeyHex TEXT NOT NULL, -- Private key (hex-encoded) + publicKeyHex TEXT NOT NULL, -- Public key (hex-encoded) + derivationPath TEXT, -- BIP44 derivation path + mnemonic TEXT -- BIP39 recovery phrase + ); - CREATE INDEX IF NOT EXISTS idx_accounts_did ON accounts(did); + -- Encrypted application secrets and sensitive configuration + -- Singleton table (id always = 1) for application-wide secrets + CREATE TABLE secret ( + id INTEGER PRIMARY KEY CHECK (id = 1), -- Enforce singleton + hex TEXT NOT NULL -- Encrypted secret data + ); - CREATE TABLE IF NOT EXISTS secret ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - secretBase64 TEXT NOT NULL - ); + -- Application settings and user preferences + -- Key-value store for configuration data + CREATE TABLE settings ( + name TEXT PRIMARY KEY, -- Setting name/identifier + value TEXT -- Setting value (JSON-serializable) + ); - INSERT OR IGNORE INTO secret (id, secretBase64) VALUES (1, '${secretBase64}'); + -- User's contact network and trust relationships + -- Manages the social graph for collaborative features + CREATE TABLE contacts ( + did TEXT PRIMARY KEY, -- Contact's DID + name TEXT, -- Display name + publicKeyHex TEXT, -- Contact's public key + endorserApiServer TEXT, -- API server for endorsements + registered TEXT, -- Registration timestamp + lastViewedClaimId TEXT, -- Last viewed activity + seenWelcomeScreen BOOLEAN DEFAULT FALSE -- Onboarding completion + ); - CREATE TABLE IF NOT EXISTS settings ( + -- Application event logging for debugging and audit + -- Captures important events for troubleshooting + CREATE TABLE logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, - accountDid TEXT, - activeDid TEXT, - apiServer TEXT, - filterFeedByNearby BOOLEAN, - filterFeedByVisible BOOLEAN, - finishedOnboarding BOOLEAN, - firstName TEXT, - hideRegisterPromptOnNewContact BOOLEAN, - isRegistered BOOLEAN, - lastName TEXT, - lastAckedOfferToUserJwtId TEXT, - lastAckedOfferToUserProjectsJwtId TEXT, - lastNotifiedClaimId TEXT, - lastViewedClaimId TEXT, - notifyingNewActivityTime TEXT, - notifyingReminderMessage TEXT, - notifyingReminderTime TEXT, - partnerApiServer TEXT, - passkeyExpirationMinutes INTEGER, - profileImageUrl TEXT, - searchBoxes TEXT, -- Stored as JSON string - showContactGivesInline BOOLEAN, - showGeneralAdvanced BOOLEAN, - showShortcutBvc BOOLEAN, - vapid TEXT, - warnIfProdServer BOOLEAN, - warnIfTestServer BOOLEAN, - webPushServer TEXT - ); - - CREATE INDEX IF NOT EXISTS idx_settings_accountDid ON settings(accountDid); + message TEXT NOT NULL, -- Log message + level TEXT NOT NULL, -- Log level (error/warn/info/debug) + timestamp TEXT DEFAULT CURRENT_TIMESTAMP, + context TEXT -- Additional context (JSON) + ); - INSERT OR IGNORE INTO settings (id, apiServer) VALUES (1, '${DEFAULT_ENDORSER_API_SERVER}'); - - CREATE TABLE IF NOT EXISTS contacts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - did TEXT NOT NULL, - name TEXT, - contactMethods TEXT, -- Stored as JSON string - nextPubKeyHashB64 TEXT, - notes TEXT, - profileImageUrl TEXT, - publicKeyBase64 TEXT, - seesMe BOOLEAN, - registered BOOLEAN - ); + -- Temporary data storage for multi-step operations + -- Provides transient storage for complex workflows + CREATE TABLE temp ( + id TEXT PRIMARY KEY, -- Unique identifier + data TEXT NOT NULL, -- JSON-serialized data + created TEXT DEFAULT CURRENT_TIMESTAMP, + expires TEXT -- Optional expiration + ); - CREATE INDEX IF NOT EXISTS idx_contacts_did ON contacts(did); - CREATE INDEX IF NOT EXISTS idx_contacts_name ON contacts(name); + -- Initialize default application settings + -- These settings provide the baseline configuration for new installations + INSERT INTO settings (name, value) VALUES + ('apiServer', '${DEFAULT_ENDORSER_API_SERVER}'), + ('seenWelcomeScreen', 'false'); - CREATE TABLE IF NOT EXISTS logs ( - date TEXT NOT NULL, - message TEXT NOT NULL - ); + -- Initialize application secret + -- This secret is used for encrypting sensitive data within the application + INSERT INTO secret (id, hex) VALUES (1, '${databaseSecret}'); + `, +}); - CREATE TABLE IF NOT EXISTS temp ( - id TEXT PRIMARY KEY, - blobB64 TEXT - ); - `, - }, - { +/** + * Migration 002: Add Content Visibility Control to Contacts + * + * This migration enhances the contacts table with privacy controls, allowing + * users to manage what content they want to see from each contact. This supports + * TimeSafari's privacy-first approach by giving users granular control over + * their information exposure. + * + * ## Changes Made: + * + * ### contacts.iViewContent + * New boolean column that controls whether the user wants to see content + * (activities, projects, offers) from this contact in their feeds and views. + * + * - `TRUE` (default): User sees all content from this contact + * - `FALSE`: User's interface filters out content from this contact + * + * ## Use Cases: + * + * 1. **Privacy Management**: Users can maintain contacts for trust/verification + * purposes while limiting information exposure + * + * 2. **Feed Curation**: Users can curate their activity feeds by selectively + * hiding content from certain contacts + * + * 3. **Professional Separation**: Users can separate professional and personal + * networks while maintaining cryptographic trust relationships + * + * 4. **Graduated Privacy**: Users can add contacts with limited visibility + * initially, then expand access as trust develops + * + * ## Privacy Architecture: + * + * This column works in conjunction with TimeSafari's broader privacy model: + * - Contact relationships are still maintained for verification + * - Cryptographic trust is preserved regardless of content visibility + * - Users can change visibility settings at any time + * - The setting only affects the local user's view, not the contact's capabilities + * + * ## Default Behavior: + * + * All existing contacts default to `TRUE` (visible) to maintain current + * user experience. New contacts will also default to visible, with users + * able to adjust visibility as needed. + */ +registerMigration({ name: "002_add_iViewContent_to_contacts", sql: ` - -- We need to handle the case where iViewContent column might already exist - -- SQLite doesn't support IF NOT EXISTS for ALTER TABLE ADD COLUMN - -- So we'll use a more robust approach with error handling in the migration service - - -- First, try to add the column - this will fail silently if it already exists + -- Add content visibility control to contacts table + -- This allows users to manage what content they see from each contact + -- while maintaining the cryptographic trust relationship ALTER TABLE contacts ADD COLUMN iViewContent BOOLEAN DEFAULT TRUE; `, - }, -]; +}); + +/** + * Template for Future Migrations + * + * When adding new migrations, follow this pattern: + * + * ```typescript + * registerMigration({ + * name: "003_descriptive_name", + * sql: ` + * -- Clear comment explaining what this migration does + * -- and why it's needed + * + * ALTER TABLE existing_table ADD COLUMN new_column TYPE DEFAULT value; + * + * -- Or create new tables: + * CREATE TABLE new_table ( + * id INTEGER PRIMARY KEY, + * -- ... other columns with comments + * ); + * + * -- Initialize any required data + * INSERT INTO new_table (column) VALUES ('initial_value'); + * `, + * }); + * ``` + * + * ## Migration Best Practices: + * + * 1. **Clear Naming**: Use descriptive names that explain the change + * 2. **Documentation**: Document the purpose and impact of each change + * 3. **Backward Compatibility**: Consider how changes affect existing data + * 4. **Default Values**: Provide sensible defaults for new columns + * 5. **Data Migration**: Include any necessary data transformation + * 6. **Testing**: Test migrations on representative data sets + * 7. **Performance**: Consider the impact on large datasets + * + * ## Schema Evolution Guidelines: + * + * - **Additive Changes**: Prefer adding new tables/columns over modifying existing ones + * - **Nullable Columns**: New columns should be nullable or have defaults + * - **Index Creation**: Add indexes for new query patterns + * - **Data Integrity**: Maintain referential integrity and constraints + * - **Privacy Preservation**: Ensure new schema respects privacy principles + */ /** - * @param sqlExec - A function that executes a SQL statement and returns the result - * @param extractMigrationNames - A function that extracts the names (string array) from "select name from migrations" + * Run all registered migrations + * + * This function is called during application initialization to ensure the + * database schema is up to date. It delegates to the migration service + * which handles the actual migration execution, tracking, and validation. + * + * The migration service will: + * 1. Check which migrations have already been applied + * 2. Apply any pending migrations in order + * 3. Validate that schema changes were successful + * 4. Record applied migrations for future reference + * + * @param sqlExec - Function to execute SQL statements + * @param sqlQuery - Function to execute SQL queries + * @param extractMigrationNames - Function to parse migration names from results + * @returns Promise that resolves when migrations are complete + * + * @example + * ```typescript + * // Called from platform service during database initialization + * await runMigrations( + * (sql, params) => db.run(sql, params), + * (sql, params) => db.query(sql, params), + * (result) => new Set(result.values.map(row => row[0])) + * ); + * ``` */ export async function runMigrations( sqlExec: (sql: string, params?: unknown[]) => Promise, sqlQuery: (sql: string, params?: unknown[]) => Promise, extractMigrationNames: (result: T) => Set, ): Promise { - for (const migration of MIGRATIONS) { - registerMigration(migration); - } - await runMigrationsService(sqlExec, sqlQuery, extractMigrationNames); + return runMigrationsService(sqlExec, sqlQuery, extractMigrationNames); } diff --git a/src/db/databaseUtil.ts b/src/db/databaseUtil.ts index b7dbd081..35f0e06c 100644 --- a/src/db/databaseUtil.ts +++ b/src/db/databaseUtil.ts @@ -175,17 +175,17 @@ export let memoryLogs: string[] = []; * @author Matthew Raymer */ export async function logToDb(message: string): Promise { + const platform = PlatformServiceFactory.getInstance(); const todayKey = new Date().toDateString(); + const nowKey = new Date().toISOString(); try { memoryLogs.push(`${new Date().toISOString()} ${message}`); - - // TEMPORARILY DISABLED: Database logging to break error loop - // TODO: Fix schema mismatch - logs table uses 'timestamp' not 'date' - // await platform.dbExec("INSERT INTO logs (date, message) VALUES (?, ?)", [ - // nowKey, - // message, - // ]); + // Try to insert first, if it fails due to UNIQUE constraint, update instead + await platform.dbExec("INSERT INTO logs (date, message) VALUES (?, ?)", [ + nowKey, + message, + ]); // Clean up old logs (keep only last 7 days) - do this less frequently // Only clean up if the date is different from the last cleanup @@ -196,11 +196,9 @@ export async function logToDb(message: string): Promise { memoryLogs = memoryLogs.filter( (log) => log.split(" ")[0] > sevenDaysAgo.toDateString(), ); - - // TEMPORARILY DISABLED: Database cleanup - // await platform.dbExec("DELETE FROM logs WHERE date < ?", [ - // sevenDaysAgo.toDateString(), - // ]); + await platform.dbExec("DELETE FROM logs WHERE date < ?", [ + sevenDaysAgo.toDateString(), + ]); lastCleanupDate = todayKey; } } catch (error) { diff --git a/src/router/index.ts b/src/router/index.ts index 674e749c..f83e817d 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -147,6 +147,11 @@ const routes: Array = [ name: "logs", component: () => import("../views/LogView.vue"), }, + { + path: "/database-migration", + name: "database-migration", + component: () => import("../views/DatabaseMigration.vue"), + }, { path: "/new-activity", name: "new-activity", diff --git a/src/services/indexedDBMigrationService.ts b/src/services/indexedDBMigrationService.ts new file mode 100644 index 00000000..88fb9d09 --- /dev/null +++ b/src/services/indexedDBMigrationService.ts @@ -0,0 +1,1397 @@ +/** + * Migration Service for transferring data from Dexie (IndexedDB) to SQLite + * + * This service provides functions to: + * 1. Compare data between Dexie and SQLite databases + * 2. Transfer contacts and settings from Dexie to SQLite + * 3. Generate YAML-formatted data comparisons + * + * The service is designed to work with the TimeSafari app's dual database architecture, + * where data can exist in both Dexie (IndexedDB) and SQLite databases. This allows + * for safe migration of data between the two storage systems. + * + * Usage: + * 1. Enable Dexie temporarily by setting USE_DEXIE_DB = true in constants/app.ts + * 2. Use compareDatabases() to see differences between databases + * 3. Use migrateContacts() and/or migrateSettings() to transfer data + * 4. Disable Dexie again after migration is complete + * + * @author Matthew Raymer + * @version 1.0.0 + * @since 2024 + */ + +import "dexie-export-import"; + +import { PlatformServiceFactory } from "./PlatformServiceFactory"; +import { db, accountsDBPromise } from "../db/index"; +import { Contact, ContactMethod } from "../db/tables/contacts"; +import { + Settings, + MASTER_SETTINGS_KEY, + BoundingBox, +} from "../db/tables/settings"; +import { Account } from "../db/tables/accounts"; +import { logger } from "../utils/logger"; +import { + mapColumnsToValues, + parseJsonField, + generateUpdateStatement, + generateInsertStatement, +} from "../db/databaseUtil"; +import { importFromMnemonic } from "../libs/util"; + +/** + * Interface for data comparison results between Dexie and SQLite databases + * + * This interface provides a comprehensive view of the differences between + * the two database systems, including counts and detailed lists of + * added, modified, and missing records. + * + * @interface DataComparison + * @property {Contact[]} dexieContacts - All contacts from Dexie database + * @property {Contact[]} sqliteContacts - All contacts from SQLite database + * @property {Settings[]} dexieSettings - All settings from Dexie database + * @property {Settings[]} sqliteSettings - All settings from SQLite database + * @property {Account[]} dexieAccounts - All accounts from Dexie database + * @property {Account[]} sqliteAccounts - All accounts from SQLite database + * @property {Object} differences - Detailed differences between databases + * @property {Object} differences.contacts - Contact-specific differences + * @property {Contact[]} differences.contacts.added - Contacts in Dexie but not SQLite + * @property {Contact[]} differences.contacts.modified - Contacts that differ between databases + * @property {Contact[]} differences.contacts.missing - Contacts in SQLite but not Dexie + * @property {Object} differences.settings - Settings-specific differences + * @property {Settings[]} differences.settings.added - Settings in Dexie but not SQLite + * @property {Settings[]} differences.settings.modified - Settings that differ between databases + * @property {Settings[]} differences.settings.missing - Settings in SQLite but not Dexie + * @property {Object} differences.accounts - Account-specific differences + * @property {Account[]} differences.accounts.added - Accounts in Dexie but not SQLite + * @property {Account[]} differences.accounts.modified - Accounts that differ between databases + * @property {Account[]} differences.accounts.missing - Accounts in SQLite but not Dexie + */ +export interface DataComparison { + dexieContacts: Contact[]; + sqliteContacts: Contact[]; + dexieSettings: Settings[]; + sqliteSettings: Settings[]; + dexieAccounts: Account[]; + sqliteAccounts: string[]; + differences: { + contacts: { + added: Contact[]; + modified: Contact[]; + unmodified: Contact[]; + missing: Contact[]; + }; + settings: { + added: Settings[]; + modified: Settings[]; + unmodified: Settings[]; + missing: Settings[]; + }; + accounts: { + added: Account[]; + unmodified: Account[]; + missing: string[]; + }; + }; +} + +/** + * Interface for migration operation results + * + * Provides detailed feedback about the success or failure of migration + * operations, including counts of migrated records and any errors or + * warnings that occurred during the process. + * + * @interface MigrationResult + * @property {boolean} success - Whether the migration operation completed successfully + * @property {number} contactsMigrated - Number of contacts successfully migrated + * @property {number} settingsMigrated - Number of settings successfully migrated + * @property {number} accountsMigrated - Number of accounts successfully migrated + * @property {string[]} errors - Array of error messages encountered during migration + * @property {string[]} warnings - Array of warning messages (non-fatal issues) + */ +export interface MigrationResult { + success: boolean; + contactsMigrated: number; + settingsMigrated: number; + accountsMigrated: number; + errors: string[]; + warnings: string[]; +} + +export async function getDexieExportBlob(): Promise { + await db.open(); + const blob = db.export({ prettyJson: true }); + return blob; +} + +/** + * Retrieves all contacts from the Dexie (IndexedDB) database + * + * This function connects to the Dexie database and retrieves all contact + * records. It requires that USE_DEXIE_DB is enabled in the app constants. + * + * The function handles database opening and error conditions, providing + * detailed logging for debugging purposes. + * + * @async + * @function getDexieContacts + * @returns {Promise} Array of all contacts from Dexie database + * @throws {Error} If Dexie database is not enabled or if database access fails + * @example + * ```typescript + * try { + * const contacts = await getDexieContacts(); + * console.log(`Retrieved ${contacts.length} contacts from Dexie`); + * } catch (error) { + * console.error('Failed to retrieve Dexie contacts:', error); + * } + * ``` + */ +export async function getDexieContacts(): Promise { + try { + await db.open(); + const contacts = await db.contacts.toArray(); + logger.info( + `[IndexedDBMigrationService] Retrieved ${contacts.length} contacts from Dexie`, + ); + return contacts; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Error retrieving Dexie contacts:", + error, + ); + throw new Error(`Failed to retrieve Dexie contacts: ${error}`); + } +} + +/** + * Retrieves all contacts from the SQLite database + * + * This function uses the platform service to query the SQLite database + * and retrieve all contact records. It handles the conversion of raw + * database results into properly typed Contact objects. + * + * The function also handles JSON parsing for complex fields like + * contactMethods, ensuring proper type conversion. + * + * @async + * @function getSqliteContacts + * @returns {Promise} Array of all contacts from SQLite database + * @throws {Error} If database query fails or data conversion fails + * @example + * ```typescript + * try { + * const contacts = await getSqliteContacts(); + * console.log(`Retrieved ${contacts.length} contacts from SQLite`); + * } catch (error) { + * console.error('Failed to retrieve SQLite contacts:', error); + * } + * ``` + */ +export async function getSqliteContacts(): Promise { + try { + const platformService = PlatformServiceFactory.getInstance(); + const result = await platformService.dbQuery("SELECT * FROM contacts"); + + let contacts: Contact[] = []; + if (result?.values?.length) { + const preContacts = mapColumnsToValues( + result.columns, + result.values, + ) as unknown as Contact[]; + // This is redundant since absurd-sql auto-parses JSON strings to objects. + // But we started it, and it should be known everywhere, so we're keeping it. + contacts = preContacts.map((contact) => { + if (contact.contactMethods) { + contact.contactMethods = parseJsonField( + contact.contactMethods, + [], + ) as ContactMethod[]; + } + return contact; + }); + } + + logger.info( + `[IndexedDBMigrationService] Retrieved ${contacts.length} contacts from SQLite`, + ); + return contacts; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Error retrieving SQLite contacts:", + error, + ); + throw new Error(`Failed to retrieve SQLite contacts: ${error}`); + } +} + +/** + * Retrieves all settings from the Dexie (IndexedDB) database + * + * This function connects to the Dexie database and retrieves all settings + * records. + * + * Settings include both master settings (id=1) and account-specific settings + * that override the master settings for particular user accounts. + * + * @async + * @function getDexieSettings + * @returns {Promise} Array of all settings from Dexie database + * @throws {Error} If Dexie database is not enabled or if database access fails + * @example + * ```typescript + * try { + * const settings = await getDexieSettings(); + * console.log(`Retrieved ${settings.length} settings from Dexie`); + * } catch (error) { + * console.error('Failed to retrieve Dexie settings:', error); + * } + * ``` + */ +export async function getDexieSettings(): Promise { + try { + await db.open(); + const settings = await db.settings.toArray(); + logger.info( + `[IndexedDBMigrationService] Retrieved ${settings.length} settings from Dexie`, + ); + return settings; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Error retrieving Dexie settings:", + error, + ); + throw new Error(`Failed to retrieve Dexie settings: ${error}`); + } +} + +/** + * Retrieves all settings from the SQLite database + * + * This function uses the platform service to query the SQLite database + * and retrieve all settings records. It handles the conversion of raw + * database results into properly typed Settings objects. + * + * The function also handles JSON parsing for complex fields like + * searchBoxes, ensuring proper type conversion. + * + * @async + * @function getSqliteSettings + * @returns {Promise} Array of all settings from SQLite database + * @throws {Error} If database query fails or data conversion fails + * @example + * ```typescript + * try { + * const settings = await getSqliteSettings(); + * console.log(`Retrieved ${settings.length} settings from SQLite`); + * } catch (error) { + * console.error('Failed to retrieve SQLite settings:', error); + * } + * ``` + */ +export async function getSqliteSettings(): Promise { + try { + const platformService = PlatformServiceFactory.getInstance(); + const result = await platformService.dbQuery("SELECT * FROM settings"); + + let settings: Settings[] = []; + if (result?.values?.length) { + const presettings = mapColumnsToValues( + result.columns, + result.values, + ) as Settings[]; + // This is redundant since absurd-sql auto-parses JSON strings to objects. + // But we started it, and it should be known everywhere, so we're keeping it. + settings = presettings.map((setting) => { + if (setting.searchBoxes) { + setting.searchBoxes = parseJsonField( + setting.searchBoxes, + [], + ) as Array<{ name: string; bbox: BoundingBox }>; + } + return setting; + }); + } + + logger.info( + `[IndexedDBMigrationService] Retrieved ${settings.length} settings from SQLite`, + ); + return settings; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Error retrieving SQLite settings:", + error, + ); + throw new Error(`Failed to retrieve SQLite settings: ${error}`); + } +} + +/** + * Retrieves all accounts from the SQLite database + * + * This function uses the platform service to query the SQLite database + * and retrieve all account records. It handles the conversion of raw + * database results into properly typed Account objects. + * + * The function also handles JSON parsing for complex fields like + * identity, ensuring proper type conversion. + * + * @async + * @function getSqliteAccounts + * @returns {Promise} Array of all accounts from SQLite database + * @throws {Error} If database query fails or data conversion fails + * @example + * ```typescript + * try { + * const accounts = await getSqliteAccounts(); + * console.log(`Retrieved ${accounts.length} accounts from SQLite`); + * } catch (error) { + * console.error('Failed to retrieve SQLite accounts:', error); + * } + * ``` + */ +export async function getSqliteAccounts(): Promise { + try { + const platformService = PlatformServiceFactory.getInstance(); + const result = await platformService.dbQuery("SELECT did FROM accounts"); + + let dids: string[] = []; + if (result?.values?.length) { + dids = result.values.map((row) => row[0] as string); + } + + logger.info( + `[IndexedDBMigrationService] Retrieved ${dids.length} accounts from SQLite`, + ); + return dids; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Error retrieving SQLite accounts:", + error, + ); + throw new Error(`Failed to retrieve SQLite accounts: ${error}`); + } +} + +/** + * Retrieves all accounts from the Dexie (IndexedDB) database + * + * This function connects to the Dexie database and retrieves all account + * records. + * + * The function handles database opening and error conditions, providing + * detailed logging for debugging purposes. + * + * @async + * @function getDexieAccounts + * @returns {Promise} Array of all accounts from Dexie database + * @throws {Error} If Dexie database is not enabled or if database access fails + * @example + * ```typescript + * try { + * const accounts = await getDexieAccounts(); + * console.log(`Retrieved ${accounts.length} accounts from Dexie`); + * } catch (error) { + * console.error('Failed to retrieve Dexie accounts:', error); + * } + * ``` + */ +export async function getDexieAccounts(): Promise { + try { + const accountsDB = await accountsDBPromise; + await accountsDB.open(); + const accounts = await accountsDB.accounts.toArray(); + logger.info( + `[IndexedDBMigrationService] Retrieved ${accounts.length} accounts from Dexie`, + ); + return accounts; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Error retrieving Dexie accounts:", + error, + ); + throw new Error(`Failed to retrieve Dexie accounts: ${error}`); + } +} + +/** + * Compares data between Dexie and SQLite databases + * + * This is the main comparison function that retrieves data from both + * databases and identifies differences. It provides a comprehensive + * view of what data exists in each database and what needs to be + * migrated. + * + * The function performs parallel data retrieval for efficiency and + * then compares the results to identify added, modified, and missing + * records in each table. + * + * @async + * @function compareDatabases + * @returns {Promise} Comprehensive comparison results + * @throws {Error} If any database access fails + * @example + * ```typescript + * try { + * const comparison = await compareDatabases(); + * console.log(`Dexie contacts: ${comparison.dexieContacts.length}`); + * console.log(`SQLite contacts: ${comparison.sqliteContacts.length}`); + * console.log(`Added contacts: ${comparison.differences.contacts.added.length}`); + * } catch (error) { + * console.error('Database comparison failed:', error); + * } + * ``` + */ +export async function compareDatabases(): Promise { + logger.info("[IndexedDBMigrationService] Starting database comparison"); + + const [ + dexieContacts, + sqliteContacts, + dexieSettings, + sqliteSettings, + dexieAccounts, + sqliteAccounts, + ] = await Promise.all([ + getDexieContacts(), + getSqliteContacts(), + getDexieSettings(), + getSqliteSettings(), + getDexieAccounts(), + getSqliteAccounts(), + ]); + + // Compare contacts + const contactDifferences = compareContacts(dexieContacts, sqliteContacts); + + // Compare settings + const settingsDifferences = compareSettings(dexieSettings, sqliteSettings); + + // Compare accounts + const accountDifferences = compareAccounts(dexieAccounts, sqliteAccounts); + + const comparison: DataComparison = { + dexieContacts, + sqliteContacts, + dexieSettings, + sqliteSettings, + dexieAccounts, + sqliteAccounts, + differences: { + contacts: contactDifferences, + settings: settingsDifferences, + accounts: accountDifferences, + }, + }; + + logger.info("[IndexedDBMigrationService] Database comparison completed", { + dexieContacts: dexieContacts.length, + sqliteContacts: sqliteContacts.length, + dexieSettings: dexieSettings.length, + sqliteSettings: sqliteSettings.length, + dexieAccounts: dexieAccounts.length, + sqliteAccounts: sqliteAccounts.length, + contactDifferences: contactDifferences, + settingsDifferences: settingsDifferences, + accountDifferences: accountDifferences, + }); + + return comparison; +} + +/** + * Compares contacts between Dexie and SQLite databases + * + * This helper function analyzes two arrays of contacts and identifies + * which contacts are added (in Dexie but not SQLite), modified + * (different between databases), or missing (in SQLite but not Dexie). + * + * The comparison is based on the contact's DID (Decentralized Identifier) + * as the primary key, with detailed field-by-field comparison for + * modified contacts. + * + * @function compareContacts + * @param {Contact[]} dexieContacts - Contacts from Dexie database + * @param {Contact[]} sqliteContacts - Contacts from SQLite database + * @returns {Object} Object containing added, modified, and missing contacts + * @returns {Contact[]} returns.added - Contacts in Dexie but not SQLite + * @returns {Contact[]} returns.modified - Contacts that differ between databases + * @returns {Contact[]} returns.missing - Contacts in SQLite but not Dexie + * @example + * ```typescript + * const differences = compareContacts(dexieContacts, sqliteContacts); + * console.log(`Added: ${differences.added.length}`); + * console.log(`Modified: ${differences.modified.length}`); + * console.log(`Missing: ${differences.missing.length}`); + * ``` + */ +function compareContacts(dexieContacts: Contact[], sqliteContacts: Contact[]) { + const added: Contact[] = []; + const modified: Contact[] = []; + const unmodified: Contact[] = []; + const missing: Contact[] = []; + + // Find contacts that exist in Dexie but not in SQLite + for (const dexieContact of dexieContacts) { + const sqliteContact = sqliteContacts.find( + (c) => c.did === dexieContact.did, + ); + if (!sqliteContact) { + added.push(dexieContact); + } else if (!contactsEqual(dexieContact, sqliteContact)) { + modified.push(dexieContact); + } else { + unmodified.push(dexieContact); + } + } + + // Find contacts that exist in SQLite but not in Dexie + for (const sqliteContact of sqliteContacts) { + const dexieContact = dexieContacts.find((c) => c.did === sqliteContact.did); + if (!dexieContact) { + missing.push(sqliteContact); + } + } + + return { added, modified, unmodified, missing }; +} + +/** + * Compares settings between Dexie and SQLite databases + * + * This helper function analyzes two arrays of settings and identifies + * which settings are added (in Dexie but not SQLite), modified + * (different between databases), or missing (in SQLite but not Dexie). + * + * The comparison is based on the setting's ID as the primary key, + * with detailed field-by-field comparison for modified settings. + * + * @function compareSettings + * @param {Settings[]} dexieSettings - Settings from Dexie database + * @param {Settings[]} sqliteSettings - Settings from SQLite database + * @returns {Object} Object containing added, modified, and missing settings + * @returns {Settings[]} returns.added - Settings in Dexie but not SQLite + * @returns {Settings[]} returns.modified - Settings that differ between databases + * @returns {Settings[]} returns.missing - Settings in SQLite but not Dexie + * @example + * ```typescript + * const differences = compareSettings(dexieSettings, sqliteSettings); + * console.log(`Added: ${differences.added.length}`); + * console.log(`Modified: ${differences.modified.length}`); + * console.log(`Missing: ${differences.missing.length}`); + * ``` + */ +function compareSettings( + dexieSettings: Settings[], + sqliteSettings: Settings[], +) { + const added: Settings[] = []; + const modified: Settings[] = []; + const unmodified: Settings[] = []; + const missing: Settings[] = []; + + // Find settings that exist in Dexie but not in SQLite + for (const dexieSetting of dexieSettings) { + const sqliteSetting = sqliteSettings.find( + (s) => s.accountDid == dexieSetting.accountDid, + ); + if (!sqliteSetting) { + added.push(dexieSetting); + } else if (!settingsEqual(dexieSetting, sqliteSetting)) { + modified.push(dexieSetting); + } else { + unmodified.push(dexieSetting); + } + } + + // Find settings that exist in SQLite but not in Dexie + for (const sqliteSetting of sqliteSettings) { + const dexieSetting = dexieSettings.find( + (s) => s.accountDid == sqliteSetting.accountDid, + ); + if (!dexieSetting) { + missing.push(sqliteSetting); + } + } + + return { added, modified, unmodified, missing }; +} + +/** + * Compares accounts between Dexie and SQLite databases + * + * This helper function analyzes two arrays of accounts and identifies + * which accounts are added (in Dexie but not SQLite), modified + * (different between databases), or missing (in SQLite but not Dexie). + * + * The comparison is based on the account's ID as the primary key, + * with detailed field-by-field comparison for modified accounts. + * + * @function compareAccounts + * @param {Account[]} dexieAccounts - Accounts from Dexie database + * @param {Account[]} sqliteDids - Accounts from SQLite database + * @returns {Object} Object containing added, modified, and missing accounts + * @returns {Account[]} returns.added - Accounts in Dexie but not SQLite + * @returns {Account[]} returns.modified - always 0 because we don't check + * @returns {string[]} returns.missing - Accounts in SQLite but not Dexie + * @example + * ```typescript + * const differences = compareAccounts(dexieAccounts, sqliteAccounts); + * console.log(`Added: ${differences.added.length}`); + * console.log(`Modified: ${differences.modified.length}`); + * console.log(`Missing: ${differences.missing.length}`); + * ``` + */ +function compareAccounts(dexieAccounts: Account[], sqliteDids: string[]) { + const added: Account[] = []; + const unmodified: Account[] = []; + const missing: string[] = []; + + // Find accounts that exist in Dexie but not in SQLite + for (const dexieAccount of dexieAccounts) { + const sqliteDid = sqliteDids.find((a) => a === dexieAccount.did); + if (!sqliteDid) { + added.push(dexieAccount); + } else { + unmodified.push(dexieAccount); + } + } + + // Find accounts that exist in SQLite but not in Dexie + for (const sqliteDid of sqliteDids) { + const dexieAccount = dexieAccounts.find((a) => a.did === sqliteDid); + if (!dexieAccount) { + missing.push(sqliteDid); + } + } + + return { added, unmodified, missing }; +} + +/** + * Compares two contacts for equality + * + * This helper function performs a deep comparison of two Contact objects + * to determine if they are identical. The comparison includes all + * relevant fields including complex objects like contactMethods. + * + * For contactMethods, the function uses JSON.stringify to compare + * the arrays, ensuring that both structure and content are identical. + * + * @function contactsEqual + * @param {Contact} contact1 - First contact to compare + * @param {Contact} contact2 - Second contact to compare + * @returns {boolean} True if contacts are identical, false otherwise + * @example + * ```typescript + * const areEqual = contactsEqual(contact1, contact2); + * if (areEqual) { + * console.log('Contacts are identical'); + * } else { + * console.log('Contacts differ'); + * } + * ``` + */ +function contactsEqual(contact1: Contact, contact2: Contact): boolean { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const ifEmpty = (arg: any, def: any) => (arg ? arg : def); + const contact1Methods = + contact1.contactMethods && + Array.isArray(contact1.contactMethods) && + contact1.contactMethods.length > 0 + ? JSON.stringify(contact1.contactMethods) + : "[]"; + const contact2Methods = + contact2.contactMethods && + Array.isArray(contact2.contactMethods) && + contact2.contactMethods.length > 0 + ? JSON.stringify(contact2.contactMethods) + : "[]"; + return ( + ifEmpty(contact1.did, "") == ifEmpty(contact2.did, "") && + ifEmpty(contact1.name, "") == ifEmpty(contact2.name, "") && + ifEmpty(contact1.notes, "") == ifEmpty(contact2.notes, "") && + ifEmpty(contact1.profileImageUrl, "") == + ifEmpty(contact2.profileImageUrl, "") && + ifEmpty(contact1.publicKeyBase64, "") == + ifEmpty(contact2.publicKeyBase64, "") && + ifEmpty(contact1.nextPubKeyHashB64, "") == + ifEmpty(contact2.nextPubKeyHashB64, "") && + !!contact1.seesMe == !!contact2.seesMe && + !!contact1.registered == !!contact2.registered && + contact1Methods == contact2Methods + ); +} + +/** + * Compares two settings for equality + * + * This helper function performs a deep comparison of two Settings objects + * to determine if they are identical. The comparison includes all + * relevant fields including complex objects like searchBoxes. + * + * For searchBoxes, the function uses JSON.stringify to compare + * the arrays, ensuring that both structure and content are identical. + * + * @function settingsEqual + * @param {Settings} settings1 - First settings to compare + * @param {Settings} settings2 - Second settings to compare + * @returns {boolean} True if settings are identical, false otherwise + * @example + * ```typescript + * const areEqual = settingsEqual(settings1, settings2); + * if (areEqual) { + * console.log('Settings are identical'); + * } else { + * console.log('Settings differ'); + * } + * ``` + */ +function settingsEqual(settings1: Settings, settings2: Settings): boolean { + return ( + settings1.id == settings2.id && + settings1.accountDid == settings2.accountDid && + settings1.activeDid == settings2.activeDid && + settings1.apiServer == settings2.apiServer && + settings1.filterFeedByNearby == settings2.filterFeedByNearby && + settings1.filterFeedByVisible == settings2.filterFeedByVisible && + settings1.finishedOnboarding == settings2.finishedOnboarding && + settings1.firstName == settings2.firstName && + settings1.hideRegisterPromptOnNewContact == + settings2.hideRegisterPromptOnNewContact && + settings1.isRegistered == settings2.isRegistered && + settings1.lastName == settings2.lastName && + settings1.lastAckedOfferToUserJwtId == + settings2.lastAckedOfferToUserJwtId && + settings1.lastAckedOfferToUserProjectsJwtId == + settings2.lastAckedOfferToUserProjectsJwtId && + settings1.lastNotifiedClaimId == settings2.lastNotifiedClaimId && + settings1.lastViewedClaimId == settings2.lastViewedClaimId && + settings1.notifyingNewActivityTime == settings2.notifyingNewActivityTime && + settings1.notifyingReminderMessage == settings2.notifyingReminderMessage && + settings1.notifyingReminderTime == settings2.notifyingReminderTime && + settings1.partnerApiServer == settings2.partnerApiServer && + settings1.passkeyExpirationMinutes == settings2.passkeyExpirationMinutes && + settings1.profileImageUrl == settings2.profileImageUrl && + settings1.showContactGivesInline == settings2.showContactGivesInline && + settings1.showGeneralAdvanced == settings2.showGeneralAdvanced && + settings1.showShortcutBvc == settings2.showShortcutBvc && + settings1.vapid == settings2.vapid && + settings1.warnIfProdServer == settings2.warnIfProdServer && + settings1.warnIfTestServer == settings2.warnIfTestServer && + settings1.webPushServer == settings2.webPushServer && + JSON.stringify(settings1.searchBoxes) == + JSON.stringify(settings2.searchBoxes) + ); +} + +/** + * Compares two accounts for equality + * + * This helper function performs a deep comparison of two Account objects + * to determine if they are identical. The comparison includes all + * relevant fields including complex objects like identity. + * + * For identity, the function uses JSON.stringify to compare + * the objects, ensuring that both structure and content are identical. + * + * @function accountsEqual + * @param {Account} account1 - First account to compare + * @param {Account} account2 - Second account to compare + * @returns {boolean} True if accounts are identical, false otherwise + * @example + * ```typescript + * const areEqual = accountsEqual(account1, account2); + * if (areEqual) { + * console.log('Accounts are identical'); + * } else { + * console.log('Accounts differ'); + * } + * ``` + */ +// +// unused +// +// function accountsEqual(account1: Account, account2: Account): boolean { +// return ( +// account1.id === account2.id && +// account1.dateCreated === account2.dateCreated && +// account1.derivationPath === account2.derivationPath && +// account1.did === account2.did && +// account1.identity === account2.identity && +// account1.mnemonic === account2.mnemonic && +// account1.passkeyCredIdHex === account2.passkeyCredIdHex && +// account1.publicKeyHex === account2.publicKeyHex +// ); +// } + +/** + * Generates YAML-formatted comparison data + * + * This function converts the database comparison results into a + * structured format that can be exported and analyzed. The output + * is actually JSON but formatted in a YAML-like structure for + * better readability. + * + * The generated data includes summary statistics, detailed differences, + * and the actual data from both databases for inspection purposes. + * + * @function generateComparisonYaml + * @param {DataComparison} comparison - The comparison results to format + * @returns {string} JSON string formatted for readability + * @example + * ```typescript + * const comparison = await compareDatabases(); + * const yaml = generateComparisonYaml(comparison); + * console.log(yaml); + * // Save to file or display in UI + * ``` + */ +export function generateComparisonYaml(comparison: DataComparison): string { + const yaml = { + summary: { + dexieContacts: comparison.dexieContacts.length, + sqliteContacts: comparison.sqliteContacts.filter((c) => c.did).length, + dexieSettings: comparison.dexieSettings.length, + sqliteSettings: comparison.sqliteSettings.filter( + (s) => s.accountDid || s.activeDid, + ).length, + dexieAccounts: comparison.dexieAccounts.length, + sqliteAccounts: comparison.sqliteAccounts.filter((a) => a).length, + }, + differences: { + contacts: { + added: comparison.differences.contacts.added.length, + modified: comparison.differences.contacts.modified.length, + unmodified: comparison.differences.contacts.unmodified.length, + missing: comparison.differences.contacts.missing.filter((c) => c.did) + .length, + }, + settings: { + added: comparison.differences.settings.added.length, + modified: comparison.differences.settings.modified.length, + unmodified: comparison.differences.settings.unmodified.length, + missing: comparison.differences.settings.missing.filter( + (s) => s.accountDid || s.activeDid, + ).length, + }, + accounts: { + added: comparison.differences.accounts.added.length, + unmodified: comparison.differences.accounts.unmodified.length, + missing: comparison.differences.accounts.missing.filter((a) => a) + .length, + }, + }, + details: { + contacts: { + dexie: comparison.dexieContacts.map((c) => ({ + did: c.did, + name: c.name || "", + contactMethods: (c.contactMethods || []).length, + })), + sqlite: comparison.sqliteContacts + .filter((c) => c.did) + .map((c) => ({ + did: c.did, + name: c.name || "", + contactMethods: (c.contactMethods || []).length, + })), + }, + settings: { + dexie: comparison.dexieSettings.map((s) => ({ + id: s.id, + type: s.id === MASTER_SETTINGS_KEY ? "master" : "account", + did: s.activeDid || s.accountDid, + isRegistered: s.isRegistered || false, + })), + sqlite: comparison.sqliteSettings + .filter((s) => s.accountDid || s.activeDid) + .map((s) => ({ + id: s.id, + type: s.id === MASTER_SETTINGS_KEY ? "master" : "account", + did: s.activeDid || s.accountDid, + isRegistered: s.isRegistered || false, + })), + }, + accounts: { + dexie: comparison.dexieAccounts.map((a) => ({ + id: a.id, + did: a.did, + dateCreated: a.dateCreated, + hasIdentity: !!a.identity, + hasMnemonic: !!a.mnemonic, + })), + sqlite: comparison.sqliteAccounts.map((a) => ({ + did: a, + })), + }, + }, + }; + + return JSON.stringify(yaml, null, 2); +} + +/** + * Migrates contacts from Dexie to SQLite database + * + * This function transfers all contacts from the Dexie database to the + * SQLite database. It handles both new contacts (INSERT) and existing + * contacts (UPDATE) based on the overwriteExisting parameter. + * + * The function processes contacts one by one to ensure data integrity + * and provides detailed logging of the migration process. It returns + * comprehensive results including success status, counts, and any + * errors or warnings encountered. + * + * @async + * @function migrateContacts + * @param {boolean} [overwriteExisting=false] - Whether to overwrite existing contacts in SQLite + * @returns {Promise} Detailed results of the migration operation + * @throws {Error} If the migration process fails completely + * @example + * ```typescript + * try { + * const result = await migrateContacts(true); // Overwrite existing + * if (result.success) { + * console.log(`Successfully migrated ${result.contactsMigrated} contacts`); + * } else { + * console.error('Migration failed:', result.errors); + * } + * } catch (error) { + * console.error('Migration process failed:', error); + * } + * ``` + */ +/** + * + * I recommend using the existing contact import view to migrate contacts. + * +export async function migrateContacts( + overwriteExisting: boolean = false, +): Promise { + logger.info("[IndexedDBMigrationService] Starting contact migration", { + overwriteExisting, + }); + + const result: MigrationResult = { + success: true, + contactsMigrated: 0, + settingsMigrated: 0, + accountsMigrated: 0, + errors: [], + warnings: [], + }; + + try { + const dexieContacts = await getDexieContacts(); + const platformService = PlatformServiceFactory.getInstance(); + + for (const contact of dexieContacts) { + try { + // Check if contact already exists + const existingResult = await platformService.dbQuery( + "SELECT did FROM contacts WHERE did = ?", + [contact.did], + ); + + if (existingResult?.values?.length) { + if (overwriteExisting) { + // Update existing contact + const { sql, params } = generateUpdateStatement( + contact as unknown as Record, + "contacts", + "did = ?", + [contact.did], + ); + await platformService.dbExec(sql, params); + result.contactsMigrated++; + logger.info(`[IndexedDBMigrationService] Updated contact: ${contact.did}`); + } else { + result.warnings.push( + `Contact ${contact.did} already exists, skipping`, + ); + } + } else { + // Insert new contact + const { sql, params } = generateInsertStatement( + contact as unknown as Record, + "contacts", + ); + await platformService.dbExec(sql, params); + result.contactsMigrated++; + logger.info(`[IndexedDBMigrationService] Added contact: ${contact.did}`); + } + } catch (error) { + const errorMsg = `Failed to migrate contact ${contact.did}: ${error}`; + logger.error("[IndexedDBMigrationService]", errorMsg); + result.errors.push(errorMsg); + result.success = false; + } + } + + logger.info("[IndexedDBMigrationService] Contact migration completed", { + contactsMigrated: result.contactsMigrated, + errors: result.errors.length, + warnings: result.warnings.length, + }); + + return result; + } catch (error) { + const errorMsg = `Contact migration failed: ${error}`; + logger.error("[IndexedDBMigrationService]", errorMsg); + result.errors.push(errorMsg); + result.success = false; + return result; + } +} + * + */ + +/** + * Migrates specific settings fields from Dexie to SQLite database + * + * This function transfers specific settings fields from the Dexie database + * to the SQLite database. It focuses on the most important user-facing + * settings: firstName, isRegistered, profileImageUrl, showShortcutBvc, + * and searchBoxes. + * + * The function handles duplicate settings by merging master settings (id=1) + * with account-specific settings (id=2) for the same DID, preferring + * the most recent values for the specified fields. + * + * @async + * @function migrateSettings + * @returns {Promise} Detailed results of the migration operation + * @throws {Error} If the migration process fails completely + * @example + * ```typescript + * try { + * const result = await migrateSettings(); + * if (result.success) { + * console.log(`Successfully migrated ${result.settingsMigrated} settings`); + * } else { + * console.error('Migration failed:', result.errors); + * } + * } catch (error) { + * console.error('Migration process failed:', error); + * } + * ``` + */ +export async function migrateSettings(): Promise { + logger.info("[IndexedDBMigrationService] Starting settings migration"); + + const result: MigrationResult = { + success: true, + contactsMigrated: 0, + settingsMigrated: 0, + accountsMigrated: 0, + errors: [], + warnings: [], + }; + + try { + const dexieSettings = await getDexieSettings(); + logger.info("[IndexedDBMigrationService] Migrating settings", { + dexieSettings: dexieSettings.length, + }); + const platformService = PlatformServiceFactory.getInstance(); + + // Create an array of promises for all settings migrations + const migrationPromises = dexieSettings.map(async (setting) => { + logger.info( + "[IndexedDBMigrationService] Starting to migrate settings", + setting, + ); + + // adjust SQL based on the accountDid key, maybe null + let conditional: string; + let preparams: unknown[]; + if (!setting.accountDid) { + conditional = "accountDid is null"; + preparams = []; + } else { + conditional = "accountDid = ?"; + preparams = [setting.accountDid]; + } + const sqliteSettingRaw = await platformService.dbQuery( + "SELECT * FROM settings WHERE " + conditional, + preparams, + ); + + logger.info( + "[IndexedDBMigrationService] Migrating one set of settings:", + { + setting, + sqliteSettingRaw, + }, + ); + if (sqliteSettingRaw?.values?.length) { + // should cover the master settings, where accountDid is null + delete setting.id; // don't conflict with the id in the sqlite database + delete setting.accountDid; // this is part of the where clause + const { sql, params } = generateUpdateStatement( + setting, + "settings", + conditional, + preparams, + ); + logger.info("[IndexedDBMigrationService] Updating settings", { + sql, + params, + }); + await platformService.dbExec(sql, params); + result.settingsMigrated++; + } else { + // insert new setting + delete setting.id; // don't conflict with the id in the sqlite database + delete setting.activeDid; // ensure we don't set the activeDid (since master settings are an update and don't hit this case) + const { sql, params } = generateInsertStatement(setting, "settings"); + await platformService.dbExec(sql, params); + result.settingsMigrated++; + } + }); + + // Wait for all migrations to complete + const updatedSettings = await Promise.all(migrationPromises); + + logger.info( + "[IndexedDBMigrationService] Finished migrating settings", + updatedSettings, + result, + ); + + return result; + } catch (error) { + logger.error( + "[IndexedDBMigrationService] Complete settings migration failed:", + error, + ); + const errorMessage = `Settings migration failed: ${error}`; + result.errors.push(errorMessage); + result.success = false; + return result; + } +} + +/** + * Migrates accounts from Dexie to SQLite database + * + * This function transfers all accounts from the Dexie database to the + * SQLite database. It handles both new accounts (INSERT) and existing + * accounts (UPDATE). + * + * For accounts with mnemonic data, the function uses importFromMnemonic + * to ensure proper key derivation and identity creation during migration. + * + * The function processes accounts one by one to ensure data integrity + * and provides detailed logging of the migration process. It returns + * comprehensive results including success status, counts, and any + * errors or warnings encountered. + * + * @async + * @function migrateAccounts + * @returns {Promise} Detailed results of the migration operation + * @throws {Error} If the migration process fails completely + * @example + * ```typescript + * try { + * const result = await migrateAccounts(); + * if (result.success) { + * console.log(`Successfully migrated ${result.accountsMigrated} accounts`); + * } else { + * console.error('Migration failed:', result.errors); + * } + * } catch (error) { + * console.error('Migration process failed:', error); + * } + * ``` + */ +export async function migrateAccounts(): Promise { + logger.info("[IndexedDBMigrationService] Starting account migration"); + + const result: MigrationResult = { + success: true, + contactsMigrated: 0, + settingsMigrated: 0, + accountsMigrated: 0, + errors: [], + warnings: [], + }; + + try { + const dexieAccounts = await getDexieAccounts(); + const platformService = PlatformServiceFactory.getInstance(); + + // Group accounts by DID and keep only the most recent one + const accountsByDid = new Map(); + dexieAccounts.forEach((account) => { + const existingAccount = accountsByDid.get(account.did); + if ( + !existingAccount || + new Date(account.dateCreated) > new Date(existingAccount.dateCreated) + ) { + accountsByDid.set(account.did, account); + if (existingAccount) { + result.warnings.push( + `Found duplicate account for DID ${account.did}, keeping most recent`, + ); + } + } + }); + + // Process each unique account + for (const [did, account] of accountsByDid.entries()) { + try { + // Check if account already exists + const existingResult = await platformService.dbQuery( + "SELECT did FROM accounts WHERE did = ?", + [did], + ); + + if (existingResult?.values?.length) { + result.warnings.push( + `Account with DID ${did} already exists, skipping`, + ); + continue; + } + if (account.mnemonic) { + await importFromMnemonic(account.mnemonic, account.derivationPath); + result.accountsMigrated++; + } else { + result.errors.push( + `Account with DID ${did} has no mnemonic, skipping`, + ); + } + + logger.info( + "[IndexedDBMigrationService] Successfully migrated account", + { + did, + dateCreated: account.dateCreated, + }, + ); + } catch (error) { + const errorMessage = `Failed to migrate account ${did}: ${error}`; + result.errors.push(errorMessage); + logger.error("[IndexedDBMigrationService] Account migration failed:", { + error, + did, + }); + } + } + + if (result.errors.length > 0) { + result.success = false; + } + + return result; + } catch (error) { + const errorMessage = `Account migration failed: ${error}`; + result.errors.push(errorMessage); + result.success = false; + logger.error( + "[IndexedDBMigrationService] Complete account migration failed:", + error, + ); + return result; + } +} + +/** + * Migrates all data from Dexie to SQLite in the proper order + * + * This function performs a complete migration of all data from Dexie to SQLite + * in the correct order to avoid foreign key constraint issues: + * 1. Accounts (foundational - contains DIDs) + * 2. Settings (references accountDid, activeDid) + * 3. ActiveDid (depends on accounts and settings) + * 4. Contacts (independent, but migrated after accounts for consistency) + * + * The migration runs within a transaction to ensure atomicity. If any step fails, + * the entire migration is rolled back. + * + * @returns Promise - Detailed result of the migration operation + */ +export async function migrateAll(): Promise { + const result: MigrationResult = { + success: false, + contactsMigrated: 0, + settingsMigrated: 0, + accountsMigrated: 0, + errors: [], + warnings: [], + }; + + try { + logger.info( + "[IndexedDBMigrationService] Starting complete migration from Dexie to SQLite", + ); + + // Step 1: Migrate Accounts (foundational) + logger.info("[IndexedDBMigrationService] Step 1: Migrating accounts..."); + const accountsResult = await migrateAccounts(); + if (!accountsResult.success) { + result.errors.push( + `Account migration failed: ${accountsResult.errors.join(", ")}`, + ); + return result; + } + result.accountsMigrated = accountsResult.accountsMigrated; + result.warnings.push(...accountsResult.warnings); + + // Step 2: Migrate Settings (depends on accounts) + logger.info("[IndexedDBMigrationService] Step 2: Migrating settings..."); + const settingsResult = await migrateSettings(); + if (!settingsResult.success) { + result.errors.push( + `Settings migration failed: ${settingsResult.errors.join(", ")}`, + ); + return result; + } + result.settingsMigrated = settingsResult.settingsMigrated; + result.warnings.push(...settingsResult.warnings); + + // Step 4: Migrate Contacts (independent, but after accounts for consistency) + // ... but which is better done through the contact import view + // logger.info("[IndexedDBMigrationService] Step 4: Migrating contacts..."); + // const contactsResult = await migrateContacts(); + // if (!contactsResult.success) { + // result.errors.push( + // `Contact migration failed: ${contactsResult.errors.join(", ")}`, + // ); + // return result; + // } + // result.contactsMigrated = contactsResult.contactsMigrated; + // result.warnings.push(...contactsResult.warnings); + + // All migrations successful + result.success = true; + const totalMigrated = + result.accountsMigrated + + result.settingsMigrated + + result.contactsMigrated; + + logger.info( + `[IndexedDBMigrationService] Complete migration successful: ${totalMigrated} total records migrated`, + { + accounts: result.accountsMigrated, + settings: result.settingsMigrated, + contacts: result.contactsMigrated, + warnings: result.warnings.length, + }, + ); + + return result; + } catch (error) { + const errorMessage = `Complete migration failed: ${error}`; + result.errors.push(errorMessage); + logger.error( + "[IndexedDBMigrationService] Complete migration failed:", + error, + ); + return result; + } +} diff --git a/src/services/migrationService.ts b/src/services/migrationService.ts index 706ff1ea..bd22115a 100644 --- a/src/services/migrationService.ts +++ b/src/services/migrationService.ts @@ -113,7 +113,7 @@ class MigrationRegistry { * Adds a migration to the list of migrations that will be applied when * runMigrations() is called. Migrations should be registered in order * of their intended execution. - * + * * @param migration - The migration to register * @throws {Error} If migration name is empty or already exists * @@ -139,7 +139,7 @@ class MigrationRegistry { /** * Get all registered migrations - * + * * Returns a copy of all migrations that have been registered with this * registry. The migrations are returned in the order they were registered. * @@ -176,11 +176,11 @@ const migrationRegistry = new MigrationRegistry(); /** * Register a migration with the migration service - * + * * This is the primary public API for registering database migrations. * Each migration should represent a single, focused schema change that * can be applied atomically. - * + * * @param migration - The migration to register * @throws {Error} If migration is invalid * @@ -339,7 +339,7 @@ async function isSchemaAlreadyPresent( /** * Run all registered migrations against the database - * + * * This is the main function that executes the migration process. It: * 1. Creates the migrations tracking table if needed * 2. Determines which migrations have already been applied @@ -350,7 +350,7 @@ async function isSchemaAlreadyPresent( * * The function is designed to be idempotent - it can be run multiple times * safely without re-applying migrations that have already been completed. - * + * * @template T - The type returned by SQL query operations * @param sqlExec - Function to execute SQL statements (INSERT, UPDATE, CREATE, etc.) * @param sqlQuery - Function to execute SQL queries (SELECT) @@ -532,14 +532,14 @@ export async function runMigrations( } else { // For other types of errors, still fail the migration console.error(`❌ [Migration] Failed to apply ${migration.name}:`, error); - logger.error( - `[MigrationService] Failed to apply migration ${migration.name}:`, - error, - ); - throw new Error(`Migration ${migration.name} failed: ${error}`); - } + logger.error( + `[MigrationService] Failed to apply migration ${migration.name}:`, + error, + ); + throw new Error(`Migration ${migration.name} failed: ${error}`); } } + } // Step 5: Final validation - verify all migrations are properly recorded console.log("\n🔍 [Migration] Final validation - checking migrations table..."); diff --git a/src/services/platforms/CapacitorPlatformService.ts b/src/services/platforms/CapacitorPlatformService.ts index e7249d36..d2d7a471 100644 --- a/src/services/platforms/CapacitorPlatformService.ts +++ b/src/services/platforms/CapacitorPlatformService.ts @@ -385,7 +385,7 @@ export class CapacitorPlatformService implements PlatformService { try { // Execute the migration process - await runMigrations(sqlExec, sqlQuery, extractMigrationNames); + await runMigrations(sqlExec, sqlQuery, extractMigrationNames); // After migrations, run integrity check to verify database state await this.verifyDatabaseIntegrity(); diff --git a/src/views/DatabaseMigration.vue b/src/views/DatabaseMigration.vue new file mode 100644 index 00000000..a2edc1c9 --- /dev/null +++ b/src/views/DatabaseMigration.vue @@ -0,0 +1,1492 @@ + + +