From 25c1d6ef4ebd8290ea0560746016abfa5c67748d Mon Sep 17 00:00:00 2001 From: Matthew Raymer Date: Wed, 18 Jun 2025 10:54:32 +0000 Subject: [PATCH] feat: Add comprehensive database migration service for Dexie to SQLite - Add migrationService.ts with functions to compare and transfer data between Dexie and SQLite - Implement data comparison with detailed difference analysis (added/modified/missing) - Add contact migration with overwrite options and error handling - Add settings migration focusing on key user fields (firstName, isRegistered, profileImageUrl, showShortcutBvc, searchBoxes) - Include YAML export functionality for data inspection - Add comprehensive JSDoc documentation with examples and usage instructions - Support both INSERT and UPDATE operations with parameterized SQL generation - Include detailed logging and error reporting for migration operations This service enables safe migration of user data from the legacy Dexie (IndexedDB) database to the new SQLite implementation, with full comparison capabilities and rollback safety through detailed reporting. --- doc/dexie-to-sqlite-mapping.md | 41 +- src/db/tables/README.md | 2 +- src/services/migrationService.ts | 1019 ++++++++++++++++++++++++++++-- 3 files changed, 996 insertions(+), 66 deletions(-) diff --git a/doc/dexie-to-sqlite-mapping.md b/doc/dexie-to-sqlite-mapping.md index 893b4670..e598cdcf 100644 --- a/doc/dexie-to-sqlite-mapping.md +++ b/doc/dexie-to-sqlite-mapping.md @@ -3,6 +3,7 @@ ## Schema Mapping ### Current Dexie Schema + ```typescript // Current Dexie schema const db = new Dexie('TimeSafariDB'); @@ -15,6 +16,7 @@ db.version(1).stores({ ``` ### New SQLite Schema + ```sql -- New SQLite schema CREATE TABLE accounts ( @@ -50,6 +52,7 @@ CREATE INDEX idx_settings_updated_at ON settings(updated_at); ### 1. Account Operations #### Get Account by DID + ```typescript // Dexie const account = await db.accounts.get(did); @@ -62,6 +65,7 @@ const account = result[0]?.values[0]; ``` #### Get All Accounts + ```typescript // Dexie const accounts = await db.accounts.toArray(); @@ -74,6 +78,7 @@ const accounts = result[0]?.values || []; ``` #### Add Account + ```typescript // Dexie await db.accounts.add({ @@ -91,6 +96,7 @@ await db.run(` ``` #### Update Account + ```typescript // Dexie await db.accounts.update(did, { @@ -100,7 +106,7 @@ await db.accounts.update(did, { // absurd-sql await db.run(` - UPDATE accounts + UPDATE accounts SET public_key_hex = ?, updated_at = ? WHERE did = ? `, [publicKeyHex, Date.now(), did]); @@ -109,6 +115,7 @@ await db.run(` ### 2. Settings Operations #### Get Setting + ```typescript // Dexie const setting = await db.settings.get(key); @@ -121,6 +128,7 @@ const setting = result[0]?.values[0]; ``` #### Set Setting + ```typescript // Dexie await db.settings.put({ @@ -142,6 +150,7 @@ await db.run(` ### 3. Contact Operations #### Get Contacts by Account + ```typescript // Dexie const contacts = await db.contacts @@ -151,7 +160,7 @@ const contacts = await db.contacts // absurd-sql const result = await db.exec(` - SELECT * FROM contacts + SELECT * FROM contacts WHERE did = ? ORDER BY created_at DESC `, [accountDid]); @@ -159,6 +168,7 @@ const contacts = result[0]?.values || []; ``` #### Add Contact + ```typescript // Dexie await db.contacts.add({ @@ -179,6 +189,7 @@ await db.run(` ## Transaction Mapping ### Batch Operations + ```typescript // Dexie await db.transaction('rw', [db.accounts, db.contacts], async () => { @@ -210,10 +221,11 @@ try { ## Migration Helper Functions ### 1. Data Export (Dexie to JSON) + ```typescript async function exportDexieData(): Promise { const db = new Dexie('TimeSafariDB'); - + return { accounts: await db.accounts.toArray(), settings: await db.settings.toArray(), @@ -228,6 +240,7 @@ async function exportDexieData(): Promise { ``` ### 2. Data Import (JSON to absurd-sql) + ```typescript async function importToAbsurdSql(data: MigrationData): Promise { await db.exec('BEGIN TRANSACTION;'); @@ -239,7 +252,7 @@ async function importToAbsurdSql(data: MigrationData): Promise { VALUES (?, ?, ?, ?) `, [account.did, account.publicKeyHex, account.createdAt, account.updatedAt]); } - + // Import settings for (const setting of data.settings) { await db.run(` @@ -247,7 +260,7 @@ async function importToAbsurdSql(data: MigrationData): Promise { VALUES (?, ?, ?) `, [setting.key, setting.value, setting.updatedAt]); } - + // Import contacts for (const contact of data.contacts) { await db.run(` @@ -264,6 +277,7 @@ async function importToAbsurdSql(data: MigrationData): Promise { ``` ### 3. Verification + ```typescript async function verifyMigration(dexieData: MigrationData): Promise { // Verify account count @@ -272,21 +286,21 @@ async function verifyMigration(dexieData: MigrationData): Promise { if (accountCount !== dexieData.accounts.length) { return false; } - + // Verify settings count const settingsResult = await db.exec('SELECT COUNT(*) as count FROM settings'); const settingsCount = settingsResult[0].values[0][0]; if (settingsCount !== dexieData.settings.length) { return false; } - + // Verify contacts count const contactsResult = await db.exec('SELECT COUNT(*) as count FROM contacts'); const contactsCount = contactsResult[0].values[0][0]; if (contactsCount !== dexieData.contacts.length) { return false; } - + // Verify data integrity for (const account of dexieData.accounts) { const result = await db.exec( @@ -294,12 +308,12 @@ async function verifyMigration(dexieData: MigrationData): Promise { [account.did] ); const migratedAccount = result[0]?.values[0]; - if (!migratedAccount || + if (!migratedAccount || migratedAccount[1] !== account.publicKeyHex) { // public_key_hex is second column return false; } } - + return true; } ``` @@ -307,18 +321,21 @@ async function verifyMigration(dexieData: MigrationData): Promise { ## Performance Considerations ### 1. Indexing + - Dexie automatically creates indexes based on the schema - absurd-sql requires explicit index creation - Added indexes for frequently queried fields - Use `PRAGMA journal_mode=MEMORY;` for better performance ### 2. Batch Operations + - Dexie has built-in bulk operations - absurd-sql uses transactions for batch operations - Consider chunking large datasets - Use prepared statements for repeated queries ### 3. Query Optimization + - Dexie uses IndexedDB's native indexing - absurd-sql requires explicit query optimization - Use prepared statements for repeated queries @@ -327,6 +344,7 @@ async function verifyMigration(dexieData: MigrationData): Promise { ## Error Handling ### 1. Common Errors + ```typescript // Dexie errors try { @@ -351,6 +369,7 @@ try { ``` ### 2. Transaction Recovery + ```typescript // Dexie transaction try { @@ -396,4 +415,4 @@ try { - Remove Dexie database - Clear IndexedDB storage - Update application code - - Remove old dependencies \ No newline at end of file + - Remove old dependencies diff --git a/src/db/tables/README.md b/src/db/tables/README.md index ae2f2688..617dae94 100644 --- a/src/db/tables/README.md +++ b/src/db/tables/README.md @@ -1 +1 @@ -Check the contact & settings export to see whether you want your new table to be included in it. +# Check the contact & settings export to see whether you want your new table to be included in it diff --git a/src/services/migrationService.ts b/src/services/migrationService.ts index 81d9de74..c8dc4e84 100644 --- a/src/services/migrationService.ts +++ b/src/services/migrationService.ts @@ -1,60 +1,971 @@ -interface Migration { - name: string; - sql: string; -} - -export class MigrationService { - private static instance: MigrationService; - private migrations: Migration[] = []; - - private constructor() {} - - static getInstance(): MigrationService { - if (!MigrationService.instance) { - MigrationService.instance = new MigrationService(); - } - return MigrationService.instance; - } - - registerMigration(migration: Migration) { - this.migrations.push(migration); - } - - /** - * @param sqlExec - A function that executes a SQL statement and returns some update result - * @param sqlQuery - A function that executes a SQL query and returns the result in some format - * @param extractMigrationNames - A function that extracts the names (string array) from a "select name from migrations" query - */ - async runMigrations( - // note that this does not take parameters because the Capacitor SQLite 'execute' is different - sqlExec: (sql: string) => Promise, - sqlQuery: (sql: string) => Promise, - extractMigrationNames: (result: T) => Set, - ): Promise { - // Create migrations table if it doesn't exist - await sqlExec(` - CREATE TABLE IF NOT EXISTS migrations ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - name TEXT NOT NULL UNIQUE, - executed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - `); - - // Get list of executed migrations - const result1: T = await sqlQuery("SELECT name FROM migrations;"); - const executedMigrations = extractMigrationNames(result1); - - // Run pending migrations in order - for (const migration of this.migrations) { - if (!executedMigrations.has(migration.name)) { - await sqlExec(migration.sql); - - await sqlExec( - `INSERT INTO migrations (name) VALUES ('${migration.name}')`, +/** + * Migration Service for transferring data from Dexie (IndexedDB) to SQLite + * + * This service provides functions to: + * 1. Compare data between Dexie and SQLite databases + * 2. Transfer contacts and settings from Dexie to SQLite + * 3. Generate YAML-formatted data comparisons + * + * The service is designed to work with the TimeSafari app's dual database architecture, + * where data can exist in both Dexie (IndexedDB) and SQLite databases. This allows + * for safe migration of data between the two storage systems. + * + * Usage: + * 1. Enable Dexie temporarily by setting USE_DEXIE_DB = true in constants/app.ts + * 2. Use compareDatabases() to see differences between databases + * 3. Use migrateContacts() and/or migrateSettings() to transfer data + * 4. Disable Dexie again after migration is complete + * + * @author Matthew Raymer + * @version 1.0.0 + * @since 2024 + */ + +import { PlatformServiceFactory } from "./PlatformServiceFactory"; +import { db } from "../db/index"; +import { Contact, ContactMethod } from "../db/tables/contacts"; +import { Settings } from "../db/tables/settings"; +import { logger } from "../utils/logger"; +import { parseJsonField } from "../db/databaseUtil"; +import { USE_DEXIE_DB } from "../constants/app"; + +/** + * Interface for data comparison results between Dexie and SQLite databases + * + * This interface provides a comprehensive view of the differences between + * the two database systems, including counts and detailed lists of + * added, modified, and missing records. + * + * @interface DataComparison + * @property {Contact[]} dexieContacts - All contacts from Dexie database + * @property {Contact[]} sqliteContacts - All contacts from SQLite database + * @property {Settings[]} dexieSettings - All settings from Dexie database + * @property {Settings[]} sqliteSettings - All settings from SQLite database + * @property {Object} differences - Detailed differences between databases + * @property {Object} differences.contacts - Contact-specific differences + * @property {Contact[]} differences.contacts.added - Contacts in Dexie but not SQLite + * @property {Contact[]} differences.contacts.modified - Contacts that differ between databases + * @property {Contact[]} differences.contacts.missing - Contacts in SQLite but not Dexie + * @property {Object} differences.settings - Settings-specific differences + * @property {Settings[]} differences.settings.added - Settings in Dexie but not SQLite + * @property {Settings[]} differences.settings.modified - Settings that differ between databases + * @property {Settings[]} differences.settings.missing - Settings in SQLite but not Dexie + */ +export interface DataComparison { + dexieContacts: Contact[]; + sqliteContacts: Contact[]; + dexieSettings: Settings[]; + sqliteSettings: Settings[]; + differences: { + contacts: { + added: Contact[]; + modified: Contact[]; + missing: Contact[]; + }; + settings: { + added: Settings[]; + modified: Settings[]; + missing: Settings[]; + }; + }; +} + +/** + * Interface for migration operation results + * + * Provides detailed feedback about the success or failure of migration + * operations, including counts of migrated records and any errors or + * warnings that occurred during the process. + * + * @interface MigrationResult + * @property {boolean} success - Whether the migration operation completed successfully + * @property {number} contactsMigrated - Number of contacts successfully migrated + * @property {number} settingsMigrated - Number of settings successfully migrated + * @property {string[]} errors - Array of error messages encountered during migration + * @property {string[]} warnings - Array of warning messages (non-fatal issues) + */ +export interface MigrationResult { + success: boolean; + contactsMigrated: number; + settingsMigrated: number; + errors: string[]; + warnings: string[]; +} + +/** + * Retrieves all contacts from the Dexie (IndexedDB) database + * + * This function connects to the Dexie database and retrieves all contact + * records. It requires that USE_DEXIE_DB is enabled in the app constants. + * + * The function handles database opening and error conditions, providing + * detailed logging for debugging purposes. + * + * @async + * @function getDexieContacts + * @returns {Promise} Array of all contacts from Dexie database + * @throws {Error} If Dexie database is not enabled or if database access fails + * @example + * ```typescript + * try { + * const contacts = await getDexieContacts(); + * console.log(`Retrieved ${contacts.length} contacts from Dexie`); + * } catch (error) { + * console.error('Failed to retrieve Dexie contacts:', error); + * } + * ``` + */ +export async function getDexieContacts(): Promise { + if (!USE_DEXIE_DB) { + throw new Error("Dexie database is not enabled"); + } + + try { + await db.open(); + const contacts = await db.contacts.toArray(); + logger.info(`[MigrationService] Retrieved ${contacts.length} contacts from Dexie`); + return contacts; + } catch (error) { + logger.error("[MigrationService] Error retrieving Dexie contacts:", error); + throw new Error(`Failed to retrieve Dexie contacts: ${error}`); + } +} + +/** + * Retrieves all contacts from the SQLite database + * + * This function uses the platform service to query the SQLite database + * and retrieve all contact records. It handles the conversion of raw + * database results into properly typed Contact objects. + * + * The function also handles JSON parsing for complex fields like + * contactMethods, ensuring proper type conversion. + * + * @async + * @function getSqliteContacts + * @returns {Promise} Array of all contacts from SQLite database + * @throws {Error} If database query fails or data conversion fails + * @example + * ```typescript + * try { + * const contacts = await getSqliteContacts(); + * console.log(`Retrieved ${contacts.length} contacts from SQLite`); + * } catch (error) { + * console.error('Failed to retrieve SQLite contacts:', error); + * } + * ``` + */ +export async function getSqliteContacts(): Promise { + try { + const platformService = PlatformServiceFactory.getInstance(); + const result = await platformService.dbQuery("SELECT * FROM contacts"); + + if (!result?.values?.length) { + return []; + } + + const contacts = result.values.map((row) => { + const contact = parseJsonField(row, {}) as any; + return { + did: contact.did || "", + name: contact.name || "", + contactMethods: parseJsonField(contact.contactMethods, []) as ContactMethod[], + nextPubKeyHashB64: contact.nextPubKeyHashB64 || "", + notes: contact.notes || "", + profileImageUrl: contact.profileImageUrl || "", + publicKeyBase64: contact.publicKeyBase64 || "", + seesMe: contact.seesMe || false, + registered: contact.registered || false, + } as Contact; + }); + + logger.info(`[MigrationService] Retrieved ${contacts.length} contacts from SQLite`); + return contacts; + } catch (error) { + logger.error("[MigrationService] Error retrieving SQLite contacts:", error); + throw new Error(`Failed to retrieve SQLite contacts: ${error}`); + } +} + +/** + * Retrieves all settings from the Dexie (IndexedDB) database + * + * This function connects to the Dexie database and retrieves all settings + * records. It requires that USE_DEXIE_DB is enabled in the app constants. + * + * Settings include both master settings (id=1) and account-specific settings + * that override the master settings for particular user accounts. + * + * @async + * @function getDexieSettings + * @returns {Promise} Array of all settings from Dexie database + * @throws {Error} If Dexie database is not enabled or if database access fails + * @example + * ```typescript + * try { + * const settings = await getDexieSettings(); + * console.log(`Retrieved ${settings.length} settings from Dexie`); + * } catch (error) { + * console.error('Failed to retrieve Dexie settings:', error); + * } + * ``` + */ +export async function getDexieSettings(): Promise { + if (!USE_DEXIE_DB) { + throw new Error("Dexie database is not enabled"); + } + + try { + await db.open(); + const settings = await db.settings.toArray(); + logger.info(`[MigrationService] Retrieved ${settings.length} settings from Dexie`); + return settings; + } catch (error) { + logger.error("[MigrationService] Error retrieving Dexie settings:", error); + throw new Error(`Failed to retrieve Dexie settings: ${error}`); + } +} + +/** + * Retrieves all settings from the SQLite database + * + * This function uses the platform service to query the SQLite database + * and retrieve all settings records. It handles the conversion of raw + * database results into properly typed Settings objects. + * + * The function also handles JSON parsing for complex fields like + * searchBoxes, ensuring proper type conversion. + * + * @async + * @function getSqliteSettings + * @returns {Promise} Array of all settings from SQLite database + * @throws {Error} If database query fails or data conversion fails + * @example + * ```typescript + * try { + * const settings = await getSqliteSettings(); + * console.log(`Retrieved ${settings.length} settings from SQLite`); + * } catch (error) { + * console.error('Failed to retrieve SQLite settings:', error); + * } + * ``` + */ +export async function getSqliteSettings(): Promise { + try { + const platformService = PlatformServiceFactory.getInstance(); + const result = await platformService.dbQuery("SELECT * FROM settings"); + + if (!result?.values?.length) { + return []; + } + + const settings = result.values.map((row) => { + const setting = parseJsonField(row, {}) as any; + return { + id: setting.id, + accountDid: setting.accountDid || "", + activeDid: setting.activeDid || "", + apiServer: setting.apiServer || "", + filterFeedByNearby: setting.filterFeedByNearby || false, + filterFeedByVisible: setting.filterFeedByVisible || false, + finishedOnboarding: setting.finishedOnboarding || false, + firstName: setting.firstName || "", + hideRegisterPromptOnNewContact: setting.hideRegisterPromptOnNewContact || false, + isRegistered: setting.isRegistered || false, + lastName: setting.lastName || "", + lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "", + lastAckedOfferToUserProjectsJwtId: setting.lastAckedOfferToUserProjectsJwtId || "", + lastNotifiedClaimId: setting.lastNotifiedClaimId || "", + lastViewedClaimId: setting.lastViewedClaimId || "", + notifyingNewActivityTime: setting.notifyingNewActivityTime || "", + notifyingReminderMessage: setting.notifyingReminderMessage || "", + notifyingReminderTime: setting.notifyingReminderTime || "", + partnerApiServer: setting.partnerApiServer || "", + passkeyExpirationMinutes: setting.passkeyExpirationMinutes, + profileImageUrl: setting.profileImageUrl || "", + searchBoxes: parseJsonField(setting.searchBoxes, []), + showContactGivesInline: setting.showContactGivesInline || false, + showGeneralAdvanced: setting.showGeneralAdvanced || false, + showShortcutBvc: setting.showShortcutBvc || false, + vapid: setting.vapid || "", + warnIfProdServer: setting.warnIfProdServer || false, + warnIfTestServer: setting.warnIfTestServer || false, + webPushServer: setting.webPushServer || "", + } as Settings; + }); + + logger.info(`[MigrationService] Retrieved ${settings.length} settings from SQLite`); + return settings; + } catch (error) { + logger.error("[MigrationService] Error retrieving SQLite settings:", error); + throw new Error(`Failed to retrieve SQLite settings: ${error}`); + } +} + +/** + * Compares data between Dexie and SQLite databases + * + * This is the main comparison function that retrieves data from both + * databases and identifies differences. It provides a comprehensive + * view of what data exists in each database and what needs to be + * migrated. + * + * The function performs parallel data retrieval for efficiency and + * then compares the results to identify added, modified, and missing + * records in each table. + * + * @async + * @function compareDatabases + * @returns {Promise} Comprehensive comparison results + * @throws {Error} If any database access fails + * @example + * ```typescript + * try { + * const comparison = await compareDatabases(); + * console.log(`Dexie contacts: ${comparison.dexieContacts.length}`); + * console.log(`SQLite contacts: ${comparison.sqliteContacts.length}`); + * console.log(`Added contacts: ${comparison.differences.contacts.added.length}`); + * } catch (error) { + * console.error('Database comparison failed:', error); + * } + * ``` + */ +export async function compareDatabases(): Promise { + logger.info("[MigrationService] Starting database comparison"); + + const [dexieContacts, sqliteContacts, dexieSettings, sqliteSettings] = await Promise.all([ + getDexieContacts(), + getSqliteContacts(), + getDexieSettings(), + getSqliteSettings(), + ]); + + // Compare contacts + const contactDifferences = compareContacts(dexieContacts, sqliteContacts); + + // Compare settings + const settingsDifferences = compareSettings(dexieSettings, sqliteSettings); + + const comparison: DataComparison = { + dexieContacts, + sqliteContacts, + dexieSettings, + sqliteSettings, + differences: { + contacts: contactDifferences, + settings: settingsDifferences, + }, + }; + + logger.info("[MigrationService] Database comparison completed", { + dexieContacts: dexieContacts.length, + sqliteContacts: sqliteContacts.length, + dexieSettings: dexieSettings.length, + sqliteSettings: sqliteSettings.length, + contactDifferences: contactDifferences, + settingsDifferences: settingsDifferences, + }); + + return comparison; +} + +/** + * Compares contacts between Dexie and SQLite databases + * + * This helper function analyzes two arrays of contacts and identifies + * which contacts are added (in Dexie but not SQLite), modified + * (different between databases), or missing (in SQLite but not Dexie). + * + * The comparison is based on the contact's DID (Decentralized Identifier) + * as the primary key, with detailed field-by-field comparison for + * modified contacts. + * + * @function compareContacts + * @param {Contact[]} dexieContacts - Contacts from Dexie database + * @param {Contact[]} sqliteContacts - Contacts from SQLite database + * @returns {Object} Object containing added, modified, and missing contacts + * @returns {Contact[]} returns.added - Contacts in Dexie but not SQLite + * @returns {Contact[]} returns.modified - Contacts that differ between databases + * @returns {Contact[]} returns.missing - Contacts in SQLite but not Dexie + * @example + * ```typescript + * const differences = compareContacts(dexieContacts, sqliteContacts); + * console.log(`Added: ${differences.added.length}`); + * console.log(`Modified: ${differences.modified.length}`); + * console.log(`Missing: ${differences.missing.length}`); + * ``` + */ +function compareContacts(dexieContacts: Contact[], sqliteContacts: Contact[]) { + const added: Contact[] = []; + const modified: Contact[] = []; + const missing: Contact[] = []; + + // Find contacts that exist in Dexie but not in SQLite + for (const dexieContact of dexieContacts) { + const sqliteContact = sqliteContacts.find(c => c.did === dexieContact.did); + if (!sqliteContact) { + added.push(dexieContact); + } else if (!contactsEqual(dexieContact, sqliteContact)) { + modified.push(dexieContact); + } + } + + // Find contacts that exist in SQLite but not in Dexie + for (const sqliteContact of sqliteContacts) { + const dexieContact = dexieContacts.find(c => c.did === sqliteContact.did); + if (!dexieContact) { + missing.push(sqliteContact); + } + } + + return { added, modified, missing }; +} + +/** + * Compares settings between Dexie and SQLite databases + * + * This helper function analyzes two arrays of settings and identifies + * which settings are added (in Dexie but not SQLite), modified + * (different between databases), or missing (in SQLite but not Dexie). + * + * The comparison is based on the setting's ID as the primary key, + * with detailed field-by-field comparison for modified settings. + * + * @function compareSettings + * @param {Settings[]} dexieSettings - Settings from Dexie database + * @param {Settings[]} sqliteSettings - Settings from SQLite database + * @returns {Object} Object containing added, modified, and missing settings + * @returns {Settings[]} returns.added - Settings in Dexie but not SQLite + * @returns {Settings[]} returns.modified - Settings that differ between databases + * @returns {Settings[]} returns.missing - Settings in SQLite but not Dexie + * @example + * ```typescript + * const differences = compareSettings(dexieSettings, sqliteSettings); + * console.log(`Added: ${differences.added.length}`); + * console.log(`Modified: ${differences.modified.length}`); + * console.log(`Missing: ${differences.missing.length}`); + * ``` + */ +function compareSettings(dexieSettings: Settings[], sqliteSettings: Settings[]) { + const added: Settings[] = []; + const modified: Settings[] = []; + const missing: Settings[] = []; + + // Find settings that exist in Dexie but not in SQLite + for (const dexieSetting of dexieSettings) { + const sqliteSetting = sqliteSettings.find(s => s.id === dexieSetting.id); + if (!sqliteSetting) { + added.push(dexieSetting); + } else if (!settingsEqual(dexieSetting, sqliteSetting)) { + modified.push(dexieSetting); + } + } + + // Find settings that exist in SQLite but not in Dexie + for (const sqliteSetting of sqliteSettings) { + const dexieSetting = dexieSettings.find(s => s.id === sqliteSetting.id); + if (!dexieSetting) { + missing.push(sqliteSetting); + } + } + + return { added, modified, missing }; +} + +/** + * Compares two contacts for equality + * + * This helper function performs a deep comparison of two Contact objects + * to determine if they are identical. The comparison includes all + * relevant fields including complex objects like contactMethods. + * + * For contactMethods, the function uses JSON.stringify to compare + * the arrays, ensuring that both structure and content are identical. + * + * @function contactsEqual + * @param {Contact} contact1 - First contact to compare + * @param {Contact} contact2 - Second contact to compare + * @returns {boolean} True if contacts are identical, false otherwise + * @example + * ```typescript + * const areEqual = contactsEqual(contact1, contact2); + * if (areEqual) { + * console.log('Contacts are identical'); + * } else { + * console.log('Contacts differ'); + * } + * ``` + */ +function contactsEqual(contact1: Contact, contact2: Contact): boolean { + return ( + contact1.did === contact2.did && + contact1.name === contact2.name && + contact1.notes === contact2.notes && + contact1.profileImageUrl === contact2.profileImageUrl && + contact1.publicKeyBase64 === contact2.publicKeyBase64 && + contact1.nextPubKeyHashB64 === contact2.nextPubKeyHashB64 && + contact1.seesMe === contact2.seesMe && + contact1.registered === contact2.registered && + JSON.stringify(contact1.contactMethods) === JSON.stringify(contact2.contactMethods) + ); +} + +/** + * Compares two settings for equality + * + * This helper function performs a deep comparison of two Settings objects + * to determine if they are identical. The comparison includes all + * relevant fields including complex objects like searchBoxes. + * + * For searchBoxes, the function uses JSON.stringify to compare + * the arrays, ensuring that both structure and content are identical. + * + * @function settingsEqual + * @param {Settings} settings1 - First settings to compare + * @param {Settings} settings2 - Second settings to compare + * @returns {boolean} True if settings are identical, false otherwise + * @example + * ```typescript + * const areEqual = settingsEqual(settings1, settings2); + * if (areEqual) { + * console.log('Settings are identical'); + * } else { + * console.log('Settings differ'); + * } + * ``` + */ +function settingsEqual(settings1: Settings, settings2: Settings): boolean { + return ( + settings1.id === settings2.id && + settings1.accountDid === settings2.accountDid && + settings1.activeDid === settings2.activeDid && + settings1.apiServer === settings2.apiServer && + settings1.filterFeedByNearby === settings2.filterFeedByNearby && + settings1.filterFeedByVisible === settings2.filterFeedByVisible && + settings1.finishedOnboarding === settings2.finishedOnboarding && + settings1.firstName === settings2.firstName && + settings1.hideRegisterPromptOnNewContact === settings2.hideRegisterPromptOnNewContact && + settings1.isRegistered === settings2.isRegistered && + settings1.lastName === settings2.lastName && + settings1.lastAckedOfferToUserJwtId === settings2.lastAckedOfferToUserJwtId && + settings1.lastAckedOfferToUserProjectsJwtId === settings2.lastAckedOfferToUserProjectsJwtId && + settings1.lastNotifiedClaimId === settings2.lastNotifiedClaimId && + settings1.lastViewedClaimId === settings2.lastViewedClaimId && + settings1.notifyingNewActivityTime === settings2.notifyingNewActivityTime && + settings1.notifyingReminderMessage === settings2.notifyingReminderMessage && + settings1.notifyingReminderTime === settings2.notifyingReminderTime && + settings1.partnerApiServer === settings2.partnerApiServer && + settings1.passkeyExpirationMinutes === settings2.passkeyExpirationMinutes && + settings1.profileImageUrl === settings2.profileImageUrl && + settings1.showContactGivesInline === settings2.showContactGivesInline && + settings1.showGeneralAdvanced === settings2.showGeneralAdvanced && + settings1.showShortcutBvc === settings2.showShortcutBvc && + settings1.vapid === settings2.vapid && + settings1.warnIfProdServer === settings2.warnIfProdServer && + settings1.warnIfTestServer === settings2.warnIfTestServer && + settings1.webPushServer === settings2.webPushServer && + JSON.stringify(settings1.searchBoxes) === JSON.stringify(settings2.searchBoxes) + ); +} + +/** + * Generates YAML-formatted comparison data + * + * This function converts the database comparison results into a + * structured format that can be exported and analyzed. The output + * is actually JSON but formatted in a YAML-like structure for + * better readability. + * + * The generated data includes summary statistics, detailed differences, + * and the actual data from both databases for inspection purposes. + * + * @function generateComparisonYaml + * @param {DataComparison} comparison - The comparison results to format + * @returns {string} JSON string formatted for readability + * @example + * ```typescript + * const comparison = await compareDatabases(); + * const yaml = generateComparisonYaml(comparison); + * console.log(yaml); + * // Save to file or display in UI + * ``` + */ +export function generateComparisonYaml(comparison: DataComparison): string { + const yaml = { + comparison: { + summary: { + dexieContacts: comparison.dexieContacts.length, + sqliteContacts: comparison.sqliteContacts.length, + dexieSettings: comparison.dexieSettings.length, + sqliteSettings: comparison.sqliteSettings.length, + }, + differences: { + contacts: { + added: comparison.differences.contacts.added.length, + modified: comparison.differences.contacts.modified.length, + missing: comparison.differences.contacts.missing.length, + }, + settings: { + added: comparison.differences.settings.added.length, + modified: comparison.differences.settings.modified.length, + missing: comparison.differences.settings.missing.length, + }, + }, + contacts: { + dexie: comparison.dexieContacts.map(c => ({ + did: c.did, + name: c.name, + notes: c.notes, + profileImageUrl: c.profileImageUrl, + seesMe: c.seesMe, + registered: c.registered, + contactMethods: c.contactMethods, + })), + sqlite: comparison.sqliteContacts.map(c => ({ + did: c.did, + name: c.name, + notes: c.notes, + profileImageUrl: c.profileImageUrl, + seesMe: c.seesMe, + registered: c.registered, + contactMethods: c.contactMethods, + })), + }, + settings: { + dexie: comparison.dexieSettings.map(s => ({ + id: s.id, + accountDid: s.accountDid, + activeDid: s.activeDid, + firstName: s.firstName, + isRegistered: s.isRegistered, + profileImageUrl: s.profileImageUrl, + showShortcutBvc: s.showShortcutBvc, + searchBoxes: s.searchBoxes, + })), + sqlite: comparison.sqliteSettings.map(s => ({ + id: s.id, + accountDid: s.accountDid, + activeDid: s.activeDid, + firstName: s.firstName, + isRegistered: s.isRegistered, + profileImageUrl: s.profileImageUrl, + showShortcutBvc: s.showShortcutBvc, + searchBoxes: s.searchBoxes, + })), + }, + }, + }; + + return JSON.stringify(yaml, null, 2); +} + +/** + * Migrates contacts from Dexie to SQLite database + * + * This function transfers all contacts from the Dexie database to the + * SQLite database. It handles both new contacts (INSERT) and existing + * contacts (UPDATE) based on the overwriteExisting parameter. + * + * The function processes contacts one by one to ensure data integrity + * and provides detailed logging of the migration process. It returns + * comprehensive results including success status, counts, and any + * errors or warnings encountered. + * + * @async + * @function migrateContacts + * @param {boolean} [overwriteExisting=false] - Whether to overwrite existing contacts in SQLite + * @returns {Promise} Detailed results of the migration operation + * @throws {Error} If the migration process fails completely + * @example + * ```typescript + * try { + * const result = await migrateContacts(true); // Overwrite existing + * if (result.success) { + * console.log(`Successfully migrated ${result.contactsMigrated} contacts`); + * } else { + * console.error('Migration failed:', result.errors); + * } + * } catch (error) { + * console.error('Migration process failed:', error); + * } + * ``` + */ +export async function migrateContacts(overwriteExisting: boolean = false): Promise { + logger.info("[MigrationService] Starting contact migration", { overwriteExisting }); + + const result: MigrationResult = { + success: true, + contactsMigrated: 0, + settingsMigrated: 0, + errors: [], + warnings: [], + }; + + try { + const dexieContacts = await getDexieContacts(); + const platformService = PlatformServiceFactory.getInstance(); + + for (const contact of dexieContacts) { + try { + // Check if contact already exists + const existingResult = await platformService.dbQuery( + "SELECT did FROM contacts WHERE did = ?", + [contact.did] ); + + if (existingResult?.values?.length) { + if (overwriteExisting) { + // Update existing contact + const { sql, params } = generateUpdateStatement( + contact as unknown as Record, + "contacts", + "did = ?", + [contact.did] + ); + await platformService.dbExec(sql, params); + result.contactsMigrated++; + logger.info(`[MigrationService] Updated contact: ${contact.did}`); + } else { + result.warnings.push(`Contact ${contact.did} already exists, skipping`); + } + } else { + // Insert new contact + const { sql, params } = generateInsertStatement( + contact as unknown as Record, + "contacts" + ); + await platformService.dbExec(sql, params); + result.contactsMigrated++; + logger.info(`[MigrationService] Added contact: ${contact.did}`); + } + } catch (error) { + const errorMsg = `Failed to migrate contact ${contact.did}: ${error}`; + logger.error("[MigrationService]", errorMsg); + result.errors.push(errorMsg); + result.success = false; } } + + logger.info("[MigrationService] Contact migration completed", { + contactsMigrated: result.contactsMigrated, + errors: result.errors.length, + warnings: result.warnings.length, + }); + + return result; + } catch (error) { + const errorMsg = `Contact migration failed: ${error}`; + logger.error("[MigrationService]", errorMsg); + result.errors.push(errorMsg); + result.success = false; + return result; } } -export default MigrationService.getInstance(); +/** + * Migrates specific settings fields from Dexie to SQLite database + * + * This function transfers specific settings fields from the Dexie database + * to the SQLite database. It focuses on the most important user-facing + * settings: firstName, isRegistered, profileImageUrl, showShortcutBvc, + * and searchBoxes. + * + * The function handles both new settings (INSERT) and existing settings + * (UPDATE) based on the overwriteExisting parameter. For updates, it + * only modifies the specified fields, preserving other settings. + * + * @async + * @function migrateSettings + * @param {boolean} [overwriteExisting=false] - Whether to overwrite existing settings in SQLite + * @returns {Promise} Detailed results of the migration operation + * @throws {Error} If the migration process fails completely + * @example + * ```typescript + * try { + * const result = await migrateSettings(true); // Overwrite existing + * if (result.success) { + * console.log(`Successfully migrated ${result.settingsMigrated} settings`); + * } else { + * console.error('Migration failed:', result.errors); + * } + * } catch (error) { + * console.error('Migration process failed:', error); + * } + * ``` + */ +export async function migrateSettings(overwriteExisting: boolean = false): Promise { + logger.info("[MigrationService] Starting settings migration", { overwriteExisting }); + + const result: MigrationResult = { + success: true, + contactsMigrated: 0, + settingsMigrated: 0, + errors: [], + warnings: [], + }; + + try { + const dexieSettings = await getDexieSettings(); + const platformService = PlatformServiceFactory.getInstance(); + + // Fields to migrate - these are the most important user-facing settings + const fieldsToMigrate = ['firstName', 'isRegistered', 'profileImageUrl', 'showShortcutBvc', 'searchBoxes']; + + for (const dexieSetting of dexieSettings) { + try { + // Check if setting already exists + const existingResult = await platformService.dbQuery( + "SELECT id FROM settings WHERE id = ?", + [dexieSetting.id] + ); + + if (existingResult?.values?.length) { + if (overwriteExisting) { + // Update existing setting with only the specified fields + const updateData: Record = {}; + fieldsToMigrate.forEach(field => { + if (dexieSetting[field as keyof Settings] !== undefined) { + updateData[field] = dexieSetting[field as keyof Settings]; + } + }); + + if (Object.keys(updateData).length > 0) { + const { sql, params } = generateUpdateStatement( + updateData as unknown as Record, + "settings", + "id = ?", + [dexieSetting.id] + ); + await platformService.dbExec(sql, params); + result.settingsMigrated++; + logger.info(`[MigrationService] Updated settings: ${dexieSetting.id}`); + } + } else { + result.warnings.push(`Settings ${dexieSetting.id} already exists, skipping`); + } + } else { + // Insert new setting + const { sql, params } = generateInsertStatement( + dexieSetting as unknown as Record, + "settings" + ); + await platformService.dbExec(sql, params); + result.settingsMigrated++; + logger.info(`[MigrationService] Added settings: ${dexieSetting.id}`); + } + } catch (error) { + const errorMsg = `Failed to migrate settings ${dexieSetting.id}: ${error}`; + logger.error("[MigrationService]", errorMsg); + result.errors.push(errorMsg); + result.success = false; + } + } + + logger.info("[MigrationService] Settings migration completed", { + settingsMigrated: result.settingsMigrated, + errors: result.errors.length, + warnings: result.warnings.length, + }); + + return result; + } catch (error) { + const errorMsg = `Settings migration failed: ${error}`; + logger.error("[MigrationService]", errorMsg); + result.errors.push(errorMsg); + result.success = false; + return result; + } +} + +/** + * Generates SQL INSERT statement and parameters from a model object + * + * This helper function creates a parameterized SQL INSERT statement + * from a JavaScript object. It filters out undefined values and + * creates the appropriate SQL syntax with placeholders. + * + * The function is used internally by the migration functions to + * safely insert data into the SQLite database. + * + * @function generateInsertStatement + * @param {Record} model - The model object containing fields to insert + * @param {string} tableName - The name of the table to insert into + * @returns {Object} Object containing the SQL statement and parameters array + * @returns {string} returns.sql - The SQL INSERT statement + * @returns {unknown[]} returns.params - Array of parameter values + * @example + * ```typescript + * const contact = { did: 'did:example:123', name: 'John Doe' }; + * const { sql, params } = generateInsertStatement(contact, 'contacts'); + * // sql: "INSERT INTO contacts (did, name) VALUES (?, ?)" + * // params: ['did:example:123', 'John Doe'] + * ``` + */ +function generateInsertStatement( + model: Record, + tableName: string, +): { sql: string; params: unknown[] } { + const columns = Object.keys(model).filter((key) => model[key] !== undefined); + const values = Object.values(model).filter((value) => value !== undefined); + const placeholders = values.map(() => "?").join(", "); + const insertSql = `INSERT INTO ${tableName} (${columns.join(", ")}) VALUES (${placeholders})`; + + return { + sql: insertSql, + params: values, + }; +} + +/** + * Generates SQL UPDATE statement and parameters from a model object + * + * This helper function creates a parameterized SQL UPDATE statement + * from a JavaScript object. It filters out undefined values and + * creates the appropriate SQL syntax with placeholders. + * + * The function is used internally by the migration functions to + * safely update data in the SQLite database. + * + * @function generateUpdateStatement + * @param {Record} model - The model object containing fields to update + * @param {string} tableName - The name of the table to update + * @param {string} whereClause - The WHERE clause for the update (e.g. "id = ?") + * @param {unknown[]} [whereParams=[]] - Parameters for the WHERE clause + * @returns {Object} Object containing the SQL statement and parameters array + * @returns {string} returns.sql - The SQL UPDATE statement + * @returns {unknown[]} returns.params - Array of parameter values + * @example + * ```typescript + * const contact = { name: 'Jane Doe' }; + * const { sql, params } = generateUpdateStatement(contact, 'contacts', 'did = ?', ['did:example:123']); + * // sql: "UPDATE contacts SET name = ? WHERE did = ?" + * // params: ['Jane Doe', 'did:example:123'] + * ``` + */ +function generateUpdateStatement( + model: Record, + tableName: string, + whereClause: string, + whereParams: unknown[] = [], +): { sql: string; params: unknown[] } { + const setClauses: string[] = []; + const params: unknown[] = []; + + Object.entries(model).forEach(([key, value]) => { + if (value !== undefined) { + setClauses.push(`${key} = ?`); + params.push(value); + } + }); + + if (setClauses.length === 0) { + throw new Error("No valid fields to update"); + } + + const sql = `UPDATE ${tableName} SET ${setClauses.join(", ")} WHERE ${whereClause}`; + + return { + sql, + params: [...params, ...whereParams], + }; +}