fix: maintain separate master/account settings in SQLite migration

- Update settings migration to maintain separate master and account records
- Use activeDid/accountDid pattern to differentiate between settings types:
  * Master settings: activeDid set, accountDid empty
  * Account settings: accountDid set, activeDid empty
- Add detailed logging for settings migration process
- Focus on migrating key fields: firstName, isRegistered, profileImageUrl,
  showShortcutBvc, and searchBoxes
- Fix issue where settings were being incorrectly merged into a single record

This change ensures the SQLite database maintains the same settings structure
as Dexie, which is required by the existing codebase.

src/services/migrationService.ts

@@ -285,25 +285,22 @@ export async function getSqliteSettings(): Promise<Settings[]> {
       return [];
     }
 
-    const settings = result.values
+    const settings = result.values.map((row) => {
-      .map((row) => {
       const setting = parseJsonField(row, {}) as Settings;
       return {
         id: setting.id,
-          accountDid: setting.accountDid || null,
+        accountDid: setting.accountDid || "",
-          activeDid: setting.activeDid || null,
+        activeDid: setting.activeDid || "",
         apiServer: setting.apiServer || "",
         filterFeedByNearby: setting.filterFeedByNearby || false,
         filterFeedByVisible: setting.filterFeedByVisible || false,
         finishedOnboarding: setting.finishedOnboarding || false,
         firstName: setting.firstName || "",
-          hideRegisterPromptOnNewContact:
+        hideRegisterPromptOnNewContact: setting.hideRegisterPromptOnNewContact || false,
-            setting.hideRegisterPromptOnNewContact || false,
         isRegistered: setting.isRegistered || false,
         lastName: setting.lastName || "",
         lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "",
-          lastAckedOfferToUserProjectsJwtId:
+        lastAckedOfferToUserProjectsJwtId: setting.lastAckedOfferToUserProjectsJwtId || "",
-            setting.lastAckedOfferToUserProjectsJwtId || "",
         lastNotifiedClaimId: setting.lastNotifiedClaimId || "",
         lastViewedClaimId: setting.lastViewedClaimId || "",
         notifyingNewActivityTime: setting.notifyingNewActivityTime || "",
@@ -317,14 +314,7 @@ export async function getSqliteSettings(): Promise<Settings[]> {
         showGeneralAdvanced: setting.showGeneralAdvanced || false,
         showShortcutBvc: setting.showShortcutBvc || false,
         vapid: setting.vapid || "",
-          warnIfProdServer: setting.warnIfProdServer || false,
-          warnIfTestServer: setting.warnIfTestServer || false,
-          webPushServer: setting.webPushServer || "",
       } as Settings;
-      })
-      .filter((setting) => {
-        // Only include settings that have either accountDid or activeDid set
-        return setting.accountDid || setting.activeDid;
     });
 
     logger.info(
@@ -1040,9 +1030,9 @@ export async function migrateContacts(
  * settings: firstName, isRegistered, profileImageUrl, showShortcutBvc,
  * and searchBoxes.
  *
- * The function handles both new settings (INSERT) and existing settings
+ * The function handles duplicate settings by merging master settings (id=1)
- * (UPDATE) based on the overwriteExisting parameter. For updates, it
+ * with account-specific settings (id=2) for the same DID, preferring
- * only modifies the specified fields, preserving other settings.
+ * the most recent values for the specified fields.
  *
  * @async
  * @function migrateSettings
@@ -1083,98 +1073,233 @@ export async function migrateSettings(
     const dexieSettings = await getDexieSettings();
     const platformService = PlatformServiceFactory.getInstance();
 
-    // Fields to migrate - these are the most important user-facing settings
+    // Group settings by DID to handle duplicates
-    const fieldsToMigrate = [
+    const settingsByDid = new Map<string, {
-      "firstName",
+      master?: Settings;
-      "isRegistered",
+      account?: Settings;
-      "profileImageUrl",
+    }>();
-      "showShortcutBvc",
-      "searchBoxes",
-    ];
 
-    for (const dexieSetting of dexieSettings) {
+    // Organize settings by DID
-      try {
+    dexieSettings.forEach(setting => {
-        // Check if setting already exists
+      const isMasterSetting = setting.id === MASTER_SETTINGS_KEY;
-        const existingResult = await platformService.dbQuery(
+      const did = isMasterSetting ? setting.activeDid : setting.accountDid;
-          "SELECT id FROM settings WHERE id = ?",
-          [dexieSetting.id],
-        );
       
-        // Prepare the data object, handling DIDs based on whether this is the master settings
+      if (!did) {
-        const settingData: Record<string, unknown> = {};
+        result.warnings.push(`Setting ${setting.id} has no DID, skipping`);
-        fieldsToMigrate.forEach((field) => {
+        return;
-          if (dexieSetting[field as keyof Settings] !== undefined) {
-            settingData[field] = dexieSetting[field as keyof Settings];
       }
-        });
 
-        // Handle DIDs based on whether this is the master settings
+      if (!settingsByDid.has(did)) {
-        if (dexieSetting.id === MASTER_SETTINGS_KEY) {
+        settingsByDid.set(did, {});
-          // Master settings should only use activeDid
-          if (dexieSetting.activeDid) {
-            settingData.activeDid = dexieSetting.activeDid;
       }
-          // Ensure accountDid is null for master settings
+
-          settingData.accountDid = null;
+      const didSettings = settingsByDid.get(did)!;
+      if (isMasterSetting) {
+        didSettings.master = setting;
+        logger.info("[MigrationService] Found master settings", { 
+          did,
+          id: setting.id,
+          firstName: setting.firstName,
+          isRegistered: setting.isRegistered,
+          profileImageUrl: setting.profileImageUrl,
+          showShortcutBvc: setting.showShortcutBvc,
+          searchBoxes: setting.searchBoxes
+        });
       } else {
-          // Non-master settings should only use accountDid
+        didSettings.account = setting;
-          if (dexieSetting.accountDid) {
+        logger.info("[MigrationService] Found account settings", { 
-            settingData.accountDid = dexieSetting.accountDid;
+          did,
-          }
+          id: setting.id,
-          // Ensure activeDid is null for non-master settings
+          firstName: setting.firstName,
-          settingData.activeDid = null;
+          isRegistered: setting.isRegistered,
+          profileImageUrl: setting.profileImageUrl,
+          showShortcutBvc: setting.showShortcutBvc,
+          searchBoxes: setting.searchBoxes
+        });
       }
+    });
 
-        if (existingResult?.values?.length) {
+    // Process each unique DID's settings
-          if (overwriteExisting) {
+    for (const [did, didSettings] of settingsByDid.entries()) {
-            // Update existing setting
+      try {
-            const { sql, params } = generateUpdateStatement(
+        // Process master settings
-              settingData,
+        if (didSettings.master) {
-              "settings",
+          const masterData = {
-              "id = ?",
+            id: MASTER_SETTINGS_KEY,
-              [dexieSetting.id],
+            activeDid: did,
+            accountDid: "", // Empty for master settings
+            apiServer: didSettings.master.apiServer || "",
+            filterFeedByNearby: didSettings.master.filterFeedByNearby || false,
+            filterFeedByVisible: didSettings.master.filterFeedByVisible || false,
+            finishedOnboarding: didSettings.master.finishedOnboarding || false,
+            firstName: didSettings.master.firstName || "",
+            hideRegisterPromptOnNewContact: didSettings.master.hideRegisterPromptOnNewContact || false,
+            isRegistered: didSettings.master.isRegistered || false,
+            lastName: didSettings.master.lastName || "",
+            profileImageUrl: didSettings.master.profileImageUrl || "",
+            searchBoxes: didSettings.master.searchBoxes || [],
+            showShortcutBvc: didSettings.master.showShortcutBvc || false
+          };
+
+          // Check if master setting exists
+          const existingMaster = await platformService.dbQuery(
+            "SELECT id FROM settings WHERE id = ? AND activeDid = ? AND accountDid = ''",
+            [MASTER_SETTINGS_KEY, did]
           );
-            await platformService.dbExec(sql, params);
+
-            result.settingsMigrated++;
+          if (existingMaster?.values?.length) {
-            logger.info(
+            logger.info("[MigrationService] Updating master settings", { did, masterData });
-              `[MigrationService] Updated settings: ${dexieSetting.id}`,
+            await platformService.dbQuery(
+              `UPDATE settings SET 
+                activeDid = ?,
+                accountDid = ?,
+                firstName = ?,
+                isRegistered = ?,
+                profileImageUrl = ?,
+                showShortcutBvc = ?,
+                searchBoxes = ?
+              WHERE id = ?`,
+              [
+                masterData.activeDid,
+                masterData.accountDid,
+                masterData.firstName,
+                masterData.isRegistered,
+                masterData.profileImageUrl,
+                masterData.showShortcutBvc,
+                JSON.stringify(masterData.searchBoxes),
+                MASTER_SETTINGS_KEY
+              ]
             );
           } else {
-            result.warnings.push(
+            logger.info("[MigrationService] Inserting master settings", { did, masterData });
-              `Settings ${dexieSetting.id} already exists, skipping`,
+            await platformService.dbQuery(
+              `INSERT INTO settings (
+                id,
+                activeDid,
+                accountDid,
+                firstName,
+                isRegistered,
+                profileImageUrl,
+                showShortcutBvc,
+                searchBoxes
+              ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+              [
+                MASTER_SETTINGS_KEY,
+                masterData.activeDid,
+                masterData.accountDid,
+                masterData.firstName,
+                masterData.isRegistered,
+                masterData.profileImageUrl,
+                masterData.showShortcutBvc,
+                JSON.stringify(masterData.searchBoxes)
+              ]
             );
           }
+          result.settingsMigrated++;
+        }
+
+        // Process account settings
+        if (didSettings.account) {
+          const accountData = {
+            id: 2, // Account settings always use id 2
+            activeDid: "", // Empty for account settings
+            accountDid: did,
+            apiServer: didSettings.account.apiServer || "",
+            filterFeedByNearby: didSettings.account.filterFeedByNearby || false,
+            filterFeedByVisible: didSettings.account.filterFeedByVisible || false,
+            finishedOnboarding: didSettings.account.finishedOnboarding || false,
+            firstName: didSettings.account.firstName || "",
+            hideRegisterPromptOnNewContact: didSettings.account.hideRegisterPromptOnNewContact || false,
+            isRegistered: didSettings.account.isRegistered || false,
+            lastName: didSettings.account.lastName || "",
+            profileImageUrl: didSettings.account.profileImageUrl || "",
+            searchBoxes: didSettings.account.searchBoxes || [],
+            showShortcutBvc: didSettings.account.showShortcutBvc || false
+          };
+
+          // Check if account setting exists
+          const existingAccount = await platformService.dbQuery(
+            "SELECT id FROM settings WHERE id = ? AND accountDid = ? AND activeDid = ''",
+            [2, did]
+          );
+
+          if (existingAccount?.values?.length) {
+            logger.info("[MigrationService] Updating account settings", { did, accountData });
+            await platformService.dbQuery(
+              `UPDATE settings SET 
+                activeDid = ?,
+                accountDid = ?,
+                firstName = ?,
+                isRegistered = ?,
+                profileImageUrl = ?,
+                showShortcutBvc = ?,
+                searchBoxes = ?
+              WHERE id = ?`,
+              [
+                accountData.activeDid,
+                accountData.accountDid,
+                accountData.firstName,
+                accountData.isRegistered,
+                accountData.profileImageUrl,
+                accountData.showShortcutBvc,
+                JSON.stringify(accountData.searchBoxes),
+                2
+              ]
+            );
           } else {
-          // Insert new setting
+            logger.info("[MigrationService] Inserting account settings", { did, accountData });
-          settingData.id = dexieSetting.id;
+            await platformService.dbQuery(
-          const { sql, params } = generateInsertStatement(
+              `INSERT INTO settings (
-            settingData,
+                id,
-            "settings",
+                activeDid,
+                accountDid,
+                firstName,
+                isRegistered,
+                profileImageUrl,
+                showShortcutBvc,
+                searchBoxes
+              ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+              [
+                2,
+                accountData.activeDid,
+                accountData.accountDid,
+                accountData.firstName,
+                accountData.isRegistered,
+                accountData.profileImageUrl,
+                accountData.showShortcutBvc,
+                JSON.stringify(accountData.searchBoxes)
+              ]
             );
-          await platformService.dbExec(sql, params);
+          }
           result.settingsMigrated++;
-          logger.info(`[MigrationService] Added settings: ${dexieSetting.id}`);
         }
+
+        logger.info("[MigrationService] Successfully migrated settings for DID", {
+          did,
+          masterMigrated: !!didSettings.master,
+          accountMigrated: !!didSettings.account
+        });
+
       } catch (error) {
-        const errorMsg = `Failed to migrate settings ${dexieSetting.id}: ${error}`;
+        const errorMessage = `Failed to migrate settings for DID ${did}: ${error}`;
-        logger.error("[MigrationService]", errorMsg);
+        result.errors.push(errorMessage);
-        result.errors.push(errorMsg);
+        logger.error("[MigrationService] Settings migration failed:", {
-        result.success = false;
+          error,
+          did
+        });
       }
     }
 
-    logger.info("[MigrationService] Settings migration completed", {
+    if (result.errors.length > 0) {
-      settingsMigrated: result.settingsMigrated,
+      result.success = false;
-      errors: result.errors.length,
+    }
-      warnings: result.warnings.length,
-    });
 
     return result;
   } catch (error) {
-    const errorMsg = `Settings migration failed: ${error}`;
+    const errorMessage = `Settings migration failed: ${error}`;
-    logger.error("[MigrationService]", errorMsg);
+    result.errors.push(errorMessage);
-    result.errors.push(errorMsg);
     result.success = false;
+    logger.error("[MigrationService] Complete settings migration failed:", error);
     return result;
   }
 }
@@ -1233,86 +1358,112 @@ export async function migrateAccounts(
     const dexieAccounts = await getDexieAccounts();
     const platformService = PlatformServiceFactory.getInstance();
 
-    for (const account of dexieAccounts) {
+    // Group accounts by DID and keep only the most recent one
+    const accountsByDid = new Map<string, Account>();
+    dexieAccounts.forEach(account => {
+      const existingAccount = accountsByDid.get(account.did);
+      if (!existingAccount || new Date(account.dateCreated) > new Date(existingAccount.dateCreated)) {
+        accountsByDid.set(account.did, account);
+        if (existingAccount) {
+          result.warnings.push(`Found duplicate account for DID ${account.did}, keeping most recent`);
+        }
+      }
+    });
+
+    // Process each unique account
+    for (const [did, account] of accountsByDid.entries()) {
       try {
         // Check if account already exists
         const existingResult = await platformService.dbQuery(
-          "SELECT id FROM accounts WHERE id = ?",
+          "SELECT did FROM accounts WHERE did = ?",
-          [account.id],
+          [did]
         );
 
-        if (existingResult?.values?.length) {
+        if (existingResult?.values?.length && !overwriteExisting) {
-          if (overwriteExisting) {
+          result.warnings.push(`Account with DID ${did} already exists, skipping`);
-            // Update existing account
+          continue;
-            const { sql, params } = generateUpdateStatement(
-              account as unknown as Record<string, unknown>,
-              "accounts",
-              "id = ?",
-              [account.id],
-            );
-            await platformService.dbExec(sql, params);
-            result.accountsMigrated++;
-            logger.info(`[MigrationService] Updated account: ${account.id}`);
-          } else {
-            result.warnings.push(
-              `Account ${account.id} already exists, skipping`,
-            );
         }
-        } else {
+
-          // For new accounts with mnemonic, use importFromMnemonic for proper key derivation
+        // Map Dexie fields to SQLite fields
-          if (account.mnemonic && account.derivationPath) {
+        const accountData = {
-            try {
+          did: account.did,
-              // Use importFromMnemonic to ensure proper key derivation and identity creation
+          dateCreated: account.dateCreated,
-              await importFromMnemonic(
+          derivationPath: account.derivationPath || "",
-                account.mnemonic,
+          identityEncrBase64: account.identity || "",
-                account.derivationPath,
+          mnemonicEncrBase64: account.mnemonic || "",
-                false, // Don't erase existing accounts during migration
+          passkeyCredIdHex: account.passkeyCredIdHex || "",
-              );
+          publicKeyHex: account.publicKeyHex || ""
-              logger.info(
+        };
-                `[MigrationService] Imported account with mnemonic: ${account.id}`,
+
-              );
+        // Insert or update the account
-            } catch (importError) {
+        if (existingResult?.values?.length) {
-              // Fall back to direct insertion if importFromMnemonic fails
+          await platformService.dbQuery(
-              logger.warn(
+            `UPDATE accounts SET 
-                `[MigrationService] importFromMnemonic failed for account ${account.id}, falling back to direct insertion: ${importError}`,
+              dateCreated = ?,
-              );
+              derivationPath = ?,
-              const { sql, params } = generateInsertStatement(
+              identityEncrBase64 = ?,
-                account as unknown as Record<string, unknown>,
+              mnemonicEncrBase64 = ?,
-                "accounts",
+              passkeyCredIdHex = ?,
+              publicKeyHex = ?
+            WHERE did = ?`,
+            [
+              accountData.dateCreated,
+              accountData.derivationPath,
+              accountData.identityEncrBase64,
+              accountData.mnemonicEncrBase64,
+              accountData.passkeyCredIdHex,
+              accountData.publicKeyHex,
+              did
+            ]
           );
-              await platformService.dbExec(sql, params);
-            }
         } else {
-            // Insert new account without mnemonic
+          await platformService.dbQuery(
-            const { sql, params } = generateInsertStatement(
+            `INSERT INTO accounts (
-              account as unknown as Record<string, unknown>,
+              did,
-              "accounts",
+              dateCreated,
+              derivationPath,
+              identityEncrBase64,
+              mnemonicEncrBase64,
+              passkeyCredIdHex,
+              publicKeyHex
+            ) VALUES (?, ?, ?, ?, ?, ?, ?)`,
+            [
+              did,
+              accountData.dateCreated,
+              accountData.derivationPath,
+              accountData.identityEncrBase64,
+              accountData.mnemonicEncrBase64,
+              accountData.passkeyCredIdHex,
+              accountData.publicKeyHex
+            ]
           );
-            await platformService.dbExec(sql, params);
         }
+
         result.accountsMigrated++;
-          logger.info(`[MigrationService] Added account: ${account.id}`);
+        logger.info("[MigrationService] Successfully migrated account", {
-        }
+          did,
+          dateCreated: account.dateCreated
+        });
       } catch (error) {
-        const errorMsg = `Failed to migrate account ${account.id}: ${error}`;
+        const errorMessage = `Failed to migrate account ${did}: ${error}`;
-        logger.error("[MigrationService]", errorMsg);
+        result.errors.push(errorMessage);
-        result.errors.push(errorMsg);
+        logger.error("[MigrationService] Account migration failed:", {
-        result.success = false;
+          error,
+          did
+        });
       }
     }
 
-    logger.info("[MigrationService] Account migration completed", {
+    if (result.errors.length > 0) {
-      accountsMigrated: result.accountsMigrated,
+      result.success = false;
-      errors: result.errors.length,
+    }
-      warnings: result.warnings.length,
-    });
 
     return result;
   } catch (error) {
-    const errorMsg = `Account migration failed: ${error}`;
+    const errorMessage = `Account migration failed: ${error}`;
-    logger.error("[MigrationService]", errorMsg);
+    result.errors.push(errorMessage);
-    result.errors.push(errorMsg);
     result.success = false;
+    logger.error("[MigrationService] Complete account migration failed:", error);
     return result;
   }
 }
@@ -1648,3 +1799,60 @@ export async function migrateAll(
     return result;
   }
 }
+
+/**
+ * Test function to verify migration of specific settings fields
+ * 
+ * This function tests the migration of the specific fields you mentioned:
+ * firstName, isRegistered, profileImageUrl, showShortcutBvc, and searchBoxes
+ * 
+ * @returns Promise<void>
+ */
+export async function testSettingsMigration(): Promise<void> {
+  logger.info("[MigrationService] Starting settings migration test");
+  
+  try {
+    // First, compare databases to see current state
+    const comparison = await compareDatabases();
+    logger.info("[MigrationService] Pre-migration comparison:", {
+      dexieSettings: comparison.dexieSettings.length,
+      sqliteSettings: comparison.sqliteSettings.length,
+      dexieAccounts: comparison.dexieAccounts.length,
+      sqliteAccounts: comparison.sqliteAccounts.length
+    });
+
+    // Run settings migration
+    const settingsResult = await migrateSettings(true);
+    logger.info("[MigrationService] Settings migration result:", settingsResult);
+
+    // Run accounts migration
+    const accountsResult = await migrateAccounts(true);
+    logger.info("[MigrationService] Accounts migration result:", accountsResult);
+
+    // Compare databases again to see changes
+    const postComparison = await compareDatabases();
+    logger.info("[MigrationService] Post-migration comparison:", {
+      dexieSettings: postComparison.dexieSettings.length,
+      sqliteSettings: postComparison.sqliteSettings.length,
+      dexieAccounts: postComparison.dexieAccounts.length,
+      sqliteAccounts: postComparison.sqliteAccounts.length
+    });
+
+    // Check if the specific fields were migrated
+    if (postComparison.sqliteSettings.length > 0) {
+      const sqliteSettings = postComparison.sqliteSettings[0];
+      logger.info("[MigrationService] Migrated settings fields:", {
+        firstName: sqliteSettings.firstName,
+        isRegistered: sqliteSettings.isRegistered,
+        profileImageUrl: sqliteSettings.profileImageUrl,
+        showShortcutBvc: sqliteSettings.showShortcutBvc,
+        searchBoxes: sqliteSettings.searchBoxes
+      });
+    }
+
+    logger.info("[MigrationService] Migration test completed successfully");
+  } catch (error) {
+    logger.error("[MigrationService] Migration test failed:", error);
+    throw error;
+  }
+}

src/views/DatabaseMigration.vue

@@ -112,6 +112,15 @@
           Migrate Accounts
         </button>
 
+        <button
+          :disabled="isLoading || !isDexieEnabled"
+          class="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-pink-600 hover:bg-pink-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-pink-500 disabled:opacity-50 disabled:cursor-not-allowed"
+          @click="testSpecificSettingsMigration"
+        >
+          <IconRenderer icon-name="test" svg-class="-ml-1 mr-3 h-5 w-5" />
+          Test Settings Migration
+        </button>
+
         <button
           :disabled="!comparison"
           class="inline-flex items-center px-4 py-2 border border-gray-300 text-sm font-medium rounded-md text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
@@ -798,6 +807,7 @@ import {
   migrateAccounts,
   migrateAll,
   generateComparisonYaml,
+  testSettingsMigration,
   type DataComparison,
   type MigrationResult,
 } from "../services/migrationService";
@@ -1214,5 +1224,33 @@ export default class DatabaseMigration extends Vue {
     this.error = "";
     this.successMessage = "";
   }
+
+  /**
+   * Tests the specific settings migration for the fields you mentioned
+   * 
+   * This method tests the migration of firstName, isRegistered, profileImageUrl,
+   * showShortcutBvc, and searchBoxes from Dexie to SQLite.
+   *
+   * @async
+   * @returns {Promise<void>}
+   */
+  async testSpecificSettingsMigration(): Promise<void> {
+    this.setLoading("Testing specific settings migration...");
+    this.clearMessages();
+
+    try {
+      await testSettingsMigration();
+      this.successMessage = "✅ Settings migration test completed successfully! Check the console for detailed logs.";
+      logger.info("[DatabaseMigration] Settings migration test completed successfully");
+      
+      // Refresh comparison data after successful test
+      this.comparison = await compareDatabases();
+    } catch (error) {
+      this.error = `Settings migration test failed: ${error}`;
+      logger.error("[DatabaseMigration] Settings migration test failed:", error);
+    } finally {
+      this.setLoading("");
+    }
+  }
 }
 </script>