feat: improve database migration comparison and UI display

- Enhanced migration service to filter empty/default records from comparisons
- Updated comparison output to exclude records without meaningful data
- Improved UI display with better visual hierarchy and count indicators
- Added security improvements to prevent sensitive data exposure
- Fixed settings migration to properly handle MASTER_SETTINGS_KEY vs account DIDs
- Updated verification display to show filtered counts and detailed differences
- Improved data formatting for contacts, settings, and accounts sections
- Added proper filtering for missing records to avoid counting empty entries

Changes:
- Filter SQLite records to only include those with actual DIDs/data
- Update comparison counts to reflect meaningful differences only
- Enhance UI with count indicators and better visual organization
- Replace sensitive data display with boolean flags (hasIdentity, hasMnemonic)
- Fix settings migration logic for proper DID field handling
- Improve verification message to show detailed breakdown by type
- Add proper filtering for missing records in all data types

Security: Prevents exposure of mnemonics, private keys, and identity data
UI/UX: Cleaner display with better information hierarchy and counts
Migration: More accurate comparison results and better debugging visibility

package-lock.json

@@ -28627,9 +28627,9 @@
       }
     },
     "node_modules/terser": {
-      "version": "5.43.0",
+      "version": "5.43.1",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.43.0.tgz",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.43.1.tgz",
-      "integrity": "sha512-CqNNxKSGKSZCunSvwKLTs8u8sGGlp27sxNZ4quGh0QeNuyHM0JSEM/clM9Mf4zUp6J+tO2gUXhgXT2YMMkwfKQ==",
+      "integrity": "sha512-+6erLbBm0+LROX2sPXlUYx/ux5PyE9K/a92Wrt6oA+WDAoFTdpHE5tCYCI5PNzq2y8df4rA+QgHLJuR4jNymsg==",
       "devOptional": true,
       "license": "BSD-2-Clause",
       "dependencies": {

src/services/migrationService.ts

@@ -24,7 +24,7 @@
 import { PlatformServiceFactory } from "./PlatformServiceFactory";
 import { db, accountsDBPromise } from "../db/index";
 import { Contact, ContactMethod } from "../db/tables/contacts";
-import { Settings } from "../db/tables/settings";
+import { Settings, MASTER_SETTINGS_KEY } from "../db/tables/settings";
 import { Account } from "../db/tables/accounts";
 import { logger } from "../utils/logger";
 import { parseJsonField } from "../db/databaseUtil";
@@ -285,12 +285,13 @@ export async function getSqliteSettings(): Promise<Settings[]> {
       return [];
     }
 
-    const settings = result.values.map((row) => {
+    const settings = result.values
+      .map((row) => {
         const setting = parseJsonField(row, {}) as Settings;
         return {
           id: setting.id,
-        accountDid: setting.accountDid || "",
+          accountDid: setting.accountDid || null,
-        activeDid: setting.activeDid || "",
+          activeDid: setting.activeDid || null,
           apiServer: setting.apiServer || "",
           filterFeedByNearby: setting.filterFeedByNearby || false,
           filterFeedByVisible: setting.filterFeedByVisible || false,
@@ -320,6 +321,10 @@ export async function getSqliteSettings(): Promise<Settings[]> {
           warnIfTestServer: setting.warnIfTestServer || false,
           webPushServer: setting.webPushServer || "",
         } as Settings;
+      })
+      .filter((setting) => {
+        // Only include settings that have either accountDid or activeDid set
+        return setting.accountDid || setting.activeDid;
       });
 
     logger.info(
@@ -837,94 +842,78 @@ function accountsEqual(account1: Account, account2: Account): boolean {
  */
 export function generateComparisonYaml(comparison: DataComparison): string {
   const yaml = {
-    comparison: {
     summary: {
       dexieContacts: comparison.dexieContacts.length,
-        sqliteContacts: comparison.sqliteContacts.length,
+      sqliteContacts: comparison.sqliteContacts.filter(c => c.did).length,
       dexieSettings: comparison.dexieSettings.length,
-        sqliteSettings: comparison.sqliteSettings.length,
+      sqliteSettings: comparison.sqliteSettings.filter(s => s.accountDid || s.activeDid).length,
       dexieAccounts: comparison.dexieAccounts.length,
-        sqliteAccounts: comparison.sqliteAccounts.length,
+      sqliteAccounts: comparison.sqliteAccounts.filter(a => a.did).length,
     },
     differences: {
       contacts: {
         added: comparison.differences.contacts.added.length,
         modified: comparison.differences.contacts.modified.length,
-          missing: comparison.differences.contacts.missing.length,
+        missing: comparison.differences.contacts.missing.filter(c => c.did).length,
       },
       settings: {
         added: comparison.differences.settings.added.length,
         modified: comparison.differences.settings.modified.length,
-          missing: comparison.differences.settings.missing.length,
+        missing: comparison.differences.settings.missing.filter(s => s.accountDid || s.activeDid).length,
       },
       accounts: {
         added: comparison.differences.accounts.added.length,
         modified: comparison.differences.accounts.modified.length,
-          missing: comparison.differences.accounts.missing.length,
+        missing: comparison.differences.accounts.missing.filter(a => a.did).length,
       },
     },
+    details: {
       contacts: {
         dexie: comparison.dexieContacts.map((c) => ({
           did: c.did,
-          name: c.name,
+          name: c.name || '<empty>',
-          notes: c.notes,
+          contactMethods: (c.contactMethods || []).length,
-          profileImageUrl: c.profileImageUrl,
-          seesMe: c.seesMe,
-          registered: c.registered,
-          contactMethods: c.contactMethods,
         })),
-        sqlite: comparison.sqliteContacts.map((c) => ({
+        sqlite: comparison.sqliteContacts
+          .filter(c => c.did)
+          .map((c) => ({
             did: c.did,
-          name: c.name,
+            name: c.name || '<empty>',
-          notes: c.notes,
+            contactMethods: (c.contactMethods || []).length,
-          profileImageUrl: c.profileImageUrl,
-          seesMe: c.seesMe,
-          registered: c.registered,
-          contactMethods: c.contactMethods,
           })),
       },
       settings: {
         dexie: comparison.dexieSettings.map((s) => ({
           id: s.id,
-          accountDid: s.accountDid,
+          type: s.id === MASTER_SETTINGS_KEY ? 'master' : 'account',
-          activeDid: s.activeDid,
+          did: s.activeDid || s.accountDid,
-          firstName: s.firstName,
+          isRegistered: s.isRegistered || false,
-          isRegistered: s.isRegistered,
-          profileImageUrl: s.profileImageUrl,
-          showShortcutBvc: s.showShortcutBvc,
-          searchBoxes: s.searchBoxes,
         })),
-        sqlite: comparison.sqliteSettings.map((s) => ({
+        sqlite: comparison.sqliteSettings
+          .filter(s => s.accountDid || s.activeDid)
+          .map((s) => ({
             id: s.id,
-          accountDid: s.accountDid,
+            type: s.id === MASTER_SETTINGS_KEY ? 'master' : 'account',
-          activeDid: s.activeDid,
+            did: s.activeDid || s.accountDid,
-          firstName: s.firstName,
+            isRegistered: s.isRegistered || false,
-          isRegistered: s.isRegistered,
-          profileImageUrl: s.profileImageUrl,
-          showShortcutBvc: s.showShortcutBvc,
-          searchBoxes: s.searchBoxes,
           })),
       },
       accounts: {
         dexie: comparison.dexieAccounts.map((a) => ({
           id: a.id,
-          dateCreated: a.dateCreated,
-          derivationPath: a.derivationPath,
           did: a.did,
-          identity: a.identity,
+          dateCreated: a.dateCreated,
-          mnemonic: a.mnemonic,
+          hasIdentity: !!a.identity,
-          passkeyCredIdHex: a.passkeyCredIdHex,
+          hasMnemonic: !!a.mnemonic,
-          publicKeyHex: a.publicKeyHex,
         })),
-        sqlite: comparison.sqliteAccounts.map((a) => ({
+        sqlite: comparison.sqliteAccounts
+          .filter(a => a.did)
+          .map((a) => ({
             id: a.id,
-          dateCreated: a.dateCreated,
-          derivationPath: a.derivationPath,
             did: a.did,
-          identity: a.identity,
+            dateCreated: a.dateCreated,
-          mnemonic: a.mnemonic,
+            hasIdentity: !!a.identity,
-          passkeyCredIdHex: a.passkeyCredIdHex,
+            hasMnemonic: !!a.mnemonic,
-          publicKeyHex: a.publicKeyHex,
           })),
       },
     },
@@ -1111,19 +1100,36 @@ export async function migrateSettings(
           [dexieSetting.id],
         );
 
-        if (existingResult?.values?.length) {
+        // Prepare the data object, handling DIDs based on whether this is the master settings
-          if (overwriteExisting) {
+        const settingData: Record<string, unknown> = {};
-            // Update existing setting with only the specified fields
-            const updateData: Record<string, unknown> = {};
         fieldsToMigrate.forEach((field) => {
           if (dexieSetting[field as keyof Settings] !== undefined) {
-                updateData[field] = dexieSetting[field as keyof Settings];
+            settingData[field] = dexieSetting[field as keyof Settings];
           }
         });
 
-            if (Object.keys(updateData).length > 0) {
+        // Handle DIDs based on whether this is the master settings
+        if (dexieSetting.id === MASTER_SETTINGS_KEY) {
+          // Master settings should only use activeDid
+          if (dexieSetting.activeDid) {
+            settingData.activeDid = dexieSetting.activeDid;
+          }
+          // Ensure accountDid is null for master settings
+          settingData.accountDid = null;
+        } else {
+          // Non-master settings should only use accountDid
+          if (dexieSetting.accountDid) {
+            settingData.accountDid = dexieSetting.accountDid;
+          }
+          // Ensure activeDid is null for non-master settings
+          settingData.activeDid = null;
+        }
+
+        if (existingResult?.values?.length) {
+          if (overwriteExisting) {
+            // Update existing setting
             const { sql, params } = generateUpdateStatement(
-                updateData as unknown as Record<string, unknown>,
+              settingData,
               "settings",
               "id = ?",
               [dexieSetting.id],
@@ -1133,7 +1139,6 @@ export async function migrateSettings(
             logger.info(
               `[MigrationService] Updated settings: ${dexieSetting.id}`,
             );
-            }
           } else {
             result.warnings.push(
               `Settings ${dexieSetting.id} already exists, skipping`,
@@ -1141,8 +1146,9 @@ export async function migrateSettings(
           }
         } else {
           // Insert new setting
+          settingData.id = dexieSetting.id;
           const { sql, params } = generateInsertStatement(
-            dexieSetting as unknown as Record<string, unknown>,
+            settingData,
             "settings",
           );
           await platformService.dbExec(sql, params);

src/views/DatabaseMigration.vue

@@ -440,17 +440,59 @@
                 class="mt-4"
               >
                 <h4 class="text-sm font-medium text-gray-900 mb-2">
-                  Added Contacts:
+                  Added Contacts ({{ comparison.differences.contacts.added.length }}):
                 </h4>
-                <div class="max-h-32 overflow-y-auto space-y-1">
+                <div class="space-y-1">
                   <div
                     v-for="contact in comparison.differences.contacts.added"
                     :key="contact.did"
                     class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
                   >
-                    {{ contact.name || "Unnamed" }} ({{
+                    <div class="font-medium">{{ contact.name || '<empty>' }}</div>
-                      contact.did.substring(0, 20)
+                    <div class="text-gray-500">{{ contact.did }}</div>
-                    }}...)
+                    <div class="text-gray-400">{{ contact.contactMethods?.length || 0 }} contact methods</div>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Modified Contacts -->
+              <div
+                v-if="comparison.differences.contacts.modified.length > 0"
+                class="mt-4"
+              >
+                <h4 class="text-sm font-medium text-gray-900 mb-2">
+                  Modified Contacts ({{ comparison.differences.contacts.modified.length }}):
+                </h4>
+                <div class="space-y-1">
+                  <div
+                    v-for="contact in comparison.differences.contacts.modified"
+                    :key="contact.did"
+                    class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
+                  >
+                    <div class="font-medium">{{ contact.name || '<empty>' }}</div>
+                    <div class="text-gray-500">{{ contact.did }}</div>
+                    <div class="text-gray-400">{{ contact.contactMethods?.length || 0 }} contact methods</div>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Missing Contacts -->
+              <div
+                v-if="comparison.differences.contacts.missing.filter(c => c.did).length > 0"
+                class="mt-4"
+              >
+                <h4 class="text-sm font-medium text-gray-900 mb-2">
+                  Missing Contacts ({{ comparison.differences.contacts.missing.filter(c => c.did).length }}):
+                </h4>
+                <div class="space-y-1">
+                  <div
+                    v-for="contact in comparison.differences.contacts.missing.filter(c => c.did)"
+                    :key="contact.did"
+                    class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
+                  >
+                    <div class="font-medium">{{ contact.name || '<empty>' }}</div>
+                    <div class="text-gray-500">{{ contact.did }}</div>
+                    <div class="text-gray-400">{{ contact.contactMethods?.length || 0 }} contact methods</div>
                   </div>
                 </div>
               </div>
@@ -521,15 +563,59 @@
                 class="mt-4"
               >
                 <h4 class="text-sm font-medium text-gray-900 mb-2">
-                  Added Settings:
+                  Added Settings ({{ comparison.differences.settings.added.length }}):
                 </h4>
-                <div class="max-h-32 overflow-y-auto space-y-1">
+                <div class="space-y-1">
                   <div
                     v-for="setting in comparison.differences.settings.added"
                     :key="setting.id"
                     class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
                   >
-                    ID: {{ setting.id }} - {{ setting.firstName || "Unnamed" }}
+                    <div class="font-medium">ID: {{ setting.id }} ({{ setting.id === 1 ? 'master' : 'account' }})</div>
+                    <div class="text-gray-500">{{ setting.activeDid || setting.accountDid }}</div>
+                    <div class="text-gray-400">Registered: {{ setting.isRegistered ? 'Yes' : 'No' }}</div>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Modified Settings -->
+              <div
+                v-if="comparison.differences.settings.modified.length > 0"
+                class="mt-4"
+              >
+                <h4 class="text-sm font-medium text-gray-900 mb-2">
+                  Modified Settings ({{ comparison.differences.settings.modified.length }}):
+                </h4>
+                <div class="space-y-1">
+                  <div
+                    v-for="setting in comparison.differences.settings.modified"
+                    :key="setting.id"
+                    class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
+                  >
+                    <div class="font-medium">ID: {{ setting.id }} ({{ setting.id === 1 ? 'master' : 'account' }})</div>
+                    <div class="text-gray-500">{{ setting.activeDid || setting.accountDid }}</div>
+                    <div class="text-gray-400">Registered: {{ setting.isRegistered ? 'Yes' : 'No' }}</div>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Missing Settings -->
+              <div
+                v-if="comparison.differences.settings.missing.filter(s => s.accountDid || s.activeDid).length > 0"
+                class="mt-4"
+              >
+                <h4 class="text-sm font-medium text-gray-900 mb-2">
+                  Missing Settings ({{ comparison.differences.settings.missing.filter(s => s.accountDid || s.activeDid).length }}):
+                </h4>
+                <div class="space-y-1">
+                  <div
+                    v-for="setting in comparison.differences.settings.missing.filter(s => s.accountDid || s.activeDid)"
+                    :key="setting.id"
+                    class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
+                  >
+                    <div class="font-medium">ID: {{ setting.id }} ({{ setting.id === 1 ? 'master' : 'account' }})</div>
+                    <div class="text-gray-500">{{ setting.activeDid || setting.accountDid }}</div>
+                    <div class="text-gray-400">Registered: {{ setting.isRegistered ? 'Yes' : 'No' }}</div>
                   </div>
                 </div>
               </div>
@@ -600,15 +686,65 @@
                 class="mt-4"
               >
                 <h4 class="text-sm font-medium text-gray-900 mb-2">
-                  Added Accounts:
+                  Added Accounts ({{ comparison.differences.accounts.added.length }}):
                 </h4>
-                <div class="max-h-32 overflow-y-auto space-y-1">
+                <div class="space-y-1">
                   <div
                     v-for="account in comparison.differences.accounts.added"
                     :key="account.id"
                     class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
                   >
-                    ID: {{ account.id }} - {{ account.did.substring(0, 20) }}...
+                    <div class="font-medium">ID: {{ account.id }}</div>
+                    <div class="text-gray-500">{{ account.did }}</div>
+                    <div class="text-gray-400">Created: {{ account.dateCreated }}</div>
+                    <div class="text-gray-400">Has Identity: {{ account.identity ? 'Yes' : 'No' }}</div>
+                    <div class="text-gray-400">Has Mnemonic: {{ account.mnemonic ? 'Yes' : 'No' }}</div>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Modified Accounts -->
+              <div
+                v-if="comparison.differences.accounts.modified.length > 0"
+                class="mt-4"
+              >
+                <h4 class="text-sm font-medium text-gray-900 mb-2">
+                  Modified Accounts ({{ comparison.differences.accounts.modified.length }}):
+                </h4>
+                <div class="space-y-1">
+                  <div
+                    v-for="account in comparison.differences.accounts.modified"
+                    :key="account.id"
+                    class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
+                  >
+                    <div class="font-medium">ID: {{ account.id }}</div>
+                    <div class="text-gray-500">{{ account.did }}</div>
+                    <div class="text-gray-400">Created: {{ account.dateCreated }}</div>
+                    <div class="text-gray-400">Has Identity: {{ account.identity ? 'Yes' : 'No' }}</div>
+                    <div class="text-gray-400">Has Mnemonic: {{ account.mnemonic ? 'Yes' : 'No' }}</div>
+                  </div>
+                </div>
+              </div>
+
+              <!-- Missing Accounts -->
+              <div
+                v-if="comparison.differences.accounts.missing.filter(a => a.did).length > 0"
+                class="mt-4"
+              >
+                <h4 class="text-sm font-medium text-gray-900 mb-2">
+                  Missing Accounts ({{ comparison.differences.accounts.missing.filter(a => a.did).length }}):
+                </h4>
+                <div class="space-y-1">
+                  <div
+                    v-for="account in comparison.differences.accounts.missing.filter(a => a.did)"
+                    :key="account.id"
+                    class="text-xs text-gray-600 bg-gray-50 p-2 rounded"
+                  >
+                    <div class="font-medium">ID: {{ account.id }}</div>
+                    <div class="text-gray-500">{{ account.did }}</div>
+                    <div class="text-gray-400">Created: {{ account.dateCreated }}</div>
+                    <div class="text-gray-400">Has Identity: {{ account.identity ? 'Yes' : 'No' }}</div>
+                    <div class="text-gray-400">Has Mnemonic: {{ account.mnemonic ? 'Yes' : 'No' }}</div>
                   </div>
                 </div>
               </div>
@@ -746,6 +882,9 @@ export default class DatabaseMigration extends Vue {
           "[DatabaseMigration] Complete migration successful",
           result,
         );
+
+        // Refresh comparison data after successful migration
+        this.comparison = await compareDatabases();
       } else {
         this.error = `Migration failed: ${result.errors.join(", ")}`;
         logger.error(
@@ -819,6 +958,9 @@ export default class DatabaseMigration extends Vue {
           "[DatabaseMigration] Contact migration completed successfully",
           result,
         );
+
+        // Refresh comparison data after successful migration
+        this.comparison = await compareDatabases();
       } else {
         this.error = `Migration failed: ${result.errors.join(", ")}`;
         logger.error(
@@ -861,6 +1003,9 @@ export default class DatabaseMigration extends Vue {
           "[DatabaseMigration] Settings migration completed successfully",
           result,
         );
+
+        // Refresh comparison data after successful migration
+        this.comparison = await compareDatabases();
       } else {
         this.error = `Migration failed: ${result.errors.join(", ")}`;
         logger.error(
@@ -904,6 +1049,9 @@ export default class DatabaseMigration extends Vue {
           "[DatabaseMigration] Account migration completed successfully",
           result,
         );
+
+        // Refresh comparison data after successful migration
+        this.comparison = await compareDatabases();
       } else {
         this.error = `Migration failed: ${result.errors.join(", ")}`;
         logger.error(
@@ -937,32 +1085,69 @@ export default class DatabaseMigration extends Vue {
     try {
       const newComparison = await compareDatabases();
 
-      // Check if there are any remaining differences
+      // Calculate differences by type for each table
-      const totalRemaining =
+      const differences = {
-        newComparison.differences.contacts.added.length +
+        contacts: {
-        newComparison.differences.contacts.modified.length +
+          added: newComparison.differences.contacts.added.length,
-        newComparison.differences.contacts.missing.length +
+          modified: newComparison.differences.contacts.modified.length,
-        newComparison.differences.settings.added.length +
+          missing: newComparison.differences.contacts.missing.length,
-        newComparison.differences.settings.modified.length +
+        },
-        newComparison.differences.settings.missing.length +
+        settings: {
-        newComparison.differences.accounts.added.length +
+          added: newComparison.differences.settings.added.length,
-        newComparison.differences.accounts.modified.length +
+          modified: newComparison.differences.settings.modified.length,
-        newComparison.differences.accounts.missing.length;
+          missing: newComparison.differences.settings.missing.length,
+        },
+        accounts: {
+          added: newComparison.differences.accounts.added.length,
+          modified: newComparison.differences.accounts.modified.length,
+          missing: newComparison.differences.accounts.missing.length,
+        },
+      };
+
+      const totalRemaining = Object.values(differences).reduce(
+        (sum, table) =>
+          sum + table.added + table.modified + table.missing,
+        0
+      );
+
+      // Build a detailed message
+      const detailMessages = [];
+      
+      if (differences.contacts.added + differences.contacts.modified + differences.contacts.missing > 0) {
+        detailMessages.push(
+          `Contacts: ${differences.contacts.added} to add, ${differences.contacts.modified} modified, ${differences.contacts.missing} missing`
+        );
+      }
+      
+      if (differences.settings.added + differences.settings.modified + differences.settings.missing > 0) {
+        detailMessages.push(
+          `Settings: ${differences.settings.added} to add, ${differences.settings.modified} modified, ${differences.settings.missing} missing`
+        );
+      }
+      
+      if (differences.accounts.added + differences.accounts.modified + differences.accounts.missing > 0) {
+        detailMessages.push(
+          `Accounts: ${differences.accounts.added} to add, ${differences.accounts.modified} modified, ${differences.accounts.missing} missing`
+        );
+      }
 
       if (totalRemaining === 0) {
         this.successMessage =
           "✅ Migration verification successful! All data has been migrated correctly.";
         logger.info(
-          "[DatabaseMigration] Migration verification successful - no differences found",
+          "[DatabaseMigration] Migration verification successful - no differences found"
         );
       } else {
-        this.successMessage = `⚠️ Migration verification completed. Found ${totalRemaining} remaining differences. Consider running additional migrations if needed.`;
+        this.successMessage = `⚠️ Migration verification completed. Found ${totalRemaining} remaining differences:\n${detailMessages.join("\n")}`;
+        if (differences.settings.modified > 0 || differences.settings.missing > 0) {
+          this.successMessage += "\n\nNote: Some settings differences may be expected due to default values in SQLite.";
+        }
         logger.warn(
           "[DatabaseMigration] Migration verification found remaining differences",
           {
             remaining: totalRemaining,
-            differences: newComparison.differences,
+            differences: differences,
-          },
+          }
         );
       }