Browse Source

fix: maintain separate master/account settings in SQLite migration

- Update settings migration to maintain separate master and account records
- Use activeDid/accountDid pattern to differentiate between settings types:
  * Master settings: activeDid set, accountDid empty
  * Account settings: accountDid set, activeDid empty
- Add detailed logging for settings migration process
- Focus on migrating key fields: firstName, isRegistered, profileImageUrl,
  showShortcutBvc, and searchBoxes
- Fix issue where settings were being incorrectly merged into a single record

This change ensures the SQLite database maintains the same settings structure
as Dexie, which is required by the existing codebase.
migrate-dexie-to-sqlite
Matthew Raymer 7 days ago
parent
commit
30de30e709
  1. 566
      src/services/migrationService.ts
  2. 38
      src/views/DatabaseMigration.vue

566
src/services/migrationService.ts

@ -285,47 +285,37 @@ export async function getSqliteSettings(): Promise<Settings[]> {
return []; return [];
} }
const settings = result.values const settings = result.values.map((row) => {
.map((row) => { const setting = parseJsonField(row, {}) as Settings;
const setting = parseJsonField(row, {}) as Settings; return {
return { id: setting.id,
id: setting.id, accountDid: setting.accountDid || "",
accountDid: setting.accountDid || null, activeDid: setting.activeDid || "",
activeDid: setting.activeDid || null, apiServer: setting.apiServer || "",
apiServer: setting.apiServer || "", filterFeedByNearby: setting.filterFeedByNearby || false,
filterFeedByNearby: setting.filterFeedByNearby || false, filterFeedByVisible: setting.filterFeedByVisible || false,
filterFeedByVisible: setting.filterFeedByVisible || false, finishedOnboarding: setting.finishedOnboarding || false,
finishedOnboarding: setting.finishedOnboarding || false, firstName: setting.firstName || "",
firstName: setting.firstName || "", hideRegisterPromptOnNewContact: setting.hideRegisterPromptOnNewContact || false,
hideRegisterPromptOnNewContact: isRegistered: setting.isRegistered || false,
setting.hideRegisterPromptOnNewContact || false, lastName: setting.lastName || "",
isRegistered: setting.isRegistered || false, lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "",
lastName: setting.lastName || "", lastAckedOfferToUserProjectsJwtId: setting.lastAckedOfferToUserProjectsJwtId || "",
lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "", lastNotifiedClaimId: setting.lastNotifiedClaimId || "",
lastAckedOfferToUserProjectsJwtId: lastViewedClaimId: setting.lastViewedClaimId || "",
setting.lastAckedOfferToUserProjectsJwtId || "", notifyingNewActivityTime: setting.notifyingNewActivityTime || "",
lastNotifiedClaimId: setting.lastNotifiedClaimId || "", notifyingReminderMessage: setting.notifyingReminderMessage || "",
lastViewedClaimId: setting.lastViewedClaimId || "", notifyingReminderTime: setting.notifyingReminderTime || "",
notifyingNewActivityTime: setting.notifyingNewActivityTime || "", partnerApiServer: setting.partnerApiServer || "",
notifyingReminderMessage: setting.notifyingReminderMessage || "", passkeyExpirationMinutes: setting.passkeyExpirationMinutes,
notifyingReminderTime: setting.notifyingReminderTime || "", profileImageUrl: setting.profileImageUrl || "",
partnerApiServer: setting.partnerApiServer || "", searchBoxes: parseJsonField(setting.searchBoxes, []),
passkeyExpirationMinutes: setting.passkeyExpirationMinutes, showContactGivesInline: setting.showContactGivesInline || false,
profileImageUrl: setting.profileImageUrl || "", showGeneralAdvanced: setting.showGeneralAdvanced || false,
searchBoxes: parseJsonField(setting.searchBoxes, []), showShortcutBvc: setting.showShortcutBvc || false,
showContactGivesInline: setting.showContactGivesInline || false, vapid: setting.vapid || "",
showGeneralAdvanced: setting.showGeneralAdvanced || false, } as Settings;
showShortcutBvc: setting.showShortcutBvc || false, });
vapid: setting.vapid || "",
warnIfProdServer: setting.warnIfProdServer || false,
warnIfTestServer: setting.warnIfTestServer || false,
webPushServer: setting.webPushServer || "",
} as Settings;
})
.filter((setting) => {
// Only include settings that have either accountDid or activeDid set
return setting.accountDid || setting.activeDid;
});
logger.info( logger.info(
`[MigrationService] Retrieved ${settings.length} settings from SQLite`, `[MigrationService] Retrieved ${settings.length} settings from SQLite`,
@ -1040,9 +1030,9 @@ export async function migrateContacts(
* settings: firstName, isRegistered, profileImageUrl, showShortcutBvc, * settings: firstName, isRegistered, profileImageUrl, showShortcutBvc,
* and searchBoxes. * and searchBoxes.
* *
* The function handles both new settings (INSERT) and existing settings * The function handles duplicate settings by merging master settings (id=1)
* (UPDATE) based on the overwriteExisting parameter. For updates, it * with account-specific settings (id=2) for the same DID, preferring
* only modifies the specified fields, preserving other settings. * the most recent values for the specified fields.
* *
* @async * @async
* @function migrateSettings * @function migrateSettings
@ -1083,98 +1073,233 @@ export async function migrateSettings(
const dexieSettings = await getDexieSettings(); const dexieSettings = await getDexieSettings();
const platformService = PlatformServiceFactory.getInstance(); const platformService = PlatformServiceFactory.getInstance();
// Fields to migrate - these are the most important user-facing settings // Group settings by DID to handle duplicates
const fieldsToMigrate = [ const settingsByDid = new Map<string, {
"firstName", master?: Settings;
"isRegistered", account?: Settings;
"profileImageUrl", }>();
"showShortcutBvc",
"searchBoxes", // Organize settings by DID
]; dexieSettings.forEach(setting => {
const isMasterSetting = setting.id === MASTER_SETTINGS_KEY;
const did = isMasterSetting ? setting.activeDid : setting.accountDid;
if (!did) {
result.warnings.push(`Setting ${setting.id} has no DID, skipping`);
return;
}
for (const dexieSetting of dexieSettings) { if (!settingsByDid.has(did)) {
try { settingsByDid.set(did, {});
// Check if setting already exists }
const existingResult = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ?",
[dexieSetting.id],
);
// Prepare the data object, handling DIDs based on whether this is the master settings const didSettings = settingsByDid.get(did)!;
const settingData: Record<string, unknown> = {}; if (isMasterSetting) {
fieldsToMigrate.forEach((field) => { didSettings.master = setting;
if (dexieSetting[field as keyof Settings] !== undefined) { logger.info("[MigrationService] Found master settings", {
settingData[field] = dexieSetting[field as keyof Settings]; did,
} id: setting.id,
firstName: setting.firstName,
isRegistered: setting.isRegistered,
profileImageUrl: setting.profileImageUrl,
showShortcutBvc: setting.showShortcutBvc,
searchBoxes: setting.searchBoxes
});
} else {
didSettings.account = setting;
logger.info("[MigrationService] Found account settings", {
did,
id: setting.id,
firstName: setting.firstName,
isRegistered: setting.isRegistered,
profileImageUrl: setting.profileImageUrl,
showShortcutBvc: setting.showShortcutBvc,
searchBoxes: setting.searchBoxes
}); });
}
});
// Handle DIDs based on whether this is the master settings // Process each unique DID's settings
if (dexieSetting.id === MASTER_SETTINGS_KEY) { for (const [did, didSettings] of settingsByDid.entries()) {
// Master settings should only use activeDid try {
if (dexieSetting.activeDid) { // Process master settings
settingData.activeDid = dexieSetting.activeDid; if (didSettings.master) {
} const masterData = {
// Ensure accountDid is null for master settings id: MASTER_SETTINGS_KEY,
settingData.accountDid = null; activeDid: did,
} else { accountDid: "", // Empty for master settings
// Non-master settings should only use accountDid apiServer: didSettings.master.apiServer || "",
if (dexieSetting.accountDid) { filterFeedByNearby: didSettings.master.filterFeedByNearby || false,
settingData.accountDid = dexieSetting.accountDid; filterFeedByVisible: didSettings.master.filterFeedByVisible || false,
finishedOnboarding: didSettings.master.finishedOnboarding || false,
firstName: didSettings.master.firstName || "",
hideRegisterPromptOnNewContact: didSettings.master.hideRegisterPromptOnNewContact || false,
isRegistered: didSettings.master.isRegistered || false,
lastName: didSettings.master.lastName || "",
profileImageUrl: didSettings.master.profileImageUrl || "",
searchBoxes: didSettings.master.searchBoxes || [],
showShortcutBvc: didSettings.master.showShortcutBvc || false
};
// Check if master setting exists
const existingMaster = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ? AND activeDid = ? AND accountDid = ''",
[MASTER_SETTINGS_KEY, did]
);
if (existingMaster?.values?.length) {
logger.info("[MigrationService] Updating master settings", { did, masterData });
await platformService.dbQuery(
`UPDATE settings SET
activeDid = ?,
accountDid = ?,
firstName = ?,
isRegistered = ?,
profileImageUrl = ?,
showShortcutBvc = ?,
searchBoxes = ?
WHERE id = ?`,
[
masterData.activeDid,
masterData.accountDid,
masterData.firstName,
masterData.isRegistered,
masterData.profileImageUrl,
masterData.showShortcutBvc,
JSON.stringify(masterData.searchBoxes),
MASTER_SETTINGS_KEY
]
);
} else {
logger.info("[MigrationService] Inserting master settings", { did, masterData });
await platformService.dbQuery(
`INSERT INTO settings (
id,
activeDid,
accountDid,
firstName,
isRegistered,
profileImageUrl,
showShortcutBvc,
searchBoxes
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[
MASTER_SETTINGS_KEY,
masterData.activeDid,
masterData.accountDid,
masterData.firstName,
masterData.isRegistered,
masterData.profileImageUrl,
masterData.showShortcutBvc,
JSON.stringify(masterData.searchBoxes)
]
);
} }
// Ensure activeDid is null for non-master settings result.settingsMigrated++;
settingData.activeDid = null;
} }
if (existingResult?.values?.length) { // Process account settings
if (overwriteExisting) { if (didSettings.account) {
// Update existing setting const accountData = {
const { sql, params } = generateUpdateStatement( id: 2, // Account settings always use id 2
settingData, activeDid: "", // Empty for account settings
"settings", accountDid: did,
"id = ?", apiServer: didSettings.account.apiServer || "",
[dexieSetting.id], filterFeedByNearby: didSettings.account.filterFeedByNearby || false,
); filterFeedByVisible: didSettings.account.filterFeedByVisible || false,
await platformService.dbExec(sql, params); finishedOnboarding: didSettings.account.finishedOnboarding || false,
result.settingsMigrated++; firstName: didSettings.account.firstName || "",
logger.info( hideRegisterPromptOnNewContact: didSettings.account.hideRegisterPromptOnNewContact || false,
`[MigrationService] Updated settings: ${dexieSetting.id}`, isRegistered: didSettings.account.isRegistered || false,
lastName: didSettings.account.lastName || "",
profileImageUrl: didSettings.account.profileImageUrl || "",
searchBoxes: didSettings.account.searchBoxes || [],
showShortcutBvc: didSettings.account.showShortcutBvc || false
};
// Check if account setting exists
const existingAccount = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ? AND accountDid = ? AND activeDid = ''",
[2, did]
);
if (existingAccount?.values?.length) {
logger.info("[MigrationService] Updating account settings", { did, accountData });
await platformService.dbQuery(
`UPDATE settings SET
activeDid = ?,
accountDid = ?,
firstName = ?,
isRegistered = ?,
profileImageUrl = ?,
showShortcutBvc = ?,
searchBoxes = ?
WHERE id = ?`,
[
accountData.activeDid,
accountData.accountDid,
accountData.firstName,
accountData.isRegistered,
accountData.profileImageUrl,
accountData.showShortcutBvc,
JSON.stringify(accountData.searchBoxes),
2
]
); );
} else { } else {
result.warnings.push( logger.info("[MigrationService] Inserting account settings", { did, accountData });
`Settings ${dexieSetting.id} already exists, skipping`, await platformService.dbQuery(
`INSERT INTO settings (
id,
activeDid,
accountDid,
firstName,
isRegistered,
profileImageUrl,
showShortcutBvc,
searchBoxes
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[
2,
accountData.activeDid,
accountData.accountDid,
accountData.firstName,
accountData.isRegistered,
accountData.profileImageUrl,
accountData.showShortcutBvc,
JSON.stringify(accountData.searchBoxes)
]
); );
} }
} else {
// Insert new setting
settingData.id = dexieSetting.id;
const { sql, params } = generateInsertStatement(
settingData,
"settings",
);
await platformService.dbExec(sql, params);
result.settingsMigrated++; result.settingsMigrated++;
logger.info(`[MigrationService] Added settings: ${dexieSetting.id}`);
} }
logger.info("[MigrationService] Successfully migrated settings for DID", {
did,
masterMigrated: !!didSettings.master,
accountMigrated: !!didSettings.account
});
} catch (error) { } catch (error) {
const errorMsg = `Failed to migrate settings ${dexieSetting.id}: ${error}`; const errorMessage = `Failed to migrate settings for DID ${did}: ${error}`;
logger.error("[MigrationService]", errorMsg); result.errors.push(errorMessage);
result.errors.push(errorMsg); logger.error("[MigrationService] Settings migration failed:", {
result.success = false; error,
did
});
} }
} }
logger.info("[MigrationService] Settings migration completed", { if (result.errors.length > 0) {
settingsMigrated: result.settingsMigrated, result.success = false;
errors: result.errors.length, }
warnings: result.warnings.length,
});
return result; return result;
} catch (error) { } catch (error) {
const errorMsg = `Settings migration failed: ${error}`; const errorMessage = `Settings migration failed: ${error}`;
logger.error("[MigrationService]", errorMsg); result.errors.push(errorMessage);
result.errors.push(errorMsg);
result.success = false; result.success = false;
logger.error("[MigrationService] Complete settings migration failed:", error);
return result; return result;
} }
} }
@ -1233,86 +1358,112 @@ export async function migrateAccounts(
const dexieAccounts = await getDexieAccounts(); const dexieAccounts = await getDexieAccounts();
const platformService = PlatformServiceFactory.getInstance(); const platformService = PlatformServiceFactory.getInstance();
for (const account of dexieAccounts) { // Group accounts by DID and keep only the most recent one
const accountsByDid = new Map<string, Account>();
dexieAccounts.forEach(account => {
const existingAccount = accountsByDid.get(account.did);
if (!existingAccount || new Date(account.dateCreated) > new Date(existingAccount.dateCreated)) {
accountsByDid.set(account.did, account);
if (existingAccount) {
result.warnings.push(`Found duplicate account for DID ${account.did}, keeping most recent`);
}
}
});
// Process each unique account
for (const [did, account] of accountsByDid.entries()) {
try { try {
// Check if account already exists // Check if account already exists
const existingResult = await platformService.dbQuery( const existingResult = await platformService.dbQuery(
"SELECT id FROM accounts WHERE id = ?", "SELECT did FROM accounts WHERE did = ?",
[account.id], [did]
); );
if (existingResult?.values?.length && !overwriteExisting) {
result.warnings.push(`Account with DID ${did} already exists, skipping`);
continue;
}
// Map Dexie fields to SQLite fields
const accountData = {
did: account.did,
dateCreated: account.dateCreated,
derivationPath: account.derivationPath || "",
identityEncrBase64: account.identity || "",
mnemonicEncrBase64: account.mnemonic || "",
passkeyCredIdHex: account.passkeyCredIdHex || "",
publicKeyHex: account.publicKeyHex || ""
};
// Insert or update the account
if (existingResult?.values?.length) { if (existingResult?.values?.length) {
if (overwriteExisting) { await platformService.dbQuery(
// Update existing account `UPDATE accounts SET
const { sql, params } = generateUpdateStatement( dateCreated = ?,
account as unknown as Record<string, unknown>, derivationPath = ?,
"accounts", identityEncrBase64 = ?,
"id = ?", mnemonicEncrBase64 = ?,
[account.id], passkeyCredIdHex = ?,
); publicKeyHex = ?
await platformService.dbExec(sql, params); WHERE did = ?`,
result.accountsMigrated++; [
logger.info(`[MigrationService] Updated account: ${account.id}`); accountData.dateCreated,
} else { accountData.derivationPath,
result.warnings.push( accountData.identityEncrBase64,
`Account ${account.id} already exists, skipping`, accountData.mnemonicEncrBase64,
); accountData.passkeyCredIdHex,
} accountData.publicKeyHex,
did
]
);
} else { } else {
// For new accounts with mnemonic, use importFromMnemonic for proper key derivation await platformService.dbQuery(
if (account.mnemonic && account.derivationPath) { `INSERT INTO accounts (
try { did,
// Use importFromMnemonic to ensure proper key derivation and identity creation dateCreated,
await importFromMnemonic( derivationPath,
account.mnemonic, identityEncrBase64,
account.derivationPath, mnemonicEncrBase64,
false, // Don't erase existing accounts during migration passkeyCredIdHex,
); publicKeyHex
logger.info( ) VALUES (?, ?, ?, ?, ?, ?, ?)`,
`[MigrationService] Imported account with mnemonic: ${account.id}`, [
); did,
} catch (importError) { accountData.dateCreated,
// Fall back to direct insertion if importFromMnemonic fails accountData.derivationPath,
logger.warn( accountData.identityEncrBase64,
`[MigrationService] importFromMnemonic failed for account ${account.id}, falling back to direct insertion: ${importError}`, accountData.mnemonicEncrBase64,
); accountData.passkeyCredIdHex,
const { sql, params } = generateInsertStatement( accountData.publicKeyHex
account as unknown as Record<string, unknown>, ]
"accounts", );
);
await platformService.dbExec(sql, params);
}
} else {
// Insert new account without mnemonic
const { sql, params } = generateInsertStatement(
account as unknown as Record<string, unknown>,
"accounts",
);
await platformService.dbExec(sql, params);
}
result.accountsMigrated++;
logger.info(`[MigrationService] Added account: ${account.id}`);
} }
result.accountsMigrated++;
logger.info("[MigrationService] Successfully migrated account", {
did,
dateCreated: account.dateCreated
});
} catch (error) { } catch (error) {
const errorMsg = `Failed to migrate account ${account.id}: ${error}`; const errorMessage = `Failed to migrate account ${did}: ${error}`;
logger.error("[MigrationService]", errorMsg); result.errors.push(errorMessage);
result.errors.push(errorMsg); logger.error("[MigrationService] Account migration failed:", {
result.success = false; error,
did
});
} }
} }
logger.info("[MigrationService] Account migration completed", { if (result.errors.length > 0) {
accountsMigrated: result.accountsMigrated, result.success = false;
errors: result.errors.length, }
warnings: result.warnings.length,
});
return result; return result;
} catch (error) { } catch (error) {
const errorMsg = `Account migration failed: ${error}`; const errorMessage = `Account migration failed: ${error}`;
logger.error("[MigrationService]", errorMsg); result.errors.push(errorMessage);
result.errors.push(errorMsg);
result.success = false; result.success = false;
logger.error("[MigrationService] Complete account migration failed:", error);
return result; return result;
} }
} }
@ -1648,3 +1799,60 @@ export async function migrateAll(
return result; return result;
} }
} }
/**
* Test function to verify migration of specific settings fields
*
* This function tests the migration of the specific fields you mentioned:
* firstName, isRegistered, profileImageUrl, showShortcutBvc, and searchBoxes
*
* @returns Promise<void>
*/
export async function testSettingsMigration(): Promise<void> {
logger.info("[MigrationService] Starting settings migration test");
try {
// First, compare databases to see current state
const comparison = await compareDatabases();
logger.info("[MigrationService] Pre-migration comparison:", {
dexieSettings: comparison.dexieSettings.length,
sqliteSettings: comparison.sqliteSettings.length,
dexieAccounts: comparison.dexieAccounts.length,
sqliteAccounts: comparison.sqliteAccounts.length
});
// Run settings migration
const settingsResult = await migrateSettings(true);
logger.info("[MigrationService] Settings migration result:", settingsResult);
// Run accounts migration
const accountsResult = await migrateAccounts(true);
logger.info("[MigrationService] Accounts migration result:", accountsResult);
// Compare databases again to see changes
const postComparison = await compareDatabases();
logger.info("[MigrationService] Post-migration comparison:", {
dexieSettings: postComparison.dexieSettings.length,
sqliteSettings: postComparison.sqliteSettings.length,
dexieAccounts: postComparison.dexieAccounts.length,
sqliteAccounts: postComparison.sqliteAccounts.length
});
// Check if the specific fields were migrated
if (postComparison.sqliteSettings.length > 0) {
const sqliteSettings = postComparison.sqliteSettings[0];
logger.info("[MigrationService] Migrated settings fields:", {
firstName: sqliteSettings.firstName,
isRegistered: sqliteSettings.isRegistered,
profileImageUrl: sqliteSettings.profileImageUrl,
showShortcutBvc: sqliteSettings.showShortcutBvc,
searchBoxes: sqliteSettings.searchBoxes
});
}
logger.info("[MigrationService] Migration test completed successfully");
} catch (error) {
logger.error("[MigrationService] Migration test failed:", error);
throw error;
}
}

38
src/views/DatabaseMigration.vue

@ -112,6 +112,15 @@
Migrate Accounts Migrate Accounts
</button> </button>
<button
:disabled="isLoading || !isDexieEnabled"
class="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-pink-600 hover:bg-pink-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-pink-500 disabled:opacity-50 disabled:cursor-not-allowed"
@click="testSpecificSettingsMigration"
>
<IconRenderer icon-name="test" svg-class="-ml-1 mr-3 h-5 w-5" />
Test Settings Migration
</button>
<button <button
:disabled="!comparison" :disabled="!comparison"
class="inline-flex items-center px-4 py-2 border border-gray-300 text-sm font-medium rounded-md text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed" class="inline-flex items-center px-4 py-2 border border-gray-300 text-sm font-medium rounded-md text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
@ -798,6 +807,7 @@ import {
migrateAccounts, migrateAccounts,
migrateAll, migrateAll,
generateComparisonYaml, generateComparisonYaml,
testSettingsMigration,
type DataComparison, type DataComparison,
type MigrationResult, type MigrationResult,
} from "../services/migrationService"; } from "../services/migrationService";
@ -1214,5 +1224,33 @@ export default class DatabaseMigration extends Vue {
this.error = ""; this.error = "";
this.successMessage = ""; this.successMessage = "";
} }
/**
* Tests the specific settings migration for the fields you mentioned
*
* This method tests the migration of firstName, isRegistered, profileImageUrl,
* showShortcutBvc, and searchBoxes from Dexie to SQLite.
*
* @async
* @returns {Promise<void>}
*/
async testSpecificSettingsMigration(): Promise<void> {
this.setLoading("Testing specific settings migration...");
this.clearMessages();
try {
await testSettingsMigration();
this.successMessage = "✅ Settings migration test completed successfully! Check the console for detailed logs.";
logger.info("[DatabaseMigration] Settings migration test completed successfully");
// Refresh comparison data after successful test
this.comparison = await compareDatabases();
} catch (error) {
this.error = `Settings migration test failed: ${error}`;
logger.error("[DatabaseMigration] Settings migration test failed:", error);
} finally {
this.setLoading("");
}
}
} }
</script> </script>

Loading…
Cancel
Save