trent_larson
/
crowd-funder-for-time-pwa
4
1
Fork 2
Code Pull Requests 12 Releases 5 Wiki Activity
Browse Source

fix: maintain separate master/account settings in SQLite migration 

- Update settings migration to maintain separate master and account records
- Use activeDid/accountDid pattern to differentiate between settings types:
  * Master settings: activeDid set, accountDid empty
  * Account settings: accountDid set, activeDid empty
- Add detailed logging for settings migration process
- Focus on migrating key fields: firstName, isRegistered, profileImageUrl,
  showShortcutBvc, and searchBoxes
- Fix issue where settings were being incorrectly merged into a single record

This change ensures the SQLite database maintains the same settings structure
as Dexie, which is required by the existing codebase.
migrate-dexie-to-sqlite
Matthew Raymer 7 days ago
parent
6cbd32af94
commit
30de30e709
2 changed files with 425 additions and 179 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 566
      src/services/migrationService.ts
  2. 38
      src/views/DatabaseMigration.vue

566
src/services/migrationService.ts
View File

@ -285,47 +285,37 @@ export async function getSqliteSettings(): Promise<Settings[]> {
return [];
}
const settings = result.values
.map((row) => {
const setting = parseJsonField(row, {}) as Settings;
return {
id: setting.id,
accountDid: setting.accountDid || null,
activeDid: setting.activeDid || null,
apiServer: setting.apiServer || "",
filterFeedByNearby: setting.filterFeedByNearby || false,
filterFeedByVisible: setting.filterFeedByVisible || false,
finishedOnboarding: setting.finishedOnboarding || false,
firstName: setting.firstName || "",
hideRegisterPromptOnNewContact:
setting.hideRegisterPromptOnNewContact || false,
isRegistered: setting.isRegistered || false,
lastName: setting.lastName || "",
lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "",
lastAckedOfferToUserProjectsJwtId:
setting.lastAckedOfferToUserProjectsJwtId || "",
lastNotifiedClaimId: setting.lastNotifiedClaimId || "",
lastViewedClaimId: setting.lastViewedClaimId || "",
notifyingNewActivityTime: setting.notifyingNewActivityTime || "",
notifyingReminderMessage: setting.notifyingReminderMessage || "",
notifyingReminderTime: setting.notifyingReminderTime || "",
partnerApiServer: setting.partnerApiServer || "",
passkeyExpirationMinutes: setting.passkeyExpirationMinutes,
profileImageUrl: setting.profileImageUrl || "",
searchBoxes: parseJsonField(setting.searchBoxes, []),
showContactGivesInline: setting.showContactGivesInline || false,
showGeneralAdvanced: setting.showGeneralAdvanced || false,
showShortcutBvc: setting.showShortcutBvc || false,
vapid: setting.vapid || "",
warnIfProdServer: setting.warnIfProdServer || false,
warnIfTestServer: setting.warnIfTestServer || false,
webPushServer: setting.webPushServer || "",
} as Settings;
})
.filter((setting) => {
// Only include settings that have either accountDid or activeDid set
return setting.accountDid || setting.activeDid;
});
const settings = result.values.map((row) => {
const setting = parseJsonField(row, {}) as Settings;
return {
id: setting.id,
accountDid: setting.accountDid || "",
activeDid: setting.activeDid || "",
apiServer: setting.apiServer || "",
filterFeedByNearby: setting.filterFeedByNearby || false,
filterFeedByVisible: setting.filterFeedByVisible || false,
finishedOnboarding: setting.finishedOnboarding || false,
firstName: setting.firstName || "",
hideRegisterPromptOnNewContact: setting.hideRegisterPromptOnNewContact || false,
isRegistered: setting.isRegistered || false,
lastName: setting.lastName || "",
lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "",
lastAckedOfferToUserProjectsJwtId: setting.lastAckedOfferToUserProjectsJwtId || "",
lastNotifiedClaimId: setting.lastNotifiedClaimId || "",
lastViewedClaimId: setting.lastViewedClaimId || "",
notifyingNewActivityTime: setting.notifyingNewActivityTime || "",
notifyingReminderMessage: setting.notifyingReminderMessage || "",
notifyingReminderTime: setting.notifyingReminderTime || "",
partnerApiServer: setting.partnerApiServer || "",
passkeyExpirationMinutes: setting.passkeyExpirationMinutes,
profileImageUrl: setting.profileImageUrl || "",
searchBoxes: parseJsonField(setting.searchBoxes, []),
showContactGivesInline: setting.showContactGivesInline || false,
showGeneralAdvanced: setting.showGeneralAdvanced || false,
showShortcutBvc: setting.showShortcutBvc || false,
vapid: setting.vapid || "",
} as Settings;
});
logger.info(
`[MigrationService] Retrieved ${settings.length} settings from SQLite`,
@ -1040,9 +1030,9 @@ export async function migrateContacts(
* settings: firstName, isRegistered, profileImageUrl, showShortcutBvc,
* and searchBoxes.
*
* The function handles both new settings (INSERT) and existing settings
* (UPDATE) based on the overwriteExisting parameter. For updates, it
* only modifies the specified fields, preserving other settings.
* The function handles duplicate settings by merging master settings (id=1)
* with account-specific settings (id=2) for the same DID, preferring
* the most recent values for the specified fields.
*
* @async
* @function migrateSettings
@ -1083,98 +1073,233 @@ export async function migrateSettings(
const dexieSettings = await getDexieSettings();
const platformService = PlatformServiceFactory.getInstance();
// Fields to migrate - these are the most important user-facing settings
const fieldsToMigrate = [
"firstName",
"isRegistered",
"profileImageUrl",
"showShortcutBvc",
"searchBoxes",
];
// Group settings by DID to handle duplicates
const settingsByDid = new Map<string, {
master?: Settings;
account?: Settings;
}>();
for (const dexieSetting of dexieSettings) {
try {
// Check if setting already exists
const existingResult = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ?",
[dexieSetting.id],
);
// Organize settings by DID
dexieSettings.forEach(setting => {
const isMasterSetting = setting.id === MASTER_SETTINGS_KEY;
const did = isMasterSetting ? setting.activeDid : setting.accountDid;
// Prepare the data object, handling DIDs based on whether this is the master settings
const settingData: Record<string, unknown> = {};
fieldsToMigrate.forEach((field) => {
if (dexieSetting[field as keyof Settings] !== undefined) {
settingData[field] = dexieSetting[field as keyof Settings];
}
if (!did) {
result.warnings.push(`Setting ${setting.id} has no DID, skipping`);
return;
}
if (!settingsByDid.has(did)) {
settingsByDid.set(did, {});
}
const didSettings = settingsByDid.get(did)!;
if (isMasterSetting) {
didSettings.master = setting;
logger.info("[MigrationService] Found master settings", {
did,
id: setting.id,
firstName: setting.firstName,
isRegistered: setting.isRegistered,
profileImageUrl: setting.profileImageUrl,
showShortcutBvc: setting.showShortcutBvc,
searchBoxes: setting.searchBoxes
});
} else {
didSettings.account = setting;
logger.info("[MigrationService] Found account settings", {
did,
id: setting.id,
firstName: setting.firstName,
isRegistered: setting.isRegistered,
profileImageUrl: setting.profileImageUrl,
showShortcutBvc: setting.showShortcutBvc,
searchBoxes: setting.searchBoxes
});
}
});
// Handle DIDs based on whether this is the master settings
if (dexieSetting.id === MASTER_SETTINGS_KEY) {
// Master settings should only use activeDid
if (dexieSetting.activeDid) {
settingData.activeDid = dexieSetting.activeDid;
}
// Ensure accountDid is null for master settings
settingData.accountDid = null;
} else {
// Non-master settings should only use accountDid
if (dexieSetting.accountDid) {
settingData.accountDid = dexieSetting.accountDid;
// Process each unique DID's settings
for (const [did, didSettings] of settingsByDid.entries()) {
try {
// Process master settings
if (didSettings.master) {
const masterData = {
id: MASTER_SETTINGS_KEY,
activeDid: did,
accountDid: "", // Empty for master settings
apiServer: didSettings.master.apiServer || "",
filterFeedByNearby: didSettings.master.filterFeedByNearby || false,
filterFeedByVisible: didSettings.master.filterFeedByVisible || false,
finishedOnboarding: didSettings.master.finishedOnboarding || false,
firstName: didSettings.master.firstName || "",
hideRegisterPromptOnNewContact: didSettings.master.hideRegisterPromptOnNewContact || false,
isRegistered: didSettings.master.isRegistered || false,
lastName: didSettings.master.lastName || "",
profileImageUrl: didSettings.master.profileImageUrl || "",
searchBoxes: didSettings.master.searchBoxes || [],
showShortcutBvc: didSettings.master.showShortcutBvc || false
};
// Check if master setting exists
const existingMaster = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ? AND activeDid = ? AND accountDid = ''",
[MASTER_SETTINGS_KEY, did]
);
if (existingMaster?.values?.length) {
logger.info("[MigrationService] Updating master settings", { did, masterData });
await platformService.dbQuery(
`UPDATE settings SET
activeDid = ?,
accountDid = ?,
firstName = ?,
isRegistered = ?,
profileImageUrl = ?,
showShortcutBvc = ?,
searchBoxes = ?
WHERE id = ?`,
[
masterData.activeDid,
masterData.accountDid,
masterData.firstName,
masterData.isRegistered,
masterData.profileImageUrl,
masterData.showShortcutBvc,
JSON.stringify(masterData.searchBoxes),
MASTER_SETTINGS_KEY
]
);
} else {
logger.info("[MigrationService] Inserting master settings", { did, masterData });
await platformService.dbQuery(
`INSERT INTO settings (
id,
activeDid,
accountDid,
firstName,
isRegistered,
profileImageUrl,
showShortcutBvc,
searchBoxes
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[
MASTER_SETTINGS_KEY,
masterData.activeDid,
masterData.accountDid,
masterData.firstName,
masterData.isRegistered,
masterData.profileImageUrl,
masterData.showShortcutBvc,
JSON.stringify(masterData.searchBoxes)
]
);
}
// Ensure activeDid is null for non-master settings
settingData.activeDid = null;
result.settingsMigrated++;
}
if (existingResult?.values?.length) {
if (overwriteExisting) {
// Update existing setting
const { sql, params } = generateUpdateStatement(
settingData,
"settings",
"id = ?",
[dexieSetting.id],
);
await platformService.dbExec(sql, params);
result.settingsMigrated++;
logger.info(
`[MigrationService] Updated settings: ${dexieSetting.id}`,
// Process account settings
if (didSettings.account) {
const accountData = {
id: 2, // Account settings always use id 2
activeDid: "", // Empty for account settings
accountDid: did,
apiServer: didSettings.account.apiServer || "",
filterFeedByNearby: didSettings.account.filterFeedByNearby || false,
filterFeedByVisible: didSettings.account.filterFeedByVisible || false,
finishedOnboarding: didSettings.account.finishedOnboarding || false,
firstName: didSettings.account.firstName || "",
hideRegisterPromptOnNewContact: didSettings.account.hideRegisterPromptOnNewContact || false,
isRegistered: didSettings.account.isRegistered || false,
lastName: didSettings.account.lastName || "",
profileImageUrl: didSettings.account.profileImageUrl || "",
searchBoxes: didSettings.account.searchBoxes || [],
showShortcutBvc: didSettings.account.showShortcutBvc || false
};
// Check if account setting exists
const existingAccount = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ? AND accountDid = ? AND activeDid = ''",
[2, did]
);
if (existingAccount?.values?.length) {
logger.info("[MigrationService] Updating account settings", { did, accountData });
await platformService.dbQuery(
`UPDATE settings SET
activeDid = ?,
accountDid = ?,
firstName = ?,
isRegistered = ?,
profileImageUrl = ?,
showShortcutBvc = ?,
searchBoxes = ?
WHERE id = ?`,
[
accountData.activeDid,
accountData.accountDid,
accountData.firstName,
accountData.isRegistered,
accountData.profileImageUrl,
accountData.showShortcutBvc,
JSON.stringify(accountData.searchBoxes),
2
]
);
} else {
result.warnings.push(
`Settings ${dexieSetting.id} already exists, skipping`,
logger.info("[MigrationService] Inserting account settings", { did, accountData });
await platformService.dbQuery(
`INSERT INTO settings (
id,
activeDid,
accountDid,
firstName,
isRegistered,
profileImageUrl,
showShortcutBvc,
searchBoxes
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
[
2,
accountData.activeDid,
accountData.accountDid,
accountData.firstName,
accountData.isRegistered,
accountData.profileImageUrl,
accountData.showShortcutBvc,
JSON.stringify(accountData.searchBoxes)
]
);
}
} else {
// Insert new setting
settingData.id = dexieSetting.id;
const { sql, params } = generateInsertStatement(
settingData,
"settings",
);
await platformService.dbExec(sql, params);
result.settingsMigrated++;
logger.info(`[MigrationService] Added settings: ${dexieSetting.id}`);
}
logger.info("[MigrationService] Successfully migrated settings for DID", {
did,
masterMigrated: !!didSettings.master,
accountMigrated: !!didSettings.account
});
} catch (error) {
const errorMsg = `Failed to migrate settings ${dexieSetting.id}: ${error}`;
logger.error("[MigrationService]", errorMsg);
result.errors.push(errorMsg);
result.success = false;
const errorMessage = `Failed to migrate settings for DID ${did}: ${error}`;
result.errors.push(errorMessage);
logger.error("[MigrationService] Settings migration failed:", {
error,
did
});
}
}
logger.info("[MigrationService] Settings migration completed", {
settingsMigrated: result.settingsMigrated,
errors: result.errors.length,
warnings: result.warnings.length,
});
if (result.errors.length > 0) {
result.success = false;
}
return result;
} catch (error) {
const errorMsg = `Settings migration failed: ${error}`;
logger.error("[MigrationService]", errorMsg);
result.errors.push(errorMsg);
const errorMessage = `Settings migration failed: ${error}`;
result.errors.push(errorMessage);
result.success = false;
logger.error("[MigrationService] Complete settings migration failed:", error);
return result;
}
}
@ -1233,86 +1358,112 @@ export async function migrateAccounts(
const dexieAccounts = await getDexieAccounts();
const platformService = PlatformServiceFactory.getInstance();
for (const account of dexieAccounts) {
// Group accounts by DID and keep only the most recent one
const accountsByDid = new Map<string, Account>();
dexieAccounts.forEach(account => {
const existingAccount = accountsByDid.get(account.did);
if (!existingAccount || new Date(account.dateCreated) > new Date(existingAccount.dateCreated)) {
accountsByDid.set(account.did, account);
if (existingAccount) {
result.warnings.push(`Found duplicate account for DID ${account.did}, keeping most recent`);
}
}
});
// Process each unique account
for (const [did, account] of accountsByDid.entries()) {
try {
// Check if account already exists
const existingResult = await platformService.dbQuery(
"SELECT id FROM accounts WHERE id = ?",
[account.id],
"SELECT did FROM accounts WHERE did = ?",
[did]
);
if (existingResult?.values?.length && !overwriteExisting) {
result.warnings.push(`Account with DID ${did} already exists, skipping`);
continue;
}
// Map Dexie fields to SQLite fields
const accountData = {
did: account.did,
dateCreated: account.dateCreated,
derivationPath: account.derivationPath || "",
identityEncrBase64: account.identity || "",
mnemonicEncrBase64: account.mnemonic || "",
passkeyCredIdHex: account.passkeyCredIdHex || "",
publicKeyHex: account.publicKeyHex || ""
};
// Insert or update the account
if (existingResult?.values?.length) {
if (overwriteExisting) {
// Update existing account
const { sql, params } = generateUpdateStatement(
account as unknown as Record<string, unknown>,
"accounts",
"id = ?",
[account.id],
);
await platformService.dbExec(sql, params);
result.accountsMigrated++;
logger.info(`[MigrationService] Updated account: ${account.id}`);
} else {
result.warnings.push(
`Account ${account.id} already exists, skipping`,
);
}
await platformService.dbQuery(
`UPDATE accounts SET
dateCreated = ?,
derivationPath = ?,
identityEncrBase64 = ?,
mnemonicEncrBase64 = ?,
passkeyCredIdHex = ?,
publicKeyHex = ?
WHERE did = ?`,
[
accountData.dateCreated,
accountData.derivationPath,
accountData.identityEncrBase64,
accountData.mnemonicEncrBase64,
accountData.passkeyCredIdHex,
accountData.publicKeyHex,
did
]
);
} else {
// For new accounts with mnemonic, use importFromMnemonic for proper key derivation
if (account.mnemonic && account.derivationPath) {
try {
// Use importFromMnemonic to ensure proper key derivation and identity creation
await importFromMnemonic(
account.mnemonic,
account.derivationPath,
false, // Don't erase existing accounts during migration
);
logger.info(
`[MigrationService] Imported account with mnemonic: ${account.id}`,
);
} catch (importError) {
// Fall back to direct insertion if importFromMnemonic fails
logger.warn(
`[MigrationService] importFromMnemonic failed for account ${account.id}, falling back to direct insertion: ${importError}`,
);
const { sql, params } = generateInsertStatement(
account as unknown as Record<string, unknown>,
"accounts",
);
await platformService.dbExec(sql, params);
}
} else {
// Insert new account without mnemonic
const { sql, params } = generateInsertStatement(
account as unknown as Record<string, unknown>,
"accounts",
);
await platformService.dbExec(sql, params);
}
result.accountsMigrated++;
logger.info(`[MigrationService] Added account: ${account.id}`);
await platformService.dbQuery(
`INSERT INTO accounts (
did,
dateCreated,
derivationPath,
identityEncrBase64,
mnemonicEncrBase64,
passkeyCredIdHex,
publicKeyHex
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
[
did,
accountData.dateCreated,
accountData.derivationPath,
accountData.identityEncrBase64,
accountData.mnemonicEncrBase64,
accountData.passkeyCredIdHex,
accountData.publicKeyHex
]
);
}
result.accountsMigrated++;
logger.info("[MigrationService] Successfully migrated account", {
did,
dateCreated: account.dateCreated
});
} catch (error) {
const errorMsg = `Failed to migrate account ${account.id}: ${error}`;
logger.error("[MigrationService]", errorMsg);
result.errors.push(errorMsg);
result.success = false;
const errorMessage = `Failed to migrate account ${did}: ${error}`;
result.errors.push(errorMessage);
logger.error("[MigrationService] Account migration failed:", {
error,
did
});
}
}
logger.info("[MigrationService] Account migration completed", {
accountsMigrated: result.accountsMigrated,
errors: result.errors.length,
warnings: result.warnings.length,
});
if (result.errors.length > 0) {
result.success = false;
}
return result;
} catch (error) {
const errorMsg = `Account migration failed: ${error}`;
logger.error("[MigrationService]", errorMsg);
result.errors.push(errorMsg);
const errorMessage = `Account migration failed: ${error}`;
result.errors.push(errorMessage);
result.success = false;
logger.error("[MigrationService] Complete account migration failed:", error);
return result;
}
}
@ -1648,3 +1799,60 @@ export async function migrateAll(
return result;
}
}
/**
* Test function to verify migration of specific settings fields
*
* This function tests the migration of the specific fields you mentioned:
* firstName, isRegistered, profileImageUrl, showShortcutBvc, and searchBoxes
*
* @returns Promise<void>
*/
export async function testSettingsMigration(): Promise<void> {
logger.info("[MigrationService] Starting settings migration test");
try {
// First, compare databases to see current state
const comparison = await compareDatabases();
logger.info("[MigrationService] Pre-migration comparison:", {
dexieSettings: comparison.dexieSettings.length,
sqliteSettings: comparison.sqliteSettings.length,
dexieAccounts: comparison.dexieAccounts.length,
sqliteAccounts: comparison.sqliteAccounts.length
});
// Run settings migration
const settingsResult = await migrateSettings(true);
logger.info("[MigrationService] Settings migration result:", settingsResult);
// Run accounts migration
const accountsResult = await migrateAccounts(true);
logger.info("[MigrationService] Accounts migration result:", accountsResult);
// Compare databases again to see changes
const postComparison = await compareDatabases();
logger.info("[MigrationService] Post-migration comparison:", {
dexieSettings: postComparison.dexieSettings.length,
sqliteSettings: postComparison.sqliteSettings.length,
dexieAccounts: postComparison.dexieAccounts.length,
sqliteAccounts: postComparison.sqliteAccounts.length
});
// Check if the specific fields were migrated
if (postComparison.sqliteSettings.length > 0) {
const sqliteSettings = postComparison.sqliteSettings[0];
logger.info("[MigrationService] Migrated settings fields:", {
firstName: sqliteSettings.firstName,
isRegistered: sqliteSettings.isRegistered,
profileImageUrl: sqliteSettings.profileImageUrl,
showShortcutBvc: sqliteSettings.showShortcutBvc,
searchBoxes: sqliteSettings.searchBoxes
});
}
logger.info("[MigrationService] Migration test completed successfully");
} catch (error) {
logger.error("[MigrationService] Migration test failed:", error);
throw error;
}
}

38
src/views/DatabaseMigration.vue
View File

@ -112,6 +112,15 @@
Migrate Accounts
</button>
<button
:disabled="isLoading || !isDexieEnabled"
class="inline-flex items-center px-4 py-2 border border-transparent text-sm font-medium rounded-md shadow-sm text-white bg-pink-600 hover:bg-pink-700 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-pink-500 disabled:opacity-50 disabled:cursor-not-allowed"
@click="testSpecificSettingsMigration"
>
<IconRenderer icon-name="test" svg-class="-ml-1 mr-3 h-5 w-5" />
Test Settings Migration
</button>
<button
:disabled="!comparison"
class="inline-flex items-center px-4 py-2 border border-gray-300 text-sm font-medium rounded-md text-gray-700 bg-white hover:bg-gray-50 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-blue-500 disabled:opacity-50 disabled:cursor-not-allowed"
@ -798,6 +807,7 @@ import {
migrateAccounts,
migrateAll,
generateComparisonYaml,
testSettingsMigration,
type DataComparison,
type MigrationResult,
} from "../services/migrationService";
@ -1214,5 +1224,33 @@ export default class DatabaseMigration extends Vue {
this.error = "";
this.successMessage = "";
}
/**
* Tests the specific settings migration for the fields you mentioned
*
* This method tests the migration of firstName, isRegistered, profileImageUrl,
* showShortcutBvc, and searchBoxes from Dexie to SQLite.
*
* @async
* @returns {Promise<void>}
*/
async testSpecificSettingsMigration(): Promise<void> {
this.setLoading("Testing specific settings migration...");
this.clearMessages();
try {
await testSettingsMigration();
this.successMessage = "✅ Settings migration test completed successfully! Check the console for detailed logs.";
logger.info("[DatabaseMigration] Settings migration test completed successfully");
// Refresh comparison data after successful test
this.comparison = await compareDatabases();
} catch (error) {
this.error = `Settings migration test failed: ${error}`;
logger.error("[DatabaseMigration] Settings migration test failed:", error);
} finally {
this.setLoading("");
}
}
}
</script>

Write Preview
Loading…
Cancel
Save