feat: Add comprehensive Database Migration UI component

- Create DatabaseMigration.vue with vue-facing-decorator and Tailwind CSS
- Add complete UI for comparing and migrating data between Dexie and SQLite
- Implement real-time loading states, error handling, and success feedback
- Add navigation link to Account page for easy access
- Include export functionality for comparison data
- Create comprehensive documentation in doc/database-migration-guide.md
- Fix all linting issues and ensure code quality standards
- Support both contact and settings migration with overwrite options
- Add visual difference analysis with summary cards and detailed breakdowns

The component provides a professional interface for the migrationService.ts,
enabling users to safely transfer data between database systems during
the transition from Dexie to SQLite storage.
This commit is contained in:
Matthew Raymer
2025-06-18 11:19:34 +00:00
parent a580ce775d
commit 9787763e4b
5 changed files with 1305 additions and 95 deletions

View File

@@ -1,21 +1,21 @@
/**
* Migration Service for transferring data from Dexie (IndexedDB) to SQLite
*
*
* This service provides functions to:
* 1. Compare data between Dexie and SQLite databases
* 2. Transfer contacts and settings from Dexie to SQLite
* 3. Generate YAML-formatted data comparisons
*
*
* The service is designed to work with the TimeSafari app's dual database architecture,
* where data can exist in both Dexie (IndexedDB) and SQLite databases. This allows
* for safe migration of data between the two storage systems.
*
*
* Usage:
* 1. Enable Dexie temporarily by setting USE_DEXIE_DB = true in constants/app.ts
* 2. Use compareDatabases() to see differences between databases
* 3. Use migrateContacts() and/or migrateSettings() to transfer data
* 4. Disable Dexie again after migration is complete
*
*
* @author Matthew Raymer
* @version 1.0.0
* @since 2024
@@ -31,11 +31,11 @@ import { USE_DEXIE_DB } from "../constants/app";
/**
* Interface for data comparison results between Dexie and SQLite databases
*
*
* This interface provides a comprehensive view of the differences between
* the two database systems, including counts and detailed lists of
* added, modified, and missing records.
*
*
* @interface DataComparison
* @property {Contact[]} dexieContacts - All contacts from Dexie database
* @property {Contact[]} sqliteContacts - All contacts from SQLite database
@@ -72,11 +72,11 @@ export interface DataComparison {
/**
* Interface for migration operation results
*
*
* Provides detailed feedback about the success or failure of migration
* operations, including counts of migrated records and any errors or
* warnings that occurred during the process.
*
*
* @interface MigrationResult
* @property {boolean} success - Whether the migration operation completed successfully
* @property {number} contactsMigrated - Number of contacts successfully migrated
@@ -94,13 +94,13 @@ export interface MigrationResult {
/**
* Retrieves all contacts from the Dexie (IndexedDB) database
*
*
* This function connects to the Dexie database and retrieves all contact
* records. It requires that USE_DEXIE_DB is enabled in the app constants.
*
*
* The function handles database opening and error conditions, providing
* detailed logging for debugging purposes.
*
*
* @async
* @function getDexieContacts
* @returns {Promise<Contact[]>} Array of all contacts from Dexie database
@@ -123,7 +123,9 @@ export async function getDexieContacts(): Promise<Contact[]> {
try {
await db.open();
const contacts = await db.contacts.toArray();
logger.info(`[MigrationService] Retrieved ${contacts.length} contacts from Dexie`);
logger.info(
`[MigrationService] Retrieved ${contacts.length} contacts from Dexie`,
);
return contacts;
} catch (error) {
logger.error("[MigrationService] Error retrieving Dexie contacts:", error);
@@ -133,14 +135,14 @@ export async function getDexieContacts(): Promise<Contact[]> {
/**
* Retrieves all contacts from the SQLite database
*
*
* This function uses the platform service to query the SQLite database
* and retrieve all contact records. It handles the conversion of raw
* database results into properly typed Contact objects.
*
*
* The function also handles JSON parsing for complex fields like
* contactMethods, ensuring proper type conversion.
*
*
* @async
* @function getSqliteContacts
* @returns {Promise<Contact[]>} Array of all contacts from SQLite database
@@ -159,7 +161,7 @@ export async function getSqliteContacts(): Promise<Contact[]> {
try {
const platformService = PlatformServiceFactory.getInstance();
const result = await platformService.dbQuery("SELECT * FROM contacts");
if (!result?.values?.length) {
return [];
}
@@ -169,7 +171,10 @@ export async function getSqliteContacts(): Promise<Contact[]> {
return {
did: contact.did || "",
name: contact.name || "",
contactMethods: parseJsonField(contact.contactMethods, []) as ContactMethod[],
contactMethods: parseJsonField(
contact.contactMethods,
[],
) as ContactMethod[],
nextPubKeyHashB64: contact.nextPubKeyHashB64 || "",
notes: contact.notes || "",
profileImageUrl: contact.profileImageUrl || "",
@@ -179,7 +184,9 @@ export async function getSqliteContacts(): Promise<Contact[]> {
} as Contact;
});
logger.info(`[MigrationService] Retrieved ${contacts.length} contacts from SQLite`);
logger.info(
`[MigrationService] Retrieved ${contacts.length} contacts from SQLite`,
);
return contacts;
} catch (error) {
logger.error("[MigrationService] Error retrieving SQLite contacts:", error);
@@ -189,13 +196,13 @@ export async function getSqliteContacts(): Promise<Contact[]> {
/**
* Retrieves all settings from the Dexie (IndexedDB) database
*
*
* This function connects to the Dexie database and retrieves all settings
* records. It requires that USE_DEXIE_DB is enabled in the app constants.
*
*
* Settings include both master settings (id=1) and account-specific settings
* that override the master settings for particular user accounts.
*
*
* @async
* @function getDexieSettings
* @returns {Promise<Settings[]>} Array of all settings from Dexie database
@@ -218,7 +225,9 @@ export async function getDexieSettings(): Promise<Settings[]> {
try {
await db.open();
const settings = await db.settings.toArray();
logger.info(`[MigrationService] Retrieved ${settings.length} settings from Dexie`);
logger.info(
`[MigrationService] Retrieved ${settings.length} settings from Dexie`,
);
return settings;
} catch (error) {
logger.error("[MigrationService] Error retrieving Dexie settings:", error);
@@ -228,14 +237,14 @@ export async function getDexieSettings(): Promise<Settings[]> {
/**
* Retrieves all settings from the SQLite database
*
*
* This function uses the platform service to query the SQLite database
* and retrieve all settings records. It handles the conversion of raw
* database results into properly typed Settings objects.
*
*
* The function also handles JSON parsing for complex fields like
* searchBoxes, ensuring proper type conversion.
*
*
* @async
* @function getSqliteSettings
* @returns {Promise<Settings[]>} Array of all settings from SQLite database
@@ -254,7 +263,7 @@ export async function getSqliteSettings(): Promise<Settings[]> {
try {
const platformService = PlatformServiceFactory.getInstance();
const result = await platformService.dbQuery("SELECT * FROM settings");
if (!result?.values?.length) {
return [];
}
@@ -270,11 +279,13 @@ export async function getSqliteSettings(): Promise<Settings[]> {
filterFeedByVisible: setting.filterFeedByVisible || false,
finishedOnboarding: setting.finishedOnboarding || false,
firstName: setting.firstName || "",
hideRegisterPromptOnNewContact: setting.hideRegisterPromptOnNewContact || false,
hideRegisterPromptOnNewContact:
setting.hideRegisterPromptOnNewContact || false,
isRegistered: setting.isRegistered || false,
lastName: setting.lastName || "",
lastAckedOfferToUserJwtId: setting.lastAckedOfferToUserJwtId || "",
lastAckedOfferToUserProjectsJwtId: setting.lastAckedOfferToUserProjectsJwtId || "",
lastAckedOfferToUserProjectsJwtId:
setting.lastAckedOfferToUserProjectsJwtId || "",
lastNotifiedClaimId: setting.lastNotifiedClaimId || "",
lastViewedClaimId: setting.lastViewedClaimId || "",
notifyingNewActivityTime: setting.notifyingNewActivityTime || "",
@@ -294,7 +305,9 @@ export async function getSqliteSettings(): Promise<Settings[]> {
} as Settings;
});
logger.info(`[MigrationService] Retrieved ${settings.length} settings from SQLite`);
logger.info(
`[MigrationService] Retrieved ${settings.length} settings from SQLite`,
);
return settings;
} catch (error) {
logger.error("[MigrationService] Error retrieving SQLite settings:", error);
@@ -304,16 +317,16 @@ export async function getSqliteSettings(): Promise<Settings[]> {
/**
* Compares data between Dexie and SQLite databases
*
*
* This is the main comparison function that retrieves data from both
* databases and identifies differences. It provides a comprehensive
* view of what data exists in each database and what needs to be
* migrated.
*
*
* The function performs parallel data retrieval for efficiency and
* then compares the results to identify added, modified, and missing
* records in each table.
*
*
* @async
* @function compareDatabases
* @returns {Promise<DataComparison>} Comprehensive comparison results
@@ -333,16 +346,17 @@ export async function getSqliteSettings(): Promise<Settings[]> {
export async function compareDatabases(): Promise<DataComparison> {
logger.info("[MigrationService] Starting database comparison");
const [dexieContacts, sqliteContacts, dexieSettings, sqliteSettings] = await Promise.all([
getDexieContacts(),
getSqliteContacts(),
getDexieSettings(),
getSqliteSettings(),
]);
const [dexieContacts, sqliteContacts, dexieSettings, sqliteSettings] =
await Promise.all([
getDexieContacts(),
getSqliteContacts(),
getDexieSettings(),
getSqliteSettings(),
]);
// Compare contacts
const contactDifferences = compareContacts(dexieContacts, sqliteContacts);
// Compare settings
const settingsDifferences = compareSettings(dexieSettings, sqliteSettings);
@@ -371,15 +385,15 @@ export async function compareDatabases(): Promise<DataComparison> {
/**
* Compares contacts between Dexie and SQLite databases
*
*
* This helper function analyzes two arrays of contacts and identifies
* which contacts are added (in Dexie but not SQLite), modified
* (different between databases), or missing (in SQLite but not Dexie).
*
*
* The comparison is based on the contact's DID (Decentralized Identifier)
* as the primary key, with detailed field-by-field comparison for
* modified contacts.
*
*
* @function compareContacts
* @param {Contact[]} dexieContacts - Contacts from Dexie database
* @param {Contact[]} sqliteContacts - Contacts from SQLite database
@@ -402,7 +416,9 @@ function compareContacts(dexieContacts: Contact[], sqliteContacts: Contact[]) {
// Find contacts that exist in Dexie but not in SQLite
for (const dexieContact of dexieContacts) {
const sqliteContact = sqliteContacts.find(c => c.did === dexieContact.did);
const sqliteContact = sqliteContacts.find(
(c) => c.did === dexieContact.did,
);
if (!sqliteContact) {
added.push(dexieContact);
} else if (!contactsEqual(dexieContact, sqliteContact)) {
@@ -412,7 +428,7 @@ function compareContacts(dexieContacts: Contact[], sqliteContacts: Contact[]) {
// Find contacts that exist in SQLite but not in Dexie
for (const sqliteContact of sqliteContacts) {
const dexieContact = dexieContacts.find(c => c.did === sqliteContact.did);
const dexieContact = dexieContacts.find((c) => c.did === sqliteContact.did);
if (!dexieContact) {
missing.push(sqliteContact);
}
@@ -423,14 +439,14 @@ function compareContacts(dexieContacts: Contact[], sqliteContacts: Contact[]) {
/**
* Compares settings between Dexie and SQLite databases
*
*
* This helper function analyzes two arrays of settings and identifies
* which settings are added (in Dexie but not SQLite), modified
* (different between databases), or missing (in SQLite but not Dexie).
*
*
* The comparison is based on the setting's ID as the primary key,
* with detailed field-by-field comparison for modified settings.
*
*
* @function compareSettings
* @param {Settings[]} dexieSettings - Settings from Dexie database
* @param {Settings[]} sqliteSettings - Settings from SQLite database
@@ -446,14 +462,17 @@ function compareContacts(dexieContacts: Contact[], sqliteContacts: Contact[]) {
* console.log(`Missing: ${differences.missing.length}`);
* ```
*/
function compareSettings(dexieSettings: Settings[], sqliteSettings: Settings[]) {
function compareSettings(
dexieSettings: Settings[],
sqliteSettings: Settings[],
) {
const added: Settings[] = [];
const modified: Settings[] = [];
const missing: Settings[] = [];
// Find settings that exist in Dexie but not in SQLite
for (const dexieSetting of dexieSettings) {
const sqliteSetting = sqliteSettings.find(s => s.id === dexieSetting.id);
const sqliteSetting = sqliteSettings.find((s) => s.id === dexieSetting.id);
if (!sqliteSetting) {
added.push(dexieSetting);
} else if (!settingsEqual(dexieSetting, sqliteSetting)) {
@@ -463,7 +482,7 @@ function compareSettings(dexieSettings: Settings[], sqliteSettings: Settings[])
// Find settings that exist in SQLite but not in Dexie
for (const sqliteSetting of sqliteSettings) {
const dexieSetting = dexieSettings.find(s => s.id === sqliteSetting.id);
const dexieSetting = dexieSettings.find((s) => s.id === sqliteSetting.id);
if (!dexieSetting) {
missing.push(sqliteSetting);
}
@@ -474,14 +493,14 @@ function compareSettings(dexieSettings: Settings[], sqliteSettings: Settings[])
/**
* Compares two contacts for equality
*
*
* This helper function performs a deep comparison of two Contact objects
* to determine if they are identical. The comparison includes all
* relevant fields including complex objects like contactMethods.
*
*
* For contactMethods, the function uses JSON.stringify to compare
* the arrays, ensuring that both structure and content are identical.
*
*
* @function contactsEqual
* @param {Contact} contact1 - First contact to compare
* @param {Contact} contact2 - Second contact to compare
@@ -506,20 +525,21 @@ function contactsEqual(contact1: Contact, contact2: Contact): boolean {
contact1.nextPubKeyHashB64 === contact2.nextPubKeyHashB64 &&
contact1.seesMe === contact2.seesMe &&
contact1.registered === contact2.registered &&
JSON.stringify(contact1.contactMethods) === JSON.stringify(contact2.contactMethods)
JSON.stringify(contact1.contactMethods) ===
JSON.stringify(contact2.contactMethods)
);
}
/**
* Compares two settings for equality
*
*
* This helper function performs a deep comparison of two Settings objects
* to determine if they are identical. The comparison includes all
* relevant fields including complex objects like searchBoxes.
*
*
* For searchBoxes, the function uses JSON.stringify to compare
* the arrays, ensuring that both structure and content are identical.
*
*
* @function settingsEqual
* @param {Settings} settings1 - First settings to compare
* @param {Settings} settings2 - Second settings to compare
@@ -544,11 +564,14 @@ function settingsEqual(settings1: Settings, settings2: Settings): boolean {
settings1.filterFeedByVisible === settings2.filterFeedByVisible &&
settings1.finishedOnboarding === settings2.finishedOnboarding &&
settings1.firstName === settings2.firstName &&
settings1.hideRegisterPromptOnNewContact === settings2.hideRegisterPromptOnNewContact &&
settings1.hideRegisterPromptOnNewContact ===
settings2.hideRegisterPromptOnNewContact &&
settings1.isRegistered === settings2.isRegistered &&
settings1.lastName === settings2.lastName &&
settings1.lastAckedOfferToUserJwtId === settings2.lastAckedOfferToUserJwtId &&
settings1.lastAckedOfferToUserProjectsJwtId === settings2.lastAckedOfferToUserProjectsJwtId &&
settings1.lastAckedOfferToUserJwtId ===
settings2.lastAckedOfferToUserJwtId &&
settings1.lastAckedOfferToUserProjectsJwtId ===
settings2.lastAckedOfferToUserProjectsJwtId &&
settings1.lastNotifiedClaimId === settings2.lastNotifiedClaimId &&
settings1.lastViewedClaimId === settings2.lastViewedClaimId &&
settings1.notifyingNewActivityTime === settings2.notifyingNewActivityTime &&
@@ -564,21 +587,22 @@ function settingsEqual(settings1: Settings, settings2: Settings): boolean {
settings1.warnIfProdServer === settings2.warnIfProdServer &&
settings1.warnIfTestServer === settings2.warnIfTestServer &&
settings1.webPushServer === settings2.webPushServer &&
JSON.stringify(settings1.searchBoxes) === JSON.stringify(settings2.searchBoxes)
JSON.stringify(settings1.searchBoxes) ===
JSON.stringify(settings2.searchBoxes)
);
}
/**
* Generates YAML-formatted comparison data
*
*
* This function converts the database comparison results into a
* structured format that can be exported and analyzed. The output
* is actually JSON but formatted in a YAML-like structure for
* better readability.
*
*
* The generated data includes summary statistics, detailed differences,
* and the actual data from both databases for inspection purposes.
*
*
* @function generateComparisonYaml
* @param {DataComparison} comparison - The comparison results to format
* @returns {string} JSON string formatted for readability
@@ -612,7 +636,7 @@ export function generateComparisonYaml(comparison: DataComparison): string {
},
},
contacts: {
dexie: comparison.dexieContacts.map(c => ({
dexie: comparison.dexieContacts.map((c) => ({
did: c.did,
name: c.name,
notes: c.notes,
@@ -621,7 +645,7 @@ export function generateComparisonYaml(comparison: DataComparison): string {
registered: c.registered,
contactMethods: c.contactMethods,
})),
sqlite: comparison.sqliteContacts.map(c => ({
sqlite: comparison.sqliteContacts.map((c) => ({
did: c.did,
name: c.name,
notes: c.notes,
@@ -632,7 +656,7 @@ export function generateComparisonYaml(comparison: DataComparison): string {
})),
},
settings: {
dexie: comparison.dexieSettings.map(s => ({
dexie: comparison.dexieSettings.map((s) => ({
id: s.id,
accountDid: s.accountDid,
activeDid: s.activeDid,
@@ -642,7 +666,7 @@ export function generateComparisonYaml(comparison: DataComparison): string {
showShortcutBvc: s.showShortcutBvc,
searchBoxes: s.searchBoxes,
})),
sqlite: comparison.sqliteSettings.map(s => ({
sqlite: comparison.sqliteSettings.map((s) => ({
id: s.id,
accountDid: s.accountDid,
activeDid: s.activeDid,
@@ -661,16 +685,16 @@ export function generateComparisonYaml(comparison: DataComparison): string {
/**
* Migrates contacts from Dexie to SQLite database
*
*
* This function transfers all contacts from the Dexie database to the
* SQLite database. It handles both new contacts (INSERT) and existing
* contacts (UPDATE) based on the overwriteExisting parameter.
*
*
* The function processes contacts one by one to ensure data integrity
* and provides detailed logging of the migration process. It returns
* comprehensive results including success status, counts, and any
* errors or warnings encountered.
*
*
* @async
* @function migrateContacts
* @param {boolean} [overwriteExisting=false] - Whether to overwrite existing contacts in SQLite
@@ -690,8 +714,12 @@ export function generateComparisonYaml(comparison: DataComparison): string {
* }
* ```
*/
export async function migrateContacts(overwriteExisting: boolean = false): Promise<MigrationResult> {
logger.info("[MigrationService] Starting contact migration", { overwriteExisting });
export async function migrateContacts(
overwriteExisting: boolean = false,
): Promise<MigrationResult> {
logger.info("[MigrationService] Starting contact migration", {
overwriteExisting,
});
const result: MigrationResult = {
success: true,
@@ -710,7 +738,7 @@ export async function migrateContacts(overwriteExisting: boolean = false): Promi
// Check if contact already exists
const existingResult = await platformService.dbQuery(
"SELECT did FROM contacts WHERE did = ?",
[contact.did]
[contact.did],
);
if (existingResult?.values?.length) {
@@ -720,19 +748,21 @@ export async function migrateContacts(overwriteExisting: boolean = false): Promi
contact as unknown as Record<string, unknown>,
"contacts",
"did = ?",
[contact.did]
[contact.did],
);
await platformService.dbExec(sql, params);
result.contactsMigrated++;
logger.info(`[MigrationService] Updated contact: ${contact.did}`);
} else {
result.warnings.push(`Contact ${contact.did} already exists, skipping`);
result.warnings.push(
`Contact ${contact.did} already exists, skipping`,
);
}
} else {
// Insert new contact
const { sql, params } = generateInsertStatement(
contact as unknown as Record<string, unknown>,
"contacts"
"contacts",
);
await platformService.dbExec(sql, params);
result.contactsMigrated++;
@@ -764,16 +794,16 @@ export async function migrateContacts(overwriteExisting: boolean = false): Promi
/**
* Migrates specific settings fields from Dexie to SQLite database
*
*
* This function transfers specific settings fields from the Dexie database
* to the SQLite database. It focuses on the most important user-facing
* settings: firstName, isRegistered, profileImageUrl, showShortcutBvc,
* and searchBoxes.
*
*
* The function handles both new settings (INSERT) and existing settings
* (UPDATE) based on the overwriteExisting parameter. For updates, it
* only modifies the specified fields, preserving other settings.
*
*
* @async
* @function migrateSettings
* @param {boolean} [overwriteExisting=false] - Whether to overwrite existing settings in SQLite
@@ -793,8 +823,12 @@ export async function migrateContacts(overwriteExisting: boolean = false): Promi
* }
* ```
*/
export async function migrateSettings(overwriteExisting: boolean = false): Promise<MigrationResult> {
logger.info("[MigrationService] Starting settings migration", { overwriteExisting });
export async function migrateSettings(
overwriteExisting: boolean = false,
): Promise<MigrationResult> {
logger.info("[MigrationService] Starting settings migration", {
overwriteExisting,
});
const result: MigrationResult = {
success: true,
@@ -809,21 +843,27 @@ export async function migrateSettings(overwriteExisting: boolean = false): Promi
const platformService = PlatformServiceFactory.getInstance();
// Fields to migrate - these are the most important user-facing settings
const fieldsToMigrate = ['firstName', 'isRegistered', 'profileImageUrl', 'showShortcutBvc', 'searchBoxes'];
const fieldsToMigrate = [
"firstName",
"isRegistered",
"profileImageUrl",
"showShortcutBvc",
"searchBoxes",
];
for (const dexieSetting of dexieSettings) {
try {
// Check if setting already exists
const existingResult = await platformService.dbQuery(
"SELECT id FROM settings WHERE id = ?",
[dexieSetting.id]
[dexieSetting.id],
);
if (existingResult?.values?.length) {
if (overwriteExisting) {
// Update existing setting with only the specified fields
const updateData: Record<string, unknown> = {};
fieldsToMigrate.forEach(field => {
fieldsToMigrate.forEach((field) => {
if (dexieSetting[field as keyof Settings] !== undefined) {
updateData[field] = dexieSetting[field as keyof Settings];
}
@@ -834,20 +874,24 @@ export async function migrateSettings(overwriteExisting: boolean = false): Promi
updateData as unknown as Record<string, unknown>,
"settings",
"id = ?",
[dexieSetting.id]
[dexieSetting.id],
);
await platformService.dbExec(sql, params);
result.settingsMigrated++;
logger.info(`[MigrationService] Updated settings: ${dexieSetting.id}`);
logger.info(
`[MigrationService] Updated settings: ${dexieSetting.id}`,
);
}
} else {
result.warnings.push(`Settings ${dexieSetting.id} already exists, skipping`);
result.warnings.push(
`Settings ${dexieSetting.id} already exists, skipping`,
);
}
} else {
// Insert new setting
const { sql, params } = generateInsertStatement(
dexieSetting as unknown as Record<string, unknown>,
"settings"
"settings",
);
await platformService.dbExec(sql, params);
result.settingsMigrated++;
@@ -879,14 +923,14 @@ export async function migrateSettings(overwriteExisting: boolean = false): Promi
/**
* Generates SQL INSERT statement and parameters from a model object
*
*
* This helper function creates a parameterized SQL INSERT statement
* from a JavaScript object. It filters out undefined values and
* creates the appropriate SQL syntax with placeholders.
*
*
* The function is used internally by the migration functions to
* safely insert data into the SQLite database.
*
*
* @function generateInsertStatement
* @param {Record<string, unknown>} model - The model object containing fields to insert
* @param {string} tableName - The name of the table to insert into
@@ -918,14 +962,14 @@ function generateInsertStatement(
/**
* Generates SQL UPDATE statement and parameters from a model object
*
*
* This helper function creates a parameterized SQL UPDATE statement
* from a JavaScript object. It filters out undefined values and
* creates the appropriate SQL syntax with placeholders.
*
*
* The function is used internally by the migration functions to
* safely update data in the SQLite database.
*
*
* @function generateUpdateStatement
* @param {Record<string, unknown>} model - The model object containing fields to update
* @param {string} tableName - The name of the table to update