forked from jsnbuchanan/crowd-funder-for-time-pwa
feat: Add comprehensive database migration service for Dexie to SQLite
- Add migrationService.ts with functions to compare and transfer data between Dexie and SQLite - Implement data comparison with detailed difference analysis (added/modified/missing) - Add contact migration with overwrite options and error handling - Add settings migration focusing on key user fields (firstName, isRegistered, profileImageUrl, showShortcutBvc, searchBoxes) - Include YAML export functionality for data inspection - Add comprehensive JSDoc documentation with examples and usage instructions - Support both INSERT and UPDATE operations with parameterized SQL generation - Include detailed logging and error reporting for migration operations This service enables safe migration of user data from the legacy Dexie (IndexedDB) database to the new SQLite implementation, with full comparison capabilities and rollback safety through detailed reporting.
This commit is contained in:
@@ -3,6 +3,7 @@
|
|||||||
## Schema Mapping
|
## Schema Mapping
|
||||||
|
|
||||||
### Current Dexie Schema
|
### Current Dexie Schema
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Current Dexie schema
|
// Current Dexie schema
|
||||||
const db = new Dexie('TimeSafariDB');
|
const db = new Dexie('TimeSafariDB');
|
||||||
@@ -15,6 +16,7 @@ db.version(1).stores({
|
|||||||
```
|
```
|
||||||
|
|
||||||
### New SQLite Schema
|
### New SQLite Schema
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
-- New SQLite schema
|
-- New SQLite schema
|
||||||
CREATE TABLE accounts (
|
CREATE TABLE accounts (
|
||||||
@@ -50,6 +52,7 @@ CREATE INDEX idx_settings_updated_at ON settings(updated_at);
|
|||||||
### 1. Account Operations
|
### 1. Account Operations
|
||||||
|
|
||||||
#### Get Account by DID
|
#### Get Account by DID
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
const account = await db.accounts.get(did);
|
const account = await db.accounts.get(did);
|
||||||
@@ -62,6 +65,7 @@ const account = result[0]?.values[0];
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### Get All Accounts
|
#### Get All Accounts
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
const accounts = await db.accounts.toArray();
|
const accounts = await db.accounts.toArray();
|
||||||
@@ -74,6 +78,7 @@ const accounts = result[0]?.values || [];
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### Add Account
|
#### Add Account
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
await db.accounts.add({
|
await db.accounts.add({
|
||||||
@@ -91,6 +96,7 @@ await db.run(`
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### Update Account
|
#### Update Account
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
await db.accounts.update(did, {
|
await db.accounts.update(did, {
|
||||||
@@ -100,7 +106,7 @@ await db.accounts.update(did, {
|
|||||||
|
|
||||||
// absurd-sql
|
// absurd-sql
|
||||||
await db.run(`
|
await db.run(`
|
||||||
UPDATE accounts
|
UPDATE accounts
|
||||||
SET public_key_hex = ?, updated_at = ?
|
SET public_key_hex = ?, updated_at = ?
|
||||||
WHERE did = ?
|
WHERE did = ?
|
||||||
`, [publicKeyHex, Date.now(), did]);
|
`, [publicKeyHex, Date.now(), did]);
|
||||||
@@ -109,6 +115,7 @@ await db.run(`
|
|||||||
### 2. Settings Operations
|
### 2. Settings Operations
|
||||||
|
|
||||||
#### Get Setting
|
#### Get Setting
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
const setting = await db.settings.get(key);
|
const setting = await db.settings.get(key);
|
||||||
@@ -121,6 +128,7 @@ const setting = result[0]?.values[0];
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### Set Setting
|
#### Set Setting
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
await db.settings.put({
|
await db.settings.put({
|
||||||
@@ -142,6 +150,7 @@ await db.run(`
|
|||||||
### 3. Contact Operations
|
### 3. Contact Operations
|
||||||
|
|
||||||
#### Get Contacts by Account
|
#### Get Contacts by Account
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
const contacts = await db.contacts
|
const contacts = await db.contacts
|
||||||
@@ -151,7 +160,7 @@ const contacts = await db.contacts
|
|||||||
|
|
||||||
// absurd-sql
|
// absurd-sql
|
||||||
const result = await db.exec(`
|
const result = await db.exec(`
|
||||||
SELECT * FROM contacts
|
SELECT * FROM contacts
|
||||||
WHERE did = ?
|
WHERE did = ?
|
||||||
ORDER BY created_at DESC
|
ORDER BY created_at DESC
|
||||||
`, [accountDid]);
|
`, [accountDid]);
|
||||||
@@ -159,6 +168,7 @@ const contacts = result[0]?.values || [];
|
|||||||
```
|
```
|
||||||
|
|
||||||
#### Add Contact
|
#### Add Contact
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
await db.contacts.add({
|
await db.contacts.add({
|
||||||
@@ -179,6 +189,7 @@ await db.run(`
|
|||||||
## Transaction Mapping
|
## Transaction Mapping
|
||||||
|
|
||||||
### Batch Operations
|
### Batch Operations
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie
|
// Dexie
|
||||||
await db.transaction('rw', [db.accounts, db.contacts], async () => {
|
await db.transaction('rw', [db.accounts, db.contacts], async () => {
|
||||||
@@ -210,10 +221,11 @@ try {
|
|||||||
## Migration Helper Functions
|
## Migration Helper Functions
|
||||||
|
|
||||||
### 1. Data Export (Dexie to JSON)
|
### 1. Data Export (Dexie to JSON)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
async function exportDexieData(): Promise<MigrationData> {
|
async function exportDexieData(): Promise<MigrationData> {
|
||||||
const db = new Dexie('TimeSafariDB');
|
const db = new Dexie('TimeSafariDB');
|
||||||
|
|
||||||
return {
|
return {
|
||||||
accounts: await db.accounts.toArray(),
|
accounts: await db.accounts.toArray(),
|
||||||
settings: await db.settings.toArray(),
|
settings: await db.settings.toArray(),
|
||||||
@@ -228,6 +240,7 @@ async function exportDexieData(): Promise<MigrationData> {
|
|||||||
```
|
```
|
||||||
|
|
||||||
### 2. Data Import (JSON to absurd-sql)
|
### 2. Data Import (JSON to absurd-sql)
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
async function importToAbsurdSql(data: MigrationData): Promise<void> {
|
async function importToAbsurdSql(data: MigrationData): Promise<void> {
|
||||||
await db.exec('BEGIN TRANSACTION;');
|
await db.exec('BEGIN TRANSACTION;');
|
||||||
@@ -239,7 +252,7 @@ async function importToAbsurdSql(data: MigrationData): Promise<void> {
|
|||||||
VALUES (?, ?, ?, ?)
|
VALUES (?, ?, ?, ?)
|
||||||
`, [account.did, account.publicKeyHex, account.createdAt, account.updatedAt]);
|
`, [account.did, account.publicKeyHex, account.createdAt, account.updatedAt]);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Import settings
|
// Import settings
|
||||||
for (const setting of data.settings) {
|
for (const setting of data.settings) {
|
||||||
await db.run(`
|
await db.run(`
|
||||||
@@ -247,7 +260,7 @@ async function importToAbsurdSql(data: MigrationData): Promise<void> {
|
|||||||
VALUES (?, ?, ?)
|
VALUES (?, ?, ?)
|
||||||
`, [setting.key, setting.value, setting.updatedAt]);
|
`, [setting.key, setting.value, setting.updatedAt]);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Import contacts
|
// Import contacts
|
||||||
for (const contact of data.contacts) {
|
for (const contact of data.contacts) {
|
||||||
await db.run(`
|
await db.run(`
|
||||||
@@ -264,6 +277,7 @@ async function importToAbsurdSql(data: MigrationData): Promise<void> {
|
|||||||
```
|
```
|
||||||
|
|
||||||
### 3. Verification
|
### 3. Verification
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
||||||
// Verify account count
|
// Verify account count
|
||||||
@@ -272,21 +286,21 @@ async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
|||||||
if (accountCount !== dexieData.accounts.length) {
|
if (accountCount !== dexieData.accounts.length) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify settings count
|
// Verify settings count
|
||||||
const settingsResult = await db.exec('SELECT COUNT(*) as count FROM settings');
|
const settingsResult = await db.exec('SELECT COUNT(*) as count FROM settings');
|
||||||
const settingsCount = settingsResult[0].values[0][0];
|
const settingsCount = settingsResult[0].values[0][0];
|
||||||
if (settingsCount !== dexieData.settings.length) {
|
if (settingsCount !== dexieData.settings.length) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify contacts count
|
// Verify contacts count
|
||||||
const contactsResult = await db.exec('SELECT COUNT(*) as count FROM contacts');
|
const contactsResult = await db.exec('SELECT COUNT(*) as count FROM contacts');
|
||||||
const contactsCount = contactsResult[0].values[0][0];
|
const contactsCount = contactsResult[0].values[0][0];
|
||||||
if (contactsCount !== dexieData.contacts.length) {
|
if (contactsCount !== dexieData.contacts.length) {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify data integrity
|
// Verify data integrity
|
||||||
for (const account of dexieData.accounts) {
|
for (const account of dexieData.accounts) {
|
||||||
const result = await db.exec(
|
const result = await db.exec(
|
||||||
@@ -294,12 +308,12 @@ async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
|||||||
[account.did]
|
[account.did]
|
||||||
);
|
);
|
||||||
const migratedAccount = result[0]?.values[0];
|
const migratedAccount = result[0]?.values[0];
|
||||||
if (!migratedAccount ||
|
if (!migratedAccount ||
|
||||||
migratedAccount[1] !== account.publicKeyHex) { // public_key_hex is second column
|
migratedAccount[1] !== account.publicKeyHex) { // public_key_hex is second column
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
@@ -307,18 +321,21 @@ async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
|||||||
## Performance Considerations
|
## Performance Considerations
|
||||||
|
|
||||||
### 1. Indexing
|
### 1. Indexing
|
||||||
|
|
||||||
- Dexie automatically creates indexes based on the schema
|
- Dexie automatically creates indexes based on the schema
|
||||||
- absurd-sql requires explicit index creation
|
- absurd-sql requires explicit index creation
|
||||||
- Added indexes for frequently queried fields
|
- Added indexes for frequently queried fields
|
||||||
- Use `PRAGMA journal_mode=MEMORY;` for better performance
|
- Use `PRAGMA journal_mode=MEMORY;` for better performance
|
||||||
|
|
||||||
### 2. Batch Operations
|
### 2. Batch Operations
|
||||||
|
|
||||||
- Dexie has built-in bulk operations
|
- Dexie has built-in bulk operations
|
||||||
- absurd-sql uses transactions for batch operations
|
- absurd-sql uses transactions for batch operations
|
||||||
- Consider chunking large datasets
|
- Consider chunking large datasets
|
||||||
- Use prepared statements for repeated queries
|
- Use prepared statements for repeated queries
|
||||||
|
|
||||||
### 3. Query Optimization
|
### 3. Query Optimization
|
||||||
|
|
||||||
- Dexie uses IndexedDB's native indexing
|
- Dexie uses IndexedDB's native indexing
|
||||||
- absurd-sql requires explicit query optimization
|
- absurd-sql requires explicit query optimization
|
||||||
- Use prepared statements for repeated queries
|
- Use prepared statements for repeated queries
|
||||||
@@ -327,6 +344,7 @@ async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
|||||||
## Error Handling
|
## Error Handling
|
||||||
|
|
||||||
### 1. Common Errors
|
### 1. Common Errors
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie errors
|
// Dexie errors
|
||||||
try {
|
try {
|
||||||
@@ -351,6 +369,7 @@ try {
|
|||||||
```
|
```
|
||||||
|
|
||||||
### 2. Transaction Recovery
|
### 2. Transaction Recovery
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Dexie transaction
|
// Dexie transaction
|
||||||
try {
|
try {
|
||||||
@@ -396,4 +415,4 @@ try {
|
|||||||
- Remove Dexie database
|
- Remove Dexie database
|
||||||
- Clear IndexedDB storage
|
- Clear IndexedDB storage
|
||||||
- Update application code
|
- Update application code
|
||||||
- Remove old dependencies
|
- Remove old dependencies
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
Check the contact & settings export to see whether you want your new table to be included in it.
|
# Check the contact & settings export to see whether you want your new table to be included in it
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user