chore: update migration documents and move to new home
This commit is contained in:
Matt Raymer
399
doc/dexie-to-sqlite-mapping.md
Normal file
399
doc/dexie-to-sqlite-mapping.md
Normal file
@@ -0,0 +1,399 @@
|
||||
# Dexie to absurd-sql Mapping Guide
|
||||
|
||||
## Schema Mapping
|
||||
|
||||
### Current Dexie Schema
|
||||
```typescript
|
||||
// Current Dexie schema
|
||||
const db = new Dexie('TimeSafariDB');
|
||||
|
||||
db.version(1).stores({
|
||||
accounts: 'did, publicKeyHex, createdAt, updatedAt',
|
||||
settings: 'key, value, updatedAt',
|
||||
contacts: 'id, did, name, createdAt, updatedAt'
|
||||
});
|
||||
```
|
||||
|
||||
### New SQLite Schema
|
||||
```sql
|
||||
-- New SQLite schema
|
||||
CREATE TABLE accounts (
|
||||
did TEXT PRIMARY KEY,
|
||||
public_key_hex TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE contacts (
|
||||
id TEXT PRIMARY KEY,
|
||||
did TEXT NOT NULL,
|
||||
name TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
FOREIGN KEY (did) REFERENCES accounts(did)
|
||||
);
|
||||
|
||||
-- Indexes for performance
|
||||
CREATE INDEX idx_accounts_created_at ON accounts(created_at);
|
||||
CREATE INDEX idx_contacts_did ON contacts(did);
|
||||
CREATE INDEX idx_settings_updated_at ON settings(updated_at);
|
||||
```
|
||||
|
||||
## Query Mapping
|
||||
|
||||
### 1. Account Operations
|
||||
|
||||
#### Get Account by DID
|
||||
```typescript
|
||||
// Dexie
|
||||
const account = await db.accounts.get(did);
|
||||
|
||||
// absurd-sql
|
||||
const result = await db.exec(`
|
||||
SELECT * FROM accounts WHERE did = ?
|
||||
`, [did]);
|
||||
const account = result[0]?.values[0];
|
||||
```
|
||||
|
||||
#### Get All Accounts
|
||||
```typescript
|
||||
// Dexie
|
||||
const accounts = await db.accounts.toArray();
|
||||
|
||||
// absurd-sql
|
||||
const result = await db.exec(`
|
||||
SELECT * FROM accounts ORDER BY created_at DESC
|
||||
`);
|
||||
const accounts = result[0]?.values || [];
|
||||
```
|
||||
|
||||
#### Add Account
|
||||
```typescript
|
||||
// Dexie
|
||||
await db.accounts.add({
|
||||
did,
|
||||
publicKeyHex,
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now()
|
||||
});
|
||||
|
||||
// absurd-sql
|
||||
await db.run(`
|
||||
INSERT INTO accounts (did, public_key_hex, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, [did, publicKeyHex, Date.now(), Date.now()]);
|
||||
```
|
||||
|
||||
#### Update Account
|
||||
```typescript
|
||||
// Dexie
|
||||
await db.accounts.update(did, {
|
||||
publicKeyHex,
|
||||
updatedAt: Date.now()
|
||||
});
|
||||
|
||||
// absurd-sql
|
||||
await db.run(`
|
||||
UPDATE accounts
|
||||
SET public_key_hex = ?, updated_at = ?
|
||||
WHERE did = ?
|
||||
`, [publicKeyHex, Date.now(), did]);
|
||||
```
|
||||
|
||||
### 2. Settings Operations
|
||||
|
||||
#### Get Setting
|
||||
```typescript
|
||||
// Dexie
|
||||
const setting = await db.settings.get(key);
|
||||
|
||||
// absurd-sql
|
||||
const result = await db.exec(`
|
||||
SELECT * FROM settings WHERE key = ?
|
||||
`, [key]);
|
||||
const setting = result[0]?.values[0];
|
||||
```
|
||||
|
||||
#### Set Setting
|
||||
```typescript
|
||||
// Dexie
|
||||
await db.settings.put({
|
||||
key,
|
||||
value,
|
||||
updatedAt: Date.now()
|
||||
});
|
||||
|
||||
// absurd-sql
|
||||
await db.run(`
|
||||
INSERT INTO settings (key, value, updated_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET
|
||||
value = excluded.value,
|
||||
updated_at = excluded.updated_at
|
||||
`, [key, value, Date.now()]);
|
||||
```
|
||||
|
||||
### 3. Contact Operations
|
||||
|
||||
#### Get Contacts by Account
|
||||
```typescript
|
||||
// Dexie
|
||||
const contacts = await db.contacts
|
||||
.where('did')
|
||||
.equals(accountDid)
|
||||
.toArray();
|
||||
|
||||
// absurd-sql
|
||||
const result = await db.exec(`
|
||||
SELECT * FROM contacts
|
||||
WHERE did = ?
|
||||
ORDER BY created_at DESC
|
||||
`, [accountDid]);
|
||||
const contacts = result[0]?.values || [];
|
||||
```
|
||||
|
||||
#### Add Contact
|
||||
```typescript
|
||||
// Dexie
|
||||
await db.contacts.add({
|
||||
id: generateId(),
|
||||
did: accountDid,
|
||||
name,
|
||||
createdAt: Date.now(),
|
||||
updatedAt: Date.now()
|
||||
});
|
||||
|
||||
// absurd-sql
|
||||
await db.run(`
|
||||
INSERT INTO contacts (id, did, name, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, [generateId(), accountDid, name, Date.now(), Date.now()]);
|
||||
```
|
||||
|
||||
## Transaction Mapping
|
||||
|
||||
### Batch Operations
|
||||
```typescript
|
||||
// Dexie
|
||||
await db.transaction('rw', [db.accounts, db.contacts], async () => {
|
||||
await db.accounts.add(account);
|
||||
await db.contacts.bulkAdd(contacts);
|
||||
});
|
||||
|
||||
// absurd-sql
|
||||
await db.exec('BEGIN TRANSACTION;');
|
||||
try {
|
||||
await db.run(`
|
||||
INSERT INTO accounts (did, public_key_hex, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, [account.did, account.publicKeyHex, account.createdAt, account.updatedAt]);
|
||||
|
||||
for (const contact of contacts) {
|
||||
await db.run(`
|
||||
INSERT INTO contacts (id, did, name, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, [contact.id, contact.did, contact.name, contact.createdAt, contact.updatedAt]);
|
||||
}
|
||||
await db.exec('COMMIT;');
|
||||
} catch (error) {
|
||||
await db.exec('ROLLBACK;');
|
||||
throw error;
|
||||
}
|
||||
```
|
||||
|
||||
## Migration Helper Functions
|
||||
|
||||
### 1. Data Export (Dexie to JSON)
|
||||
```typescript
|
||||
async function exportDexieData(): Promise<MigrationData> {
|
||||
const db = new Dexie('TimeSafariDB');
|
||||
|
||||
return {
|
||||
accounts: await db.accounts.toArray(),
|
||||
settings: await db.settings.toArray(),
|
||||
contacts: await db.contacts.toArray(),
|
||||
metadata: {
|
||||
version: '1.0.0',
|
||||
timestamp: Date.now(),
|
||||
dexieVersion: Dexie.version
|
||||
}
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Data Import (JSON to absurd-sql)
|
||||
```typescript
|
||||
async function importToAbsurdSql(data: MigrationData): Promise<void> {
|
||||
await db.exec('BEGIN TRANSACTION;');
|
||||
try {
|
||||
// Import accounts
|
||||
for (const account of data.accounts) {
|
||||
await db.run(`
|
||||
INSERT INTO accounts (did, public_key_hex, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, [account.did, account.publicKeyHex, account.createdAt, account.updatedAt]);
|
||||
}
|
||||
|
||||
// Import settings
|
||||
for (const setting of data.settings) {
|
||||
await db.run(`
|
||||
INSERT INTO settings (key, value, updated_at)
|
||||
VALUES (?, ?, ?)
|
||||
`, [setting.key, setting.value, setting.updatedAt]);
|
||||
}
|
||||
|
||||
// Import contacts
|
||||
for (const contact of data.contacts) {
|
||||
await db.run(`
|
||||
INSERT INTO contacts (id, did, name, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, [contact.id, contact.did, contact.name, contact.createdAt, contact.updatedAt]);
|
||||
}
|
||||
await db.exec('COMMIT;');
|
||||
} catch (error) {
|
||||
await db.exec('ROLLBACK;');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Verification
|
||||
```typescript
|
||||
async function verifyMigration(dexieData: MigrationData): Promise<boolean> {
|
||||
// Verify account count
|
||||
const accountResult = await db.exec('SELECT COUNT(*) as count FROM accounts');
|
||||
const accountCount = accountResult[0].values[0][0];
|
||||
if (accountCount !== dexieData.accounts.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Verify settings count
|
||||
const settingsResult = await db.exec('SELECT COUNT(*) as count FROM settings');
|
||||
const settingsCount = settingsResult[0].values[0][0];
|
||||
if (settingsCount !== dexieData.settings.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Verify contacts count
|
||||
const contactsResult = await db.exec('SELECT COUNT(*) as count FROM contacts');
|
||||
const contactsCount = contactsResult[0].values[0][0];
|
||||
if (contactsCount !== dexieData.contacts.length) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Verify data integrity
|
||||
for (const account of dexieData.accounts) {
|
||||
const result = await db.exec(
|
||||
'SELECT * FROM accounts WHERE did = ?',
|
||||
[account.did]
|
||||
);
|
||||
const migratedAccount = result[0]?.values[0];
|
||||
if (!migratedAccount ||
|
||||
migratedAccount[1] !== account.publicKeyHex) { // public_key_hex is second column
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
### 1. Indexing
|
||||
- Dexie automatically creates indexes based on the schema
|
||||
- absurd-sql requires explicit index creation
|
||||
- Added indexes for frequently queried fields
|
||||
- Use `PRAGMA journal_mode=MEMORY;` for better performance
|
||||
|
||||
### 2. Batch Operations
|
||||
- Dexie has built-in bulk operations
|
||||
- absurd-sql uses transactions for batch operations
|
||||
- Consider chunking large datasets
|
||||
- Use prepared statements for repeated queries
|
||||
|
||||
### 3. Query Optimization
|
||||
- Dexie uses IndexedDB's native indexing
|
||||
- absurd-sql requires explicit query optimization
|
||||
- Use prepared statements for repeated queries
|
||||
- Consider using `PRAGMA synchronous=NORMAL;` for better performance
|
||||
|
||||
## Error Handling
|
||||
|
||||
### 1. Common Errors
|
||||
```typescript
|
||||
// Dexie errors
|
||||
try {
|
||||
await db.accounts.add(account);
|
||||
} catch (error) {
|
||||
if (error instanceof Dexie.ConstraintError) {
|
||||
// Handle duplicate key
|
||||
}
|
||||
}
|
||||
|
||||
// absurd-sql errors
|
||||
try {
|
||||
await db.run(`
|
||||
INSERT INTO accounts (did, public_key_hex, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, [account.did, account.publicKeyHex, account.createdAt, account.updatedAt]);
|
||||
} catch (error) {
|
||||
if (error.message.includes('UNIQUE constraint failed')) {
|
||||
// Handle duplicate key
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Transaction Recovery
|
||||
```typescript
|
||||
// Dexie transaction
|
||||
try {
|
||||
await db.transaction('rw', db.accounts, async () => {
|
||||
// Operations
|
||||
});
|
||||
} catch (error) {
|
||||
// Dexie automatically rolls back
|
||||
}
|
||||
|
||||
// absurd-sql transaction
|
||||
try {
|
||||
await db.exec('BEGIN TRANSACTION;');
|
||||
// Operations
|
||||
await db.exec('COMMIT;');
|
||||
} catch (error) {
|
||||
await db.exec('ROLLBACK;');
|
||||
throw error;
|
||||
}
|
||||
```
|
||||
|
||||
## Migration Strategy
|
||||
|
||||
1. **Preparation**
|
||||
- Export all Dexie data
|
||||
- Verify data integrity
|
||||
- Create SQLite schema
|
||||
- Setup indexes
|
||||
|
||||
2. **Migration**
|
||||
- Import data in transactions
|
||||
- Verify each batch
|
||||
- Handle errors gracefully
|
||||
- Maintain backup
|
||||
|
||||
3. **Verification**
|
||||
- Compare record counts
|
||||
- Verify data integrity
|
||||
- Test common queries
|
||||
- Validate relationships
|
||||
|
||||
4. **Cleanup**
|
||||
- Remove Dexie database
|
||||
- Clear IndexedDB storage
|
||||
- Update application code
|
||||
- Remove old dependencies
|
||||
613
doc/migration-to-wa-sqlite.md
Normal file
613
doc/migration-to-wa-sqlite.md
Normal file
@@ -0,0 +1,613 @@
|
||||
# Migration Guide: Dexie to absurd-sql
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines the migration process from Dexie.js to absurd-sql for the TimeSafari app's storage implementation. The migration aims to provide a consistent SQLite-based storage solution across all platforms while maintaining data integrity and ensuring a smooth transition for users.
|
||||
|
||||
## Migration Goals
|
||||
|
||||
1. **Data Integrity**
|
||||
- Preserve all existing data
|
||||
- Maintain data relationships
|
||||
- Ensure data consistency
|
||||
|
||||
2. **Performance**
|
||||
- Improve query performance
|
||||
- Reduce storage overhead
|
||||
- Optimize for platform-specific features
|
||||
|
||||
3. **Security**
|
||||
- Maintain or improve encryption
|
||||
- Preserve access controls
|
||||
- Enhance data protection
|
||||
|
||||
4. **User Experience**
|
||||
- Zero data loss
|
||||
- Minimal downtime
|
||||
- Automatic migration where possible
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. **Backup Requirements**
|
||||
```typescript
|
||||
interface MigrationBackup {
|
||||
timestamp: number;
|
||||
accounts: Account[];
|
||||
settings: Setting[];
|
||||
contacts: Contact[];
|
||||
metadata: {
|
||||
version: string;
|
||||
platform: string;
|
||||
dexieVersion: string;
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
2. **Dependencies**
|
||||
```json
|
||||
{
|
||||
"@jlongster/sql.js": "^1.8.0",
|
||||
"absurd-sql": "^1.8.0"
|
||||
}
|
||||
```
|
||||
|
||||
3. **Storage Requirements**
|
||||
- Sufficient IndexedDB quota
|
||||
- Available disk space for SQLite
|
||||
- Backup storage space
|
||||
|
||||
4. **Platform Support**
|
||||
- Web: Modern browser with IndexedDB support
|
||||
- iOS: iOS 13+ with SQLite support
|
||||
- Android: Android 5+ with SQLite support
|
||||
- Electron: Latest version with SQLite support
|
||||
|
||||
## Migration Process
|
||||
|
||||
### 1. Preparation
|
||||
|
||||
```typescript
|
||||
// src/services/storage/migration/MigrationService.ts
|
||||
import initSqlJs from '@jlongster/sql.js';
|
||||
import { SQLiteFS } from 'absurd-sql';
|
||||
import IndexedDBBackend from 'absurd-sql/dist/indexeddb-backend';
|
||||
|
||||
export class MigrationService {
|
||||
private static instance: MigrationService;
|
||||
private backup: MigrationBackup | null = null;
|
||||
private sql: any = null;
|
||||
private db: any = null;
|
||||
|
||||
async prepare(): Promise<void> {
|
||||
try {
|
||||
// 1. Check prerequisites
|
||||
await this.checkPrerequisites();
|
||||
|
||||
// 2. Create backup
|
||||
this.backup = await this.createBackup();
|
||||
|
||||
// 3. Verify backup integrity
|
||||
await this.verifyBackup();
|
||||
|
||||
// 4. Initialize absurd-sql
|
||||
await this.initializeAbsurdSql();
|
||||
} catch (error) {
|
||||
throw new StorageError(
|
||||
'Migration preparation failed',
|
||||
StorageErrorCodes.MIGRATION_FAILED,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async initializeAbsurdSql(): Promise<void> {
|
||||
// Initialize SQL.js
|
||||
this.sql = await initSqlJs({
|
||||
locateFile: (file: string) => {
|
||||
return new URL(`/node_modules/@jlongster/sql.js/dist/${file}`, import.meta.url).href;
|
||||
}
|
||||
});
|
||||
|
||||
// Setup SQLiteFS with IndexedDB backend
|
||||
const sqlFS = new SQLiteFS(this.sql.FS, new IndexedDBBackend());
|
||||
this.sql.register_for_idb(sqlFS);
|
||||
|
||||
// Create and mount filesystem
|
||||
this.sql.FS.mkdir('/sql');
|
||||
this.sql.FS.mount(sqlFS, {}, '/sql');
|
||||
|
||||
// Open database
|
||||
const path = '/sql/db.sqlite';
|
||||
if (typeof SharedArrayBuffer === 'undefined') {
|
||||
let stream = this.sql.FS.open(path, 'a+');
|
||||
await stream.node.contents.readIfFallback();
|
||||
this.sql.FS.close(stream);
|
||||
}
|
||||
|
||||
this.db = new this.sql.Database(path, { filename: true });
|
||||
if (!this.db) {
|
||||
throw new StorageError(
|
||||
'Database initialization failed',
|
||||
StorageErrorCodes.INITIALIZATION_FAILED
|
||||
);
|
||||
}
|
||||
|
||||
// Configure database
|
||||
await this.db.exec(`PRAGMA journal_mode=MEMORY;`);
|
||||
}
|
||||
|
||||
private async checkPrerequisites(): Promise<void> {
|
||||
// Check IndexedDB availability
|
||||
if (!window.indexedDB) {
|
||||
throw new StorageError(
|
||||
'IndexedDB not available',
|
||||
StorageErrorCodes.INITIALIZATION_FAILED
|
||||
);
|
||||
}
|
||||
|
||||
// Check storage quota
|
||||
const quota = await navigator.storage.estimate();
|
||||
if (quota.quota && quota.usage && quota.usage > quota.quota * 0.9) {
|
||||
throw new StorageError(
|
||||
'Insufficient storage space',
|
||||
StorageErrorCodes.STORAGE_FULL
|
||||
);
|
||||
}
|
||||
|
||||
// Check platform support
|
||||
const capabilities = await PlatformDetection.getCapabilities();
|
||||
if (!capabilities.hasFileSystem) {
|
||||
throw new StorageError(
|
||||
'Platform does not support required features',
|
||||
StorageErrorCodes.INITIALIZATION_FAILED
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async createBackup(): Promise<MigrationBackup> {
|
||||
const dexieDB = new Dexie('TimeSafariDB');
|
||||
|
||||
return {
|
||||
timestamp: Date.now(),
|
||||
accounts: await dexieDB.accounts.toArray(),
|
||||
settings: await dexieDB.settings.toArray(),
|
||||
contacts: await dexieDB.contacts.toArray(),
|
||||
metadata: {
|
||||
version: '1.0.0',
|
||||
platform: await PlatformDetection.getPlatform(),
|
||||
dexieVersion: Dexie.version
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Data Migration
|
||||
|
||||
```typescript
|
||||
// src/services/storage/migration/DataMigration.ts
|
||||
export class DataMigration {
|
||||
async migrate(backup: MigrationBackup): Promise<void> {
|
||||
try {
|
||||
// 1. Create new database schema
|
||||
await this.createSchema();
|
||||
|
||||
// 2. Migrate accounts
|
||||
await this.migrateAccounts(backup.accounts);
|
||||
|
||||
// 3. Migrate settings
|
||||
await this.migrateSettings(backup.settings);
|
||||
|
||||
// 4. Migrate contacts
|
||||
await this.migrateContacts(backup.contacts);
|
||||
|
||||
// 5. Verify migration
|
||||
await this.verifyMigration(backup);
|
||||
} catch (error) {
|
||||
// 6. Handle failure
|
||||
await this.handleMigrationFailure(error, backup);
|
||||
}
|
||||
}
|
||||
|
||||
private async migrateAccounts(accounts: Account[]): Promise<void> {
|
||||
// Use transaction for atomicity
|
||||
await this.db.exec('BEGIN TRANSACTION;');
|
||||
try {
|
||||
for (const account of accounts) {
|
||||
await this.db.run(`
|
||||
INSERT INTO accounts (did, public_key_hex, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?)
|
||||
`, [
|
||||
account.did,
|
||||
account.publicKeyHex,
|
||||
account.createdAt,
|
||||
account.updatedAt
|
||||
]);
|
||||
}
|
||||
await this.db.exec('COMMIT;');
|
||||
} catch (error) {
|
||||
await this.db.exec('ROLLBACK;');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async verifyMigration(backup: MigrationBackup): Promise<void> {
|
||||
// Verify account count
|
||||
const result = await this.db.exec('SELECT COUNT(*) as count FROM accounts');
|
||||
const accountCount = result[0].values[0][0];
|
||||
|
||||
if (accountCount !== backup.accounts.length) {
|
||||
throw new StorageError(
|
||||
'Account count mismatch',
|
||||
StorageErrorCodes.VERIFICATION_FAILED
|
||||
);
|
||||
}
|
||||
|
||||
// Verify data integrity
|
||||
await this.verifyDataIntegrity(backup);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Rollback Strategy
|
||||
|
||||
```typescript
|
||||
// src/services/storage/migration/RollbackService.ts
|
||||
export class RollbackService {
|
||||
async rollback(backup: MigrationBackup): Promise<void> {
|
||||
try {
|
||||
// 1. Stop all database operations
|
||||
await this.stopDatabaseOperations();
|
||||
|
||||
// 2. Restore from backup
|
||||
await this.restoreFromBackup(backup);
|
||||
|
||||
// 3. Verify restoration
|
||||
await this.verifyRestoration(backup);
|
||||
|
||||
// 4. Clean up absurd-sql
|
||||
await this.cleanupAbsurdSql();
|
||||
} catch (error) {
|
||||
throw new StorageError(
|
||||
'Rollback failed',
|
||||
StorageErrorCodes.ROLLBACK_FAILED,
|
||||
error
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async restoreFromBackup(backup: MigrationBackup): Promise<void> {
|
||||
const dexieDB = new Dexie('TimeSafariDB');
|
||||
|
||||
// Restore accounts
|
||||
await dexieDB.accounts.bulkPut(backup.accounts);
|
||||
|
||||
// Restore settings
|
||||
await dexieDB.settings.bulkPut(backup.settings);
|
||||
|
||||
// Restore contacts
|
||||
await dexieDB.contacts.bulkPut(backup.contacts);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Migration UI
|
||||
|
||||
```vue
|
||||
<!-- src/components/MigrationProgress.vue -->
|
||||
<template>
|
||||
<div class="migration-progress">
|
||||
<h2>Database Migration</h2>
|
||||
|
||||
<div class="progress-container">
|
||||
<div class="progress-bar" :style="{ width: `${progress}%` }" />
|
||||
<div class="progress-text">{{ progress }}%</div>
|
||||
</div>
|
||||
|
||||
<div class="status-message">{{ statusMessage }}</div>
|
||||
|
||||
<div v-if="error" class="error-message">
|
||||
{{ error }}
|
||||
<button @click="retryMigration">Retry</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { ref, onMounted } from 'vue';
|
||||
import { MigrationService } from '@/services/storage/migration/MigrationService';
|
||||
|
||||
const progress = ref(0);
|
||||
const statusMessage = ref('Preparing migration...');
|
||||
const error = ref<string | null>(null);
|
||||
|
||||
const migrationService = MigrationService.getInstance();
|
||||
|
||||
async function startMigration() {
|
||||
try {
|
||||
// 1. Preparation
|
||||
statusMessage.value = 'Creating backup...';
|
||||
await migrationService.prepare();
|
||||
progress.value = 20;
|
||||
|
||||
// 2. Data migration
|
||||
statusMessage.value = 'Migrating data...';
|
||||
await migrationService.migrate();
|
||||
progress.value = 80;
|
||||
|
||||
// 3. Verification
|
||||
statusMessage.value = 'Verifying migration...';
|
||||
await migrationService.verify();
|
||||
progress.value = 100;
|
||||
|
||||
statusMessage.value = 'Migration completed successfully!';
|
||||
} catch (err) {
|
||||
error.value = err instanceof Error ? err.message : 'Migration failed';
|
||||
statusMessage.value = 'Migration failed';
|
||||
}
|
||||
}
|
||||
|
||||
async function retryMigration() {
|
||||
error.value = null;
|
||||
progress.value = 0;
|
||||
await startMigration();
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
startMigration();
|
||||
});
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.migration-progress {
|
||||
padding: 2rem;
|
||||
max-width: 600px;
|
||||
margin: 0 auto;
|
||||
}
|
||||
|
||||
.progress-container {
|
||||
position: relative;
|
||||
height: 20px;
|
||||
background: #eee;
|
||||
border-radius: 10px;
|
||||
overflow: hidden;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
position: absolute;
|
||||
height: 100%;
|
||||
background: #4CAF50;
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
position: absolute;
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
line-height: 20px;
|
||||
color: #000;
|
||||
}
|
||||
|
||||
.status-message {
|
||||
text-align: center;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
.error-message {
|
||||
color: #f44336;
|
||||
text-align: center;
|
||||
margin: 1rem 0;
|
||||
}
|
||||
|
||||
button {
|
||||
margin-top: 1rem;
|
||||
padding: 0.5rem 1rem;
|
||||
background: #2196F3;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
button:hover {
|
||||
background: #1976D2;
|
||||
}
|
||||
</style>
|
||||
```
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
1. **Unit Tests**
|
||||
```typescript
|
||||
// src/services/storage/migration/__tests__/MigrationService.spec.ts
|
||||
describe('MigrationService', () => {
|
||||
it('should initialize absurd-sql correctly', async () => {
|
||||
const service = MigrationService.getInstance();
|
||||
await service.initializeAbsurdSql();
|
||||
|
||||
expect(service.isInitialized()).toBe(true);
|
||||
expect(service.getDatabase()).toBeDefined();
|
||||
});
|
||||
|
||||
it('should create valid backup', async () => {
|
||||
const service = MigrationService.getInstance();
|
||||
const backup = await service.createBackup();
|
||||
|
||||
expect(backup).toBeDefined();
|
||||
expect(backup.accounts).toBeInstanceOf(Array);
|
||||
expect(backup.settings).toBeInstanceOf(Array);
|
||||
expect(backup.contacts).toBeInstanceOf(Array);
|
||||
});
|
||||
|
||||
it('should migrate data correctly', async () => {
|
||||
const service = MigrationService.getInstance();
|
||||
const backup = await service.createBackup();
|
||||
|
||||
await service.migrate(backup);
|
||||
|
||||
// Verify migration
|
||||
const accounts = await service.getMigratedAccounts();
|
||||
expect(accounts).toHaveLength(backup.accounts.length);
|
||||
});
|
||||
|
||||
it('should handle rollback correctly', async () => {
|
||||
const service = MigrationService.getInstance();
|
||||
const backup = await service.createBackup();
|
||||
|
||||
// Simulate failed migration
|
||||
await service.migrate(backup);
|
||||
await service.simulateFailure();
|
||||
|
||||
// Perform rollback
|
||||
await service.rollback(backup);
|
||||
|
||||
// Verify rollback
|
||||
const accounts = await service.getOriginalAccounts();
|
||||
expect(accounts).toHaveLength(backup.accounts.length);
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
2. **Integration Tests**
|
||||
```typescript
|
||||
// src/services/storage/migration/__tests__/integration/Migration.spec.ts
|
||||
describe('Migration Integration', () => {
|
||||
it('should handle concurrent access during migration', async () => {
|
||||
const service = MigrationService.getInstance();
|
||||
|
||||
// Start migration
|
||||
const migrationPromise = service.migrate();
|
||||
|
||||
// Simulate concurrent access
|
||||
const accessPromises = Array(5).fill(null).map(() =>
|
||||
service.getAccount('did:test:123')
|
||||
);
|
||||
|
||||
// Wait for all operations
|
||||
const [migrationResult, ...accessResults] = await Promise.allSettled([
|
||||
migrationPromise,
|
||||
...accessPromises
|
||||
]);
|
||||
|
||||
// Verify results
|
||||
expect(migrationResult.status).toBe('fulfilled');
|
||||
expect(accessResults.some(r => r.status === 'rejected')).toBe(true);
|
||||
});
|
||||
|
||||
it('should maintain data integrity during platform transition', async () => {
|
||||
const service = MigrationService.getInstance();
|
||||
|
||||
// Simulate platform change
|
||||
await service.simulatePlatformChange();
|
||||
|
||||
// Verify data
|
||||
const accounts = await service.getAllAccounts();
|
||||
const settings = await service.getAllSettings();
|
||||
const contacts = await service.getAllContacts();
|
||||
|
||||
expect(accounts).toBeDefined();
|
||||
expect(settings).toBeDefined();
|
||||
expect(contacts).toBeDefined();
|
||||
});
|
||||
});
|
||||
```
|
||||
|
||||
## Success Criteria
|
||||
|
||||
1. **Data Integrity**
|
||||
- [ ] All accounts migrated successfully
|
||||
- [ ] All settings preserved
|
||||
- [ ] All contacts transferred
|
||||
- [ ] No data corruption
|
||||
|
||||
2. **Performance**
|
||||
- [ ] Migration completes within acceptable time
|
||||
- [ ] No significant performance degradation
|
||||
- [ ] Efficient storage usage
|
||||
- [ ] Smooth user experience
|
||||
|
||||
3. **Security**
|
||||
- [ ] Encrypted data remains secure
|
||||
- [ ] Access controls maintained
|
||||
- [ ] No sensitive data exposure
|
||||
- [ ] Secure backup process
|
||||
|
||||
4. **User Experience**
|
||||
- [ ] Clear migration progress
|
||||
- [ ] Informative error messages
|
||||
- [ ] Automatic recovery from failures
|
||||
- [ ] No data loss
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
1. **Automatic Rollback**
|
||||
- Triggered by migration failure
|
||||
- Restores from verified backup
|
||||
- Maintains data consistency
|
||||
- Logs rollback reason
|
||||
|
||||
2. **Manual Rollback**
|
||||
- Available through settings
|
||||
- Requires user confirmation
|
||||
- Preserves backup data
|
||||
- Provides rollback status
|
||||
|
||||
3. **Emergency Recovery**
|
||||
- Manual backup restoration
|
||||
- Database repair tools
|
||||
- Data recovery procedures
|
||||
- Support contact information
|
||||
|
||||
## Post-Migration
|
||||
|
||||
1. **Verification**
|
||||
- Data integrity checks
|
||||
- Performance monitoring
|
||||
- Error rate tracking
|
||||
- User feedback collection
|
||||
|
||||
2. **Cleanup**
|
||||
- Remove old database
|
||||
- Clear migration artifacts
|
||||
- Update application state
|
||||
- Archive backup data
|
||||
|
||||
3. **Monitoring**
|
||||
- Track migration success rate
|
||||
- Monitor performance metrics
|
||||
- Collect error reports
|
||||
- Gather user feedback
|
||||
|
||||
## Support
|
||||
|
||||
For assistance with migration:
|
||||
1. Check the troubleshooting guide
|
||||
2. Review error logs
|
||||
3. Contact support team
|
||||
4. Submit issue report
|
||||
|
||||
## Timeline
|
||||
|
||||
1. **Preparation Phase** (1 week)
|
||||
- Backup system implementation
|
||||
- Migration service development
|
||||
- Testing framework setup
|
||||
|
||||
2. **Testing Phase** (2 weeks)
|
||||
- Unit testing
|
||||
- Integration testing
|
||||
- Performance testing
|
||||
- Security testing
|
||||
|
||||
3. **Deployment Phase** (1 week)
|
||||
- Staged rollout
|
||||
- Monitoring
|
||||
- Support preparation
|
||||
- Documentation updates
|
||||
|
||||
4. **Post-Deployment** (2 weeks)
|
||||
- Monitoring
|
||||
- Bug fixes
|
||||
- Performance optimization
|
||||
- User feedback collection
|
||||
1310
doc/secure-storage-implementation.md
Normal file
1310
doc/secure-storage-implementation.md
Normal file
File diff suppressed because it is too large
Load Diff
323
doc/storage-implementation-checklist.md
Normal file
323
doc/storage-implementation-checklist.md
Normal file
@@ -0,0 +1,323 @@
|
||||
# Storage Implementation Checklist
|
||||
|
||||
## Core Services
|
||||
|
||||
### 1. Storage Service Layer
|
||||
- [ ] Create base `StorageService` interface
|
||||
- [ ] Define common methods for all platforms
|
||||
- [ ] Add platform-specific method signatures
|
||||
- [ ] Include error handling types
|
||||
- [ ] Add migration support methods
|
||||
|
||||
- [ ] Implement platform-specific services
|
||||
- [ ] `WebSQLiteService` (absurd-sql)
|
||||
- [ ] Database initialization
|
||||
- [ ] VFS setup with IndexedDB backend
|
||||
- [ ] Connection management
|
||||
- [ ] Query builder
|
||||
- [ ] `NativeSQLiteService` (iOS/Android)
|
||||
- [ ] SQLCipher integration
|
||||
- [ ] Native bridge setup
|
||||
- [ ] File system access
|
||||
- [ ] `ElectronSQLiteService`
|
||||
- [ ] Node SQLite integration
|
||||
- [ ] IPC communication
|
||||
- [ ] File system access
|
||||
|
||||
### 2. Migration Services
|
||||
- [ ] Implement `MigrationService`
|
||||
- [ ] Backup creation
|
||||
- [ ] Data verification
|
||||
- [ ] Rollback procedures
|
||||
- [ ] Progress tracking
|
||||
- [ ] Create `MigrationUI` components
|
||||
- [ ] Progress indicators
|
||||
- [ ] Error handling
|
||||
- [ ] User notifications
|
||||
- [ ] Manual triggers
|
||||
|
||||
### 3. Security Layer
|
||||
- [ ] Implement `EncryptionService`
|
||||
- [ ] Key management
|
||||
- [ ] Encryption/decryption
|
||||
- [ ] Secure storage
|
||||
- [ ] Add `BiometricService`
|
||||
- [ ] Platform detection
|
||||
- [ ] Authentication flow
|
||||
- [ ] Fallback mechanisms
|
||||
|
||||
## Platform-Specific Implementation
|
||||
|
||||
### Web Platform
|
||||
- [ ] Setup absurd-sql
|
||||
- [ ] Install dependencies
|
||||
```json
|
||||
{
|
||||
"@jlongster/sql.js": "^1.8.0",
|
||||
"absurd-sql": "^1.8.0"
|
||||
}
|
||||
```
|
||||
- [ ] Configure VFS with IndexedDB backend
|
||||
- [ ] Setup worker threads
|
||||
- [ ] Implement connection pooling
|
||||
- [ ] Configure database pragmas
|
||||
```sql
|
||||
PRAGMA journal_mode=MEMORY;
|
||||
PRAGMA synchronous=NORMAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
PRAGMA busy_timeout=5000;
|
||||
```
|
||||
|
||||
- [ ] Update build configuration
|
||||
- [ ] Modify `vite.config.ts`
|
||||
- [ ] Add worker configuration
|
||||
- [ ] Update chunk splitting
|
||||
- [ ] Configure asset handling
|
||||
|
||||
- [ ] Implement IndexedDB fallback
|
||||
- [ ] Create fallback service
|
||||
- [ ] Add data synchronization
|
||||
- [ ] Handle quota exceeded
|
||||
- [ ] Implement atomic operations
|
||||
|
||||
### iOS Platform
|
||||
- [ ] Setup SQLCipher
|
||||
- [ ] Install pod dependencies
|
||||
- [ ] Configure encryption
|
||||
- [ ] Setup keychain access
|
||||
- [ ] Implement secure storage
|
||||
|
||||
- [ ] Update Capacitor config
|
||||
- [ ] Modify `capacitor.config.ts`
|
||||
- [ ] Add iOS permissions
|
||||
- [ ] Configure backup
|
||||
- [ ] Setup app groups
|
||||
|
||||
### Android Platform
|
||||
- [ ] Setup SQLCipher
|
||||
- [ ] Add Gradle dependencies
|
||||
- [ ] Configure encryption
|
||||
- [ ] Setup keystore
|
||||
- [ ] Implement secure storage
|
||||
|
||||
- [ ] Update Capacitor config
|
||||
- [ ] Modify `capacitor.config.ts`
|
||||
- [ ] Add Android permissions
|
||||
- [ ] Configure backup
|
||||
- [ ] Setup file provider
|
||||
|
||||
### Electron Platform
|
||||
- [ ] Setup Node SQLite
|
||||
- [ ] Install dependencies
|
||||
- [ ] Configure IPC
|
||||
- [ ] Setup file system access
|
||||
- [ ] Implement secure storage
|
||||
|
||||
- [ ] Update Electron config
|
||||
- [ ] Modify `electron.config.ts`
|
||||
- [ ] Add security policies
|
||||
- [ ] Configure file access
|
||||
- [ ] Setup auto-updates
|
||||
|
||||
## Data Models and Types
|
||||
|
||||
### 1. Database Schema
|
||||
- [ ] Define tables
|
||||
```sql
|
||||
-- Accounts table
|
||||
CREATE TABLE accounts (
|
||||
did TEXT PRIMARY KEY,
|
||||
public_key_hex TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Settings table
|
||||
CREATE TABLE settings (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Contacts table
|
||||
CREATE TABLE contacts (
|
||||
id TEXT PRIMARY KEY,
|
||||
did TEXT NOT NULL,
|
||||
name TEXT,
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
FOREIGN KEY (did) REFERENCES accounts(did)
|
||||
);
|
||||
|
||||
-- Indexes for performance
|
||||
CREATE INDEX idx_accounts_created_at ON accounts(created_at);
|
||||
CREATE INDEX idx_contacts_did ON contacts(did);
|
||||
CREATE INDEX idx_settings_updated_at ON settings(updated_at);
|
||||
```
|
||||
|
||||
- [ ] Create indexes
|
||||
- [ ] Define constraints
|
||||
- [ ] Add triggers
|
||||
- [ ] Setup migrations
|
||||
|
||||
### 2. Type Definitions
|
||||
- [ ] Create interfaces
|
||||
```typescript
|
||||
interface Account {
|
||||
did: string;
|
||||
publicKeyHex: string;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
interface Setting {
|
||||
key: string;
|
||||
value: string;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
interface Contact {
|
||||
id: string;
|
||||
did: string;
|
||||
name?: string;
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Add validation
|
||||
- [ ] Create DTOs
|
||||
- [ ] Define enums
|
||||
- [ ] Add type guards
|
||||
|
||||
## UI Components
|
||||
|
||||
### 1. Migration UI
|
||||
- [ ] Create components
|
||||
- [ ] `MigrationProgress.vue`
|
||||
- [ ] `MigrationError.vue`
|
||||
- [ ] `MigrationSettings.vue`
|
||||
- [ ] `MigrationStatus.vue`
|
||||
|
||||
### 2. Settings UI
|
||||
- [ ] Update components
|
||||
- [ ] Add storage settings
|
||||
- [ ] Add migration controls
|
||||
- [ ] Add backup options
|
||||
- [ ] Add security settings
|
||||
|
||||
### 3. Error Handling UI
|
||||
- [ ] Create components
|
||||
- [ ] `StorageError.vue`
|
||||
- [ ] `QuotaExceeded.vue`
|
||||
- [ ] `MigrationFailed.vue`
|
||||
- [ ] `RecoveryOptions.vue`
|
||||
|
||||
## Testing
|
||||
|
||||
### 1. Unit Tests
|
||||
- [ ] Test services
|
||||
- [ ] Storage service tests
|
||||
- [ ] Migration service tests
|
||||
- [ ] Security service tests
|
||||
- [ ] Platform detection tests
|
||||
|
||||
### 2. Integration Tests
|
||||
- [ ] Test migrations
|
||||
- [ ] Web platform tests
|
||||
- [ ] iOS platform tests
|
||||
- [ ] Android platform tests
|
||||
- [ ] Electron platform tests
|
||||
|
||||
### 3. E2E Tests
|
||||
- [ ] Test workflows
|
||||
- [ ] Account management
|
||||
- [ ] Settings management
|
||||
- [ ] Contact management
|
||||
- [ ] Migration process
|
||||
|
||||
## Documentation
|
||||
|
||||
### 1. Technical Documentation
|
||||
- [ ] Update architecture docs
|
||||
- [ ] Add API documentation
|
||||
- [ ] Create migration guides
|
||||
- [ ] Document security measures
|
||||
|
||||
### 2. User Documentation
|
||||
- [ ] Update user guides
|
||||
- [ ] Add troubleshooting guides
|
||||
- [ ] Create FAQ
|
||||
- [ ] Document new features
|
||||
|
||||
## Deployment
|
||||
|
||||
### 1. Build Process
|
||||
- [ ] Update build scripts
|
||||
- [ ] Add platform-specific builds
|
||||
- [ ] Configure CI/CD
|
||||
- [ ] Setup automated testing
|
||||
|
||||
### 2. Release Process
|
||||
- [ ] Create release checklist
|
||||
- [ ] Add version management
|
||||
- [ ] Setup rollback procedures
|
||||
- [ ] Configure monitoring
|
||||
|
||||
## Monitoring and Analytics
|
||||
|
||||
### 1. Error Tracking
|
||||
- [ ] Setup error logging
|
||||
- [ ] Add performance monitoring
|
||||
- [ ] Configure alerts
|
||||
- [ ] Create dashboards
|
||||
|
||||
### 2. Usage Analytics
|
||||
- [ ] Add storage metrics
|
||||
- [ ] Track migration success
|
||||
- [ ] Monitor performance
|
||||
- [ ] Collect user feedback
|
||||
|
||||
## Security Audit
|
||||
|
||||
### 1. Code Review
|
||||
- [ ] Review encryption
|
||||
- [ ] Check access controls
|
||||
- [ ] Verify data handling
|
||||
- [ ] Audit dependencies
|
||||
|
||||
### 2. Penetration Testing
|
||||
- [ ] Test data access
|
||||
- [ ] Verify encryption
|
||||
- [ ] Check authentication
|
||||
- [ ] Review permissions
|
||||
|
||||
## Success Criteria
|
||||
|
||||
### 1. Performance
|
||||
- [ ] Query response time < 100ms
|
||||
- [ ] Migration time < 5s per 1000 records
|
||||
- [ ] Storage overhead < 10%
|
||||
- [ ] Memory usage < 50MB
|
||||
- [ ] Atomic operations complete successfully
|
||||
- [ ] Transaction performance meets requirements
|
||||
|
||||
### 2. Reliability
|
||||
- [ ] 99.9% uptime
|
||||
- [ ] Zero data loss
|
||||
- [ ] Automatic recovery
|
||||
- [ ] Backup verification
|
||||
- [ ] Transaction atomicity
|
||||
- [ ] Data consistency
|
||||
|
||||
### 3. Security
|
||||
- [ ] AES-256 encryption
|
||||
- [ ] Secure key storage
|
||||
- [ ] Access control
|
||||
- [ ] Audit logging
|
||||
|
||||
### 4. User Experience
|
||||
- [ ] Smooth migration
|
||||
- [ ] Clear error messages
|
||||
- [ ] Progress indicators
|
||||
- [ ] Recovery options
|
||||
Reference in New Issue
Block a user