add encryption for the two SQL columns, replace basic DB utils, add USE_DEXIE_DB flag, and start adding SQL everywhere

2025-05-26 19:03:20 -06:00
parent 6e005de65a
commit 3f46e3817a
8 changed files with 467 additions and 27 deletions
--- a/doc/secure-storage-implementation.md
+++ b/doc/secure-storage-implementation.md
@@ -144,6 +144,104 @@ try {
 }
 ```

+#### A. Modifying Code
+
+When converting from Dexie.js to SQL-based implementation, follow these patterns:
+
+1. **Database Access Pattern**
+   ```typescript
+   // Before (Dexie)
+   const result = await db.table.where("field").equals(value).first();
+   
+   // After (SQL)
+   const platform = PlatformServiceFactory.getInstance();
+   const result = await platform.dbQuery(
+     "SELECT * FROM table WHERE field = ?",
+     [value]
+   );
+   ```
+
+2. **Update Operations**
+   ```typescript
+   // Before (Dexie)
+   await db.table.where("id").equals(id).modify(changes);
+   
+   // After (SQL)
+   const { sql, params } = generateUpdateStatement(
+     changes,
+     "table",
+     "id = ?",
+     [id]
+   );
+   await platform.dbExec(sql, params);
+   ```
+
+3. **Insert Operations**
+   ```typescript
+   // Before (Dexie)
+   await db.table.add(item);
+   
+   // After (SQL)
+   const columns = Object.keys(item);
+   const values = Object.values(item);
+   const placeholders = values.map(() => '?').join(', ');
+   const sql = `INSERT INTO table (${columns.join(', ')}) VALUES (${placeholders})`;
+   await platform.dbExec(sql, values);
+   ```
+
+4. **Delete Operations**
+   ```typescript
+   // Before (Dexie)
+   await db.table.where("id").equals(id).delete();
+   
+   // After (SQL)
+   await platform.dbExec("DELETE FROM table WHERE id = ?", [id]);
+   ```
+
+5. **Result Processing**
+   ```typescript
+   // Before (Dexie)
+   const items = await db.table.toArray();
+   
+   // After (SQL)
+   const result = await platform.dbQuery("SELECT * FROM table");
+   const items = mapColumnsToValues(result.columns, result.values);
+   ```
+
+Key Considerations:
+1. Always use parameterized queries to prevent SQL injection
+2. Use the `generateUpdateStatement` helper for update operations
+3. Use the `mapColumnsToValues` helper for processing query results
+4. Handle transactions explicitly for batch operations
+5. Use appropriate error handling with the StorageError class
+6. Consider platform-specific capabilities when implementing features
+
+Example Migration:
+```typescript
+// Before (Dexie)
+export async function updateSettings(settings: Settings): Promise<void> {
+  await db.settings.put(settings);
+}
+
+// After (SQL)
+export async function updateSettings(settings: Settings): Promise<void> {
+  const platform = PlatformServiceFactory.getInstance();
+  const { sql, params } = generateUpdateStatement(
+    settings,
+    "settings",
+    "id = ?",
+    [settings.id]
+  );
+  await platform.dbExec(sql, params);
+}
+```
+
+Remember to:
+- Create database access code to use the platform service, putting it in front of the Dexie version
+- Instead of removing Dexie-specific code, keep it; if we use the results of the query, then check the USE_DEXIE_DB from app.ts and if it's true then use that instead of the SQL code. 
+- Test thoroughly after migration
+- Consider data migration needs, and warn if there are any potential migration problems
+
 ### 3. Platform Detection

 ```typescript
--- a/src/constants/app.ts
+++ b/src/constants/app.ts
@@ -50,6 +50,8 @@ export const IMAGE_TYPE_PROFILE = "profile";
 export const PASSKEYS_ENABLED =
  !!import.meta.env.VITE_PASSKEYS_ENABLED || false;

+export const USE_DEXIE_DB = true;
+
 /**
 * The possible values for "group" and "type" are in App.vue.
 * Some of this comes from the notiwind package, some is custom.
--- a/src/db-sql/migration.ts
+++ b/src/db-sql/migration.ts
@@ -1,5 +1,31 @@
 import migrationService from "../services/migrationService";
 import type { QueryExecResult, SqlValue } from "../interfaces/database";
+import { DEFAULT_ENDORSER_API_SERVER } from "@/constants/app";
+
+// Generate a random secret for the secret table
+
+// It's not really secure to maintain the secret next to the user's data.
+// However, until we have better hooks into a real wallet or reliable secure
+// storage, we'll do this for user convenience. As they sign more records
+// and integrate with more people, they'll value it more and want to be more
+// secure, so we'll prompt them to take steps to back it up, properly encrypt,
+// etc. At the beginning, we'll prompt for a password, then we'll prompt for a
+// PWA so it's not in a browser... and then we hope to be integrated with a
+// real wallet or something else more secure.
+
+// One might ask: why encrypt at all? We figure a basic encryption is better
+// than none. Plus, we expect to support their own password or keystore or
+// external wallet as better signing options in the future, so it's gonna be
+// important to have the structure where each account access might require
+// user action.
+
+// (Once upon a time we stored the secret in localStorage, but it frequently
+// got erased, even though the IndexedDB still had the identity data. This
+// ended up throwing lots of errors to the user... and they'd end up in a state
+// where they couldn't take action because they couldn't unlock that identity.)
+
+const randomBytes = crypto.getRandomValues(new Uint8Array(32));
+const secret = btoa(String.fromCharCode(...randomBytes));

 // Each migration can include multiple SQL statements (with semicolons)
 const MIGRATIONS = [
@@ -12,8 +38,8 @@ const MIGRATIONS = [
        dateCreated TEXT NOT NULL,
        derivationPath TEXT,
        did TEXT NOT NULL,
-        identity TEXT,
-        mnemonic TEXT,
+        identityEncrBase64 TEXT, -- encrypted & base64-encoded
+        mnemonicEncrBase64 TEXT, -- encrypted & base64-encoded
        passkeyCredIdHex TEXT,
        publicKeyHex TEXT NOT NULL
      );
@@ -22,9 +48,11 @@ const MIGRATIONS = [

      CREATE TABLE IF NOT EXISTS secret (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
-        secret TEXT NOT NULL
+        secretBase64 TEXT NOT NULL
      );

+      INSERT INTO secret (id, secretBase64) VALUES (1, '${secret}');
+
      CREATE TABLE IF NOT EXISTS settings (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        accountDid TEXT,
@@ -59,6 +87,8 @@ const MIGRATIONS = [

      CREATE INDEX IF NOT EXISTS idx_settings_accountDid ON settings(accountDid);

+      INSERT INTO settings (id, apiServer) VALUES (1, '${DEFAULT_ENDORSER_API_SERVER}');
+
      CREATE TABLE IF NOT EXISTS contacts (
        id INTEGER PRIMARY KEY AUTOINCREMENT,
        did TEXT NOT NULL,
--- a/src/db/databaseUtil.ts
+++ b/src/db/databaseUtil.ts
@@ -0,0 +1,196 @@
+import { PlatformServiceFactory } from "@/services/PlatformServiceFactory";
+import { MASTER_SETTINGS_KEY, Settings } from "./tables/settings";
+import { logger } from "@/utils/logger";
+import { DEFAULT_ENDORSER_API_SERVER } from "@/constants/app";
+
+export async function updateDefaultSettings(
+  settingsChanges: Settings,
+): Promise<boolean> {
+  delete settingsChanges.accountDid; // just in case
+  // ensure there is no "id" that would override the key
+  delete settingsChanges.id;
+  try {
+    const platformService = PlatformServiceFactory.getInstance();
+    const { sql, params } = generateUpdateStatement(settingsChanges, "settings", "id = ?", [MASTER_SETTINGS_KEY]);
+    const result = await platformService.dbExec(sql, params);
+    return result.changes === 1;
+  } catch (error) {
+    logger.error("Error updating default settings:", error);
+    if (error instanceof Error) {
+      throw error; // Re-throw if it's already an Error with a message
+    } else {
+      throw new Error(
+        `Failed to update settings. We recommend you try again or restart the app.`,
+      );
+    }
+  }
+}
+
+const DEFAULT_SETTINGS: Settings = {
+  id: MASTER_SETTINGS_KEY,
+  activeDid: undefined,
+  apiServer: DEFAULT_ENDORSER_API_SERVER,
+};
+
+// retrieves default settings
+export async function retrieveSettingsForDefaultAccount(): Promise<Settings> {
+  const platform = PlatformServiceFactory.getInstance();
+  const result = await platform.dbQuery("SELECT * FROM settings WHERE id = ?", MASTER_SETTINGS_KEY)
+  if (!result) {
+    return DEFAULT_SETTINGS;
+  } else {
+    return mapColumnsToValues(result.columns, result.values)[0] as Settings;
+  }
+}
+
+export async function retrieveSettingsForActiveAccount(): Promise<Settings> {
+  const defaultSettings = await retrieveSettingsForDefaultAccount();
+  if (!defaultSettings.activeDid) {
+    return defaultSettings;
+  } else {
+    const platform = PlatformServiceFactory.getInstance();
+    const result = await platform.dbQuery(
+      "SELECT * FROM settings WHERE accountDid = ?",
+      [defaultSettings.activeDid]
+    );
+    const overrideSettings = result ? mapColumnsToValues(result.columns, result.values)[0] as Settings : {};
+    return { ...defaultSettings, ...overrideSettings };
+  }
+}
+
+export async function updateAccountSettings(
+  accountDid: string,
+  settingsChanges: Settings,
+): Promise<boolean> {
+  settingsChanges.accountDid = accountDid;
+  delete settingsChanges.id; // key off account, not ID
+
+  const platform = PlatformServiceFactory.getInstance();
+  
+  // First try to update existing record
+  const { sql: updateSql, params: updateParams } = generateUpdateStatement(
+    settingsChanges,
+    "settings",
+    "accountDid = ?",
+    [accountDid]
+  );
+  
+  const updateResult = await platform.dbExec(updateSql, updateParams);
+  
+  // If no record was updated, insert a new one
+  if (updateResult.changes === 1) {
+    return true;
+  } else {
+    const columns = Object.keys(settingsChanges);
+    const values = Object.values(settingsChanges);
+    const placeholders = values.map(() => '?').join(', ');
+    
+    const insertSql = `INSERT INTO settings (${columns.join(', ')}) VALUES (${placeholders})`;
+    const result = await platform.dbExec(insertSql, values);
+    
+    return result.changes === 1;
+  }
+}
+
+export async function logToDb(message: string): Promise<void> {
+  const platform = PlatformServiceFactory.getInstance();
+  const todayKey = new Date().toDateString();
+  
+  // Check if we have any logs for today
+  const result = await platform.dbQuery(
+    "SELECT message FROM logs WHERE date = ?",
+    [todayKey]
+  );
+  
+  if (!result || result.values.length === 0) {
+    // If no logs for today, clear all previous logs
+    await platform.dbExec("DELETE FROM logs");
+    
+    // Insert new log
+    const fullMessage = `${new Date().toISOString()} ${message}`;
+    await platform.dbExec(
+      "INSERT INTO logs (date, message) VALUES (?, ?)",
+      [todayKey, fullMessage]
+    );
+  } else {
+    // Append to existing log
+    const prevMessages = result.values[0][0] as string;
+    const fullMessage = `${prevMessages}\n${new Date().toISOString()} ${message}`;
+    
+    await platform.dbExec(
+      "UPDATE logs SET message = ? WHERE date = ?",
+      [fullMessage, todayKey]
+    );
+  }
+}
+
+// similar method is in the sw_scripts/additional-scripts.js file
+export async function logConsoleAndDb(
+  message: string,
+  isError = false,
+): Promise<void> {
+  if (isError) {
+    logger.error(`${new Date().toISOString()} ${message}`);
+  } else {
+    logger.log(`${new Date().toISOString()} ${message}`);
+  }
+  await logToDb(message);
+}
+
+/**
+ * Generates an SQL UPDATE statement and parameters from a model object.
+ * @param model The model object containing fields to update
+ * @param tableName The name of the table to update
+ * @param whereClause The WHERE clause for the update (e.g. "id = ?")
+ * @param whereParams Parameters for the WHERE clause
+ * @returns Object containing the SQL statement and parameters array
+ */
+function generateUpdateStatement(
+  model: Record<string, any>,
+  tableName: string,
+  whereClause: string,
+  whereParams: any[] = []
+): { sql: string; params: any[] } {
+  // Filter out undefined/null values and create SET clause
+  const setClauses: string[] = [];
+  const params: any[] = [];
+
+  Object.entries(model).forEach(([key, value]) => {
+    if (value !== undefined) {
+      setClauses.push(`${key} = ?`);
+      params.push(value);
+    }
+  });
+
+  if (setClauses.length === 0) {
+    throw new Error('No valid fields to update');
+  }
+
+  const sql = `UPDATE ${tableName} SET ${setClauses.join(', ')} WHERE ${whereClause}`;
+  
+  return {
+    sql,
+    params: [...params, ...whereParams]
+  };
+}
+
+/**
+ * Maps an array of column names to an array of value arrays, creating objects where each column name
+ * is mapped to its corresponding value.
+ * @param columns Array of column names to use as object keys
+ * @param values Array of value arrays, where each inner array corresponds to one row of data
+ * @returns Array of objects where each object maps column names to their corresponding values
+ */
+export function mapColumnsToValues(
+  columns: string[],
+  values: any[][]
+): Record<string, any>[] {
+  return values.map(row => {
+    const obj: Record<string, any> = {};
+    columns.forEach((column, index) => {
+      obj[column] = row[index];
+    });
+    return obj;
+  });
+}
+
--- a/src/db/index.ts
+++ b/src/db/index.ts
@@ -26,8 +26,8 @@ type NonsensitiveTables = {
 };

 // Using 'unknown' instead of 'any' for stricter typing and to avoid TypeScript warnings
-export type SecretDexie<T extends unknown = SecretTable> = BaseDexie & T;
-export type SensitiveDexie<T extends unknown = SensitiveTables> = BaseDexie & T;
+type SecretDexie<T extends unknown = SecretTable> = BaseDexie & T;
+type SensitiveDexie<T extends unknown = SensitiveTables> = BaseDexie & T;
 export type NonsensitiveDexie<T extends unknown = NonsensitiveTables> =
  BaseDexie & T;

--- a/src/libs/crypto/index.ts
+++ b/src/libs/crypto/index.ts
@@ -273,7 +273,7 @@ export async function decryptMessage(encryptedJson: string, password: string) {
 }

 // Test function to verify encryption/decryption
-export async function testEncryptionDecryption() {
+export async function testMessageEncryptionDecryption() {
  try {
    const testMessage = "Hello, this is a test message! 🚀";
    const testPassword = "myTestPassword123";
@@ -299,9 +299,102 @@ export async function testEncryptionDecryption() {
    logger.log("\nTesting with wrong password...");
    try {
      await decryptMessage(encrypted, "wrongPassword");
-      logger.log("Should not reach here");
+      logger.log("Incorrectly decrypted with wrong password ❌");
    } catch (error) {
-      logger.log("Correctly failed with wrong password ✅");
+      logger.log("Correctly failed to decrypt with wrong password ✅");
+    }
+
+    return success;
+  } catch (error) {
+    logger.error("Test failed with error:", error);
+    return false;
+  }
+}
+
+// Simple encryption/decryption using Node's crypto
+export async function simpleEncrypt(text: string, secret: string): Promise<string> {
+  const iv = crypto.getRandomValues(new Uint8Array(16));
+  
+  // Derive a 256-bit key from the secret using SHA-256
+  const keyData = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(secret));
+  const key = await crypto.subtle.importKey(
+    'raw',
+    keyData,
+    { name: 'AES-GCM' },
+    false,
+    ['encrypt']
+  );
+
+  const encrypted = await crypto.subtle.encrypt(
+    { name: 'AES-GCM', iv },
+    key,
+    new TextEncoder().encode(text)
+  );
+
+  // Combine IV and encrypted data
+  const result = new Uint8Array(iv.length + encrypted.byteLength);
+  result.set(iv);
+  result.set(new Uint8Array(encrypted), iv.length);
+
+  return btoa(String.fromCharCode(...result));
+}
+
+export async function simpleDecrypt(encryptedText: string, secret: string): Promise<string> {
+  const data = Uint8Array.from(atob(encryptedText), c => c.charCodeAt(0));
+  
+  // Extract IV and encrypted data
+  const iv = data.slice(0, 16);
+  const encrypted = data.slice(16);
+
+  // Derive the same 256-bit key from the secret using SHA-256
+  const keyData = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(secret));
+  const key = await crypto.subtle.importKey(
+    'raw',
+    keyData,
+    { name: 'AES-GCM' },
+    false,
+    ['decrypt']
+  );
+
+  const decrypted = await crypto.subtle.decrypt(
+    { name: 'AES-GCM', iv },
+    key,
+    encrypted
+  );
+
+  return new TextDecoder().decode(decrypted);
+}
+
+// Test function for simple encryption/decryption
+export async function testSimpleEncryptionDecryption() {
+  try {
+    const testMessage = "Hello, this is a test message! 🚀";
+    const testSecret = "myTestSecret123";
+
+    logger.log("Original message:", testMessage);
+
+    // Test encryption
+    logger.log("Encrypting...");
+    const encrypted = await simpleEncrypt(testMessage, testSecret);
+    logger.log("Encrypted result:", encrypted);
+
+    // Test decryption
+    logger.log("Decrypting...");
+    const decrypted = await simpleDecrypt(encrypted, testSecret);
+    logger.log("Decrypted result:", decrypted);
+
+    // Verify
+    const success = testMessage === decrypted;
+    logger.log("Test " + (success ? "PASSED ✅" : "FAILED ❌"));
+    logger.log("Messages match:", success);
+
+    // Test with wrong secret
+    logger.log("\nTesting with wrong secret...");
+    try {
+      await simpleDecrypt(encrypted, "wrongSecret");
+      logger.log("Incorrectly decrypted with wrong secret ❌");
+    } catch (error) {
+      logger.log("Correctly failed to decrypt with wrong secret ✅");
    }

    return success;
--- a/src/libs/util.ts
+++ b/src/libs/util.ts
@@ -5,7 +5,7 @@ import { Buffer } from "buffer";
 import * as R from "ramda";
 import { useClipboard } from "@vueuse/core";

-import { DEFAULT_PUSH_SERVER, NotificationIface } from "../constants/app";
+import { DEFAULT_PUSH_SERVER, NotificationIface, USE_DEXIE_DB } from "../constants/app";
 import {
  accountsDBPromise,
  retrieveSettingsForActiveAccount,
@@ -14,6 +14,7 @@ import {
 } from "../db/index";
 import { Account } from "../db/tables/accounts";
 import { Contact } from "../db/tables/contacts";
+import * as databaseUtil from "../db/databaseUtil";
 import { DEFAULT_PASSKEY_EXPIRATION_MINUTES } from "../db/tables/settings";
 import { deriveAddress, generateSeed, newIdentifier } from "../libs/crypto";
 import * as serverUtil from "../libs/endorserServer";
@@ -541,16 +542,6 @@ export const generateSaveAndActivateIdentity = async (): Promise<string> => {

  // one of the few times we use accountsDBPromise directly; try to avoid more usage
  try {
-    const accountsDB = await accountsDBPromise;
-    await accountsDB.accounts.add({
-      dateCreated: new Date().toISOString(),
-      derivationPath: derivationPath,
-      did: newId.did,
-      identity: identity,
-      mnemonic: mnemonic,
-      publicKeyHex: newId.keys[0].publicKeyHex,
-    });
-
    // add to the new sql db
    const platformService = PlatformServiceFactory.getInstance();
    await platformService.dbExec(
@@ -565,15 +556,31 @@ export const generateSaveAndActivateIdentity = async (): Promise<string> => {
        newId.keys[0].publicKeyHex,
      ],
    );
+    await databaseUtil.updateDefaultSettings({ activeDid: newId.did });
+
+    if (USE_DEXIE_DB) {
+      const accountsDB = await accountsDBPromise;
+      await accountsDB.accounts.add({
+        dateCreated: new Date().toISOString(),
+        derivationPath: derivationPath,
+        did: newId.did,
+        identity: identity,
+        mnemonic: mnemonic,
+        publicKeyHex: newId.keys[0].publicKeyHex,
+      });

      await updateDefaultSettings({ activeDid: newId.did });
+    }
  } catch (error) {
    logger.error("Failed to update default settings:", error);
    throw new Error(
      "Failed to set default settings. Please try again or restart the app.",
    );
  }
+  await databaseUtil.updateAccountSettings(newId.did, { isRegistered: false });
+  if (USE_DEXIE_DB) {
    await updateAccountSettings(newId.did, { isRegistered: false });
+  }
  return newId.did;
 };

--- a/src/views/TestView.vue
+++ b/src/views/TestView.vue
@@ -285,11 +285,20 @@
      <div>
        <button
          class="font-bold capitalize bg-slate-500 text-white px-3 py-2 rounded-md mr-2"
-          @click="testEncryptionDecryption()"
+          @click="testMessageEncryptionDecryption()"
        >
-          Run Test
+          Run Test for Message Encryption/Decryption
        </button>
-        Result: {{ encryptionTestResult }}
+        Result: {{ messageEncryptionTestResult }}
+      </div>
+      <div>
+        <button
+          class="font-bold capitalize bg-slate-500 text-white px-3 py-2 rounded-md mr-2"
+          @click="testSimpleEncryptionDecryption()"
+        >
+          Run Test for Simple Encryption/Decryption
+        </button>
+        Result: {{ simpleEncryptionTestResult }}
      </div>
    </div>
  </section>
@@ -341,7 +350,8 @@ export default class Help extends Vue {
  $router!: Router;

  // for encryption/decryption
-  encryptionTestResult?: boolean;
+  messageEncryptionTestResult?: boolean;
+  simpleEncryptionTestResult?: boolean;

  // for file import
  fileName?: string;
@@ -434,8 +444,12 @@ export default class Help extends Vue {
    this.credIdHex = account.passkeyCredIdHex;
  }

-  public async testEncryptionDecryption() {
-    this.encryptionTestResult = await cryptoLib.testEncryptionDecryption();
+  public async testMessageEncryptionDecryption() {
+    this.messageEncryptionTestResult = await cryptoLib.testMessageEncryptionDecryption();
+  }
+
+  public async testSimpleEncryptionDecryption() {
+    this.simpleEncryptionTestResult = await cryptoLib.testSimpleEncryptionDecryption();
  }

  public async createJwtSimplewebauthn() {