Merge branch 'sql-absurd-sql-back'

src/db/databaseUtil.ts

@@ -0,0 +1,330 @@
+/**
+ * This file is the SQL replacement of the index.ts file in the db directory.
+ * That file will eventually be deleted.
+ */
+
+import { PlatformServiceFactory } from "@/services/PlatformServiceFactory";
+import { MASTER_SETTINGS_KEY, Settings } from "./tables/settings";
+import { logger } from "@/utils/logger";
+import { DEFAULT_ENDORSER_API_SERVER } from "@/constants/app";
+import { QueryExecResult } from "@/interfaces/database";
+
+export async function updateDefaultSettings(
+  settingsChanges: Settings,
+): Promise<boolean> {
+  delete settingsChanges.accountDid; // just in case
+  // ensure there is no "id" that would override the key
+  delete settingsChanges.id;
+  try {
+    const platformService = PlatformServiceFactory.getInstance();
+    const { sql, params } = generateUpdateStatement(
+      settingsChanges,
+      "settings",
+      "id = ?",
+      [MASTER_SETTINGS_KEY],
+    );
+    const result = await platformService.dbExec(sql, params);
+    return result.changes === 1;
+  } catch (error) {
+    logger.error("Error updating default settings:", error);
+    if (error instanceof Error) {
+      throw error; // Re-throw if it's already an Error with a message
+    } else {
+      throw new Error(
+        `Failed to update settings. We recommend you try again or restart the app.`,
+      );
+    }
+  }
+}
+
+export async function updateAccountSettings(
+  accountDid: string,
+  settingsChanges: Settings,
+): Promise<boolean> {
+  settingsChanges.accountDid = accountDid;
+  delete settingsChanges.id; // key off account, not ID
+
+  const platform = PlatformServiceFactory.getInstance();
+
+  // First try to update existing record
+  const { sql: updateSql, params: updateParams } = generateUpdateStatement(
+    settingsChanges,
+    "settings",
+    "accountDid = ?",
+    [accountDid],
+  );
+
+  const updateResult = await platform.dbExec(updateSql, updateParams);
+
+  // If no record was updated, insert a new one
+  if (updateResult.changes === 1) {
+    return true;
+  } else {
+    const columns = Object.keys(settingsChanges);
+    const values = Object.values(settingsChanges);
+    const placeholders = values.map(() => "?").join(", ");
+
+    const insertSql = `INSERT INTO settings (${columns.join(", ")}) VALUES (${placeholders})`;
+    const result = await platform.dbExec(insertSql, values);
+
+    return result.changes === 1;
+  }
+}
+
+const DEFAULT_SETTINGS: Settings = {
+  id: MASTER_SETTINGS_KEY,
+  activeDid: undefined,
+  apiServer: DEFAULT_ENDORSER_API_SERVER,
+};
+
+// retrieves default settings
+export async function retrieveSettingsForDefaultAccount(): Promise<Settings> {
+  const platform = PlatformServiceFactory.getInstance();
+  const sql = "SELECT * FROM settings WHERE id = ?";
+  const result = await platform.dbQuery(sql, [MASTER_SETTINGS_KEY]);
+  if (!result) {
+    return DEFAULT_SETTINGS;
+  } else {
+    const settings = mapColumnsToValues(
+      result.columns,
+      result.values,
+    )[0] as Settings;
+    if (settings.searchBoxes) {
+      // @ts-expect-error - the searchBoxes field is a string in the DB
+      settings.searchBoxes = JSON.parse(settings.searchBoxes);
+    }
+    return settings;
+  }
+}
+
+/**
+ * Retrieves settings for the active account, merging with default settings
+ *
+ * @returns Promise<Settings> Combined settings with account-specific overrides
+ * @throws Will log specific errors for debugging but returns default settings on failure
+ */
+export async function retrieveSettingsForActiveAccount(): Promise<Settings> {
+  try {
+    // Get default settings first
+    const defaultSettings = await retrieveSettingsForDefaultAccount();
+
+    // If no active DID, return defaults
+    if (!defaultSettings.activeDid) {
+      logConsoleAndDb(
+        "[databaseUtil] No active DID found, returning default settings",
+      );
+      return defaultSettings;
+    }
+
+    // Get account-specific settings
+    try {
+      const platform = PlatformServiceFactory.getInstance();
+      const result = await platform.dbQuery(
+        "SELECT * FROM settings WHERE accountDid = ?",
+        [defaultSettings.activeDid],
+      );
+
+      if (!result?.values?.length) {
+        logConsoleAndDb(
+          `[databaseUtil] No account-specific settings found for ${defaultSettings.activeDid}`,
+        );
+        return defaultSettings;
+      }
+
+      // Map and filter settings
+      const overrideSettings = mapColumnsToValues(
+        result.columns,
+        result.values,
+      )[0] as Settings;
+      const overrideSettingsFiltered = Object.fromEntries(
+        Object.entries(overrideSettings).filter(([_, v]) => v !== null),
+      );
+
+      // Merge settings
+      const settings = { ...defaultSettings, ...overrideSettingsFiltered };
+
+      // Handle searchBoxes parsing
+      if (settings.searchBoxes) {
+        try {
+          // @ts-expect-error - the searchBoxes field is a string in the DB
+          settings.searchBoxes = JSON.parse(settings.searchBoxes);
+        } catch (error) {
+          logConsoleAndDb(
+            `[databaseUtil] Failed to parse searchBoxes for ${defaultSettings.activeDid}: ${error}`,
+            true,
+          );
+          // Reset to empty array on parse failure
+          settings.searchBoxes = [];
+        }
+      }
+
+      return settings;
+    } catch (error) {
+      logConsoleAndDb(
+        `[databaseUtil] Failed to retrieve account settings for ${defaultSettings.activeDid}: ${error}`,
+        true,
+      );
+      // Return defaults on error
+      return defaultSettings;
+    }
+  } catch (error) {
+    logConsoleAndDb(
+      `[databaseUtil] Failed to retrieve default settings: ${error}`,
+      true,
+    );
+    // Return minimal default settings on complete failure
+    return {
+      id: MASTER_SETTINGS_KEY,
+      activeDid: undefined,
+      apiServer: DEFAULT_ENDORSER_API_SERVER,
+    };
+  }
+}
+
+let lastCleanupDate: string | null = null;
+export let memoryLogs: string[] = [];
+
+/**
+ * Logs a message to the database with proper handling of concurrent writes
+ * @param message - The message to log
+ * @author Matthew Raymer
+ */
+export async function logToDb(message: string): Promise<void> {
+  const platform = PlatformServiceFactory.getInstance();
+  const todayKey = new Date().toDateString();
+  const nowKey = new Date().toISOString();
+
+  try {
+    memoryLogs.push(`${new Date().toISOString()} ${message}`);
+    // Try to insert first, if it fails due to UNIQUE constraint, update instead
+    await platform.dbExec("INSERT INTO logs (date, message) VALUES (?, ?)", [
+      nowKey,
+      message,
+    ]);
+
+    // Clean up old logs (keep only last 7 days) - do this less frequently
+    // Only clean up if the date is different from the last cleanup
+    if (!lastCleanupDate || lastCleanupDate !== todayKey) {
+      const sevenDaysAgo = new Date(
+        new Date().getTime() - 7 * 24 * 60 * 60 * 1000,
+      );
+      memoryLogs = memoryLogs.filter(
+        (log) => log.split(" ")[0] > sevenDaysAgo.toDateString(),
+      );
+      await platform.dbExec("DELETE FROM logs WHERE date < ?", [
+        sevenDaysAgo.toDateString(),
+      ]);
+      lastCleanupDate = todayKey;
+    }
+  } catch (error) {
+    // Log to console as fallback
+    // eslint-disable-next-line no-console
+    console.error(
+      "Error logging to database:",
+      error,
+      " ... for original message:",
+      message,
+    );
+  }
+}
+
+// similar method is in the sw_scripts/additional-scripts.js file
+export async function logConsoleAndDb(
+  message: string,
+  isError = false,
+): Promise<void> {
+  if (isError) {
+    logger.error(`${new Date().toISOString()} ${message}`);
+  } else {
+    logger.log(`${new Date().toISOString()} ${message}`);
+  }
+  await logToDb(message);
+}
+
+/**
+ * Generates an SQL INSERT statement and parameters from a model object.
+ * @param model The model object containing fields to update
+ * @param tableName The name of the table to update
+ * @returns Object containing the SQL statement and parameters array
+ */
+export function generateInsertStatement(
+  model: Record<string, unknown>,
+  tableName: string,
+): { sql: string; params: unknown[] } {
+  const columns = Object.keys(model).filter((key) => model[key] !== undefined);
+  const values = Object.values(model).filter((value) => value !== undefined);
+  const placeholders = values.map(() => "?").join(", ");
+  const insertSql = `INSERT INTO ${tableName} (${columns.join(", ")}) VALUES (${placeholders})`;
+  return {
+    sql: insertSql,
+    params: values,
+  };
+}
+
+/**
+ * Generates an SQL UPDATE statement and parameters from a model object.
+ * @param model The model object containing fields to update
+ * @param tableName The name of the table to update
+ * @param whereClause The WHERE clause for the update (e.g. "id = ?")
+ * @param whereParams Parameters for the WHERE clause
+ * @returns Object containing the SQL statement and parameters array
+ */
+export function generateUpdateStatement(
+  model: Record<string, unknown>,
+  tableName: string,
+  whereClause: string,
+  whereParams: unknown[] = [],
+): { sql: string; params: unknown[] } {
+  // Filter out undefined/null values and create SET clause
+  const setClauses: string[] = [];
+  const params: unknown[] = [];
+
+  Object.entries(model).forEach(([key, value]) => {
+    if (value !== undefined) {
+      setClauses.push(`${key} = ?`);
+      params.push(value);
+    }
+  });
+
+  if (setClauses.length === 0) {
+    throw new Error("No valid fields to update");
+  }
+
+  const sql = `UPDATE ${tableName} SET ${setClauses.join(", ")} WHERE ${whereClause}`;
+
+  return {
+    sql,
+    params: [...params, ...whereParams],
+  };
+}
+
+export function mapQueryResultToValues(
+  record: QueryExecResult | undefined,
+): Array<Record<string, unknown>> {
+  if (!record) {
+    return [];
+  }
+  return mapColumnsToValues(record.columns, record.values) as Array<
+    Record<string, unknown>
+  >;
+}
+
+/**
+ * Maps an array of column names to an array of value arrays, creating objects where each column name
+ * is mapped to its corresponding value.
+ * @param columns Array of column names to use as object keys
+ * @param values Array of value arrays, where each inner array corresponds to one row of data
+ * @returns Array of objects where each object maps column names to their corresponding values
+ */
+export function mapColumnsToValues(
+  columns: string[],
+  values: unknown[][],
+): Array<Record<string, unknown>> {
+  return values.map((row) => {
+    const obj: Record<string, unknown> = {};
+    columns.forEach((column, index) => {
+      obj[column] = row[index];
+    });
+    return obj;
+  });
+}

src/db/index.ts

@@ -1,3 +1,9 @@
+/**
+ * This is the original IndexedDB version of the database.
+ * It will eventually be replaced fully by the SQL version in databaseUtil.ts.
+ * Turn this on or off with the USE_DEXIE_DB constant in constants/app.ts.
+ */
+
 import BaseDexie, { Table } from "dexie";
 import { encrypted, Encryption } from "@pvermeer/dexie-encrypted-addon";
 import * as R from "ramda";
@@ -26,8 +32,8 @@ type NonsensitiveTables = {
 };
 
 // Using 'unknown' instead of 'any' for stricter typing and to avoid TypeScript warnings
-export type SecretDexie<T extends unknown = SecretTable> = BaseDexie & T;
-export type SensitiveDexie<T extends unknown = SensitiveTables> = BaseDexie & T;
+type SecretDexie<T extends unknown = SecretTable> = BaseDexie & T;
+type SensitiveDexie<T extends unknown = SensitiveTables> = BaseDexie & T;
 export type NonsensitiveDexie<T extends unknown = NonsensitiveTables> =
   BaseDexie & T;
 
@@ -90,7 +96,7 @@ db.on("populate", async () => {
   try {
     await db.settings.add(DEFAULT_SETTINGS);
   } catch (error) {
-    console.error(
+    logger.error(
       "Error populating the database with default settings:",
       error,
     );
@@ -99,12 +105,12 @@ db.on("populate", async () => {
 
 // Helper function to safely open the database with retries
 async function safeOpenDatabase(retries = 1, delay = 500): Promise<void> {
-  // console.log("Starting safeOpenDatabase with retries:", retries);
+  // logger.log("Starting safeOpenDatabase with retries:", retries);
   for (let i = 0; i < retries; i++) {
     try {
-      // console.log(`Attempt ${i + 1}: Checking if database is open...`);
+      // logger.log(`Attempt ${i + 1}: Checking if database is open...`);
       if (!db.isOpen()) {
-        // console.log(`Attempt ${i + 1}: Database is closed, attempting to open...`);
+        // logger.log(`Attempt ${i + 1}: Database is closed, attempting to open...`);
 
         // Create a promise that rejects after 5 seconds
         const timeoutPromise = new Promise((_, reject) => {
@@ -113,19 +119,19 @@ async function safeOpenDatabase(retries = 1, delay = 500): Promise<void> {
 
         // Race between the open operation and the timeout
         const openPromise = db.open();
-        // console.log(`Attempt ${i + 1}: Waiting for db.open() promise...`);
+        // logger.log(`Attempt ${i + 1}: Waiting for db.open() promise...`);
         await Promise.race([openPromise, timeoutPromise]);
 
         // If we get here, the open succeeded
-        // console.log(`Attempt ${i + 1}: Database opened successfully`);
+        // logger.log(`Attempt ${i + 1}: Database opened successfully`);
         return;
       }
-      // console.log(`Attempt ${i + 1}: Database was already open`);
+      // logger.log(`Attempt ${i + 1}: Database was already open`);
       return;
     } catch (error) {
-      console.error(`Attempt ${i + 1}: Database open failed:`, error);
+      logger.error(`Attempt ${i + 1}: Database open failed:`, error);
       if (i < retries - 1) {
-        console.log(`Attempt ${i + 1}: Waiting ${delay}ms before retry...`);
+        logger.log(`Attempt ${i + 1}: Waiting ${delay}ms before retry...`);
         await new Promise((resolve) => setTimeout(resolve, delay));
       } else {
         throw error;
@@ -142,16 +148,14 @@ export async function updateDefaultSettings(
   delete settingsChanges.id;
   try {
     try {
-      // console.log("Database state before open:", db.isOpen() ? "open" : "closed");
-      // console.log("Database name:", db.name);
-      // console.log("Database version:", db.verno);
+      // logger.log("Database state before open:", db.isOpen() ? "open" : "closed");
+      // logger.log("Database name:", db.name);
+      // logger.log("Database version:", db.verno);
       await safeOpenDatabase();
     } catch (openError: unknown) {
-      console.error("Failed to open database:", openError);
-      const errorMessage =
-        openError instanceof Error ? openError.message : String(openError);
+      logger.error("Failed to open database:", openError, String(openError));
       throw new Error(
-        `Database connection failed: ${errorMessage}. Please try again or restart the app.`,
+        `The database connection failed. We recommend you try again or restart the app.`,
       );
     }
     const result = await db.settings.update(
@@ -160,11 +164,13 @@ export async function updateDefaultSettings(
     );
     return result;
   } catch (error) {
-    console.error("Error updating default settings:", error);
+    logger.error("Error updating default settings:", error);
     if (error instanceof Error) {
       throw error; // Re-throw if it's already an Error with a message
     } else {
-      throw new Error(`Failed to update settings: ${error}`);
+      throw new Error(
+        `Failed to update settings. We recommend you try again or restart the app.`,
+      );
     }
   }
 }

src/db/tables/accounts.ts

@@ -45,6 +45,12 @@ export type Account = {
   publicKeyHex: string;
 };
 
+// When finished with USE_DEXIE_DB, move these fields to Account and move identity and mnemonic here.
+export type AccountEncrypted = Account & {
+  identityEncrBase64: string;
+  mnemonicEncrBase64: string;
+};
+
 /**
  * Schema for the accounts table in the database.
  * Fields starting with a $ character are encrypted.