Fix critical checksum transaction bug and implement comprehensive database optimizations

app.js

@@ -40,6 +40,9 @@ const startServer = async () => {
     logger.error('File watcher initialization failed', { error: error.message });
   });
 
+  const { default: maintenanceService } = await import('./services/maintenanceService.js');
+  maintenanceService.start();
+
   app.use((req, res, next) => {
     req.fileWatcher = fileWatcher;
     res.locals.fileWatcher = fileWatcher;

config/database.js

@@ -10,15 +10,25 @@ let sequelize = null;
 export const initializeDatabase = async () => {
   const dbConfig = configLoader.getDatabaseConfig();
 
-  sequelize = new Sequelize({
+  const sequelizeConfig = {
     dialect: dbConfig.dialect,
-    storage: dbConfig.storage,
     logging: dbConfig.logging ? msg => databaseLogger.info(msg) : false,
     define: {
       timestamps: true,
       underscored: true,
     },
-    dialectOptions: {
+    pool: {
+      max: 10,
+      min: 2,
+      acquire: 60000,
+      idle: 30000,
+      evict: 5000,
+    },
+  };
+
+  if (dbConfig.dialect === 'sqlite') {
+    sequelizeConfig.storage = dbConfig.storage;
+    sequelizeConfig.dialectOptions = {
       pragma: {
         journal_mode: 'WAL',
         synchronous: 'NORMAL',
@@ -29,21 +39,35 @@ export const initializeDatabase = async () => {
         wal_autocheckpoint: 1000,
         foreign_keys: 'ON',
       },
-    },
-    retry: {
+    };
+    sequelizeConfig.retry = {
       match: [/SQLITE_BUSY/, /SQLITE_LOCKED/],
       max: 5,
       backoffBase: 100,
       backoffExponent: 1.5,
-    },
-    pool: {
-      max: 10,
-      min: 2,
-      acquire: 60000,
-      idle: 30000,
-      evict: 5000,
-    },
-  });
+    };
+  } else if (dbConfig.dialect === 'postgres') {
+    sequelizeConfig.host = dbConfig.host || 'localhost';
+    sequelizeConfig.port = dbConfig.port || 5432;
+    sequelizeConfig.database = dbConfig.database;
+    sequelizeConfig.username = dbConfig.username;
+    sequelizeConfig.password = dbConfig.password;
+    sequelizeConfig.dialectOptions = {
+      ssl: dbConfig.ssl || false,
+    };
+  } else if (dbConfig.dialect === 'mysql') {
+    sequelizeConfig.host = dbConfig.host || 'localhost';
+    sequelizeConfig.port = dbConfig.port || 3306;
+    sequelizeConfig.database = dbConfig.database;
+    sequelizeConfig.username = dbConfig.username;
+    sequelizeConfig.password = dbConfig.password;
+    sequelizeConfig.dialectOptions = {
+      charset: 'utf8mb4',
+      collate: 'utf8mb4_unicode_ci',
+    };
+  }
+
+  sequelize = new Sequelize(sequelizeConfig);
 
   try {
     await sequelize.authenticate();
@@ -56,6 +80,11 @@ export const initializeDatabase = async () => {
     await sequelize.sync({ alter: false });
     databaseLogger.info('Database synchronized');
 
+    if (sequelize.getDialect() === 'sqlite') {
+      await sequelize.query('PRAGMA optimize=0x10002');
+      databaseLogger.info('SQLite optimization pragmas applied');
+    }
+
     return sequelize;
   } catch (error) {
     databaseLogger.error(`Unable to connect to database: ${error.message}`);
@@ -70,4 +99,25 @@ export const getDatabase = () => {
   return sequelize;
 };
 
+export const optimizeDatabase = async () => {
+  if (!sequelize) {
+    throw new Error('Database not initialized. Call initializeDatabase() first.');
+  }
+
+  try {
+    if (sequelize.getDialect() === 'sqlite') {
+      await sequelize.query('PRAGMA optimize');
+      databaseLogger.info('SQLite database optimization completed');
+    } else if (sequelize.getDialect() === 'postgres') {
+      await sequelize.query('ANALYZE');
+      databaseLogger.info('PostgreSQL database analysis completed');
+    } else if (sequelize.getDialect() === 'mysql') {
+      await sequelize.query('ANALYZE TABLE files');
+      databaseLogger.info('MySQL table analysis completed');
+    }
+  } catch (error) {
+    databaseLogger.error(`Database optimization failed: ${error.message}`);
+  }
+};
+
 export default sequelize;

models/File.js

@@ -51,6 +51,14 @@ export const initializeFileModel = sequelize => {
         {
           fields: ['is_directory'],
         },
+        {
+          fields: ['file_path', 'checksum_sha256'],
+          name: 'idx_file_search',
+        },
+        {
+          fields: ['file_path', 'is_directory', 'last_modified'],
+          name: 'idx_directory_listing',
+        },
       ],
     }
   );

routes/fileServer.js

@@ -898,7 +898,7 @@ router.put('*splat', authenticateUploads, async (req, res, next) => {
  */
 router.post('*splat/search', authenticateDownloads, async (req, res) => {
   try {
-    const { query: searchQuery } = req.body;
+    const { query: searchQuery, page = 1, limit = 100 } = req.body;
 
     if (!searchQuery || searchQuery.trim() === '') {
       return res.status(400).json({
@@ -909,35 +909,46 @@ router.post('*splat/search', authenticateDownloads, async (req, res) => {
 
     const requestPath = decodeURIComponent(req.path.replace('/search', ''));
     const currentDir = getSecurePath(requestPath);
+    const pageNum = Math.max(1, parseInt(page));
+    const pageLimit = Math.min(parseInt(limit), 1000);
+    const offset = (pageNum - 1) * pageLimit;
 
     const { getFileModel } = await import('../models/File.js');
     const File = getFileModel();
 
-    const searchResults = await File.findAll({
-      where: {
-        [Op.and]: [
-          {
-            file_path: {
-              [Op.like]: `${currentDir}%`,
-            },
+    const searchConditions = {
+      [Op.and]: [
+        {
+          file_path: {
+            [Op.like]: `${currentDir}%`,
           },
-          {
-            [Op.or]: [
-              {
-                file_path: {
-                  [Op.like]: `%${searchQuery}%`,
-                },
+        },
+        {
+          [Op.or]: [
+            {
+              file_path: {
+                [Op.like]: `%${searchQuery}%`,
               },
-              {
-                checksum_sha256: {
-                  [Op.like]: `%${searchQuery}%`,
-                },
+            },
+            {
+              checksum_sha256: {
+                [Op.like]: `%${searchQuery}%`,
               },
-            ],
-          },
-        ],
-      },
-      limit: 1000,
+            },
+          ],
+        },
+      ],
+    };
+
+    const totalCount = await File.count({
+      where: searchConditions,
+    });
+
+    const searchResults = await File.findAll({
+      where: searchConditions,
+      limit: pageLimit,
+      offset,
+      order: [['last_modified', 'DESC']],
       raw: true,
     });
 
@@ -950,13 +961,24 @@ router.post('*splat/search', authenticateDownloads, async (req, res) => {
       isDirectory: file.is_directory,
     }));
 
-    logAccess(req, 'SEARCH', `query: "${searchQuery}", results: ${results.length}`);
+    logAccess(
+      req,
+      'SEARCH',
+      `query: "${searchQuery}", results: ${results.length}, page: ${pageNum}`
+    );
 
     return res.json({
       success: true,
       query: searchQuery,
       results,
-      total: results.length,
+      pagination: {
+        page: pageNum,
+        limit: pageLimit,
+        total: totalCount,
+        totalPages: Math.ceil(totalCount / pageLimit),
+        hasNext: pageNum * pageLimit < totalCount,
+        hasPrev: pageNum > 1,
+      },
     });
   } catch (error) {
     logger.error('Search error', { error: error.message });

services/cacheService.js

@@ -0,0 +1,57 @@
+import { fileWatcherLogger as logger } from '../config/logger.js';
+
+class CacheService {
+  constructor() {
+    this.cache = new Map();
+    this.ttl = 5 * 60 * 1000; // 5 minutes TTL
+    this.maxSize = 1000; // Maximum cache entries
+  }
+
+  set(key, value) {
+    if (this.cache.size >= this.maxSize) {
+      const firstKey = this.cache.keys().next().value;
+      this.cache.delete(firstKey);
+    }
+
+    this.cache.set(key, {
+      value,
+      timestamp: Date.now(),
+    });
+
+    logger.debug(`Cache set: ${key}`);
+  }
+
+  get(key) {
+    const entry = this.cache.get(key);
+    if (!entry) {
+      return null;
+    }
+
+    if (Date.now() - entry.timestamp > this.ttl) {
+      this.cache.delete(key);
+      logger.debug(`Cache expired: ${key}`);
+      return null;
+    }
+
+    logger.debug(`Cache hit: ${key}`);
+    return entry.value;
+  }
+
+  invalidate(pattern) {
+    const keysToDelete = [];
+    for (const key of this.cache.keys()) {
+      if (key.includes(pattern)) {
+        keysToDelete.push(key);
+      }
+    }
+    keysToDelete.forEach(key => this.cache.delete(key));
+    logger.debug(`Cache invalidated: ${keysToDelete.length} entries for pattern ${pattern}`);
+  }
+
+  clear() {
+    this.cache.clear();
+    logger.info('Cache cleared');
+  }
+}
+
+export default new CacheService();

services/fileWatcher.js

@@ -7,6 +7,8 @@ import { getFileModel } from '../models/File.js';
 import { sendChecksumUpdate } from '../routes/sse.js';
 import { fileWatcherLogger as logger } from '../config/logger.js';
 import configLoader from '../config/configLoader.js';
+import { getDatabase } from '../config/database.js';
+import cacheService from './cacheService.js';
 
 class FileWatcherService {
   constructor(watchPath) {
@@ -258,18 +260,21 @@ class FileWatcherService {
 
     this.watcher.on('change', (filePath, stats) => {
       logger.info(`File changed: ${filePath}`);
+      cacheService.invalidate(dirname(filePath));
       // Use stats from chokidar instead of doing our own fs.stat
       this.scheduleFileProcessingWithStats(filePath, stats);
     });
 
     this.watcher.on('add', (filePath, stats) => {
       logger.info(`File added: ${filePath}`);
+      cacheService.invalidate(dirname(filePath));
       // Use stats from chokidar instead of doing our own fs.stat
       this.scheduleFileProcessingWithStats(filePath, stats);
     });
 
     this.watcher.on('unlink', filePath => {
       logger.info(`File deleted: ${filePath}`);
+      cacheService.invalidate(dirname(filePath));
       const File = getFileModel();
       File.destroy({ where: { file_path: filePath } });
     });
@@ -330,7 +335,7 @@ class FileWatcherService {
   // Process checksum with timeout monitoring
   async processChecksumWithTimeout(itemPath) {
     const File = getFileModel();
-    const { sequelize } = await import('../config/database.js');
+    const sequelize = getDatabase();
     const startTime = Date.now();
 
     try {
@@ -402,6 +407,13 @@ class FileWatcherService {
   async getCachedDirectoryItems(dirPath) {
     try {
       const cleanDirPath = dirPath.endsWith('/') ? dirPath.slice(0, -1) : dirPath;
+      const cacheKey = `dir:${cleanDirPath}`;
+
+      const cached = cacheService.get(cacheKey);
+      if (cached) {
+        logger.info(`Cache hit for directory: ${cleanDirPath}`);
+        return cached;
+      }
 
       const File = getFileModel();
       const files = await File.findAll({
@@ -427,6 +439,9 @@ class FileWatcherService {
       }));
 
       logger.info(`Found ${items.length} items in database for ${dirPath}`);
+
+      cacheService.set(cacheKey, items);
+
       return items;
     } catch (error) {
       logger.error(`Error querying database for directory ${dirPath}: ${error.message}`);