diff --git a/app.js b/app.js index 912aa59..5877a18 100644 --- a/app.js +++ b/app.js @@ -40,6 +40,9 @@ const startServer = async () => { logger.error('File watcher initialization failed', { error: error.message }); }); + const { default: maintenanceService } = await import('./services/maintenanceService.js'); + maintenanceService.start(); + app.use((req, res, next) => { req.fileWatcher = fileWatcher; res.locals.fileWatcher = fileWatcher; diff --git a/config/database.js b/config/database.js index a841d3d..57682c9 100644 --- a/config/database.js +++ b/config/database.js @@ -10,15 +10,25 @@ let sequelize = null; export const initializeDatabase = async () => { const dbConfig = configLoader.getDatabaseConfig(); - sequelize = new Sequelize({ + const sequelizeConfig = { dialect: dbConfig.dialect, - storage: dbConfig.storage, logging: dbConfig.logging ? msg => databaseLogger.info(msg) : false, define: { timestamps: true, underscored: true, }, - dialectOptions: { + pool: { + max: 10, + min: 2, + acquire: 60000, + idle: 30000, + evict: 5000, + }, + }; + + if (dbConfig.dialect === 'sqlite') { + sequelizeConfig.storage = dbConfig.storage; + sequelizeConfig.dialectOptions = { pragma: { journal_mode: 'WAL', synchronous: 'NORMAL', @@ -29,21 +39,35 @@ export const initializeDatabase = async () => { wal_autocheckpoint: 1000, foreign_keys: 'ON', }, - }, - retry: { + }; + sequelizeConfig.retry = { match: [/SQLITE_BUSY/, /SQLITE_LOCKED/], max: 5, backoffBase: 100, backoffExponent: 1.5, - }, - pool: { - max: 10, - min: 2, - acquire: 60000, - idle: 30000, - evict: 5000, - }, - }); + }; + } else if (dbConfig.dialect === 'postgres') { + sequelizeConfig.host = dbConfig.host || 'localhost'; + sequelizeConfig.port = dbConfig.port || 5432; + sequelizeConfig.database = dbConfig.database; + sequelizeConfig.username = dbConfig.username; + sequelizeConfig.password = dbConfig.password; + sequelizeConfig.dialectOptions = { + ssl: dbConfig.ssl || false, + }; + } else if (dbConfig.dialect === 'mysql') { + sequelizeConfig.host = dbConfig.host || 'localhost'; + sequelizeConfig.port = dbConfig.port || 3306; + sequelizeConfig.database = dbConfig.database; + sequelizeConfig.username = dbConfig.username; + sequelizeConfig.password = dbConfig.password; + sequelizeConfig.dialectOptions = { + charset: 'utf8mb4', + collate: 'utf8mb4_unicode_ci', + }; + } + + sequelize = new Sequelize(sequelizeConfig); try { await sequelize.authenticate(); @@ -56,6 +80,11 @@ export const initializeDatabase = async () => { await sequelize.sync({ alter: false }); databaseLogger.info('Database synchronized'); + if (sequelize.getDialect() === 'sqlite') { + await sequelize.query('PRAGMA optimize=0x10002'); + databaseLogger.info('SQLite optimization pragmas applied'); + } + return sequelize; } catch (error) { databaseLogger.error(`Unable to connect to database: ${error.message}`); @@ -70,4 +99,25 @@ export const getDatabase = () => { return sequelize; }; +export const optimizeDatabase = async () => { + if (!sequelize) { + throw new Error('Database not initialized. Call initializeDatabase() first.'); + } + + try { + if (sequelize.getDialect() === 'sqlite') { + await sequelize.query('PRAGMA optimize'); + databaseLogger.info('SQLite database optimization completed'); + } else if (sequelize.getDialect() === 'postgres') { + await sequelize.query('ANALYZE'); + databaseLogger.info('PostgreSQL database analysis completed'); + } else if (sequelize.getDialect() === 'mysql') { + await sequelize.query('ANALYZE TABLE files'); + databaseLogger.info('MySQL table analysis completed'); + } + } catch (error) { + databaseLogger.error(`Database optimization failed: ${error.message}`); + } +}; + export default sequelize; diff --git a/models/File.js b/models/File.js index 6a30e05..62695ec 100644 --- a/models/File.js +++ b/models/File.js @@ -51,6 +51,14 @@ export const initializeFileModel = sequelize => { { fields: ['is_directory'], }, + { + fields: ['file_path', 'checksum_sha256'], + name: 'idx_file_search', + }, + { + fields: ['file_path', 'is_directory', 'last_modified'], + name: 'idx_directory_listing', + }, ], } ); diff --git a/routes/fileServer.js b/routes/fileServer.js index 8c6543c..ef17afa 100644 --- a/routes/fileServer.js +++ b/routes/fileServer.js @@ -898,7 +898,7 @@ router.put('*splat', authenticateUploads, async (req, res, next) => { */ router.post('*splat/search', authenticateDownloads, async (req, res) => { try { - const { query: searchQuery } = req.body; + const { query: searchQuery, page = 1, limit = 100 } = req.body; if (!searchQuery || searchQuery.trim() === '') { return res.status(400).json({ @@ -909,35 +909,46 @@ router.post('*splat/search', authenticateDownloads, async (req, res) => { const requestPath = decodeURIComponent(req.path.replace('/search', '')); const currentDir = getSecurePath(requestPath); + const pageNum = Math.max(1, parseInt(page)); + const pageLimit = Math.min(parseInt(limit), 1000); + const offset = (pageNum - 1) * pageLimit; const { getFileModel } = await import('../models/File.js'); const File = getFileModel(); - const searchResults = await File.findAll({ - where: { - [Op.and]: [ - { - file_path: { - [Op.like]: `${currentDir}%`, - }, + const searchConditions = { + [Op.and]: [ + { + file_path: { + [Op.like]: `${currentDir}%`, }, - { - [Op.or]: [ - { - file_path: { - [Op.like]: `%${searchQuery}%`, - }, + }, + { + [Op.or]: [ + { + file_path: { + [Op.like]: `%${searchQuery}%`, }, - { - checksum_sha256: { - [Op.like]: `%${searchQuery}%`, - }, + }, + { + checksum_sha256: { + [Op.like]: `%${searchQuery}%`, }, - ], - }, - ], - }, - limit: 1000, + }, + ], + }, + ], + }; + + const totalCount = await File.count({ + where: searchConditions, + }); + + const searchResults = await File.findAll({ + where: searchConditions, + limit: pageLimit, + offset, + order: [['last_modified', 'DESC']], raw: true, }); @@ -950,13 +961,24 @@ router.post('*splat/search', authenticateDownloads, async (req, res) => { isDirectory: file.is_directory, })); - logAccess(req, 'SEARCH', `query: "${searchQuery}", results: ${results.length}`); + logAccess( + req, + 'SEARCH', + `query: "${searchQuery}", results: ${results.length}, page: ${pageNum}` + ); return res.json({ success: true, query: searchQuery, results, - total: results.length, + pagination: { + page: pageNum, + limit: pageLimit, + total: totalCount, + totalPages: Math.ceil(totalCount / pageLimit), + hasNext: pageNum * pageLimit < totalCount, + hasPrev: pageNum > 1, + }, }); } catch (error) { logger.error('Search error', { error: error.message }); diff --git a/services/cacheService.js b/services/cacheService.js new file mode 100644 index 0000000..d0655b1 --- /dev/null +++ b/services/cacheService.js @@ -0,0 +1,57 @@ +import { fileWatcherLogger as logger } from '../config/logger.js'; + +class CacheService { + constructor() { + this.cache = new Map(); + this.ttl = 5 * 60 * 1000; // 5 minutes TTL + this.maxSize = 1000; // Maximum cache entries + } + + set(key, value) { + if (this.cache.size >= this.maxSize) { + const firstKey = this.cache.keys().next().value; + this.cache.delete(firstKey); + } + + this.cache.set(key, { + value, + timestamp: Date.now(), + }); + + logger.debug(`Cache set: ${key}`); + } + + get(key) { + const entry = this.cache.get(key); + if (!entry) { + return null; + } + + if (Date.now() - entry.timestamp > this.ttl) { + this.cache.delete(key); + logger.debug(`Cache expired: ${key}`); + return null; + } + + logger.debug(`Cache hit: ${key}`); + return entry.value; + } + + invalidate(pattern) { + const keysToDelete = []; + for (const key of this.cache.keys()) { + if (key.includes(pattern)) { + keysToDelete.push(key); + } + } + keysToDelete.forEach(key => this.cache.delete(key)); + logger.debug(`Cache invalidated: ${keysToDelete.length} entries for pattern ${pattern}`); + } + + clear() { + this.cache.clear(); + logger.info('Cache cleared'); + } +} + +export default new CacheService(); diff --git a/services/fileWatcher.js b/services/fileWatcher.js index 92e726d..81975a1 100644 --- a/services/fileWatcher.js +++ b/services/fileWatcher.js @@ -7,6 +7,8 @@ import { getFileModel } from '../models/File.js'; import { sendChecksumUpdate } from '../routes/sse.js'; import { fileWatcherLogger as logger } from '../config/logger.js'; import configLoader from '../config/configLoader.js'; +import { getDatabase } from '../config/database.js'; +import cacheService from './cacheService.js'; class FileWatcherService { constructor(watchPath) { @@ -258,18 +260,21 @@ class FileWatcherService { this.watcher.on('change', (filePath, stats) => { logger.info(`File changed: ${filePath}`); + cacheService.invalidate(dirname(filePath)); // Use stats from chokidar instead of doing our own fs.stat this.scheduleFileProcessingWithStats(filePath, stats); }); this.watcher.on('add', (filePath, stats) => { logger.info(`File added: ${filePath}`); + cacheService.invalidate(dirname(filePath)); // Use stats from chokidar instead of doing our own fs.stat this.scheduleFileProcessingWithStats(filePath, stats); }); this.watcher.on('unlink', filePath => { logger.info(`File deleted: ${filePath}`); + cacheService.invalidate(dirname(filePath)); const File = getFileModel(); File.destroy({ where: { file_path: filePath } }); }); @@ -330,7 +335,7 @@ class FileWatcherService { // Process checksum with timeout monitoring async processChecksumWithTimeout(itemPath) { const File = getFileModel(); - const { sequelize } = await import('../config/database.js'); + const sequelize = getDatabase(); const startTime = Date.now(); try { @@ -402,6 +407,13 @@ class FileWatcherService { async getCachedDirectoryItems(dirPath) { try { const cleanDirPath = dirPath.endsWith('/') ? dirPath.slice(0, -1) : dirPath; + const cacheKey = `dir:${cleanDirPath}`; + + const cached = cacheService.get(cacheKey); + if (cached) { + logger.info(`Cache hit for directory: ${cleanDirPath}`); + return cached; + } const File = getFileModel(); const files = await File.findAll({ @@ -427,6 +439,9 @@ class FileWatcherService { })); logger.info(`Found ${items.length} items in database for ${dirPath}`); + + cacheService.set(cacheKey, items); + return items; } catch (error) { logger.error(`Error querying database for directory ${dirPath}: ${error.message}`); diff --git a/services/maintenanceService.js b/services/maintenanceService.js new file mode 100644 index 0000000..d438bfd --- /dev/null +++ b/services/maintenanceService.js @@ -0,0 +1,64 @@ +import { optimizeDatabase } from '../config/database.js'; +import { fileWatcherLogger as logger } from '../config/logger.js'; + +class MaintenanceService { + constructor() { + this.intervalId = null; + this.isRunning = false; + } + + start() { + if (this.intervalId) { + logger.warn('Database maintenance scheduler already running'); + return; + } + + this.intervalId = setInterval( + async () => { + if (this.isRunning) { + logger.warn('Database maintenance already in progress, skipping'); + return; + } + + try { + this.isRunning = true; + logger.info('Starting scheduled database maintenance'); + await optimizeDatabase(); + logger.info('Scheduled database maintenance completed'); + } catch (error) { + logger.error(`Scheduled database maintenance failed: ${error.message}`); + } finally { + this.isRunning = false; + } + }, + 24 * 60 * 60 * 1000 + ); + + logger.info('Database maintenance scheduler started (24 hour interval)'); + } + + stop() { + if (this.intervalId) { + clearInterval(this.intervalId); + this.intervalId = null; + logger.info('Database maintenance scheduler stopped'); + } + } + + async runMaintenance() { + if (this.isRunning) { + throw new Error('Database maintenance already in progress'); + } + + try { + this.isRunning = true; + logger.info('Starting manual database maintenance'); + await optimizeDatabase(); + logger.info('Manual database maintenance completed'); + } finally { + this.isRunning = false; + } + } +} + +export default new MaintenanceService();