Files
cim_summary/backend/src/services/uploadProgressService.ts
Jon 5bad434a27 feat: Complete Task 6 - File Upload Backend Infrastructure
Backend File Upload System:
- Implemented comprehensive multer middleware with file validation
- Created file storage service supporting local filesystem and S3
- Added upload progress tracking with real-time status updates
- Built file cleanup utilities and error handling
- Integrated with document routes for complete upload workflow

Key Features:
- PDF file validation (type, size, extension)
- User-specific file storage directories
- Unique filename generation with timestamps
- Comprehensive error handling for all upload scenarios
- Upload progress tracking with estimated time remaining
- File storage statistics and cleanup utilities

API Endpoints:
- POST /api/documents - Upload and process documents
- GET /api/documents/upload/:uploadId/progress - Track upload progress
- Enhanced document CRUD operations with file management
- Proper authentication and authorization checks

Testing:
- Comprehensive unit tests for upload middleware (7 tests)
- File storage service tests (18 tests)
- All existing tests still passing (117 backend + 25 frontend)
- Total test coverage: 142 tests

Dependencies Added:
- multer for file upload handling
- uuid for unique upload ID generation

Ready for Task 7: Document Processing Pipeline
2025-07-27 13:40:27 -04:00

267 lines
6.5 KiB
TypeScript

import { EventEmitter } from 'events';
import { logger } from '../utils/logger';
export interface UploadProgress {
uploadId: string;
userId: string;
filename: string;
totalSize: number;
uploadedSize: number;
percentage: number;
status: 'uploading' | 'processing' | 'completed' | 'failed';
error?: string;
startTime: Date;
lastUpdate: Date;
estimatedTimeRemaining?: number;
}
export interface UploadEvent {
type: 'progress' | 'complete' | 'error';
uploadId: string;
data: any;
}
class UploadProgressService extends EventEmitter {
private uploads: Map<string, UploadProgress> = new Map();
private cleanupInterval: NodeJS.Timeout | null = null;
constructor() {
super();
this.startCleanupInterval();
}
/**
* Start tracking an upload
*/
startTracking(uploadId: string, userId: string, filename: string, totalSize: number): void {
const upload: UploadProgress = {
uploadId,
userId,
filename,
totalSize,
uploadedSize: 0,
percentage: 0,
status: 'uploading',
startTime: new Date(),
lastUpdate: new Date(),
};
this.uploads.set(uploadId, upload);
logger.info(`Started tracking upload: ${uploadId}`, {
userId,
filename,
totalSize,
});
this.emit('upload:started', upload);
}
/**
* Update upload progress
*/
updateProgress(uploadId: string, uploadedSize: number): void {
const upload = this.uploads.get(uploadId);
if (!upload) {
logger.warn(`Upload not found for progress update: ${uploadId}`);
return;
}
upload.uploadedSize = uploadedSize;
upload.percentage = Math.round((uploadedSize / upload.totalSize) * 100);
upload.lastUpdate = new Date();
// Calculate estimated time remaining
const elapsed = Date.now() - upload.startTime.getTime();
if (uploadedSize > 0 && elapsed > 0) {
const bytesPerMs = uploadedSize / elapsed;
const remainingBytes = upload.totalSize - uploadedSize;
upload.estimatedTimeRemaining = Math.round(remainingBytes / bytesPerMs);
}
logger.debug(`Upload progress updated: ${uploadId}`, {
percentage: upload.percentage,
uploadedSize,
totalSize: upload.totalSize,
});
this.emit('upload:progress', upload);
}
/**
* Mark upload as processing
*/
markProcessing(uploadId: string): void {
const upload = this.uploads.get(uploadId);
if (!upload) {
logger.warn(`Upload not found for processing update: ${uploadId}`);
return;
}
upload.status = 'processing';
upload.lastUpdate = new Date();
logger.info(`Upload marked as processing: ${uploadId}`);
this.emit('upload:processing', upload);
}
/**
* Mark upload as completed
*/
markCompleted(uploadId: string): void {
const upload = this.uploads.get(uploadId);
if (!upload) {
logger.warn(`Upload not found for completion update: ${uploadId}`);
return;
}
upload.status = 'completed';
upload.uploadedSize = upload.totalSize;
upload.percentage = 100;
upload.lastUpdate = new Date();
logger.info(`Upload completed: ${uploadId}`, {
duration: Date.now() - upload.startTime.getTime(),
});
this.emit('upload:completed', upload);
}
/**
* Mark upload as failed
*/
markFailed(uploadId: string, error: string): void {
const upload = this.uploads.get(uploadId);
if (!upload) {
logger.warn(`Upload not found for failure update: ${uploadId}`);
return;
}
upload.status = 'failed';
upload.error = error;
upload.lastUpdate = new Date();
logger.error(`Upload failed: ${uploadId}`, {
error,
duration: Date.now() - upload.startTime.getTime(),
});
this.emit('upload:failed', upload);
}
/**
* Get upload progress
*/
getProgress(uploadId: string): UploadProgress | null {
return this.uploads.get(uploadId) || null;
}
/**
* Get all uploads for a user
*/
getUserUploads(userId: string): UploadProgress[] {
return Array.from(this.uploads.values()).filter(
upload => upload.userId === userId
);
}
/**
* Get all active uploads
*/
getActiveUploads(): UploadProgress[] {
return Array.from(this.uploads.values()).filter(
upload => upload.status === 'uploading' || upload.status === 'processing'
);
}
/**
* Remove upload from tracking
*/
removeUpload(uploadId: string): boolean {
const upload = this.uploads.get(uploadId);
if (!upload) {
return false;
}
this.uploads.delete(uploadId);
logger.info(`Removed upload from tracking: ${uploadId}`);
this.emit('upload:removed', upload);
return true;
}
/**
* Get upload statistics
*/
getStats(): {
total: number;
uploading: number;
processing: number;
completed: number;
failed: number;
} {
const uploads = Array.from(this.uploads.values());
return {
total: uploads.length,
uploading: uploads.filter(u => u.status === 'uploading').length,
processing: uploads.filter(u => u.status === 'processing').length,
completed: uploads.filter(u => u.status === 'completed').length,
failed: uploads.filter(u => u.status === 'failed').length,
};
}
/**
* Start cleanup interval to remove old completed uploads
*/
private startCleanupInterval(): void {
this.cleanupInterval = setInterval(() => {
this.cleanupOldUploads();
}, 5 * 60 * 1000); // Clean up every 5 minutes
}
/**
* Clean up old completed uploads (older than 1 hour)
*/
private cleanupOldUploads(): void {
const cutoffTime = Date.now() - (60 * 60 * 1000); // 1 hour
const uploadsToRemove: string[] = [];
for (const [uploadId, upload] of this.uploads.entries()) {
if (
(upload.status === 'completed' || upload.status === 'failed') &&
upload.lastUpdate.getTime() < cutoffTime
) {
uploadsToRemove.push(uploadId);
}
}
uploadsToRemove.forEach(uploadId => {
this.removeUpload(uploadId);
});
if (uploadsToRemove.length > 0) {
logger.info(`Cleaned up ${uploadsToRemove.length} old uploads`);
}
}
/**
* Stop the service and cleanup
*/
stop(): void {
if (this.cleanupInterval) {
clearInterval(this.cleanupInterval);
this.cleanupInterval = null;
}
this.uploads.clear();
this.removeAllListeners();
logger.info('Upload progress service stopped');
}
}
export const uploadProgressService = new UploadProgressService();
export default uploadProgressService;