Backend File Upload System: - Implemented comprehensive multer middleware with file validation - Created file storage service supporting local filesystem and S3 - Added upload progress tracking with real-time status updates - Built file cleanup utilities and error handling - Integrated with document routes for complete upload workflow Key Features: - PDF file validation (type, size, extension) - User-specific file storage directories - Unique filename generation with timestamps - Comprehensive error handling for all upload scenarios - Upload progress tracking with estimated time remaining - File storage statistics and cleanup utilities API Endpoints: - POST /api/documents - Upload and process documents - GET /api/documents/upload/:uploadId/progress - Track upload progress - Enhanced document CRUD operations with file management - Proper authentication and authorization checks Testing: - Comprehensive unit tests for upload middleware (7 tests) - File storage service tests (18 tests) - All existing tests still passing (117 backend + 25 frontend) - Total test coverage: 142 tests Dependencies Added: - multer for file upload handling - uuid for unique upload ID generation Ready for Task 7: Document Processing Pipeline
267 lines
6.5 KiB
TypeScript
267 lines
6.5 KiB
TypeScript
import { EventEmitter } from 'events';
|
|
import { logger } from '../utils/logger';
|
|
|
|
export interface UploadProgress {
|
|
uploadId: string;
|
|
userId: string;
|
|
filename: string;
|
|
totalSize: number;
|
|
uploadedSize: number;
|
|
percentage: number;
|
|
status: 'uploading' | 'processing' | 'completed' | 'failed';
|
|
error?: string;
|
|
startTime: Date;
|
|
lastUpdate: Date;
|
|
estimatedTimeRemaining?: number;
|
|
}
|
|
|
|
export interface UploadEvent {
|
|
type: 'progress' | 'complete' | 'error';
|
|
uploadId: string;
|
|
data: any;
|
|
}
|
|
|
|
class UploadProgressService extends EventEmitter {
|
|
private uploads: Map<string, UploadProgress> = new Map();
|
|
private cleanupInterval: NodeJS.Timeout | null = null;
|
|
|
|
constructor() {
|
|
super();
|
|
this.startCleanupInterval();
|
|
}
|
|
|
|
/**
|
|
* Start tracking an upload
|
|
*/
|
|
startTracking(uploadId: string, userId: string, filename: string, totalSize: number): void {
|
|
const upload: UploadProgress = {
|
|
uploadId,
|
|
userId,
|
|
filename,
|
|
totalSize,
|
|
uploadedSize: 0,
|
|
percentage: 0,
|
|
status: 'uploading',
|
|
startTime: new Date(),
|
|
lastUpdate: new Date(),
|
|
};
|
|
|
|
this.uploads.set(uploadId, upload);
|
|
|
|
logger.info(`Started tracking upload: ${uploadId}`, {
|
|
userId,
|
|
filename,
|
|
totalSize,
|
|
});
|
|
|
|
this.emit('upload:started', upload);
|
|
}
|
|
|
|
/**
|
|
* Update upload progress
|
|
*/
|
|
updateProgress(uploadId: string, uploadedSize: number): void {
|
|
const upload = this.uploads.get(uploadId);
|
|
if (!upload) {
|
|
logger.warn(`Upload not found for progress update: ${uploadId}`);
|
|
return;
|
|
}
|
|
|
|
upload.uploadedSize = uploadedSize;
|
|
upload.percentage = Math.round((uploadedSize / upload.totalSize) * 100);
|
|
upload.lastUpdate = new Date();
|
|
|
|
// Calculate estimated time remaining
|
|
const elapsed = Date.now() - upload.startTime.getTime();
|
|
if (uploadedSize > 0 && elapsed > 0) {
|
|
const bytesPerMs = uploadedSize / elapsed;
|
|
const remainingBytes = upload.totalSize - uploadedSize;
|
|
upload.estimatedTimeRemaining = Math.round(remainingBytes / bytesPerMs);
|
|
}
|
|
|
|
logger.debug(`Upload progress updated: ${uploadId}`, {
|
|
percentage: upload.percentage,
|
|
uploadedSize,
|
|
totalSize: upload.totalSize,
|
|
});
|
|
|
|
this.emit('upload:progress', upload);
|
|
}
|
|
|
|
/**
|
|
* Mark upload as processing
|
|
*/
|
|
markProcessing(uploadId: string): void {
|
|
const upload = this.uploads.get(uploadId);
|
|
if (!upload) {
|
|
logger.warn(`Upload not found for processing update: ${uploadId}`);
|
|
return;
|
|
}
|
|
|
|
upload.status = 'processing';
|
|
upload.lastUpdate = new Date();
|
|
|
|
logger.info(`Upload marked as processing: ${uploadId}`);
|
|
|
|
this.emit('upload:processing', upload);
|
|
}
|
|
|
|
/**
|
|
* Mark upload as completed
|
|
*/
|
|
markCompleted(uploadId: string): void {
|
|
const upload = this.uploads.get(uploadId);
|
|
if (!upload) {
|
|
logger.warn(`Upload not found for completion update: ${uploadId}`);
|
|
return;
|
|
}
|
|
|
|
upload.status = 'completed';
|
|
upload.uploadedSize = upload.totalSize;
|
|
upload.percentage = 100;
|
|
upload.lastUpdate = new Date();
|
|
|
|
logger.info(`Upload completed: ${uploadId}`, {
|
|
duration: Date.now() - upload.startTime.getTime(),
|
|
});
|
|
|
|
this.emit('upload:completed', upload);
|
|
}
|
|
|
|
/**
|
|
* Mark upload as failed
|
|
*/
|
|
markFailed(uploadId: string, error: string): void {
|
|
const upload = this.uploads.get(uploadId);
|
|
if (!upload) {
|
|
logger.warn(`Upload not found for failure update: ${uploadId}`);
|
|
return;
|
|
}
|
|
|
|
upload.status = 'failed';
|
|
upload.error = error;
|
|
upload.lastUpdate = new Date();
|
|
|
|
logger.error(`Upload failed: ${uploadId}`, {
|
|
error,
|
|
duration: Date.now() - upload.startTime.getTime(),
|
|
});
|
|
|
|
this.emit('upload:failed', upload);
|
|
}
|
|
|
|
/**
|
|
* Get upload progress
|
|
*/
|
|
getProgress(uploadId: string): UploadProgress | null {
|
|
return this.uploads.get(uploadId) || null;
|
|
}
|
|
|
|
/**
|
|
* Get all uploads for a user
|
|
*/
|
|
getUserUploads(userId: string): UploadProgress[] {
|
|
return Array.from(this.uploads.values()).filter(
|
|
upload => upload.userId === userId
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Get all active uploads
|
|
*/
|
|
getActiveUploads(): UploadProgress[] {
|
|
return Array.from(this.uploads.values()).filter(
|
|
upload => upload.status === 'uploading' || upload.status === 'processing'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Remove upload from tracking
|
|
*/
|
|
removeUpload(uploadId: string): boolean {
|
|
const upload = this.uploads.get(uploadId);
|
|
if (!upload) {
|
|
return false;
|
|
}
|
|
|
|
this.uploads.delete(uploadId);
|
|
|
|
logger.info(`Removed upload from tracking: ${uploadId}`);
|
|
|
|
this.emit('upload:removed', upload);
|
|
return true;
|
|
}
|
|
|
|
/**
|
|
* Get upload statistics
|
|
*/
|
|
getStats(): {
|
|
total: number;
|
|
uploading: number;
|
|
processing: number;
|
|
completed: number;
|
|
failed: number;
|
|
} {
|
|
const uploads = Array.from(this.uploads.values());
|
|
|
|
return {
|
|
total: uploads.length,
|
|
uploading: uploads.filter(u => u.status === 'uploading').length,
|
|
processing: uploads.filter(u => u.status === 'processing').length,
|
|
completed: uploads.filter(u => u.status === 'completed').length,
|
|
failed: uploads.filter(u => u.status === 'failed').length,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Start cleanup interval to remove old completed uploads
|
|
*/
|
|
private startCleanupInterval(): void {
|
|
this.cleanupInterval = setInterval(() => {
|
|
this.cleanupOldUploads();
|
|
}, 5 * 60 * 1000); // Clean up every 5 minutes
|
|
}
|
|
|
|
/**
|
|
* Clean up old completed uploads (older than 1 hour)
|
|
*/
|
|
private cleanupOldUploads(): void {
|
|
const cutoffTime = Date.now() - (60 * 60 * 1000); // 1 hour
|
|
const uploadsToRemove: string[] = [];
|
|
|
|
for (const [uploadId, upload] of this.uploads.entries()) {
|
|
if (
|
|
(upload.status === 'completed' || upload.status === 'failed') &&
|
|
upload.lastUpdate.getTime() < cutoffTime
|
|
) {
|
|
uploadsToRemove.push(uploadId);
|
|
}
|
|
}
|
|
|
|
uploadsToRemove.forEach(uploadId => {
|
|
this.removeUpload(uploadId);
|
|
});
|
|
|
|
if (uploadsToRemove.length > 0) {
|
|
logger.info(`Cleaned up ${uploadsToRemove.length} old uploads`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Stop the service and cleanup
|
|
*/
|
|
stop(): void {
|
|
if (this.cleanupInterval) {
|
|
clearInterval(this.cleanupInterval);
|
|
this.cleanupInterval = null;
|
|
}
|
|
|
|
this.uploads.clear();
|
|
this.removeAllListeners();
|
|
|
|
logger.info('Upload progress service stopped');
|
|
}
|
|
}
|
|
|
|
export const uploadProgressService = new UploadProgressService();
|
|
export default uploadProgressService;
|