288 lines
11 KiB
TypeScript
288 lines
11 KiB
TypeScript
import dotenv from 'dotenv';
|
|
import Joi from 'joi';
|
|
|
|
// Load environment variables
|
|
dotenv.config();
|
|
|
|
// Environment validation schema
|
|
const envSchema = Joi.object({
|
|
NODE_ENV: Joi.string().valid('development', 'production', 'test').default('development'),
|
|
PORT: Joi.number().default(5000),
|
|
|
|
// Database - Made optional for Firebase deployment with Supabase
|
|
DATABASE_URL: Joi.string().allow('').default(''),
|
|
DB_HOST: Joi.string().default('localhost'),
|
|
DB_PORT: Joi.number().default(5432),
|
|
DB_NAME: Joi.string().allow('').default(''),
|
|
DB_USER: Joi.string().allow('').default(''),
|
|
DB_PASSWORD: Joi.string().allow('').default(''),
|
|
|
|
// Supabase Configuration
|
|
SUPABASE_URL: Joi.string().allow('').optional(),
|
|
SUPABASE_ANON_KEY: Joi.string().allow('').optional(),
|
|
SUPABASE_SERVICE_KEY: Joi.string().allow('').optional(),
|
|
|
|
// Redis
|
|
REDIS_URL: Joi.string().default('redis://localhost:6379'),
|
|
REDIS_HOST: Joi.string().default('localhost'),
|
|
REDIS_PORT: Joi.number().default(6379),
|
|
|
|
// JWT - Optional for Firebase Auth
|
|
JWT_SECRET: Joi.string().default('default-jwt-secret-change-in-production'),
|
|
JWT_EXPIRES_IN: Joi.string().default('1h'),
|
|
JWT_REFRESH_SECRET: Joi.string().default('default-refresh-secret-change-in-production'),
|
|
JWT_REFRESH_EXPIRES_IN: Joi.string().default('7d'),
|
|
|
|
// File Upload
|
|
MAX_FILE_SIZE: Joi.number().default(104857600), // 100MB
|
|
UPLOAD_DIR: Joi.string().default('uploads'),
|
|
ALLOWED_FILE_TYPES: Joi.string().default('application/pdf'),
|
|
|
|
// LLM
|
|
LLM_PROVIDER: Joi.string().valid('openai', 'anthropic').default('openai'),
|
|
OPENAI_API_KEY: Joi.string().when('LLM_PROVIDER', {
|
|
is: 'openai',
|
|
then: Joi.string().required(),
|
|
otherwise: Joi.string().allow('').optional()
|
|
}),
|
|
ANTHROPIC_API_KEY: Joi.string().when('LLM_PROVIDER', {
|
|
is: 'anthropic',
|
|
then: Joi.string().required(),
|
|
otherwise: Joi.string().allow('').optional()
|
|
}),
|
|
LLM_MODEL: Joi.string().default('gpt-4'),
|
|
LLM_MAX_TOKENS: Joi.number().default(3500),
|
|
LLM_TEMPERATURE: Joi.number().min(0).max(2).default(0.1),
|
|
LLM_PROMPT_BUFFER: Joi.number().default(500),
|
|
|
|
// Storage
|
|
STORAGE_TYPE: Joi.string().valid('local', 's3').default('local'),
|
|
AWS_ACCESS_KEY_ID: Joi.string().when('STORAGE_TYPE', {
|
|
is: 's3',
|
|
then: Joi.required(),
|
|
otherwise: Joi.optional()
|
|
}),
|
|
AWS_SECRET_ACCESS_KEY: Joi.string().when('STORAGE_TYPE', {
|
|
is: 's3',
|
|
then: Joi.required(),
|
|
otherwise: Joi.optional()
|
|
}),
|
|
AWS_REGION: Joi.string().when('STORAGE_TYPE', {
|
|
is: 's3',
|
|
then: Joi.required(),
|
|
otherwise: Joi.optional()
|
|
}),
|
|
AWS_S3_BUCKET: Joi.string().when('STORAGE_TYPE', {
|
|
is: 's3',
|
|
then: Joi.required(),
|
|
otherwise: Joi.optional()
|
|
}),
|
|
|
|
// Security
|
|
BCRYPT_ROUNDS: Joi.number().default(12),
|
|
RATE_LIMIT_WINDOW_MS: Joi.number().default(900000), // 15 minutes
|
|
RATE_LIMIT_MAX_REQUESTS: Joi.number().default(100),
|
|
|
|
// Logging
|
|
LOG_LEVEL: Joi.string().valid('error', 'warn', 'info', 'debug').default('info'),
|
|
LOG_FILE: Joi.string().default('logs/app.log'),
|
|
|
|
// Processing Strategy
|
|
PROCESSING_STRATEGY: Joi.string().valid('chunking', 'rag', 'agentic_rag').default('chunking'),
|
|
ENABLE_RAG_PROCESSING: Joi.boolean().default(false),
|
|
ENABLE_PROCESSING_COMPARISON: Joi.boolean().default(false),
|
|
|
|
// Agentic RAG Configuration
|
|
AGENTIC_RAG_ENABLED: Joi.boolean().default(false),
|
|
AGENTIC_RAG_MAX_AGENTS: Joi.number().default(6),
|
|
AGENTIC_RAG_PARALLEL_PROCESSING: Joi.boolean().default(true),
|
|
AGENTIC_RAG_VALIDATION_STRICT: Joi.boolean().default(true),
|
|
AGENTIC_RAG_RETRY_ATTEMPTS: Joi.number().default(3),
|
|
AGENTIC_RAG_TIMEOUT_PER_AGENT: Joi.number().default(60000),
|
|
|
|
// Agent-Specific Configuration
|
|
AGENT_DOCUMENT_UNDERSTANDING_ENABLED: Joi.boolean().default(true),
|
|
AGENT_FINANCIAL_ANALYSIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_MARKET_ANALYSIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_INVESTMENT_THESIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_SYNTHESIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_VALIDATION_ENABLED: Joi.boolean().default(true),
|
|
|
|
// Quality Control
|
|
AGENTIC_RAG_QUALITY_THRESHOLD: Joi.number().min(0).max(1).default(0.8),
|
|
AGENTIC_RAG_COMPLETENESS_THRESHOLD: Joi.number().min(0).max(1).default(0.9),
|
|
AGENTIC_RAG_CONSISTENCY_CHECK: Joi.boolean().default(true),
|
|
|
|
// Monitoring and Logging
|
|
AGENTIC_RAG_DETAILED_LOGGING: Joi.boolean().default(true),
|
|
AGENTIC_RAG_PERFORMANCE_TRACKING: Joi.boolean().default(true),
|
|
AGENTIC_RAG_ERROR_REPORTING: Joi.boolean().default(true),
|
|
}).unknown();
|
|
|
|
// Validate environment variables
|
|
const { error, value: envVars } = envSchema.validate(process.env);
|
|
|
|
if (error) {
|
|
throw new Error(`Config validation error: ${error.message}`);
|
|
}
|
|
|
|
// Export validated configuration
|
|
export const config = {
|
|
env: envVars.NODE_ENV,
|
|
nodeEnv: envVars.NODE_ENV,
|
|
port: envVars.PORT,
|
|
frontendUrl: process.env['FRONTEND_URL'] || 'http://localhost:3000',
|
|
|
|
database: {
|
|
url: envVars.DATABASE_URL,
|
|
host: envVars.DB_HOST,
|
|
port: envVars.DB_PORT,
|
|
name: envVars.DB_NAME,
|
|
user: envVars.DB_USER,
|
|
password: envVars.DB_PASSWORD,
|
|
},
|
|
|
|
supabase: {
|
|
url: envVars.SUPABASE_URL,
|
|
anonKey: envVars.SUPABASE_ANON_KEY,
|
|
serviceKey: envVars.SUPABASE_SERVICE_KEY,
|
|
},
|
|
|
|
redis: {
|
|
url: envVars.REDIS_URL,
|
|
host: envVars.REDIS_HOST,
|
|
port: envVars.REDIS_PORT,
|
|
},
|
|
|
|
jwt: {
|
|
secret: envVars.JWT_SECRET,
|
|
expiresIn: envVars.JWT_EXPIRES_IN,
|
|
refreshSecret: envVars.JWT_REFRESH_SECRET,
|
|
refreshExpiresIn: envVars.JWT_REFRESH_EXPIRES_IN,
|
|
},
|
|
|
|
upload: {
|
|
maxFileSize: envVars.MAX_FILE_SIZE,
|
|
uploadDir: envVars.UPLOAD_DIR,
|
|
allowedFileTypes: envVars.ALLOWED_FILE_TYPES.split(','),
|
|
},
|
|
|
|
llm: {
|
|
provider: envVars['LLM_PROVIDER'] || 'anthropic', // Default to Claude for cost efficiency
|
|
|
|
// Anthropic Configuration (Primary)
|
|
anthropicApiKey: envVars['ANTHROPIC_API_KEY'],
|
|
|
|
// OpenAI Configuration (Fallback)
|
|
openaiApiKey: envVars['OPENAI_API_KEY'],
|
|
|
|
// Model Selection - Hybrid approach optimized for different tasks
|
|
model: envVars['LLM_MODEL'] || 'claude-3-7-sonnet-20250219', // Primary model for analysis
|
|
fastModel: envVars['LLM_FAST_MODEL'] || 'claude-3-5-haiku-20241022', // Fast model for cost optimization
|
|
fallbackModel: envVars['LLM_FALLBACK_MODEL'] || 'gpt-4.5-preview-2025-02-27', // Fallback for creativity
|
|
|
|
// Task-specific model selection
|
|
financialModel: envVars['LLM_FINANCIAL_MODEL'] || 'claude-3-7-sonnet-20250219', // Best for financial analysis
|
|
creativeModel: envVars['LLM_CREATIVE_MODEL'] || 'gpt-4.5-preview-2025-02-27', // Best for creative content
|
|
reasoningModel: envVars['LLM_REASONING_MODEL'] || 'claude-3-7-sonnet-20250219', // Best for complex reasoning
|
|
|
|
// Token Limits - Optimized for CIM documents with hierarchical processing
|
|
maxTokens: parseInt(envVars['LLM_MAX_TOKENS'] || '4000'), // Output tokens (increased for better analysis)
|
|
maxInputTokens: parseInt(envVars['LLM_MAX_INPUT_TOKENS'] || '200000'), // Input tokens (increased for larger context)
|
|
chunkSize: parseInt(envVars['LLM_CHUNK_SIZE'] || '15000'), // Chunk size for section analysis (increased from 4000)
|
|
promptBuffer: parseInt(envVars['LLM_PROMPT_BUFFER'] || '1000'), // Buffer for prompt tokens (increased)
|
|
|
|
// Processing Configuration
|
|
temperature: parseFloat(envVars['LLM_TEMPERATURE'] || '0.1'), // Low temperature for consistent output
|
|
timeoutMs: parseInt(envVars['LLM_TIMEOUT_MS'] || '180000'), // 3 minutes timeout (increased for complex analysis)
|
|
|
|
// Cost Optimization
|
|
enableCostOptimization: envVars['LLM_ENABLE_COST_OPTIMIZATION'] === 'true',
|
|
maxCostPerDocument: parseFloat(envVars['LLM_MAX_COST_PER_DOCUMENT'] || '3.00'), // Max $3 per document (increased for better quality)
|
|
useFastModelForSimpleTasks: envVars['LLM_USE_FAST_MODEL_FOR_SIMPLE_TASKS'] === 'true',
|
|
|
|
// Hybrid approach settings
|
|
enableHybridApproach: envVars['LLM_ENABLE_HYBRID_APPROACH'] === 'true',
|
|
useClaudeForFinancial: envVars['LLM_USE_CLAUDE_FOR_FINANCIAL'] === 'true',
|
|
useGPTForCreative: envVars['LLM_USE_GPT_FOR_CREATIVE'] === 'true',
|
|
},
|
|
|
|
storage: {
|
|
type: envVars.STORAGE_TYPE,
|
|
aws: {
|
|
accessKeyId: envVars.AWS_ACCESS_KEY_ID,
|
|
secretAccessKey: envVars.AWS_SECRET_ACCESS_KEY,
|
|
region: envVars.AWS_REGION,
|
|
bucket: envVars.AWS_S3_BUCKET,
|
|
},
|
|
},
|
|
|
|
security: {
|
|
bcryptRounds: envVars.BCRYPT_ROUNDS,
|
|
rateLimit: {
|
|
windowMs: envVars.RATE_LIMIT_WINDOW_MS,
|
|
maxRequests: envVars.RATE_LIMIT_MAX_REQUESTS,
|
|
},
|
|
},
|
|
|
|
logging: {
|
|
level: envVars.LOG_LEVEL,
|
|
file: envVars.LOG_FILE,
|
|
},
|
|
|
|
// Processing Strategy
|
|
processingStrategy: envVars['PROCESSING_STRATEGY'] || 'agentic_rag', // 'chunking' | 'rag' | 'agentic_rag'
|
|
enableRAGProcessing: envVars['ENABLE_RAG_PROCESSING'] === 'true',
|
|
enableProcessingComparison: envVars['ENABLE_PROCESSING_COMPARISON'] === 'true',
|
|
|
|
// Agentic RAG Configuration
|
|
agenticRag: {
|
|
enabled: envVars.AGENTIC_RAG_ENABLED,
|
|
maxAgents: parseInt(envVars.AGENTIC_RAG_MAX_AGENTS || '6'),
|
|
parallelProcessing: envVars.AGENTIC_RAG_PARALLEL_PROCESSING,
|
|
validationStrict: envVars.AGENTIC_RAG_VALIDATION_STRICT,
|
|
retryAttempts: parseInt(envVars.AGENTIC_RAG_RETRY_ATTEMPTS || '3'),
|
|
timeoutPerAgent: parseInt(envVars.AGENTIC_RAG_TIMEOUT_PER_AGENT || '60000'),
|
|
},
|
|
|
|
// Agent-Specific Configuration
|
|
agentSpecific: {
|
|
documentUnderstandingEnabled: envVars['AGENT_DOCUMENT_UNDERSTANDING_ENABLED'] === 'true',
|
|
financialAnalysisEnabled: envVars['AGENT_FINANCIAL_ANALYSIS_ENABLED'] === 'true',
|
|
marketAnalysisEnabled: envVars['AGENT_MARKET_ANALYSIS_ENABLED'] === 'true',
|
|
investmentThesisEnabled: envVars['AGENT_INVESTMENT_THESIS_ENABLED'] === 'true',
|
|
synthesisEnabled: envVars['AGENT_SYNTHESIS_ENABLED'] === 'true',
|
|
validationEnabled: envVars['AGENT_VALIDATION_ENABLED'] === 'true',
|
|
},
|
|
|
|
// Quality Control
|
|
qualityControl: {
|
|
qualityThreshold: parseFloat(envVars['AGENTIC_RAG_QUALITY_THRESHOLD'] || '0.8'),
|
|
completenessThreshold: parseFloat(envVars['AGENTIC_RAG_COMPLETENESS_THRESHOLD'] || '0.9'),
|
|
consistencyCheck: envVars['AGENTIC_RAG_CONSISTENCY_CHECK'] === 'true',
|
|
},
|
|
|
|
// Monitoring and Logging
|
|
monitoringAndLogging: {
|
|
detailedLogging: envVars['AGENTIC_RAG_DETAILED_LOGGING'] === 'true',
|
|
performanceTracking: envVars['AGENTIC_RAG_PERFORMANCE_TRACKING'] === 'true',
|
|
errorReporting: envVars['AGENTIC_RAG_ERROR_REPORTING'] === 'true',
|
|
},
|
|
|
|
// Vector Database Configuration
|
|
vector: {
|
|
provider: envVars['VECTOR_PROVIDER'] || 'supabase', // 'pinecone' | 'pgvector' | 'chroma' | 'supabase'
|
|
|
|
// Pinecone Configuration
|
|
pineconeApiKey: envVars['PINECONE_API_KEY'],
|
|
pineconeIndex: envVars['PINECONE_INDEX'],
|
|
|
|
// Chroma Configuration
|
|
chromaUrl: envVars['CHROMA_URL'] || 'http://localhost:8000',
|
|
|
|
// pgvector uses existing PostgreSQL connection
|
|
// No additional configuration needed
|
|
},
|
|
};
|
|
|
|
export default config; |