274 lines
11 KiB
TypeScript
274 lines
11 KiB
TypeScript
import dotenv from 'dotenv';
|
|
import Joi from 'joi';
|
|
|
|
// Load environment variables
|
|
dotenv.config();
|
|
|
|
// Environment validation schema
|
|
const envSchema = Joi.object({
|
|
NODE_ENV: Joi.string().valid('development', 'production', 'test').default('development'),
|
|
PORT: Joi.number().default(5000),
|
|
|
|
// Supabase Configuration (Required for cloud-only architecture)
|
|
SUPABASE_URL: Joi.string().required(),
|
|
SUPABASE_ANON_KEY: Joi.string().required(),
|
|
SUPABASE_SERVICE_KEY: Joi.string().required(),
|
|
|
|
// Google Cloud Configuration (Required)
|
|
GCLOUD_PROJECT_ID: Joi.string().required(),
|
|
DOCUMENT_AI_LOCATION: Joi.string().default('us'),
|
|
DOCUMENT_AI_PROCESSOR_ID: Joi.string().required(),
|
|
GCS_BUCKET_NAME: Joi.string().required(),
|
|
DOCUMENT_AI_OUTPUT_BUCKET_NAME: Joi.string().required(),
|
|
GOOGLE_APPLICATION_CREDENTIALS: Joi.string().default('./serviceAccountKey.json'),
|
|
|
|
// Vector Database Configuration
|
|
VECTOR_PROVIDER: Joi.string().valid('supabase', 'pinecone').default('supabase'),
|
|
|
|
// Pinecone Configuration (optional, only if using Pinecone)
|
|
PINECONE_API_KEY: Joi.string().when('VECTOR_PROVIDER', {
|
|
is: 'pinecone',
|
|
then: Joi.string().required(),
|
|
otherwise: Joi.string().allow('').optional()
|
|
}),
|
|
PINECONE_INDEX: Joi.string().when('VECTOR_PROVIDER', {
|
|
is: 'pinecone',
|
|
then: Joi.string().required(),
|
|
otherwise: Joi.string().allow('').optional()
|
|
}),
|
|
|
|
// JWT - Optional for Firebase Auth
|
|
JWT_SECRET: Joi.string().default('default-jwt-secret-change-in-production'),
|
|
JWT_EXPIRES_IN: Joi.string().default('1h'),
|
|
JWT_REFRESH_SECRET: Joi.string().default('default-refresh-secret-change-in-production'),
|
|
JWT_REFRESH_EXPIRES_IN: Joi.string().default('7d'),
|
|
|
|
// File Upload Configuration (Cloud-only)
|
|
MAX_FILE_SIZE: Joi.number().default(104857600), // 100MB
|
|
ALLOWED_FILE_TYPES: Joi.string().default('application/pdf'),
|
|
|
|
// LLM
|
|
LLM_PROVIDER: Joi.string().valid('openai', 'anthropic').default('openai'),
|
|
OPENAI_API_KEY: Joi.string().when('LLM_PROVIDER', {
|
|
is: 'openai',
|
|
then: Joi.string().required(),
|
|
otherwise: Joi.string().allow('').optional()
|
|
}),
|
|
ANTHROPIC_API_KEY: Joi.string().when('LLM_PROVIDER', {
|
|
is: 'anthropic',
|
|
then: Joi.string().required(),
|
|
otherwise: Joi.string().allow('').optional()
|
|
}),
|
|
LLM_MODEL: Joi.string().default('gpt-4'),
|
|
LLM_MAX_TOKENS: Joi.number().default(3500),
|
|
LLM_TEMPERATURE: Joi.number().min(0).max(2).default(0.1),
|
|
LLM_PROMPT_BUFFER: Joi.number().default(500),
|
|
|
|
// Security
|
|
BCRYPT_ROUNDS: Joi.number().default(12),
|
|
RATE_LIMIT_WINDOW_MS: Joi.number().default(900000), // 15 minutes
|
|
RATE_LIMIT_MAX_REQUESTS: Joi.number().default(100),
|
|
|
|
// Logging
|
|
LOG_LEVEL: Joi.string().valid('error', 'warn', 'info', 'debug').default('info'),
|
|
LOG_FILE: Joi.string().default('logs/app.log'),
|
|
|
|
// Processing Strategy
|
|
PROCESSING_STRATEGY: Joi.string().valid('document_ai_agentic_rag').default('document_ai_agentic_rag'),
|
|
|
|
// Agentic RAG Configuration
|
|
AGENTIC_RAG_ENABLED: Joi.boolean().default(false),
|
|
AGENTIC_RAG_MAX_AGENTS: Joi.number().default(6),
|
|
AGENTIC_RAG_PARALLEL_PROCESSING: Joi.boolean().default(true),
|
|
AGENTIC_RAG_VALIDATION_STRICT: Joi.boolean().default(true),
|
|
AGENTIC_RAG_RETRY_ATTEMPTS: Joi.number().default(3),
|
|
AGENTIC_RAG_TIMEOUT_PER_AGENT: Joi.number().default(60000),
|
|
|
|
// Agent-Specific Configuration
|
|
AGENT_DOCUMENT_UNDERSTANDING_ENABLED: Joi.boolean().default(true),
|
|
AGENT_FINANCIAL_ANALYSIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_MARKET_ANALYSIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_INVESTMENT_THESIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_SYNTHESIS_ENABLED: Joi.boolean().default(true),
|
|
AGENT_VALIDATION_ENABLED: Joi.boolean().default(true),
|
|
|
|
// Quality Control
|
|
AGENTIC_RAG_QUALITY_THRESHOLD: Joi.number().min(0).max(1).default(0.8),
|
|
AGENTIC_RAG_COMPLETENESS_THRESHOLD: Joi.number().min(0).max(1).default(0.9),
|
|
AGENTIC_RAG_CONSISTENCY_CHECK: Joi.boolean().default(true),
|
|
|
|
// Monitoring and Logging
|
|
AGENTIC_RAG_DETAILED_LOGGING: Joi.boolean().default(true),
|
|
AGENTIC_RAG_PERFORMANCE_TRACKING: Joi.boolean().default(true),
|
|
AGENTIC_RAG_ERROR_REPORTING: Joi.boolean().default(true),
|
|
}).unknown();
|
|
|
|
// Validate environment variables
|
|
const { error, value: envVars } = envSchema.validate(process.env);
|
|
|
|
if (error) {
|
|
// In a serverless environment (like Firebase Functions or Cloud Run),
|
|
// environment variables are often injected at runtime, not from a .env file.
|
|
// Therefore, we log a warning instead of throwing a fatal error.
|
|
// Throwing an error would cause the container to crash on startup
|
|
// before the runtime has a chance to provide the necessary variables.
|
|
console.warn(`[Config Validation Warning] ${error.message}`);
|
|
}
|
|
|
|
// Export validated configuration
|
|
export const config = {
|
|
env: envVars.NODE_ENV,
|
|
nodeEnv: envVars.NODE_ENV,
|
|
port: envVars.PORT,
|
|
frontendUrl: process.env['FRONTEND_URL'] || 'http://localhost:3000',
|
|
|
|
supabase: {
|
|
url: envVars.SUPABASE_URL,
|
|
anonKey: envVars.SUPABASE_ANON_KEY,
|
|
serviceKey: envVars.SUPABASE_SERVICE_KEY,
|
|
},
|
|
|
|
// Google Cloud Configuration
|
|
googleCloud: {
|
|
projectId: envVars.GCLOUD_PROJECT_ID,
|
|
documentAiLocation: envVars.DOCUMENT_AI_LOCATION,
|
|
documentAiProcessorId: envVars.DOCUMENT_AI_PROCESSOR_ID,
|
|
gcsBucketName: envVars.GCS_BUCKET_NAME,
|
|
documentAiOutputBucketName: envVars.DOCUMENT_AI_OUTPUT_BUCKET_NAME,
|
|
applicationCredentials: envVars.GOOGLE_APPLICATION_CREDENTIALS,
|
|
},
|
|
|
|
jwt: {
|
|
secret: envVars.JWT_SECRET,
|
|
expiresIn: envVars.JWT_EXPIRES_IN,
|
|
refreshSecret: envVars.JWT_REFRESH_SECRET,
|
|
refreshExpiresIn: envVars.JWT_REFRESH_EXPIRES_IN,
|
|
},
|
|
|
|
upload: {
|
|
maxFileSize: envVars.MAX_FILE_SIZE,
|
|
allowedFileTypes: envVars.ALLOWED_FILE_TYPES.split(','),
|
|
// Cloud-only: No local upload directory needed
|
|
uploadDir: '/tmp/uploads', // Temporary directory for file processing
|
|
},
|
|
|
|
llm: {
|
|
provider: envVars['LLM_PROVIDER'] || 'anthropic', // Default to Claude for cost efficiency
|
|
|
|
// Anthropic Configuration (Primary)
|
|
anthropicApiKey: envVars['ANTHROPIC_API_KEY'],
|
|
|
|
// OpenAI Configuration (Fallback)
|
|
openaiApiKey: envVars['OPENAI_API_KEY'],
|
|
|
|
// Model Selection - Hybrid approach optimized for different tasks
|
|
model: envVars['LLM_MODEL'] || 'claude-3-7-sonnet-20250219', // Primary model for analysis
|
|
fastModel: envVars['LLM_FAST_MODEL'] || 'claude-3-5-haiku-20241022', // Fast model for cost optimization
|
|
fallbackModel: envVars['LLM_FALLBACK_MODEL'] || 'gpt-4.5-preview-2025-02-27', // Fallback for creativity
|
|
|
|
// Task-specific model selection
|
|
financialModel: envVars['LLM_FINANCIAL_MODEL'] || 'claude-3-7-sonnet-20250219', // Best for financial analysis
|
|
creativeModel: envVars['LLM_CREATIVE_MODEL'] || 'gpt-4.5-preview-2025-02-27', // Best for creative content
|
|
reasoningModel: envVars['LLM_REASONING_MODEL'] || 'claude-3-7-sonnet-20250219', // Best for complex reasoning
|
|
|
|
// Token Limits - Optimized for CIM documents with hierarchical processing
|
|
maxTokens: parseInt(envVars['LLM_MAX_TOKENS'] || '4000'), // Output tokens (increased for better analysis)
|
|
maxInputTokens: parseInt(envVars['LLM_MAX_INPUT_TOKENS'] || '200000'), // Input tokens (increased for larger context)
|
|
chunkSize: parseInt(envVars['LLM_CHUNK_SIZE'] || '15000'), // Chunk size for section analysis (increased from 4000)
|
|
promptBuffer: parseInt(envVars['LLM_PROMPT_BUFFER'] || '1000'), // Buffer for prompt tokens (increased)
|
|
|
|
// Processing Configuration
|
|
temperature: parseFloat(envVars['LLM_TEMPERATURE'] || '0.1'), // Low temperature for consistent output
|
|
timeoutMs: parseInt(envVars['LLM_TIMEOUT_MS'] || '180000'), // 3 minutes timeout (increased for complex analysis)
|
|
|
|
// Cost Optimization
|
|
enableCostOptimization: envVars['LLM_ENABLE_COST_OPTIMIZATION'] === 'true',
|
|
maxCostPerDocument: parseFloat(envVars['LLM_MAX_COST_PER_DOCUMENT'] || '3.00'), // Max $3 per document (increased for better quality)
|
|
useFastModelForSimpleTasks: envVars['LLM_USE_FAST_MODEL_FOR_SIMPLE_TASKS'] === 'true',
|
|
|
|
// Hybrid approach settings
|
|
enableHybridApproach: envVars['LLM_ENABLE_HYBRID_APPROACH'] === 'true',
|
|
useClaudeForFinancial: envVars['LLM_USE_CLAUDE_FOR_FINANCIAL'] === 'true',
|
|
useGPTForCreative: envVars['LLM_USE_GPT_FOR_CREATIVE'] === 'true',
|
|
},
|
|
|
|
security: {
|
|
bcryptRounds: envVars.BCRYPT_ROUNDS,
|
|
rateLimit: {
|
|
windowMs: envVars.RATE_LIMIT_WINDOW_MS,
|
|
maxRequests: envVars.RATE_LIMIT_MAX_REQUESTS,
|
|
},
|
|
},
|
|
|
|
logging: {
|
|
level: envVars.LOG_LEVEL,
|
|
file: envVars.LOG_FILE,
|
|
},
|
|
|
|
// Processing Strategy
|
|
processingStrategy: envVars['PROCESSING_STRATEGY'] || 'agentic_rag', // 'chunking' | 'rag' | 'agentic_rag'
|
|
enableRAGProcessing: envVars['ENABLE_RAG_PROCESSING'] === 'true',
|
|
enableProcessingComparison: envVars['ENABLE_PROCESSING_COMPARISON'] === 'true',
|
|
|
|
// Agentic RAG Configuration
|
|
agenticRag: {
|
|
enabled: envVars.AGENTIC_RAG_ENABLED,
|
|
maxAgents: parseInt(envVars.AGENTIC_RAG_MAX_AGENTS || '6'),
|
|
parallelProcessing: envVars.AGENTIC_RAG_PARALLEL_PROCESSING,
|
|
validationStrict: envVars.AGENTIC_RAG_VALIDATION_STRICT,
|
|
retryAttempts: parseInt(envVars.AGENTIC_RAG_RETRY_ATTEMPTS || '3'),
|
|
timeoutPerAgent: parseInt(envVars.AGENTIC_RAG_TIMEOUT_PER_AGENT || '60000'),
|
|
},
|
|
|
|
// Agent-Specific Configuration
|
|
agentSpecific: {
|
|
documentUnderstandingEnabled: envVars['AGENT_DOCUMENT_UNDERSTANDING_ENABLED'] === 'true',
|
|
financialAnalysisEnabled: envVars['AGENT_FINANCIAL_ANALYSIS_ENABLED'] === 'true',
|
|
marketAnalysisEnabled: envVars['AGENT_MARKET_ANALYSIS_ENABLED'] === 'true',
|
|
investmentThesisEnabled: envVars['AGENT_INVESTMENT_THESIS_ENABLED'] === 'true',
|
|
synthesisEnabled: envVars['AGENT_SYNTHESIS_ENABLED'] === 'true',
|
|
validationEnabled: envVars['AGENT_VALIDATION_ENABLED'] === 'true',
|
|
},
|
|
|
|
// Quality Control
|
|
qualityControl: {
|
|
qualityThreshold: parseFloat(envVars['AGENTIC_RAG_QUALITY_THRESHOLD'] || '0.8'),
|
|
completenessThreshold: parseFloat(envVars['AGENTIC_RAG_COMPLETENESS_THRESHOLD'] || '0.9'),
|
|
consistencyCheck: envVars['AGENTIC_RAG_CONSISTENCY_CHECK'] === 'true',
|
|
},
|
|
|
|
// Monitoring and Logging
|
|
monitoringAndLogging: {
|
|
detailedLogging: envVars['AGENTIC_RAG_DETAILED_LOGGING'] === 'true',
|
|
performanceTracking: envVars['AGENTIC_RAG_PERFORMANCE_TRACKING'] === 'true',
|
|
errorReporting: envVars['AGENTIC_RAG_ERROR_REPORTING'] === 'true',
|
|
},
|
|
|
|
// Vector Database Configuration (Cloud-only)
|
|
vector: {
|
|
provider: envVars['VECTOR_PROVIDER'] || 'supabase', // 'pinecone' | 'supabase'
|
|
|
|
// Pinecone Configuration (if used)
|
|
pineconeApiKey: envVars['PINECONE_API_KEY'],
|
|
pineconeIndex: envVars['PINECONE_INDEX'],
|
|
},
|
|
|
|
// Legacy database configuration (for compatibility - using Supabase)
|
|
database: {
|
|
url: envVars.SUPABASE_URL,
|
|
host: 'db.supabase.co',
|
|
port: 5432,
|
|
name: 'postgres',
|
|
user: 'postgres',
|
|
password: envVars.SUPABASE_SERVICE_KEY,
|
|
},
|
|
|
|
// Legacy Redis configuration (for compatibility - using in-memory or cloud Redis)
|
|
redis: {
|
|
url: process.env['REDIS_URL'] || 'redis://localhost:6379',
|
|
host: 'localhost',
|
|
port: 6379,
|
|
},
|
|
};
|
|
|
|
export default config; |