import { logger } from '../utils/logger';
import { llmService } from './llmService';
import { config } from '../config/env';
import { CIMReview } from './llmSchemas';
import {
  AgentExecution,
  AgenticRAGSession,
  AgenticRAGResult,
  AgenticRAGConfig
} from '../models/agenticTypes';
import { agenticRAGDatabaseService } from './agenticRAGDatabaseService';
import { vectorDocumentProcessor } from './vectorDocumentProcessor';
import { loadAndParseTemplate, IReviewTemplate, IFormSection, IFormField } from '../utils/templateParser';
import { extractFinancials, CleanedFinancials } from '../utils/financialExtractor';


// Best Practice: Centralized error types
enum AgenticRAGErrorType {
  AGENT_EXECUTION_FAILED = 'AGENT_EXECUTION_FAILED',
  VALIDATION_FAILED = 'VALIDATION_FAILED',
  TIMEOUT_ERROR = 'TIMEOUT_ERROR',
  RATE_LIMIT_ERROR = 'RATE_LIMIT_ERROR',
  INVALID_RESPONSE = 'INVALID_RESPONSE',
  DATABASE_ERROR = 'DATABASE_ERROR',
  CONFIGURATION_ERROR = 'CONFIGURATION_ERROR',
  TEMPLATE_ERROR = 'TEMPLATE_ERROR'
}

// Best Practice: Custom error class with context
class AgenticRAGError extends Error {
  constructor(
    message: string,
    public type: AgenticRAGErrorType,
    public agentName?: string,
    public retryable: boolean = false,
    public context?: any,
    public originalError?: Error
  ) {
    super(message);
    this.name = 'AgenticRAGError';
    
    // Best Practice: Preserve stack trace
    if (originalError?.stack) {
      this.stack = originalError.stack;
    }
  }
}

// Best Practice: Safe serialization utility
class SafeSerializer {
  static serialize(data: any): any {
    if (data === null || data === undefined) {
      return null;
    }
    
    if (typeof data === 'string' || typeof data === 'number' || typeof data === 'boolean') {
      return data;
    }
    
    if (data instanceof Date) {
      return data.toISOString();
    }
    
    if (Array.isArray(data)) {
      return data.map(item => this.serialize(item));
    }
    
    if (typeof data === 'object') {
      // Handle circular references and complex objects
      const seen = new WeakSet();
      return this.serializeObject(data, seen);
    }
    
    return String(data);
  }
  
  private static serializeObject(obj: any, seen: WeakSet<any>): any {
    if (seen.has(obj)) {
      return '[Circular Reference]';
    }
    
    seen.add(obj);
    
    const result: any = {};
    
    for (const [key, value] of Object.entries(obj)) {
      try {
        // Skip functions and symbols
        if (typeof value === 'function' || typeof value === 'symbol') {
          continue;
        }
        
        // Skip undefined values
        if (value === undefined) {
          continue;
        }
        
        result[key] = this.serialize(value);
      } catch (error) {
        logger.warn('Failed to serialize object property', { key, error: error instanceof Error ? error.message : 'Unknown error' });
        result[key] = '[Serialization Error]';
      }
    }
    
    return result;
  }
  
  static safeStringify(data: any): string {
    try {
      const serialized = this.serialize(data);
      return JSON.stringify(serialized);
    } catch (error) {
      logger.error('Failed to stringify data', { error: error instanceof Error ? error.message : 'Unknown error' });
      return JSON.stringify({ error: 'Serialization failed', originalType: typeof data });
    }
  }
  
  static safeParse(jsonString: string): any {
    try {
      return JSON.parse(jsonString);
    } catch (error) {
      logger.error('Failed to parse JSON string', { error: error instanceof Error ? error.message : 'Unknown error' });
      return { error: 'JSON parsing failed', originalString: jsonString.substring(0, 100) + '...' };
    }
  }
}

// Best Practice: Configuration validation
class ConfigurationValidator {
  static validateAgenticRAGConfig(config: AgenticRAGConfig): void {
    const errors: string[] = [];
    
    if (config.maxAgents <= 0 || config.maxAgents > 10) {
      errors.push('maxAgents must be between 1 and 10');
    }
    
    if (config.retryAttempts < 0 || config.retryAttempts > 5) {
      errors.push('retryAttempts must be between 0 and 5');
    }
    
    if (config.timeoutPerAgent < 1000 || config.timeoutPerAgent > 300000) {
      errors.push('timeoutPerAgent must be between 1 and 300 seconds');
    }
    
    if (config.qualityThreshold < 0 || config.qualityThreshold > 1) {
      errors.push('qualityThreshold must be between 0 and 1');
    }
    
    if (errors.length > 0) {
      throw new AgenticRAGError(
        `Invalid configuration: ${errors.join(', ')}`,
        AgenticRAGErrorType.CONFIGURATION_ERROR,
        undefined,
        false,
        { errors }
      );
    }
  }
}

// Best Practice: Dependency injection interface


interface ISessionManager {
  createSession(documentId: string, userId: string, strategy: string): Promise<AgenticRAGSession>;
  updateSession(sessionId: string, updates: Partial<AgenticRAGSession>): Promise<void>;
  createExecution(sessionId: string, agentName: string, inputData: any): Promise<AgentExecution>;
}

class AgenticRAGSessionManager implements ISessionManager {
  async createSession(
    documentId: string, 
    userId: string, 
    strategy: string
  ): Promise<AgenticRAGSession> {
    return await agenticRAGDatabaseService.createSessionWithTransaction(documentId, userId, strategy);
  }
  
  async updateSession(
    sessionId: string, 
    updates: Partial<AgenticRAGSession>
  ): Promise<void> {
    await agenticRAGDatabaseService.updateSessionWithMetrics(sessionId, updates);
  }
  
  async createExecution(
    sessionId: string, 
    agentName: string, 
    inputData: any
  ): Promise<AgentExecution> {
    return await agenticRAGDatabaseService.createExecutionWithTransaction(sessionId, agentName, inputData);
  }
}

// Main processor with configuration validation and health monitoring
class AgenticRAGProcessor {
  private sessionManager: AgenticRAGSessionManager;
  private isInitialized: boolean = false;

  constructor() {
    // Best Practice: Validate configuration at startup
    try {
      // Validate the configuration structure we have
      ConfigurationValidator.validateAgenticRAGConfig({
        enabled: config.agenticRag.enabled,
        maxAgents: config.agenticRag.maxAgents,
        parallelProcessing: config.agenticRag.parallelProcessing,
        validationStrict: config.agenticRag.validationStrict,
        retryAttempts: config.agenticRag.retryAttempts,
        timeoutPerAgent: config.agenticRag.timeoutPerAgent,
        qualityThreshold: config.qualityControl.qualityThreshold,
        completenessThreshold: config.qualityControl.completenessThreshold,
        consistencyCheck: config.qualityControl.consistencyCheck,
        detailedLogging: config.monitoringAndLogging.detailedLogging,
        performanceTracking: config.monitoringAndLogging.performanceTracking,
        errorReporting: config.monitoringAndLogging.errorReporting
      });
      this.sessionManager = new AgenticRAGSessionManager();
      this.isInitialized = true;
      
      logger.info('Agentic RAG Processor initialized successfully', {
        maxAgents: config.agenticRag.maxAgents,
        parallelProcessing: config.agenticRag.parallelProcessing,
        retryAttempts: config.agenticRag.retryAttempts
      });
    } catch (error) {
      logger.error('Failed to initialize Agentic RAG Processor', { error });
      throw error;
    }
  }

  /**
   * Process CIM document using the new multi-phase agentic RAG approach
   */
  async processDocument(text: string, documentId: string, userId: string): Promise<AgenticRAGResult> {
    const startTime = Date.now();

    if (!this.isInitialized) {
      throw new AgenticRAGError('Processor not initialized', AgenticRAGErrorType.CONFIGURATION_ERROR);
    }
    if (!text || !documentId || !userId) {
      throw new AgenticRAGError('Text, document ID, and user ID are required', AgenticRAGErrorType.CONFIGURATION_ERROR);
    }

    logger.info('Starting agentic RAG processing...', { documentId, userId });

    const session = await this.sessionManager.createSession(documentId, userId, 'agentic_rag');
    
    try {
      await this.sessionManager.updateSession(session.id, { status: 'processing' });

      // Phase 0: Load Template
      const reviewTemplate = await this.loadTemplate(session.id);

      // Phase 0.5: Document Vectorization (Critical for accurate retrieval)
      await this.executePhase0_DocumentVectorization(text, documentId, session.id);

      // Phase 1: Structured Data Extraction
      const structuredData = await this.executePhase1_StructuredDataExtraction(text, documentId, session.id);

      // Phase 2: Iterative, Section-by-Section Analysis
      const analysisResults = await this.executePhase2_IterativeAnalysis(documentId, session.id, reviewTemplate, structuredData);
      
      // Phase 3: Thesis Generation and Gap Analysis
      const thesisAndQuestions = await this.executePhase3_ThesisAndGapAnalysis(session.id, structuredData, analysisResults);

      // Combine all results
      const finalResult = { ...structuredData, ...analysisResults, ...thesisAndQuestions };

      const processingTime = Date.now() - startTime;
      await this.sessionManager.updateSession(session.id, {
        status: 'completed',
        processingTimeMs: processingTime,
        finalResult: SafeSerializer.serialize(finalResult),
        completedAt: new Date()
      });

      logger.info('Agentic RAG processing completed successfully.', { sessionId: session.id, processingTime });

      return {
        success: true,
        summary: this.convertToMarkdown(finalResult as any),
        analysisData: SafeSerializer.serialize(finalResult),
        reasoningSteps: [], // This can be populated by the sub-methods
        processingTime,
        apiCalls: 0, // This should be tracked in a refactored execution engine
        totalCost: 0,
        qualityMetrics: [],
        sessionId: session.id
      };

    } catch (error) {
      const processingTime = Date.now() - startTime;
      const agenticError = error instanceof AgenticRAGError ? error : new AgenticRAGError(
        error instanceof Error ? error.message : 'Unknown error',
        AgenticRAGErrorType.AGENT_EXECUTION_FAILED,
        'main_processor',
        false,
        {},
        error instanceof Error ? error : undefined
      );

      await this.sessionManager.updateSession(session.id, {
        status: 'failed',
        processingTimeMs: processingTime,
        completedAt: new Date()
      });

      logger.error('Agentic RAG processing failed', { sessionId: session.id, error: agenticError });
      
      throw agenticError;
    }
  }

  private async loadTemplate(sessionId: string): Promise<IReviewTemplate> {
    const execution = await this.sessionManager.createExecution(sessionId, 'load_template', {});
    try {
      const template = await loadAndParseTemplate();
      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'completed', outputData: { success: true } });
      return template;
    } catch (e) {
      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'failed', errorMessage: (e as Error).message });
      throw new AgenticRAGError('Failed to load or parse review template', AgenticRAGErrorType.TEMPLATE_ERROR, 'load_template', false, {}, e as Error);
    }
  }

  private async executePhase1_StructuredDataExtraction(text: string, documentId: string, sessionId: string): Promise<any> {
    const execution = await this.sessionManager.createExecution(sessionId, 'phase1_extraction', { documentId });
    logger.info('Executing Phase 1: Structured Data Extraction', { sessionId });

    try {
      // 1. Extract Financials
      const financials = extractFinancials(text);
      if (!financials) {
        logger.warn('Could not extract structured financial table.', { sessionId });
        // Decide if this is a critical failure or not. For now, we'll continue.
      }
      
      // 2. Extract Deal Overview (Example of a targeted LLM call)
      const dealOverviewPrompt = this.buildDealOverviewPrompt();
      const dealOverviewResult = await llmService.processCIMDocument(dealOverviewPrompt, text.substring(0, 15000), { agentName: 'deal_overview_extractor' });
      
      if (!dealOverviewResult.success) {
        throw new Error('Failed to extract deal overview from LLM.');
      }

      const outputData = {
        financialSummary: financials,
        dealOverview: dealOverviewResult.jsonOutput?.dealOverview,
      };

      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'completed', outputData: SafeSerializer.serialize(outputData) });
      return outputData;

    } catch (e) {
      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'failed', errorMessage: (e as Error).message });
      throw new AgenticRAGError('Phase 1 failed', AgenticRAGErrorType.AGENT_EXECUTION_FAILED, 'phase1_extraction', false, {}, e as Error);
    }
  }

  private async executePhase2_IterativeAnalysis(documentId: string, sessionId: string, template: IReviewTemplate, structuredData: any): Promise<any> {
    const execution = await this.sessionManager.createExecution(sessionId, 'phase2_iterative_analysis', { documentId });
    logger.info('Executing Phase 2: Iterative Analysis with Parallelism and Intelligent Query Generation', { sessionId });
    
    const analysisPromises: Promise<any>[] = [];

    for (const section of template.sections) {
      // Skip sections already handled in Phase 1 or other special sections
      if (section.title.includes('Deal Overview') || section.title.includes('Financial Summary')) {
        continue;
      }

      for (const field of section.fields) {
        analysisPromises.push(
          this.analyzeSingleField(section, field, documentId, structuredData)
        );
      }
    }

    try {
      const settledResults = await Promise.allSettled(analysisPromises);
      const finalResults: any = {};

      settledResults.forEach(result => {
        if (result.status === 'fulfilled') {
          const { sectionTitle, fieldLabel, analysis } = result.value;
          if (!finalResults[sectionTitle]) {
            finalResults[sectionTitle] = {};
          }
          finalResults[sectionTitle][fieldLabel] = analysis;
        } else {
          // Log the rejected promise error
          logger.error('Field analysis failed in Promise.allSettled', {
            error: result.reason,
            sessionId 
          });
          // Optionally, you can still mark it as failed in the final output
          // This requires passing section/field info in the error
        }
      });

      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'completed', outputData: SafeSerializer.serialize(finalResults) });
      return finalResults;

    } catch (e) {
      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'failed', errorMessage: (e as Error).message });
      throw new AgenticRAGError('Phase 2 failed', AgenticRAGErrorType.AGENT_EXECUTION_FAILED, 'phase2_iterative_analysis', false, {}, e as Error);
    }
  }

  private async analyzeSingleField(section: IFormSection, field: IFormField, documentId: string, structuredData: any): Promise<{sectionTitle: string, fieldLabel: string, analysis: any}> {
    try {
      // Step 1: Generate intelligent search queries for the field
      const searchQueries = await this.generateSearchQueriesForField(section, field);

      // Step 2: Execute enhanced vector searches for all generated queries
      const searchPromises = searchQueries.map(query => 
        vectorDocumentProcessor.searchRelevantContent(query, { 
          documentId, 
          limit: 5, // Increased for better context
          similarityThreshold: 0.75, // Higher threshold for precision
          prioritizeFinancial: this.isFinancialField(section, field),
          boostImportance: true
        })
      );
      const searchResults = await Promise.all(searchPromises);
      const relevantChunks = [...new Set(searchResults.flat().map((c: any) => c.chunkContent))]; // Deduplicate chunks
      const context = relevantChunks.join('\n---\n');

      if (!context) {
        logger.warn('No context found from vector search for field', { section: section.title, field: field.label });
        return { sectionTitle: section.title, fieldLabel: field.label, analysis: 'No relevant information found in document.' };
      }

      // Step 3: Build the final analysis prompt and get the answer
      const fieldPrompt = this.buildIterativeFieldPrompt(section, field, structuredData);
      const fieldResult = await llmService.processCIMDocument(fieldPrompt, context, { agentName: `field_analyzer_${field.label}` });

      let analysis = 'Analysis failed for this field.';
      if (fieldResult.success && fieldResult.jsonOutput && (fieldResult.jsonOutput as any).analysis) {
        analysis = (fieldResult.jsonOutput as any).analysis;
      } else {
        logger.warn('LLM analysis failed for field', { section: section.title, field: field.label, result: fieldResult });
      }
      
      return { sectionTitle: section.title, fieldLabel: field.label, analysis };

    } catch (error) {
      logger.error('Error analyzing single field', { section: section.title, field: field.label, error });
      // Re-throw to be caught by Promise.allSettled
      throw error;
    }
  }

  private async generateSearchQueriesForField(section: IFormSection, field: IFormField): Promise<string[]> {
    const prompt = `You are a research analyst preparing to analyze a section of a report.\n    Your goal is to find information about the field "${field.label}" within the broader section "${section.title}".\n    The purpose is to understand: "${field.details || section.purpose}".\n\n    Generate 3 specific, targeted questions you would use to search a document for this information. The questions should be distinct and cover different angles of the topic.\n    \n    Return ONLY a JSON array of strings. Example: ["What is the company's annual revenue?", "How has revenue grown over the past three years?"]`;

    try {
      const result = await llmService.processCIMDocument(prompt, '', { agentName: 'query_generator' });
      if (result.success && Array.isArray(result.jsonOutput) && result.jsonOutput.length > 0) {
        return result.jsonOutput as string[];
      }
      logger.warn('Failed to generate intelligent search queries, falling back to generic query.', { section: section.title, field: field.label });
    } catch (error) {
      logger.error('Error generating search queries', { error });
    }

    // Fallback to a generic query if LLM-based generation fails
    return [`${section.title} ${field.label}`];
  }

  private async executePhase3_ThesisAndGapAnalysis(sessionId: string, structuredData: any, analysisResults: any): Promise<any> {
    const execution = await this.sessionManager.createExecution(sessionId, 'phase3_thesis_and_gaps', {});
    logger.info('Executing Phase 3: Thesis Generation and Gap Analysis', { sessionId });

    try {
      // 1. Consolidate all available data
      const fullContext = {
        structuredData,
        qualitativeAnalysis: analysisResults
      };

      // 2. Perform the rule-based fund strategy alignment check
      const alignment = this.checkFundStrategyAlignment(structuredData);

      // 3. Build and execute the master synthesis prompt
      const prompt = this.buildThesisAndQuestionsPrompt(fullContext);
      const result = await llmService.processCIMDocument(prompt, '', { agentName: 'thesis_generator' });

      if (!result.success || !result.jsonOutput) {
        throw new Error('LLM failed to generate thesis and questions.');
      }

      const thesisData = result.jsonOutput;
      // Inject the definitive, rule-based alignment check into the final output
      if (thesisData.preliminaryInvestmentThesis) {
          thesisData.preliminaryInvestmentThesis.alignmentWithFundStrategy = alignment;
      }

      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'completed', outputData: SafeSerializer.serialize(thesisData) });
      return thesisData;

    } catch (e) {
      await agenticRAGDatabaseService.updateExecutionWithTransaction(execution.id, { status: 'failed', errorMessage: (e as Error).message });
      throw new AgenticRAGError('Phase 3 failed', AgenticRAGErrorType.AGENT_EXECUTION_FAILED, 'phase3_thesis_and_gaps', false, {}, e as Error);
    }
  }

  private checkFundStrategyAlignment(structuredData: any): string {
    const reasons: string[] = [];
    let aligns = true;

    // Rule 1: EBITDA check
    const financials: CleanedFinancials | null = structuredData.financialSummary;
    const ebitdaMetric = financials?.metrics.find(m => m.name.toLowerCase().includes('ebitda'));
    const lastPeriodEbitda = ebitdaMetric?.values[ebitdaMetric.values.length - 1];
    if (lastPeriodEbitda && lastPeriodEbitda < 5000000) {
      aligns = false;
      reasons.push(`EBITDA of ~${(lastPeriodEbitda / 1000000).toFixed(1)}M is below the $5M target.`);
    } else if (!lastPeriodEbitda) {
        reasons.push('EBITDA could not be definitively determined.');
    }

    // Rule 2: Industry check
    const industry = structuredData.dealOverview?.industrySector?.toLowerCase() || '';
    if (!industry.includes('consumer') && !industry.includes('industrial')) {
      aligns = false;
      reasons.push(`Industry "${structuredData.dealOverview?.industrySector || 'Unknown'}" is not in the target consumer or industrial end markets.`);
    }

    // Final alignment string
    if (aligns && reasons.length === 0) {
        return "Aligns with fund strategy: EBITDA is above $5M and the industry is in a target sector.";
    }
    return `Does not align with fund strategy. ${reasons.join(' ')}`;
  }

  private buildThesisAndQuestionsPrompt(fullContext: any): string {
    return `You are a senior partner at a private equity firm. You have been presented with a full analysis of a potential investment. Your task is to synthesize this information into a final recommendation.

    Here is all the information you have:
    ${JSON.stringify(fullContext, null, 2)}

    Based on the entirety of the data provided, perform the following tasks and return the output as a single, valid JSON object with the specified keys:

    1.  **preliminaryInvestmentThesis**: Create a concise investment thesis.
        -   keyAttractions: What are the 3-4 most compelling reasons to invest in this company? (e.g., market leadership, strong margins, recurring revenue).
        -   potentialRisks: What are the 3-4 most significant risks or concerns? (e.g., customer concentration, cyclical industry, management turnover).
        -   valueCreationLevers: How could a PE firm create value here? (e.g., M&A, operational improvements, professionalizing sales).

    2.  **keyQuestionsNextSteps**: Based on your analysis, identify gaps and define the next steps.
        -   criticalQuestions: What are the top 5-7 critical questions that remain unanswered by the CIM?
        -   keyMissingInfo: What specific information is missing that is crucial for the next stage of diligence?
        -   recommendation: Based on everything, what is your preliminary recommendation? (e.g., "Proceed with initial diligence", "Pass", "Hold for more information").
        -   rationale: A brief (1-2 sentence) rationale for your recommendation.

    Respond with ONLY the JSON object. Do not include any other text or explanation.
    `;
  }

  private buildDealOverviewPrompt(): string {
    return `You are an analyst. From the first few pages of the provided CIM text, extract the following information. Respond with ONLY a JSON object.
    {
      "dealOverview": {
        "targetCompanyName": "...",
        "industrySector": "...",
        "geography": "...",
        "dealSource": "...",
        "transactionType": "...",
        "statedReasonForSale": "..."
      }
    }`;
  }

  private buildIterativeFieldPrompt(section: IFormSection, field: IFormField, structuredData: any): string {
    return `You are a private equity analyst. Your current task is to fill out one specific field in an investment memo.
    
    SECTION: "${section.title}"
    FIELD: "${field.label}"
    FIELD PURPOSE: "${field.details || section.purpose}"

    Previously extracted data for context:
    ${JSON.stringify(structuredData, null, 2)}

    Based ONLY on the provided text snippets from the CIM, provide a concise and factual analysis for the field "${field.label}".
    Do not use outside knowledge. Respond with ONLY a JSON object with a single key "analysis".
    
    {
      "analysis": "..."
    }`;
  }

  /**
   * Convert analysis to markdown
   */
  private convertToMarkdown(analysis: CIMReview): string {
    if (!analysis || typeof analysis !== 'object') {
      return `# BPCP CIM Review: Analysis Unavailable`;
    }

    return `# BPCP CIM Review: ${analysis.dealOverview?.targetCompanyName || 'Unknown Company'}
    
    ... markdown conversion logic ...
    `;
  }

  /**
   * Phase 0.5: Advanced Document Vectorization with Intelligent Chunking
   * This is critical for accurate retrieval in subsequent phases
   */
  private async executePhase0_DocumentVectorization(text: string, documentId: string, sessionId: string): Promise<void> {
    logger.info('Starting comprehensive document vectorization', { documentId, sessionId });
    
    try {
      // Strategy 1: Stream processing for large documents
      const MAX_TEXT_SIZE = 50000; // 50KB chunks to prevent memory issues
      const chunks: Array<{
        content: string;
        chunkIndex: number;
        startPosition: number;
        endPosition: number;
        sectionType?: string;
      }> = [];

      if (text.length > MAX_TEXT_SIZE) {
        logger.info('Large document detected, using streaming chunking', { 
          documentId, 
          textLength: text.length,
          estimatedChunks: Math.ceil(text.length / MAX_TEXT_SIZE)
        });
        
        // Stream processing for large documents
        let chunkIndex = 0;
        let position = 0;
        
        while (position < text.length) {
          // Force garbage collection between chunks
          if (global.gc) {
            global.gc();
          }
          
          const chunkSize = Math.min(MAX_TEXT_SIZE, text.length - position);
          let chunkEnd = position + chunkSize;
          
          // Try to end at sentence boundary
          if (chunkEnd < text.length) {
            const sentenceEnd = this.findSentenceBoundary(text, chunkEnd);
            if (sentenceEnd > position + 1000) { // Ensure minimum chunk size
              chunkEnd = sentenceEnd;
            }
          }
          
          const chunkText = text.substring(position, chunkEnd);
          
          // Detect section type for this chunk
          const sectionType = this.identifySectionType(chunkText);
          
          chunks.push({
            content: chunkText,
            chunkIndex: chunkIndex++,
            startPosition: position,
            endPosition: chunkEnd,
            sectionType
          });
          
          position = chunkEnd;
          
          // Log progress for large documents
          if (chunkIndex % 10 === 0) {
            logger.info('Vectorization progress', { 
              documentId, 
              chunkIndex, 
              progress: Math.round((position / text.length) * 100) + '%' 
            });
          }
        }
      } else {
        // For smaller documents, use the original intelligent chunking
        chunks.push(...await this.createIntelligentChunks(text, documentId));
      }
      
      // Strategy 2: Process chunks in batches to manage memory
      const BATCH_SIZE = 5; // Process 5 chunks at a time
      const enrichedChunks: Array<{
        content: string;
        chunkIndex: number;
        startPosition: number;
        endPosition: number;
        sectionType?: string;
        metadata: {
          hasFinancialData: boolean;
          hasMetrics: boolean;
          keyTerms: string[];
          importance: 'high' | 'medium' | 'low';
          conceptDensity: number;
        };
      }> = [];
      
      for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
        const batch = chunks.slice(i, i + BATCH_SIZE);
        
        // Process batch
        const batchPromises = batch.map(async (chunk) => {
          const metadata = {
            hasFinancialData: this.containsFinancialData(chunk.content),
            hasMetrics: this.containsMetrics(chunk.content),
            keyTerms: this.extractKeyTerms(chunk.content),
            importance: this.calculateImportance(chunk.content, chunk.sectionType),
            conceptDensity: this.calculateConceptDensity(chunk.content)
          };
          
          return {
            ...chunk,
            metadata
          };
        });
        
        const batchResults = await Promise.all(batchPromises);
        enrichedChunks.push(...batchResults);
        
        // Force garbage collection after each batch
        if (global.gc) {
          global.gc();
        }
        
        // Log batch progress
        logger.info('Enriched chunk batch', { 
          documentId, 
          batchNumber: Math.floor(i / BATCH_SIZE) + 1,
          totalBatches: Math.ceil(chunks.length / BATCH_SIZE),
          processedChunks: enrichedChunks.length
        });
      }
      
      // Strategy 3: Store chunks in batches to prevent memory buildup
      const STORE_BATCH_SIZE = 3;
      for (let i = 0; i < enrichedChunks.length; i += STORE_BATCH_SIZE) {
        const storeBatch = enrichedChunks.slice(i, i + STORE_BATCH_SIZE);
        
        await vectorDocumentProcessor.storeDocumentChunks(storeBatch, {
          documentId,
          indexingStrategy: 'hierarchical',
          similarity_threshold: 0.8,
          enable_hybrid_search: true
        });
        
        // Force garbage collection after storing each batch
        if (global.gc) {
          global.gc();
        }
        
        logger.info('Stored chunk batch', { 
          documentId, 
          batchNumber: Math.floor(i / STORE_BATCH_SIZE) + 1,
          totalBatches: Math.ceil(enrichedChunks.length / STORE_BATCH_SIZE),
          storedChunks: Math.min(i + STORE_BATCH_SIZE, enrichedChunks.length)
        });
      }
      
      logger.info('Document vectorization completed successfully', { 
        documentId, 
        sessionId,
        chunksCreated: enrichedChunks.length,
        avgChunkSize: Math.round(enrichedChunks.reduce((sum: number, c: any) => sum + c.content.length, 0) / enrichedChunks.length),
        totalTextLength: text.length
      });
      
    } catch (error) {
      logger.error('Document vectorization failed', { documentId, sessionId, error });
      throw new AgenticRAGError(
        'Failed to vectorize document for retrieval',
        AgenticRAGErrorType.DATABASE_ERROR,
        'vectorization_engine',
        true,
        { documentId, sessionId },
        error instanceof Error ? error : undefined
      );
    }
  }

  /**
   * Create intelligent chunks with semantic boundaries and optimal overlap
   */
  private async createIntelligentChunks(text: string, documentId: string): Promise<Array<{
    content: string;
    chunkIndex: number;
    startPosition: number;
    endPosition: number;
    sectionType?: string;
  }>> {
    const chunks: Array<{
      content: string;
      chunkIndex: number;
      startPosition: number;
      endPosition: number;
      sectionType?: string;
    }> = [];

    // Configuration for optimal CIM document processing
    const CHUNK_SIZE = 1000; // Optimal for financial documents
    const OVERLAP_SIZE = 200; // 20% overlap for context preservation
    const MIN_CHUNK_SIZE = 300; // Minimum meaningful chunk size

    // Strategy 1: Detect section boundaries (headers, page breaks, etc.)
    const sectionBoundaries = this.detectSectionBoundaries(text);
    
    // Strategy 2: Split on semantic boundaries first
    const semanticSections = this.splitOnSemanticBoundaries(text, sectionBoundaries);
    
    let chunkIndex = 0;
    let globalPosition = 0;

    for (const section of semanticSections) {
      const sectionText = section.content;
      const sectionType = section.type;
      
      // If section is small enough, keep it as one chunk
      if (sectionText.length <= CHUNK_SIZE) {
        chunks.push({
          content: sectionText,
          chunkIndex: chunkIndex++,
          startPosition: globalPosition,
          endPosition: globalPosition + sectionText.length,
          sectionType
        });
        globalPosition += sectionText.length;
        continue;
      }
      
      // For larger sections, create overlapping chunks
      let sectionPosition = 0;
      const sectionStart = globalPosition;
      
      while (sectionPosition < sectionText.length) {
        const remainingText = sectionText.length - sectionPosition;
        const chunkSize = Math.min(CHUNK_SIZE, remainingText);
        
        // Adjust chunk end to sentence boundary if possible
        let chunkEnd = sectionPosition + chunkSize;
        if (chunkEnd < sectionText.length) {
          const sentenceEnd = this.findSentenceBoundary(sectionText, chunkEnd);
          if (sentenceEnd > sectionPosition + MIN_CHUNK_SIZE) {
            chunkEnd = sentenceEnd;
          }
        }
        
        const chunkContent = sectionText.substring(sectionPosition, chunkEnd);
        
        chunks.push({
          content: chunkContent.trim(),
          chunkIndex: chunkIndex++,
          startPosition: sectionStart + sectionPosition,
          endPosition: sectionStart + chunkEnd,
          sectionType
        });
        
        // Move to next chunk with overlap
        sectionPosition = chunkEnd - OVERLAP_SIZE;
        if (sectionPosition < 0) sectionPosition = chunkEnd;
      }
      
      globalPosition += sectionText.length;
    }

    logger.info('Intelligent chunking completed', {
      documentId,
      totalChunks: chunks.length,
              avgChunkSize: Math.round(chunks.reduce((sum: number, c: any) => sum + c.content.length, 0) / chunks.length),
      sectionTypes: [...new Set(chunks.map(c => c.sectionType).filter(Boolean))]
    });

    return chunks;
  }

  // /**
  //  * Enrich chunks with metadata for enhanced retrieval
  //  */
  // private async enrichChunksWithMetadata(chunks: Array<{
  //   content: string;
  //   chunkIndex: number;
  //   startPosition: number;
  //   endPosition: number;
  //   sectionType?: string;
  // }>): Promise<Array<{
  //   content: string;
  //   chunkIndex: number;
  //   startPosition: number;
  //   endPosition: number;
  //   sectionType?: string;
  //   metadata: {
  //     hasFinancialData: boolean;
  //     hasMetrics: boolean;
  //     keyTerms: string[];
  //     importance: 'high' | 'medium' | 'low';
  //     conceptDensity: number;
  //   };
  // }>> {
  //   const enrichedChunks = [];

  //   for (const chunk of chunks) {
  //     // Analyze chunk content for metadata
  //     const hasFinancialData = this.containsFinancialData(chunk.content);
  //     const hasMetrics = this.containsMetrics(chunk.content);
  //     const keyTerms = this.extractKeyTerms(chunk.content);
  //     const importance = this.calculateImportance(chunk.content, chunk.sectionType);
  //     const conceptDensity = this.calculateConceptDensity(chunk.content);

  //     enrichedChunks.push({
  //       ...chunk,
  //       metadata: {
  //         hasFinancialData,
  //         hasMetrics,
  //         keyTerms,
  //         importance,
  //         conceptDensity
  //       }
  //     });
  //   }

  //   return enrichedChunks;
  // }

  /**
   * Detect section boundaries in CIM documents
   */
  private detectSectionBoundaries(text: string): number[] {
    const boundaries: number[] = [0];
    
    // Common CIM section patterns
    const sectionPatterns = [
      /^(EXECUTIVE SUMMARY|COMPANY OVERVIEW|BUSINESS DESCRIPTION)/im,
      /^(FINANCIAL PERFORMANCE|FINANCIAL ANALYSIS|HISTORICAL FINANCIALS)/im,
      /^(MARKET ANALYSIS|INDUSTRY OVERVIEW|COMPETITIVE LANDSCAPE)/im,
      /^(MANAGEMENT TEAM|LEADERSHIP|KEY PERSONNEL)/im,
      /^(INVESTMENT HIGHLIGHTS|GROWTH OPPORTUNITIES)/im,
      /^(APPENDIX|FINANCIAL STATEMENTS|SUPPORTING DOCUMENTS)/im
    ];

    const lines = text.split('\n');
    let position = 0;

    for (let i = 0; i < lines.length; i++) {
      const line = (lines[i] || '').trim();
      
      // Check for section headers
      if (sectionPatterns.some(pattern => pattern.test(line))) {
        boundaries.push(position);
      }
      
      // Check for page breaks or significant whitespace
      if (line === '' && i > 0 && i < lines.length - 1) {
        const nextNonEmpty = lines.slice(i + 1).findIndex(l => l.trim() !== '');
        if (nextNonEmpty > 2) { // Multiple empty lines suggest section break
          boundaries.push(position);
        }
      }
      
      position += (lines[i] || '').length + 1; // +1 for newline
    }

    boundaries.push(text.length);
    return [...new Set(boundaries)].sort((a, b) => a - b);
  }

  /**
   * Split text on semantic boundaries
   */
  private splitOnSemanticBoundaries(text: string, boundaries: number[]): Array<{
    content: string;
    type: string;
  }> {
    const sections: Array<{ content: string; type: string }> = [];
    
    for (let i = 0; i < boundaries.length - 1; i++) {
      const start = boundaries[i] || 0;
      const end = boundaries[i + 1] || text.length;
      const content = text.substring(start, end).trim();
      
      if (content.length > 50) { // Filter out tiny sections
        const type = this.identifySectionType(content);
        sections.push({ content, type });
      }
    }
    
    return sections;
  }

  /**
   * Identify section type based on content
   */
  private identifySectionType(content: string): string {
    const firstLines = content.split('\n').slice(0, 3).join(' ').toLowerCase();
    
    if (/executive summary|overview|introduction/i.test(firstLines)) return 'executive_summary';
    if (/financial|revenue|ebitda|cash flow/i.test(firstLines)) return 'financial';
    if (/market|industry|competitive|sector/i.test(firstLines)) return 'market_analysis';
    if (/management|team|leadership|personnel/i.test(firstLines)) return 'management';
    if (/growth|opportunity|strategy|expansion/i.test(firstLines)) return 'growth_strategy';
    if (/risk|challenge|concern/i.test(firstLines)) return 'risk_analysis';
    
    return 'general';
  }

  /**
   * Find optimal sentence boundary for chunk splitting
   */
  private findSentenceBoundary(text: string, position: number): number {
    const searchWindow = 100; // Look 100 chars back for sentence end
    const searchStart = Math.max(0, position - searchWindow);
    
    for (let i = position; i >= searchStart; i--) {
      const char = text[i];
      if (char === '.' || char === '!' || char === '?') {
        // Make sure it's actually end of sentence, not abbreviation
        if (i < text.length - 1 && /\s/.test(text[i + 1] || '')) {
          return i + 1;
        }
      }
    }
    
    return position; // Fallback to original position
  }

  /**
   * Check if chunk contains financial data
   */
  private containsFinancialData(content: string): boolean {
    const financialPatterns = [
      /\$[\d,]+(?:\.\d{2})?(?:[kmb])?/i, // Currency amounts
      /\d+(?:\.\d+)?%/, // Percentages
      /revenue|ebitda|cash flow|profit|margin|roi|irr/i,
      /\d{4}\s*(fy|fiscal year|year ended)/i // Fiscal years
    ];
    
    return financialPatterns.some(pattern => pattern.test(content));
  }

  /**
   * Check if chunk contains metrics
   */
  private containsMetrics(content: string): boolean {
    const metricPatterns = [
      /\d+(?:\.\d+)?\s*(?:million|billion|thousand|m|b|k)/i,
      /\d+(?:\.\d+)?x/i, // Multiples
      /growth|increase|decrease|change/i
    ];
    
    return metricPatterns.some(pattern => pattern.test(content));
  }

  /**
   * Extract key terms from chunk
   */
  private extractKeyTerms(content: string): string[] {
    // Simple key term extraction - could be enhanced with NLP
    const keyTermPatterns = [
      /\b[A-Z][a-z]+ [A-Z][a-z]+\b/g, // Proper nouns (likely company/person names)
      /\b(?:EBITDA|ROI|IRR|CAGR|SaaS|B2B|B2C)\b/gi, // Business acronyms
      /\b\d+(?:\.\d+)?%\b/g, // Percentages
      /\$[\d,]+(?:\.\d{2})?(?:[kmb])?/gi // Currency amounts
    ];
    
    const terms: string[] = [];
    keyTermPatterns.forEach(pattern => {
      const matches = content.match(pattern) || [];
      terms.push(...matches);
    });
    
    return [...new Set(terms)].slice(0, 10); // Top 10 unique terms
  }

  /**
   * Calculate importance score for chunk
   */
  private calculateImportance(content: string, sectionType?: string): 'high' | 'medium' | 'low' {
    let score = 0;
    
    // Section type scoring
    if (sectionType === 'executive_summary') score += 3;
    else if (sectionType === 'financial') score += 2;
    else if (sectionType === 'market_analysis') score += 2;
    else score += 1;
    
    // Content analysis scoring
    if (this.containsFinancialData(content)) score += 2;
    if (this.containsMetrics(content)) score += 1;
    if (/key|important|critical|significant/i.test(content)) score += 1;
    
    if (score >= 5) return 'high';
    if (score >= 3) return 'medium';
    return 'low';
  }

  /**
   * Calculate concept density (information richness)
   */
  private calculateConceptDensity(content: string): number {
    const words = content.split(/\s+/).length;
    const concepts = this.extractKeyTerms(content).length;
    const financialElements = (content.match(/\$[\d,]+|\d+%|\d+(?:\.\d+)?[kmb]/gi) || []).length;
    
    return Math.min(1.0, (concepts + financialElements) / Math.max(words / 100, 1));
  }

  /**
   * Determine if a field is financial-related for search prioritization
   */
  private isFinancialField(section: IFormSection, field: IFormField): boolean {
    const fieldText = `${section.title} ${field.label}`.toLowerCase();
    return /financial|revenue|ebitda|profit|margin|cash|debt|cost|expense|income|sales/i.test(fieldText);
  }
  
  // Best Practice: Graceful shutdown
  async shutdown(): Promise<void> {
    logger.info('Shutting down Agentic RAG Processor');
    this.isInitialized = false;
  }
}

export const agenticRAGProcessor = new AgenticRAGProcessor(); 