import fs from 'fs';
import path from 'path';
import { logger } from '../utils/logger';
import { fileStorageService } from './fileStorageService';
import { DocumentModel } from '../models/DocumentModel';
import { ProcessingJobModel } from '../models/ProcessingJobModel';
import { llmService } from './llmService';
import { pdfGenerationService } from './pdfGenerationService';
import { config } from '../config/env';
import { uploadProgressService } from './uploadProgressService';
import { CIMReview } from './llmSchemas';

export interface ProcessingStep {
  name: string;
  status: 'pending' | 'processing' | 'completed' | 'failed';
  startTime?: Date;
  endTime?: Date;
  error?: string;
  metadata?: Record<string, any>;
}

export interface ProcessingResult {
  success: boolean;
  jobId: string;
  documentId: string;
  steps: ProcessingStep[];
  extractedText?: string;
  summary?: string;
  analysis?: Record<string, any>;
  error?: string;
}

export interface ProcessingOptions {
  extractText?: boolean;
  generateSummary?: boolean;
  performAnalysis?: boolean;
  maxTextLength?: number;
  chunkSize?: number;
  strategy?: string;
}

class DocumentProcessingService {
  private readonly defaultOptions: ProcessingOptions = {
    extractText: true,
    generateSummary: true,
    performAnalysis: true,
    maxTextLength: 100000, // 100KB limit
    chunkSize: 4000, // 4KB chunks for processing
  };

  /**
   * Process a document through the complete pipeline
   */
  async processDocument(
    documentId: string,
    userId: string,
    options: ProcessingOptions = {}
  ): Promise<ProcessingResult> {
    const mergedOptions = { ...this.defaultOptions, ...options };
    const jobId = `job_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
    
    logger.info(`Starting document processing: ${documentId}`, {
      documentId,
      jobId,
      userId,
      options: mergedOptions,
      timestamp: new Date().toISOString(),
    });

    // Initialize progress tracking
    uploadProgressService.initializeProgress(documentId, jobId);

    const steps: ProcessingStep[] = [
      { name: 'validation', status: 'pending' },
      { name: 'text_extraction', status: 'pending' },
      { name: 'analysis', status: 'pending' },
      { name: 'summary_generation', status: 'pending' },
      { name: 'storage', status: 'pending' },
    ];

    let extractedText: string | undefined;
    let analysis: Record<string, any> | undefined;
    let summary: string | undefined;
    // Removed unused variable
    let markdownPath: string | undefined;
    let pdfPath: string | undefined;

    try {
      // Create processing job record
      await this.createProcessingJob(jobId, documentId);

      // Step 1: Validation
      uploadProgressService.updateProgress(documentId, 'validation', 10, 'Validating document...');
      await this.executeStep(steps, 'validation', async () => {
        await this.validateDocument(documentId, userId);
      });

      // Step 2: Text Extraction
      uploadProgressService.updateProgress(documentId, 'text_extraction', 20, 'Extracting text from document...');
      await this.executeStep(steps, 'text_extraction', async () => {
        extractedText = await this.extractTextFromPDF(documentId);
        uploadProgressService.updateProgress(documentId, 'text_extraction', 100, `Text extraction completed (${extractedText.length} characters)`);
      });

      // Step 3: Analysis
      uploadProgressService.updateProgress(documentId, 'analysis', 40, 'Analyzing document content...');
      await this.executeStep(steps, 'analysis', async () => {
        if (extractedText && mergedOptions.performAnalysis) {
          analysis = await this.analyzeDocument(extractedText);
          uploadProgressService.updateProgress(documentId, 'analysis', 100, 'Document analysis completed');
        }
      });

      // Step 4: Summary Generation
      uploadProgressService.updateProgress(documentId, 'summary_generation', 60, 'Generating summary...');
      await this.executeStep(steps, 'summary_generation', async () => {
        if (extractedText && mergedOptions.generateSummary) {
          const summaryResult = await this.generateSummary(documentId, extractedText, analysis || {});
          summary = summaryResult.summary;
          analysis = summaryResult.analysisData;
          
          // Generate markdown file
          const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
          markdownPath = `uploads/summaries/${documentId}_${timestamp}.md`;
          pdfPath = `uploads/summaries/${documentId}_${timestamp}.pdf`;
          
          logger.info('Saving markdown file', {
            documentId,
            markdownPath,
            summaryLength: summary.length
          });
          
          // Save markdown file
          await this.saveMarkdownFile(markdownPath, summary);
          
          logger.info('Markdown file saved successfully', {
            documentId,
            markdownPath
          });
          
          // Generate PDF from markdown
          logger.info('Generating PDF from markdown', {
            documentId,
            pdfPath,
            summaryLength: summary.length
          });
          
          const pdfGenerated = await pdfGenerationService.generatePDFFromMarkdown(
            summary,
            path.join(config.upload.uploadDir, pdfPath)
          );
          
          if (!pdfGenerated) {
            logger.warn('PDF generation failed, continuing without PDF', {
              documentId,
              pdfPath
            });
          } else {
            logger.info('PDF generated successfully', {
              documentId,
              pdfPath
            });
          }

          uploadProgressService.updateProgress(documentId, 'summary_generation', 100, 'Summary generation completed');

          return {
            summaryLength: summary.length,
            markdownPath,
            pdfPath: pdfGenerated ? pdfPath : '',
          };
        }
        return null;
      });

      // Step 5: Storage
      uploadProgressService.updateProgress(documentId, 'storage', 90, 'Saving processing results...');
      await this.executeStep(steps, 'storage', async () => {
        logger.info('Starting storage step', {
          documentId,
          hasExtractedText: !!extractedText,
          hasSummary: !!summary,
          hasAnalysis: !!analysis,
          summaryLength: summary?.length || 0
        });
        
        try {
          const storageResult = await this.storeProcessingResults(documentId, {
            extractedText: extractedText || '',
            summary: summary || '',
            analysis: analysis as CIMReview || {} as CIMReview,
            processingSteps: steps,
            markdownPath: markdownPath || '',
            pdfPath: pdfPath || '',
          });
          
          logger.info('Storage step completed successfully', {
            documentId,
            storageResult
          });
          
          return storageResult;
        } catch (error) {
          logger.error('Storage step failed', {
            documentId,
            error: error instanceof Error ? error.message : 'Unknown error'
          });
          throw error;
        }
      });

      logger.info('All processing steps completed, updating job status', {
        documentId,
        jobId,
        stepsCompleted: steps.filter(s => s.status === 'completed').length,
        totalSteps: steps.length
      });

      // Update job status to completed
      await this.updateProcessingJob(jobId, 'completed');

      // Clean up the original uploaded file after successful processing
      await this.cleanupOriginalFile(documentId);

      // Mark progress as completed
      uploadProgressService.markCompleted(documentId, 'Document processing completed successfully');

      logger.info(`Document processing completed: ${documentId}`, {
        jobId,
        documentId,
        userId,
        processingTime: this.calculateProcessingTime(steps),
        summaryLength: summary?.length || 0
      });

      return {
        success: true,
        jobId,
        documentId,
        steps,
        extractedText: extractedText || '',
        summary: summary || '',
        analysis: analysis || {},
      };
    } catch (error) {
      const errorMessage = error instanceof Error ? error.message : 'Unknown error';
      
      logger.error(`Document processing failed: ${documentId}`, {
        jobId,
        documentId,
        userId,
        error: errorMessage,
        steps: steps.map(s => ({ name: s.name, status: s.status, error: s.error })),
      });

      // Update job status to failed
      await this.updateProcessingJob(jobId, 'failed');

              // Only clean up the original uploaded file if this is the final attempt
        // (not a retry) to avoid cleaning up files that might be needed for retries
        const job = await ProcessingJobModel.findByJobId(jobId);
        if (job && (job as any).attempts >= 3) {
          await this.cleanupOriginalFile(documentId);
        }

      // Mark progress as failed
      uploadProgressService.markError(documentId, errorMessage);

      return {
        success: false,
        jobId,
        documentId,
        steps,
        error: errorMessage,
      };
    }
  }

  /**
   * Execute a processing step with error handling
   */
  private async executeStep(
    steps: ProcessingStep[],
    stepName: string,
    stepFunction: () => Promise<any>
  ): Promise<void> {
    const step = steps.find(s => s.name === stepName);
    if (!step) {
      throw new Error(`Step ${stepName} not found`);
    }

    try {
      step.status = 'processing';
      step.startTime = new Date();

      logger.info(`Executing processing step: ${stepName}`);

      const result = await stepFunction();

      step.status = 'completed';
      step.endTime = new Date();
      step.metadata = result;

      logger.info(`Processing step completed: ${stepName}`, {
        duration: step.endTime.getTime() - step.startTime!.getTime(),
      });
    } catch (error) {
      step.status = 'failed';
      step.endTime = new Date();
      step.error = error instanceof Error ? error.message : 'Unknown error';

      logger.error(`Processing step failed: ${stepName}`, {
        error: step.error,
      });

      throw error;
    }
  }

  /**
   * Validate document exists and user has access
   */
  private async validateDocument(documentId: string, userId: string): Promise<void> {
    const document = await DocumentModel.findById(documentId);
    
    if (!document) {
      throw new Error('Document not found');
    }

    if (document.user_id !== userId) {
      throw new Error('Access denied');
    }

    if (!document.file_path) {
      throw new Error('Document file not found');
    }

    const fileExists = await fileStorageService.fileExists(document.file_path);
    if (!fileExists) {
      throw new Error('Document file not accessible');
    }

    logger.info(`Document validation passed: ${documentId}`);
  }

  /**
   * Extract text from PDF file
   */
  private async extractTextFromPDF(documentId: string): Promise<string> {
    const document = await DocumentModel.findById(documentId);
    if (!document || !document.file_path) {
      throw new Error('Document file not found');
    }

    try {
      const fileBuffer = await fileStorageService.getFile(document.file_path);
      if (!fileBuffer) {
        throw new Error('Could not read document file');
      }

      const filePath = document.file_path;
      let extractedText: string;

      // Check file extension to determine processing method
      if (filePath.toLowerCase().endsWith('.pdf')) {
        // Use pdf-parse for actual PDF text extraction
        const pdfParse = require('pdf-parse');
        const data = await pdfParse(fileBuffer);
        extractedText = data.text;
        
        logger.info(`PDF text extraction completed: ${documentId}`, {
          textLength: extractedText.length,
          fileSize: fileBuffer.length,
          pages: data.numpages,
        });
      } else {
        // For text files, read the content directly
        extractedText = fileBuffer.toString('utf-8');
        
        logger.info(`Text file extraction completed: ${documentId}`, {
          textLength: extractedText.length,
          fileSize: fileBuffer.length,
          fileType: path.extname(filePath),
        });
      }

      return extractedText;
    } catch (error) {
      logger.error(`Text extraction failed: ${documentId}`, error);
      throw new Error(`Text extraction failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }
  }

  /**
   * Analyze extracted text for key information
   */
  private async analyzeDocument(text: string): Promise<Record<string, any>> {
    try {
      // Enhanced document analysis with LLM integration
      const analysis: Record<string, any> = {
        wordCount: text.split(/\s+/).length,
        characterCount: text.length,
        paragraphCount: text.split(/\n\s*\n/).length,
        estimatedReadingTime: Math.ceil(text.split(/\s+/).length / 200), // 200 words per minute
        language: this.detectLanguage(text),
        hasFinancialData: this.detectFinancialContent(text),
        hasTechnicalData: this.detectTechnicalContent(text),
        documentType: this.detectDocumentType(text),
        keyTopics: this.extractKeyTopics(text),
        sentiment: this.analyzeSentiment(text),
        complexity: this.assessComplexity(text),
        tokenEstimate: this.estimateTokenCount(text),
      };

      logger.info('Document analysis completed', analysis);
      return analysis;
    } catch (error) {
      logger.error('Document analysis failed', error);
      throw new Error(`Document analysis failed: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }
  }

  /**
   * Generate summary from extracted text using LLM with hierarchical processing
   */
  private async generateSummary(documentId: string, text: string, analysis: Record<string, any>): Promise<{ summary: string; analysisData: CIMReview }> {
    try {
      // Update document status to processing_llm
      await this.updateDocumentStatus(documentId, 'processing_llm');
      
      logger.info('Starting hierarchical summary generation process', {
        textLength: text.length,
        analysisKeys: Object.keys(analysis || {})
      });

      // Estimate tokens and determine processing strategy
      const tokenEstimate = this.estimateTokenCount(text);
      const maxTokens = config.llm.maxTokens;
      const threshold = maxTokens - config.llm.promptBuffer;
      const needsChunking = tokenEstimate > threshold;

      logger.info('Token analysis completed', {
        tokenEstimate,
        maxTokens,
        threshold,
        needsChunking
      });

      if (needsChunking) {
        // Use hierarchical processing for large documents
        return await this.processLargeDocumentHierarchically(documentId, text, analysis);
      } else {
        // Process entire document in a single pass
        const llmResult = await llmService.processCIMDocument(text, '', analysis || {});
        
        if (!llmResult.success || !llmResult.jsonOutput) {
          throw new Error(llmResult.error || 'LLM processing failed to return valid JSON.');
        }

        return {
          summary: this.convertJsonToMarkdown(llmResult.jsonOutput),
          analysisData: llmResult.jsonOutput
        };
      }
    } catch (error) {
      logger.error('Summary generation failed', {
        documentId,
        error: error instanceof Error ? error.message : 'Unknown error'
      });
      throw error;
    }
  }

  /**
   * Process large documents using hierarchical approach
   */
  private async processLargeDocumentHierarchically(documentId: string, text: string, analysis: Record<string, any>): Promise<{ summary: string; analysisData: CIMReview }> {
    logger.info('Starting hierarchical processing for large document');
    
    // Step 1: High-level document overview (first 20% of document)
    uploadProgressService.updateProgress(documentId, 'summary_generation', 65, 'Analyzing document structure...');
    
    const overviewChunk = this.extractDocumentOverview(text);
    const overviewResult = await llmService.processCIMDocument(overviewChunk, '', { overviewMode: true });
    
    if (!overviewResult.success || !overviewResult.jsonOutput) {
      throw new Error('Failed to generate document overview');
    }

    // Step 2: Section-specific analysis with larger chunks
    uploadProgressService.updateProgress(documentId, 'summary_generation', 75, 'Analyzing document sections...');
    
    const sections = this.extractDocumentSections(text);
    const sectionResults: any[] = [];
    
    for (let i = 0; i < sections.length; i++) {
      const section = sections[i];
      if (!section) continue;
      
      logger.info(`Processing section ${i + 1}/${sections.length}`, {
        sectionType: section.type,
        sectionLength: section.content.length
      });
      
      uploadProgressService.updateProgress(documentId, 'summary_generation', 75 + ((i + 1) / sections.length) * 15, `Analyzing ${section.type} section...`);
      
      try {
        const sectionResult = await llmService.processCIMDocument(
          section.content, 
          '', 
          { 
            sectionType: section.type,
            overview: overviewResult.jsonOutput,
            analysis 
          }
        );
        
        if (sectionResult.success && sectionResult.jsonOutput) {
          sectionResults.push({
            type: section.type,
            data: sectionResult.jsonOutput
          });
        }
      } catch (error) {
        logger.warn(`Section ${section.type} processing failed, continuing with other sections`, {
          error: error instanceof Error ? error.message : 'Unknown error'
        });
      }
    }

    // Step 3: Synthesize results
    uploadProgressService.updateProgress(documentId, 'summary_generation', 95, 'Synthesizing analysis...');
    
    const synthesizedResult = await this.synthesizeSectionResults(
      overviewResult.jsonOutput,
      sectionResults,
      text
    );

    return {
      summary: this.convertJsonToMarkdown(synthesizedResult),
      analysisData: synthesizedResult
    };
  }

  /**
   * Extract document overview (first portion with key information)
   */
  private extractDocumentOverview(text: string): string {
    // Extract first 30% of document which typically contains executive summary, company overview
    const overviewLength = Math.floor(text.length * 0.3);
    return text.substring(0, overviewLength);
  }

  /**
   * Extract logical sections from document
   */
  private extractDocumentSections(text: string): Array<{ type: string; content: string }> {
    const sections: Array<{ type: string; content: string }> = [];
    
    // Split document into logical sections based on headers
    const sectionHeaders = [
      { pattern: /(?:^|\n)(?:executive\s+summary|overview|introduction)/i, type: 'overview' },
      { pattern: /(?:^|\n)(?:business\s+description|company\s+overview|operations)/i, type: 'business' },
      { pattern: /(?:^|\n)(?:market\s+analysis|industry\s+analysis|competitive\s+landscape)/i, type: 'market' },
      { pattern: /(?:^|\n)(?:financial\s+(?:overview|summary|performance)|financials)/i, type: 'financial' },
      { pattern: /(?:^|\n)(?:management\s+(?:team|overview)|leadership)/i, type: 'management' },
      { pattern: /(?:^|\n)(?:investment\s+(?:thesis|opportunity|highlights))/i, type: 'investment' },
    ];

    let currentSection = { type: 'overview', content: text };
    let remainingText = text;

    for (const header of sectionHeaders) {
      const match = remainingText.match(header.pattern);
      if (match) {
        // Save previous section
        if (currentSection.content.length > 1000) { // Only include substantial sections
          sections.push(currentSection);
        }
        
        // Start new section
        const startIndex = match.index!;
        currentSection = {
          type: header.type,
          content: remainingText.substring(startIndex)
        };
        remainingText = remainingText.substring(startIndex);
      }
    }

    // Add final section
    if (currentSection.content.length > 1000) {
      sections.push(currentSection);
    }

    // If no sections found, use intelligent chunking
    if (sections.length === 0) {
      const chunks = this.chunkText(text, 15000); // Larger chunks for section analysis
      return chunks.map((chunk, index) => ({
        type: `section_${index + 1}`,
        content: chunk
      }));
    }

    return sections;
  }

  /**
   * Synthesize results from different sections
   */
  private async synthesizeSectionResults(
    overview: CIMReview,
    sectionResults: Array<{ type: string; data: any }>,
    fullText: string
  ): Promise<CIMReview> {
    // Combine all section data
    const combinedData = { ...overview };
    
    for (const section of sectionResults) {
      // Merge section data into combined structure
      this.mergeSectionData(combinedData, section.data, section.type);
    }

    // Final synthesis pass with full context
    const synthesisResult = await llmService.processCIMDocument(
      JSON.stringify(combinedData),
      '',
      { 
        synthesisMode: true,
        fullText: fullText.substring(0, 10000) // Include sample of full text for context
      }
    );

    if (synthesisResult.success && synthesisResult.jsonOutput) {
      return synthesisResult.jsonOutput;
    }

    return combinedData;
  }

  /**
   * Merge section data into main structure
   */
  private mergeSectionData(mainData: CIMReview, sectionData: any, sectionType: string): void {
    switch (sectionType) {
      case 'business':
        if (sectionData.businessDescription) {
          mainData.businessDescription = {
            ...mainData.businessDescription,
            ...sectionData.businessDescription
          };
        }
        break;
      case 'market':
        if (sectionData.marketIndustryAnalysis) {
          mainData.marketIndustryAnalysis = {
            ...mainData.marketIndustryAnalysis,
            ...sectionData.marketIndustryAnalysis
          };
        }
        break;
      case 'financial':
        if (sectionData.financialSummary) {
          mainData.financialSummary = {
            ...mainData.financialSummary,
            ...sectionData.financialSummary
          };
        }
        break;
      case 'management':
        if (sectionData.managementTeamOverview) {
          mainData.managementTeamOverview = {
            ...mainData.managementTeamOverview,
            ...sectionData.managementTeamOverview
          };
        }
        break;
      case 'investment':
        if (sectionData.preliminaryInvestmentThesis) {
          mainData.preliminaryInvestmentThesis = {
            ...mainData.preliminaryInvestmentThesis,
            ...sectionData.preliminaryInvestmentThesis
          };
        }
        break;
    }
  }

  /**
   * Update document status during processing
   */
  private async updateDocumentStatus(documentId: string, status: string): Promise<void> {
    try {
      const updateData: any = { status };
      
      if (status === 'processing_llm') {
        updateData.processing_started_at = new Date();
      }
      
      const updated = await DocumentModel.updateById(documentId, updateData);
      if (!updated) {
        logger.warn(`Failed to update document status: ${documentId} - ${status}`);
      } else {
        logger.info(`Document status updated: ${documentId} - ${status}`);
      }
    } catch (error) {
      logger.error(`Failed to update document status: ${documentId} - ${status}`, error);
    }
  }

  /**
   * Store processing results in database
   */
  private async storeProcessingResults(
    documentId: string,
    results: {
      extractedText?: string;
      summary?: string;
      analysis?: CIMReview;
      processingSteps: ProcessingStep[];
      markdownPath?: string;
      pdfPath?: string;
    }
  ): Promise<void> {
    try {
      const updateFields: any = {
        status: 'completed',
        processing_completed_at: new Date(),
      };

      if (results.extractedText) {
        updateFields.extracted_text = results.extractedText;
      }

      if (results.summary) {
        updateFields.generated_summary = results.summary;
      }

      if (results.analysis) {
        updateFields.analysis_data = results.analysis;
      }

      if (results.markdownPath) {
        updateFields.summary_markdown_path = results.markdownPath;
      }

      if (results.pdfPath) {
        updateFields.summary_pdf_path = results.pdfPath;
      }

      const updated = await DocumentModel.updateById(documentId, updateFields);
      if (!updated) {
        throw new Error('Failed to update document with processing results');
      }

      logger.info(`Processing results stored: ${documentId}`);
    } catch (error) {
      logger.error(`Failed to store processing results: ${documentId}`, error);
      throw new Error(`Failed to store processing results: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }
  }

  /**
   * Create processing job record
   */
  private async createProcessingJob(
    jobId: string,
    documentId: string
  ): Promise<void> {
    try {
      await ProcessingJobModel.create({
        job_id: jobId,
        document_id: documentId,
        type: 'llm_processing', // Default type
      });

      logger.info(`Processing job created: ${jobId}`);
    } catch (error) {
      logger.error(`Failed to create processing job: ${jobId}`, error);
      throw error;
    }
  }

  /**
   * Update processing job status
   */
  private async updateProcessingJob(
    jobId: string,
    status: string
  ): Promise<void> {
    // Note: Job queue service manages jobs in memory, database jobs are separate
    // This method is kept for potential future integration but currently disabled
    // to avoid warnings about missing job_id values in database
    logger.debug(`Processing job status update (in-memory): ${jobId} -> ${status}`);
  }

  /**
   * Calculate total processing time
   */
  private calculateProcessingTime(steps: ProcessingStep[]): number {
    const completedSteps = steps.filter(s => s.startTime && s.endTime);
    if (completedSteps.length === 0) return 0;

    const startTime = Math.min(...completedSteps.map(s => s.startTime!.getTime()));
    const endTime = Math.max(...completedSteps.map(s => s.endTime!.getTime()));

    return endTime - startTime;
  }

  /**
   * Detect language of the text
   */
  private detectLanguage(text: string): string {
    // Simple language detection based on common words
    const lowerText = text.toLowerCase();
    
    if (lowerText.includes('the') && lowerText.includes('and') && lowerText.includes('of')) {
      return 'en';
    }
    
    // Add more language detection logic as needed
    return 'en'; // Default to English
  }

  /**
   * Detect financial content in text
   */
  private detectFinancialContent(text: string): boolean {
    const financialKeywords = [
      'financial', 'revenue', 'profit', 'ebitda', 'margin', 'cash flow',
      'balance sheet', 'income statement', 'assets', 'liabilities',
      'equity', 'debt', 'investment', 'return', 'valuation'
    ];
    
    const lowerText = text.toLowerCase();
    return financialKeywords.some(keyword => lowerText.includes(keyword));
  }

  /**
   * Detect technical content in text
   */
  private detectTechnicalContent(text: string): boolean {
    const technicalKeywords = [
      'technical', 'specification', 'requirements', 'architecture',
      'system', 'technology', 'software', 'hardware', 'protocol',
      'algorithm', 'data', 'analysis', 'methodology'
    ];
    
    const lowerText = text.toLowerCase();
    return technicalKeywords.some(keyword => lowerText.includes(keyword));
  }

  /**
   * Extract key topics from text
   */
  private extractKeyTopics(text: string): string[] {
    const topics: string[] = [];
    const lowerText = text.toLowerCase();
    
    // Extract potential topics based on common patterns
    if (lowerText.includes('financial')) topics.push('Financial Analysis');
    if (lowerText.includes('market')) topics.push('Market Analysis');
    if (lowerText.includes('competitive')) topics.push('Competitive Landscape');
    if (lowerText.includes('technology')) topics.push('Technology');
    if (lowerText.includes('operations')) topics.push('Operations');
    if (lowerText.includes('management')) topics.push('Management');
    
    return topics.slice(0, 5); // Return top 5 topics
  }

  /**
   * Analyze sentiment of the text
   */
  private analyzeSentiment(text: string): string {
    const positiveWords = ['growth', 'increase', 'positive', 'strong', 'excellent', 'opportunity'];
    const negativeWords = ['decline', 'decrease', 'negative', 'weak', 'risk', 'challenge'];
    
    const lowerText = text.toLowerCase();
    const positiveCount = positiveWords.filter(word => lowerText.includes(word)).length;
    const negativeCount = negativeWords.filter(word => lowerText.includes(word)).length;
    
    if (positiveCount > negativeCount) return 'positive';
    if (negativeCount > positiveCount) return 'negative';
    return 'neutral';
  }

  /**
   * Assess complexity of the text
   */
  private assessComplexity(text: string): string {
    const words = text.split(/\s+/);
    const avgWordLength = words.reduce((sum: number, word: string) => sum + word.length, 0) / words.length;
    const sentenceCount = text.split(/[.!?]+/).length;
    const avgSentenceLength = words.length / sentenceCount;
    
    if (avgWordLength > 6 || avgSentenceLength > 20) return 'high';
    if (avgWordLength > 5 || avgSentenceLength > 15) return 'medium';
    return 'low';
  }

  private convertJsonToMarkdown(data: CIMReview): string {
    let markdown = `# BPCP CIM Review Template: ${data.dealOverview.targetCompanyName}\n\n`;

    // (A) Deal Overview
    markdown += `## (A) Deal Overview\n\n`;
    markdown += `- **Target Company Name:** ${data.dealOverview.targetCompanyName}\n`;
    markdown += `- **Industry/Sector:** ${data.dealOverview.industrySector}\n`;
    markdown += `- **Geography (HQ & Key Operations):** ${data.dealOverview.geography}\n`;
    markdown += `- **Deal Source:** ${data.dealOverview.dealSource}\n`;
    markdown += `- **Transaction Type:** ${data.dealOverview.transactionType}\n`;
    markdown += `- **Date CIM Received:** ${data.dealOverview.dateCIMReceived}\n`;
    markdown += `- **Date Reviewed:** ${data.dealOverview.dateReviewed}\n`;
    markdown += `- **Reviewer(s):** ${data.dealOverview.reviewers}\n`;
    markdown += `- **CIM Page Count:** ${data.dealOverview.cimPageCount}\n`;
    markdown += `- **Stated Reason for Sale (if provided):** ${data.dealOverview.statedReasonForSale}\n\n`;

    // (B) Business Description
    markdown += `## (B) Business Description\n\n`;
    markdown += `- **Core Operations Summary (3-5 sentences):** ${data.businessDescription.coreOperationsSummary}\n`;
    markdown += `- **Key Products/Services & Revenue Mix (Est. % if available):** ${data.businessDescription.keyProductsServices}\n`;
    markdown += `- **Unique Value Proposition (UVP) / Why Customers Buy:** ${data.businessDescription.uniqueValueProposition}\n`;
    markdown += `- **Customer Base Overview:**\n`;
    markdown += `  - **Key Customer Segments/Types:** ${data.businessDescription.customerBaseOverview.keyCustomerSegments}\n`;
    markdown += `  - **Customer Concentration Risk (Top 5 and/or Top 10 Customers as % Revenue - if stated/inferable):** ${data.businessDescription.customerBaseOverview.customerConcentrationRisk}\n`;
    markdown += `  - **Typical Contract Length / Recurring Revenue % (if applicable):** ${data.businessDescription.customerBaseOverview.typicalContractLength}\n`;
    markdown += `- **Key Supplier Overview (if critical & mentioned):**\n`;
    markdown += `  - **Dependence/Concentration Risk:** ${data.businessDescription.keySupplierOverview.dependenceConcentrationRisk}\n\n`;

    // (C) Market & Industry Analysis
    markdown += `## (C) Market & Industry Analysis\n\n`;
    markdown += `- **Estimated Market Size (TAM/SAM - if provided):** ${data.marketIndustryAnalysis.estimatedMarketSize}\n`;
    markdown += `- **Estimated Market Growth Rate (% CAGR - Historical & Projected):** ${data.marketIndustryAnalysis.estimatedMarketGrowthRate}\n`;
    markdown += `- **Key Industry Trends & Drivers (Tailwinds/Headwinds):** ${data.marketIndustryAnalysis.keyIndustryTrends}\n`;
    markdown += `- **Competitive Landscape:**\n`;
    markdown += `  - **Key Competitors Identified:** ${data.marketIndustryAnalysis.competitiveLandscape.keyCompetitors}\n`;
    markdown += `  - **Target's Stated Market Position/Rank:** ${data.marketIndustryAnalysis.competitiveLandscape.targetMarketPosition}\n`;
    markdown += `  - **Basis of Competition:** ${data.marketIndustryAnalysis.competitiveLandscape.basisOfCompetition}\n`;
    markdown += `- **Barriers to Entry / Competitive Moat (Stated/Inferred):** ${data.marketIndustryAnalysis.barriersToEntry}\n\n`;

    // (D) Financial Summary
    markdown += `## (D) Financial Summary\n\n`;
    markdown += `### Key Historical Financials\n\n`;
    markdown += `| Metric | FY-3 (or earliest avail.) | FY-2 | FY-1 | LTM (Last Twelve Months) |\n`;
    markdown += `| :--- | :---: | :---: | :---: | :---: |\n`;
    
    // Generate table rows from the financials data structure
    const metricsToDisplay: Array<{ name: string; field: keyof typeof data.financialSummary.financials.fy3 }> = [
      { name: 'Revenue', field: 'revenue' },
      { name: '_Revenue Growth (%)_', field: 'revenueGrowth' },
      { name: 'Gross Profit (if avail.)', field: 'grossProfit' },
      { name: '_Gross Margin (%)_', field: 'grossMargin' },
      { name: 'EBITDA (Note Adjustments)', field: 'ebitda' },
      { name: '_EBITDA Margin (%)_', field: 'ebitdaMargin' }
    ];
    
    metricsToDisplay.forEach(metric => {
      markdown += `| ${metric.name} | ${data.financialSummary.financials.fy3[metric.field] || 'N/A'} | ${data.financialSummary.financials.fy2[metric.field] || 'N/A'} | ${data.financialSummary.financials.fy1[metric.field] || 'N/A'} | ${data.financialSummary.financials.ltm[metric.field] || 'N/A'} |\n`;
    });
    
    markdown += `\n`;

    markdown += `### Key Financial Notes & Observations\n\n`;
    markdown += `- **Quality of Earnings/Adjustments (Initial Impression):** ${data.financialSummary.qualityOfEarnings}\n`;
    markdown += `- **Revenue Growth Drivers (Stated):** ${data.financialSummary.revenueGrowthDrivers}\n`;
    markdown += `- **Margin Stability/Trend Analysis:** ${data.financialSummary.marginStabilityAnalysis}\n`;
    markdown += `- **Capital Expenditures (Approx. LTM % of Revenue):** ${data.financialSummary.capitalExpenditures}\n`;
    markdown += `- **Working Capital Intensity (Impression):** ${data.financialSummary.workingCapitalIntensity}\n`;
    markdown += `- **Free Cash Flow (FCF) Proxy Quality (Impression):** ${data.financialSummary.freeCashFlowQuality}\n\n`;

    // (E) Management Team Overview
    markdown += `## (E) Management Team Overview\n\n`;
    markdown += `- **Key Leaders Identified (CEO, CFO, COO, Head of Sales, etc.):** ${data.managementTeamOverview.keyLeaders}\n`;
    markdown += `- **Initial Assessment of Quality/Experience (Based on Bios):** ${data.managementTeamOverview.managementQualityAssessment}\n`;
    markdown += `- **Management's Stated Post-Transaction Role/Intentions (if mentioned):** ${data.managementTeamOverview.postTransactionIntentions}\n`;
    markdown += `- **Organizational Structure Overview (Impression):** ${data.managementTeamOverview.organizationalStructure}\n\n`;

    // (F) Preliminary Investment Thesis
    markdown += `## (F) Preliminary Investment Thesis\n\n`;
    markdown += `- **Key Attractions / Strengths (Why Invest?):** ${data.preliminaryInvestmentThesis.keyAttractions}\n`;
    markdown += `- **Potential Risks / Concerns (Why Not Invest?):** ${data.preliminaryInvestmentThesis.potentialRisks}\n`;
    markdown += `- **Initial Value Creation Levers (How PE Adds Value):** ${data.preliminaryInvestmentThesis.valueCreationLevers}\n`;
    markdown += `- **Alignment with Fund Strategy:** ${data.preliminaryInvestmentThesis.alignmentWithFundStrategy}\n\n`;

    // (G) Key Questions & Next Steps
    markdown += `## (G) Key Questions & Next Steps\n\n`;
    markdown += `- **Critical Questions Arising from CIM Review:** ${data.keyQuestionsNextSteps.criticalQuestions}\n`;
    markdown += `- **Key Missing Information / Areas for Diligence Focus:** ${data.keyQuestionsNextSteps.missingInformation}\n`;
    markdown += `- **Preliminary Recommendation:** ${data.keyQuestionsNextSteps.preliminaryRecommendation}\n`;
    markdown += `- **Rationale for Recommendation (Brief):** ${data.keyQuestionsNextSteps.rationaleForRecommendation}\n`;
    markdown += `- **Proposed Next Steps:** ${data.keyQuestionsNextSteps.proposedNextSteps}\n\n`;

    return markdown;
  }

  /**
   * Get default template (fallback if BPCP template not found)
   */
  private getDefaultTemplate(): string {
    // This can be simplified as the template is now embodied in the JSON schema
    return 'Provide a comprehensive analysis of the CIM document in the required JSON format.';
  }

  // eslint-disable-next-line @typescript-eslint/no-unused-vars
  // @ts-ignore
  private async combineChunkResults(chunkResults: any[]): Promise<{ summary: string; analysisData: CIMReview }> {
    const combinedJson = this.mergeJsonObjects(chunkResults.map(r => r.jsonOutput));
    
    // Final refinement step
    const finalResult = await llmService.processCIMDocument(JSON.stringify(combinedJson), '', { refinementMode: true });

    if (!finalResult.success || !finalResult.jsonOutput) {
      logger.warn('Final refinement step failed, using combined JSON without refinement.');
      return {
        summary: this.convertJsonToMarkdown(combinedJson),
        analysisData: combinedJson
      };
    }
    
    return {
      summary: this.convertJsonToMarkdown(finalResult.jsonOutput),
      analysisData: finalResult.jsonOutput
    };
  }

  private mergeJsonObjects(objects: CIMReview[]): CIMReview {
    if (!objects || objects.length === 0) {
      // This should not happen if we have successful chunk results, but as a safeguard:
      throw new Error("Cannot merge empty array of JSON objects.");
    }

    // This is a simplified merge. A more sophisticated version would handle conflicts.
    const base = JSON.parse(JSON.stringify(objects[0])); // Deep copy to avoid mutation issues

    for (let i = 1; i < objects.length; i++) {
      const obj = objects[i];
      if (obj) {
        // Simple merge - later objects overwrite earlier ones for most fields
        Object.assign(base.dealOverview, obj.dealOverview);
        Object.assign(base.businessDescription, obj.businessDescription);
        Object.assign(base.marketIndustryAnalysis, obj.marketIndustryAnalysis);
        Object.assign(base.financialSummary, obj.financialSummary);
        Object.assign(base.managementTeamOverview, obj.managementTeamOverview);
        Object.assign(base.preliminaryInvestmentThesis, obj.preliminaryInvestmentThesis);
        Object.assign(base.keyQuestionsNextSteps, obj.keyQuestionsNextSteps);
      }
    }
    return base;
  }

  // Removed unused function parseAllChunkSections

  // Removed unused function parseCIMSections

  // Removed unused function getSectionKey

  /**
   * Merge CIM sections from multiple chunks
   */
  // Removed unused function mergeCIMSections

  // Removed unused function mergeSectionContent

  /**
   * Build the final combined markdown from merged sections
   */
  // Removed unused function buildCombinedMarkdown

  // Removed unused function getSectionTitle

  /**
   * Save markdown file
   */
  private async saveMarkdownFile(filePath: string, content: string): Promise<void> {
    try {
      const fullPath = path.join(config.upload.uploadDir, filePath);
      const dir = path.dirname(fullPath);
      
      if (!fs.existsSync(dir)) {
        fs.mkdirSync(dir, { recursive: true });
      }
      
      fs.writeFileSync(fullPath, content, 'utf-8');
      logger.info(`Markdown file saved: ${filePath}`);
    } catch (error) {
      logger.error(`Failed to save markdown file: ${filePath}`, error);
      throw new Error(`Failed to save markdown file: ${error instanceof Error ? error.message : 'Unknown error'}`);
    }
  }

  /**
   * Detect document type based on content
   */
  private detectDocumentType(text: string): string {
    const lowerText = text.toLowerCase();
    
    if (lowerText.includes('financial') || lowerText.includes('revenue') || lowerText.includes('profit')) {
      return 'financial_report';
    }
    
    if (lowerText.includes('technical') || lowerText.includes('specification') || lowerText.includes('requirements')) {
      return 'technical_document';
    }
    
    if (lowerText.includes('contract') || lowerText.includes('agreement') || lowerText.includes('legal')) {
      return 'legal_document';
    }
    
    return 'general_document';
  }

  /**
   * Estimate token count for text
   */
  private estimateTokenCount(text: string): number {
    // Rough estimation: 1 token ≈ 4 characters for English text
    return Math.ceil(text.length / 4);
  }

  /**
   * Chunk text for processing with intelligent boundaries and overlap
   */
  private chunkText(text: string, maxTokens: number = 4000): string[] {
    const chunks: string[] = [];
    const estimatedTokens = this.estimateTokenCount(text);
    
    if (estimatedTokens <= maxTokens) {
      return [text];
    }

    // Calculate overlap size (20% of max tokens for continuity)
    const overlapTokens = Math.floor(maxTokens * 0.2);
    const overlapChars = overlapTokens * 4; // Rough conversion back to characters
    
    // Split by paragraphs first
    const paragraphs = text.split(/\n\s*\n/).filter(p => p.trim());
    
    if (paragraphs.length === 0) {
      // If no paragraphs, split by sentences
      const sentences = text.split(/[.!?]+/).filter(s => s.trim());
      return this.chunkBySentences(sentences, maxTokens, overlapChars);
    }
    
    let currentChunk = '';
    let currentTokens = 0;
    
    for (let i = 0; i < paragraphs.length; i++) {
      const paragraph = paragraphs[i];
      if (!paragraph) continue;
      const paragraphTokens = this.estimateTokenCount(paragraph);
      
      // Check if adding this paragraph would exceed the limit
      if (currentTokens + paragraphTokens > maxTokens && currentChunk) {
        // Current chunk is full, save it
        chunks.push(currentChunk.trim());
        
        // Start new chunk with overlap from the end of previous chunk
        if (chunks.length > 0 && overlapChars > 0) {
          const previousChunk = chunks[chunks.length - 1];
          if (!previousChunk) continue;
          const overlapText = previousChunk.slice(-overlapChars);
          
          // Find the last complete paragraph in the overlap
          const overlapParagraphs = overlapText.split(/\n\s*\n/);
          if (overlapParagraphs.length > 1) {
            const lastCompleteParagraph = overlapParagraphs[overlapParagraphs.length - 1];
            currentChunk = lastCompleteParagraph + '\n\n';
            currentTokens = this.estimateTokenCount(currentChunk);
          } else {
            currentChunk = '';
            currentTokens = 0;
          }
        } else {
          currentChunk = '';
          currentTokens = 0;
        }
      }
      
      // Add paragraph to current chunk
      if (currentChunk) {
        currentChunk += '\n\n' + paragraph;
      } else {
        currentChunk = paragraph || '';
      }
      currentTokens += paragraphTokens;
    }
    
    // Add the final chunk if it has content
    if (currentChunk.trim()) {
      chunks.push(currentChunk.trim());
    }
    
    // Ensure we don't have empty chunks
    const validChunks = chunks.filter(chunk => chunk.trim().length > 0);
    
    logger.info('Text chunking completed', {
      originalLength: text.length,
      estimatedTokens,
      maxTokens,
      overlapTokens,
      paragraphs: paragraphs.length,
      chunks: validChunks.length,
      chunkSizes: validChunks.map(chunk => chunk.length)
    });
    
    return validChunks;
  }

  /**
   * Chunk text by sentences when paragraph chunking isn't suitable
   */
  private chunkBySentences(sentences: string[], maxTokens: number, overlapChars: number): string[] {
    const chunks: string[] = [];
    let currentChunk = '';
    let currentTokens = 0;
    
    for (let i = 0; i < sentences.length; i++) {
      const sentence = sentences[i];
      if (!sentence) continue;
      const sentenceTokens = this.estimateTokenCount(sentence);
      
      if (currentTokens + sentenceTokens > maxTokens && currentChunk) {
        chunks.push(currentChunk.trim());
        
        // Add overlap from previous chunk
        if (chunks.length > 0 && overlapChars > 0) {
          const previousChunk = chunks[chunks.length - 1];
          if (!previousChunk) continue;
          const overlapText = previousChunk.slice(-overlapChars);
          
          // Find the last complete sentence in the overlap
          const overlapSentences = overlapText.split(/[.!?]+/);
          if (overlapSentences.length > 1) {
            const lastCompleteSentence = overlapSentences[overlapSentences.length - 1];
            currentChunk = lastCompleteSentence + '. ';
            currentTokens = this.estimateTokenCount(currentChunk);
          } else {
            currentChunk = '';
            currentTokens = 0;
          }
        } else {
          currentChunk = '';
          currentTokens = 0;
        }
      }
      
      if (currentChunk) {
        currentChunk += sentence + '. ';
      } else {
        currentChunk = sentence + '. ';
      }
      currentTokens += sentenceTokens;
    }
    
    if (currentChunk.trim()) {
      chunks.push(currentChunk.trim());
    }
    
    return chunks.filter(chunk => chunk.trim().length > 0);
  }

  /**
   * Refine the combined summary using LLM for better coherence and completeness
   */
  // Removed unused function refineCombinedSummary

  /**
   * Get processing job status
   */
  async getProcessingJobStatus(jobId: string): Promise<any> {
    try {
      const job = await ProcessingJobModel.findByJobId(jobId);
      return job;
    } catch (error) {
      logger.error(`Failed to get processing job status: ${jobId}`, error);
      throw error;
    }
  }

  /**
   * Get document processing history
   */
  async getDocumentProcessingHistory(documentId: string): Promise<any[]> {
    try {
      const jobs = await ProcessingJobModel.findByDocumentId(documentId);
      return jobs;
    } catch (error) {
      logger.error(`Failed to get document processing history: ${documentId}`, error);
      throw error;
    }
  }

  /**
   * Regenerate summary for an existing document
   */
  async regenerateSummary(documentId: string): Promise<void> {
    try {
      logger.info('Starting summary regeneration', { documentId });
      
      // Get the document
      const document = await DocumentModel.findById(documentId);
      if (!document) {
        throw new Error('Document not found');
      }
      
      if (!document.extracted_text) {
        throw new Error('Document has no extracted text to regenerate summary from');
      }
      
      // Update status to processing
      await this.updateDocumentStatus(documentId, 'processing_llm');
      
      // Load template
      const templatePath = path.join(process.cwd(), '..', 'BPCP CIM REVIEW TEMPLATE.md');
      let template = '';
      
      try {
        template = fs.readFileSync(templatePath, 'utf-8');
        logger.info('BPCP template loaded successfully', { templateLength: template.length });
      } catch (error) {
        logger.warn('Could not load BPCP template, using default template', { error: error instanceof Error ? error.message : 'Unknown error' });
        template = this.getDefaultTemplate();
      }
      
      // Generate new summary
      const summaryResult = await this.generateSummary(documentId, document.extracted_text, {});
      const newSummary = summaryResult.summary;
      const newAnalysisData = summaryResult.analysisData;
      
      // Save new markdown file
      const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
      const markdownPath = `uploads/summaries/${documentId}_${timestamp}.md`;
      const fullMarkdownPath = path.join(process.cwd(), markdownPath);
      
      await this.saveMarkdownFile(fullMarkdownPath, newSummary);
      
      // Generate PDF
      const pdfPath = markdownPath.replace('.md', '.pdf');
      const fullPdfPath = path.join(process.cwd(), pdfPath);
      
      await pdfGenerationService.generatePDFFromMarkdown(newSummary, fullPdfPath);
      
      // Update document with new summary
      const updateData = {
        generated_summary: newSummary,
        analysis_data: newAnalysisData,
        summary_markdown_path: markdownPath,
        summary_pdf_path: pdfPath,
        status: 'completed' as const,
        processing_completed_at: new Date()
      };
      
      const updated = await DocumentModel.updateById(documentId, updateData);
      if (!updated) {
        throw new Error('Failed to update document with new summary');
      }
      
      logger.info('Summary regeneration completed successfully', {
        documentId,
        newSummaryLength: newSummary.length,
        markdownPath,
        pdfPath
      });
      
    } catch (error) {
      logger.error('Summary regeneration failed', {
        documentId,
        error: error instanceof Error ? error.message : 'Unknown error'
      });
      
      // Update status to failed
      await this.updateDocumentStatus(documentId, 'failed');
      throw error;
    }
  }

  /**
   * Clean up the original uploaded file after successful processing
   */
  private async cleanupOriginalFile(documentId: string): Promise<void> {
    try {
      const document = await DocumentModel.findById(documentId);
      if (!document || !document.file_path) {
        logger.warn(`No file path found for document: ${documentId}`);
        return;
      }

      // Check if file exists before attempting to delete
      if (await fileStorageService.fileExists(document.file_path)) {
        await fileStorageService.deleteFile(document.file_path);
        logger.info(`Cleaned up original uploaded file: ${document.file_path}`);
      } else {
        logger.warn(`Original file not found for cleanup: ${document.file_path}`);
      }
    } catch (error) {
      logger.error(`Failed to cleanup original file: ${documentId}`, error);
      // Don't throw error - cleanup failure shouldn't fail the entire process
    }
  }
}

export const documentProcessingService = new DocumentProcessingService();
export default documentProcessingService; 