diff --git a/backend/src/models/VectorDatabaseModel.ts b/backend/src/models/VectorDatabaseModel.ts index b0e27aa..573e58b 100644 --- a/backend/src/models/VectorDatabaseModel.ts +++ b/backend/src/models/VectorDatabaseModel.ts @@ -1,4 +1,3 @@ -import { Pool } from 'pg'; import { v4 as uuidv4 } from 'uuid'; import { logger } from '../utils/logger'; import pool from '../config/database'; @@ -68,6 +67,9 @@ export class VectorDatabaseModel { }); } + // Format embedding properly for pgvector - must be a JSON array string + const embeddingString = JSON.stringify(embeddingArray); + await client.query(` INSERT INTO document_chunks ( id, document_id, content, metadata, embedding, @@ -85,7 +87,7 @@ export class VectorDatabaseModel { chunk.documentId, chunk.content, JSON.stringify(chunk.metadata), - embeddingArray, // Pass as array, pgvector will handle the conversion + embeddingString, // Pass as JSON string for pgvector chunk.chunkIndex, chunk.section, chunk.pageNumber diff --git a/backend/src/services/jobQueueService.ts b/backend/src/services/jobQueueService.ts index 10b0659..66f64c9 100644 --- a/backend/src/services/jobQueueService.ts +++ b/backend/src/services/jobQueueService.ts @@ -1,5 +1,6 @@ import { EventEmitter } from 'events'; import { logger } from '../utils/logger'; +import { config } from '../config/env'; import { ProcessingOptions } from './documentProcessingService'; import { unifiedDocumentProcessor } from './unifiedDocumentProcessor'; @@ -209,8 +210,8 @@ class JobQueueService extends EventEmitter { await this.updateJobStatus(job.id, 'processing'); // Use unified processor for strategy-aware processing - const strategy = options?.strategy || 'chunking'; - logger.info('Processing document job with strategy', { documentId, strategy, jobId: job.id }); + const strategy = options?.strategy || config.processingStrategy; + logger.info('Processing document job with strategy', { documentId, strategy, jobId: job.id, configStrategy: config.processingStrategy }); const result = await unifiedDocumentProcessor.processDocument( documentId, diff --git a/backend/src/services/llmService.ts b/backend/src/services/llmService.ts index 48ed43d..62c0616 100644 --- a/backend/src/services/llmService.ts +++ b/backend/src/services/llmService.ts @@ -466,17 +466,60 @@ IMPORTANT: Replace all placeholder text with actual information from the CIM doc return JSON.parse(codeBlockMatch[1]); } - // If that fails, fall back to finding the first and last curly braces + // If that fails, try to find the largest valid JSON object const startIndex = content.indexOf('{'); - const endIndex = content.lastIndexOf('}'); - if (startIndex === -1 || endIndex === -1) { + if (startIndex === -1) { throw new Error('No JSON object found in response'); } + // Try to find the complete JSON object by matching braces + let braceCount = 0; + let endIndex = -1; + + for (let i = startIndex; i < content.length; i++) { + if (content[i] === '{') { + braceCount++; + } else if (content[i] === '}') { + braceCount--; + if (braceCount === 0) { + endIndex = i; + break; + } + } + } + + if (endIndex === -1) { + // If we can't find a complete JSON object, try to extract what we have + // and attempt to complete it + const partialJson = content.substring(startIndex); + logger.warn('Attempting to recover from truncated JSON response', { + contentLength: content.length, + partialJsonLength: partialJson.length + }); + + // Try to find the last complete object or array + const lastCompleteMatch = partialJson.match(/(\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\})/); + if (lastCompleteMatch && lastCompleteMatch[1]) { + return JSON.parse(lastCompleteMatch[1]); + } + + // If that fails, try to find the last complete key-value pair + const lastPairMatch = partialJson.match(/(\{[^{}]*"[^"]*"\s*:\s*"[^"]*"[^{}]*\})/); + if (lastPairMatch && lastPairMatch[1]) { + return JSON.parse(lastPairMatch[1]); + } + + throw new Error('Unable to extract valid JSON from truncated response'); + } + const jsonString = content.substring(startIndex, endIndex + 1); return JSON.parse(jsonString); } catch (error) { - logger.error('Failed to extract JSON from LLM response', { error, content: content.substring(0, 500) }); + logger.error('Failed to extract JSON from LLM response', { + error, + contentLength: content.length, + contentPreview: content.substring(0, 1000) + }); throw new Error(`JSON extraction failed: ${error instanceof Error ? error.message : 'Unknown error'}`); } } diff --git a/backend/src/services/optimizedAgenticRAGProcessor.ts b/backend/src/services/optimizedAgenticRAGProcessor.ts index 7f83b3d..ffa86cd 100644 --- a/backend/src/services/optimizedAgenticRAGProcessor.ts +++ b/backend/src/services/optimizedAgenticRAGProcessor.ts @@ -1,6 +1,8 @@ import { logger } from '../utils/logger'; import { vectorDatabaseService } from './vectorDatabaseService'; import { VectorDatabaseModel } from '../models/VectorDatabaseModel'; +import { llmService } from './llmService'; +import { CIMReview } from './llmSchemas'; interface ProcessingChunk { id: string; @@ -18,6 +20,10 @@ interface ProcessingResult { processingTime: number; averageChunkSize: number; memoryUsage: number; + summary?: string; + analysisData?: CIMReview; + success: boolean; + error?: string; } export class OptimizedAgenticRAGProcessor { @@ -56,6 +62,10 @@ export class OptimizedAgenticRAGProcessor { // Step 3: Store chunks with optimized batching await this.storeChunksOptimized(processedChunks, documentId); + // Step 4: Generate LLM analysis using the vectorized chunks + logger.info(`Starting LLM analysis for document: ${documentId}`); + const llmResult = await this.generateLLMAnalysis(documentId, text, processedChunks); + const processingTime = Date.now() - startTime; const finalMemory = process.memoryUsage().heapUsed; const memoryUsage = finalMemory - initialMemory; @@ -65,7 +75,10 @@ export class OptimizedAgenticRAGProcessor { processedChunks: processedChunks.length, processingTime, averageChunkSize: Math.round(processedChunks.reduce((sum, c) => sum + c.content.length, 0) / processedChunks.length), - memoryUsage: Math.round(memoryUsage / 1024 / 1024) // MB + memoryUsage: Math.round(memoryUsage / 1024 / 1024), // MB + success: true, + summary: llmResult.summary, + analysisData: llmResult.analysisData }; logger.info(`Optimized processing completed for document: ${documentId}`, result); @@ -433,6 +446,34 @@ export class OptimizedAgenticRAGProcessor { return chunksWithEmbeddings; } + + /** + * Generate LLM analysis using the vectorized chunks + */ + private async generateLLMAnalysis( + documentId: string, + text: string, + chunks: ProcessingChunk[] + ): Promise<{ summary: string; analysisData: CIMReview }> { + try { + logger.info(`Generating LLM analysis for document: ${documentId} with ${chunks.length} chunks`); + + // Use the existing LLM service to generate CIM review + const result = await llmService.processCIMDocument(text, 'BPCP CIM Review Template'); + + return { + summary: 'Document processed with optimized agentic RAG', + analysisData: result.jsonOutput || {} as CIMReview + }; + } catch (error) { + logger.error(`Failed to generate LLM analysis for document: ${documentId}`, error); + // Return default values if LLM analysis fails + return { + summary: 'Document processed with optimized agentic RAG (LLM analysis failed)', + analysisData: {} as CIMReview + }; + } + } } export const optimizedAgenticRAGProcessor = new OptimizedAgenticRAGProcessor(); \ No newline at end of file diff --git a/backend/src/services/unifiedDocumentProcessor.ts b/backend/src/services/unifiedDocumentProcessor.ts index 7ea0a69..fbed16a 100644 --- a/backend/src/services/unifiedDocumentProcessor.ts +++ b/backend/src/services/unifiedDocumentProcessor.ts @@ -154,16 +154,15 @@ class UnifiedDocumentProcessor { } ); - // For now, return a basic result since the optimized processor focuses on vectorization - // In a full implementation, you would also run the LLM analysis on the vectorized chunks + // Return the complete result from the optimized processor return { - success: true, - summary: `Document successfully processed with optimized agentic RAG. Created ${optimizedResult.processedChunks} chunks with ${optimizedResult.averageChunkSize} average size.`, - analysisData: {} as CIMReview, // Would be populated with actual analysis + success: optimizedResult.success, + summary: optimizedResult.summary || `Document successfully processed with optimized agentic RAG. Created ${optimizedResult.processedChunks} chunks with ${optimizedResult.averageChunkSize} average size.`, + analysisData: optimizedResult.analysisData || {} as CIMReview, processingStrategy: 'optimized_agentic_rag', processingTime: optimizedResult.processingTime, apiCalls: Math.ceil(optimizedResult.processedChunks / 5), // Estimate API calls - error: undefined + error: optimizedResult.error }; } catch (error) { logger.error('Optimized agentic RAG processing failed', { documentId, error }); diff --git a/backend/src/services/vectorDatabaseService.ts b/backend/src/services/vectorDatabaseService.ts index df6203b..7f74ec2 100644 --- a/backend/src/services/vectorDatabaseService.ts +++ b/backend/src/services/vectorDatabaseService.ts @@ -100,22 +100,27 @@ class VectorDatabaseService { return cached.embedding; } - // Use OpenAI embeddings for production-quality results - if (config.llm.provider === 'openai' && config.llm.openaiApiKey) { - const embedding = await this.generateOpenAIEmbeddings(text); - // Cache the result - this.semanticCache.set(cacheKey, { embedding, timestamp: Date.now() }); - return embedding; - } + // Use OpenAI embeddings by default (more reliable than custom Claude embeddings) + let embedding: number[]; - // Fallback to Claude embeddings approach - const embedding = await this.generateClaudeEmbeddings(text); + if (config.llm.openaiApiKey) { + embedding = await this.generateOpenAIEmbeddings(text); + } else if (config.llm.anthropicApiKey) { + embedding = await this.generateClaudeEmbeddings(text); + } else { + throw new Error('No API key available for embedding generation'); + } + // Cache the result - this.semanticCache.set(cacheKey, { embedding, timestamp: Date.now() }); + this.semanticCache.set(cacheKey, { + embedding, + timestamp: Date.now() + }); + return embedding; } catch (error) { logger.error('Failed to generate embeddings', error); - throw new Error('Embedding generation failed'); + throw error; } } diff --git a/backend/src/services/vectorDocumentProcessor.ts b/backend/src/services/vectorDocumentProcessor.ts index 1ff28b1..8b60aea 100644 --- a/backend/src/services/vectorDocumentProcessor.ts +++ b/backend/src/services/vectorDocumentProcessor.ts @@ -472,8 +472,8 @@ Return only a JSON array of indices in order of relevance: [1, 3, 2, ...]`; }); if (result.success && typeof result.jsonOutput === 'object') { - const ranking = result.jsonOutput as number[]; - if (Array.isArray(ranking)) { + const ranking = Array.isArray(result.jsonOutput) ? result.jsonOutput as number[] : null; + if (ranking) { // Apply the ranking const reranked = ranking .map(index => candidates[index - 1]) // Convert 1-based to 0-based diff --git a/test-llm-processing.js b/test-llm-processing.js index 33a7881..105f0cf 100644 --- a/test-llm-processing.js +++ b/test-llm-processing.js @@ -1,99 +1,49 @@ -const fs = require('fs'); -const path = require('path'); +const axios = require('axios'); -// Test the LLM processing with our sample CIM content -const sampleCIMContent = `# Confidential Information Memorandum -## TechStart Solutions Inc. +async function testLLMProcessing() { + try { + console.log('🚀 Testing LLM Processing for STAX CIM...'); + + // First, authenticate to get a valid token + const loginResponse = await axios.post('http://localhost:5000/api/auth/login', { + email: 'test@stax-processing.com', + password: 'TestPass123!' + }); + + console.log('✅ Authentication successful'); + console.log('Login response structure:', Object.keys(loginResponse.data)); + + const token = loginResponse.data.data?.tokens?.accessToken; + console.log('Token:', token ? 'Received' : 'Not received'); + + if (!token) { + console.error('No token received from login'); + return; + } + + // Document ID that's already in the system + const documentId = '0876b7f4-0899-4eb0-b2c6-434ec4e7a46d'; + + // Trigger LLM processing + const response = await axios.post(`http://localhost:5000/api/documents/${documentId}/process`, { + processingType: 'llm', + template: 'BPCP CIM Review Template' + }, { + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + } + }); + + console.log('✅ LLM Processing triggered successfully'); + console.log('Response:', response.data); + + } catch (error) { + console.error('❌ Error:', error.response?.data || error.message); + if (error.response?.data) { + console.error('Full error response:', JSON.stringify(error.response.data, null, 2)); + } + } +} -### Executive Summary -TechStart Solutions Inc. is a rapidly growing SaaS company specializing in AI-powered business intelligence tools. The company has achieved 300% year-over-year growth and is seeking $15M in Series B funding to expand its product portfolio and enter new markets. - -### Company Overview -- **Founded**: 2020 -- **Headquarters**: San Francisco, CA -- **Employees**: 85 (45 engineers, 25 sales, 15 operations) -- **Revenue**: $8.2M (2023), $2.1M (2022), $500K (2021) -- **Customers**: 1,200+ enterprise clients -- **Market Cap**: $45M (pre-money valuation) - -### Business Model -- **Primary Revenue**: SaaS subscriptions (85% of revenue) -- **Secondary Revenue**: Professional services (10%), API licensing (5%) -- **Average Contract Value**: $45,000 annually -- **Customer Retention Rate**: 94% -- **Gross Margin**: 78% - -### Market Opportunity -- **Total Addressable Market**: $45B -- **Serviceable Addressable Market**: $2.8B -- **Target Market**: Mid-market enterprises (500-5,000 employees) -- **Competitive Landscape**: 15 major competitors, 3 direct competitors - -### Financial Highlights -**Revenue Growth**: -- 2021: $500K -- 2022: $2.1M (320% growth) -- 2023: $8.2M (290% growth) -- 2024 (projected): $18M (120% growth) - -**Key Metrics**: -- Monthly Recurring Revenue: $683K -- Annual Recurring Revenue: $8.2M -- Customer Acquisition Cost: $12,000 -- Lifetime Value: $180,000 -- Payback Period: 8 months - -### Use of Funds -- **Product Development**: $8M (53%) -- **Sales & Marketing**: $4M (27%) -- **Operations**: $2M (13%) -- **Working Capital**: $1M (7%) - -### Management Team -- **CEO**: Sarah Johnson (ex-Google, 15 years experience) -- **CTO**: Michael Chen (ex-Microsoft, PhD Computer Science) -- **CFO**: David Rodriguez (ex-Salesforce, CPA) -- **VP Sales**: Lisa Thompson (ex-Oracle, 12 years experience) - -### Risk Factors -- Dependency on key personnel -- Competition from larger tech companies -- Economic downturn impact on SaaS spending -- Regulatory changes in data privacy -- Technology obsolescence - -### Investment Terms -- **Round**: Series B -- **Amount**: $15M -- **Valuation**: $45M pre-money, $60M post-money -- **Structure**: Preferred equity -- **Board Seats**: 2 seats for investors -- **Exit Strategy**: IPO in 3-5 years or strategic acquisition`; - -console.log('🚀 Testing LLM Processing with Real CIM Document'); -console.log('================================================'); -console.log(''); -console.log('📄 Sample CIM Content Length:', sampleCIMContent.length, 'characters'); -console.log('📊 Estimated Tokens:', Math.ceil(sampleCIMContent.length / 4)); -console.log(''); -console.log('🔧 Next Steps:'); -console.log('1. Open http://localhost:3000 in your browser'); -console.log('2. Go to the Upload tab'); -console.log('3. Upload test-cim-sample.pdf'); -console.log('4. Watch the real-time LLM processing'); -console.log('5. View the generated CIM analysis'); -console.log(''); -console.log('📋 Expected LLM Processing Steps:'); -console.log('- PDF text extraction'); -console.log('- Part 1: CIM Data Extraction (Deal Overview, Business Description, etc.)'); -console.log('- Part 2: Investment Analysis (Key Considerations, Risk Factors, etc.)'); -console.log('- Markdown output generation'); -console.log('- CIM Review Template population'); -console.log(''); -console.log('💡 The system will use your configured API keys to:'); -console.log('- Extract structured data from the CIM'); -console.log('- Generate investment analysis'); -console.log('- Create a comprehensive review template'); -console.log('- Provide actionable insights for investment decisions'); -console.log(''); -console.log('🎯 Ready to test! Open the frontend and upload the PDF.'); \ No newline at end of file +testLLMProcessing(); \ No newline at end of file