diff --git a/TODO_AND_OPTIMIZATIONS.md b/TODO_AND_OPTIMIZATIONS.md index 568db06..8600785 100644 --- a/TODO_AND_OPTIMIZATIONS.md +++ b/TODO_AND_OPTIMIZATIONS.md @@ -8,6 +8,8 @@ - **Infrastructure deployment checklist**: Update `DEPLOYMENT_GUIDE.md` with the exact Firebase/GCP commands used to fetch secrets and run Sonnet validation so future deploys stay reproducible. - **Runtime upgrade**: Migrate Firebase Functions from Node.js 20 to a supported runtime well before the 2026‑10‑30 decommission date (warning surfaced during deploy). - **`firebase-functions` dependency bump**: Upgrade the project to the latest `firebase-functions` package and address any breaking changes on the next development pass. +- **Document viewer KPIs missing after Project Panther run**: `Project Panther - Confidential Information Memorandum_vBluePoint.pdf` → `Revenue/EBITDA/Employees/Founded` surfaced as "Not specified in CIM" even though the CIM has numeric tables. Trace `optimizedAgenticRAGProcessor` → `dealOverview` mapper to ensure summary metrics populate the dashboard cards and add a regression test for this doc. +- **10+ minute processing latency regression**: The same Project Panther run (doc ID `document-55c4a6e2-8c08-4734-87f6-24407cea50ac.pdf`) took ~10 minutes end-to-end. Instrument each pipeline phase (PDF chunking, Document AI, RAG passes, financial parser) so we can see where time is lost, then cap slow stages (e.g., GCS upload retries, three Anthropic fallbacks) before the next deploy. ## Optimization Backlog (ordered by Accuracy → Speed → Cost benefit vs. implementation risk) 1. **Deterministic financial parser enhancements** (status: partially addressed). Continue improving token alignment (multi-row tables, negative numbers) to reduce dependence on LLM retries. Risk: low, limited to parser module. diff --git a/backend/package.json b/backend/package.json index 7c86cc8..0eb63df 100644 --- a/backend/package.json +++ b/backend/package.json @@ -37,7 +37,8 @@ "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:pipeline": "ts-node src/scripts/test-complete-pipeline.ts", - "check:pipeline": "ts-node src/scripts/check-pipeline-readiness.ts" + "check:pipeline": "ts-node src/scripts/check-pipeline-readiness.ts", + "logs:cloud": "ts-node src/scripts/fetch-cloud-run-logs.ts" }, "dependencies": { "@anthropic-ai/sdk": "^0.57.0", @@ -88,4 +89,4 @@ "ts-node": "^10.9.2", "vitest": "^2.1.0" } -} \ No newline at end of file +} diff --git a/backend/src/config/env.ts b/backend/src/config/env.ts index 074fec7..83fb448 100644 --- a/backend/src/config/env.ts +++ b/backend/src/config/env.ts @@ -308,16 +308,15 @@ export const config = { openrouterApiKey: process.env['OPENROUTER_API_KEY'] || envVars['OPENROUTER_API_KEY'], openrouterUseBYOK: envVars['OPENROUTER_USE_BYOK'] === 'true', // Use BYOK (Bring Your Own Key) - // Model Selection - Unified on Claude Sonnet 4 (May 2025 release) - // Claude Sonnet 4 20250514 is the currently supported, non-deprecated variant - // This keeps multi-pass extraction aligned with the same reasoning model across passes - model: envVars['LLM_MODEL'] || 'claude-sonnet-4-20250514', // Primary model (Claude Sonnet 4) - fastModel: envVars['LLM_FAST_MODEL'] || 'claude-sonnet-4-20250514', // Fast model aligned with Sonnet 4 + // Model Selection - Default to Anthropic Claude 4.6/4.5 family (current production tier) + // Override via env vars if specific dated versions are required + model: envVars['LLM_MODEL'] || 'claude-sonnet-4-6', // Primary reasoning model (Sonnet 4.6) + fastModel: envVars['LLM_FAST_MODEL'] || 'claude-haiku-4-5', // Lower-cost/faster variant (Haiku 4.5) fallbackModel: envVars['LLM_FALLBACK_MODEL'] || 'gpt-4o', // Fallback for creativity // Task-specific model selection - // Use Sonnet 4 for financial extraction to avoid deprecated Haiku endpoints - financialModel: envVars['LLM_FINANCIAL_MODEL'] || 'claude-sonnet-4-20250514', // Financial extraction model (Claude Sonnet 4) + // Use Haiku 4.5 for financial extraction by default (override via env to use Sonnet/Opus) + financialModel: envVars['LLM_FINANCIAL_MODEL'] || 'claude-haiku-4-5', creativeModel: envVars['LLM_CREATIVE_MODEL'] || 'gpt-4o', // Best for creative content reasoningModel: envVars['LLM_REASONING_MODEL'] || 'claude-opus-4-1-20250805', // Best for complex reasoning (Opus 4.1) @@ -329,7 +328,7 @@ export const config = { // Processing Configuration temperature: parseFloat(envVars['LLM_TEMPERATURE'] || '0.1'), // Low temperature for consistent output - timeoutMs: parseInt(envVars['LLM_TIMEOUT_MS'] || '180000'), // 3 minutes timeout (increased for complex analysis) + timeoutMs: parseInt(envVars['LLM_TIMEOUT_MS'] || '360000'), // 6 minutes timeout for complex CIM analysis // Cost Optimization enableCostOptimization: envVars['LLM_ENABLE_COST_OPTIMIZATION'] === 'true', diff --git a/backend/src/controllers/documentController.ts b/backend/src/controllers/documentController.ts index ca048e8..f513615 100644 --- a/backend/src/controllers/documentController.ts +++ b/backend/src/controllers/documentController.ts @@ -5,6 +5,7 @@ import { fileStorageService } from '../services/fileStorageService'; import { uploadProgressService } from '../services/uploadProgressService'; import { uploadMonitoringService } from '../services/uploadMonitoringService'; import { config } from '../config/env'; +import { ensureApplicationDefaultCredentials, getGoogleClientOptions } from '../utils/googleServiceAccount'; export const documentController = { async getUploadUrl(req: Request, res: Response): Promise { @@ -510,8 +511,9 @@ export const documentController = { // Get GCS bucket and save PDF buffer const { Storage } = await import('@google-cloud/storage'); - const storage = new Storage(); - const bucket = storage.bucket(process.env.GCS_BUCKET_NAME || 'cim-summarizer-uploads'); + ensureApplicationDefaultCredentials(); + const storage = new Storage(getGoogleClientOptions() as any); + const bucket = storage.bucket(config.googleCloud.gcsBucketName || process.env.GCS_BUCKET_NAME || 'cim-summarizer-uploads'); const file = bucket.file(pdfPath); await file.save(pdfBuffer, { diff --git a/backend/src/scripts/fetch-cloud-run-logs.ts b/backend/src/scripts/fetch-cloud-run-logs.ts new file mode 100644 index 0000000..2e50aa9 --- /dev/null +++ b/backend/src/scripts/fetch-cloud-run-logs.ts @@ -0,0 +1,217 @@ +/** + * Utility script to pull recent Cloud Run / Firebase Function logs via gcloud. + * + * Usage examples: + * npx ts-node src/scripts/fetch-cloud-run-logs.ts --documentId=21aa62a4-... --minutes=180 + * npm run logs:cloud -- --service=api --limit=50 + * + * Requirements: + * - gcloud CLI installed and authenticated. + * - Access to the project configured in config/googleCloud.projectId. + */ + +import { spawnSync } from 'child_process'; +import { config } from '../config/env'; + +interface LogOptions { + service?: string; + functionName?: string; + region: string; + limit: number; + minutes: number; + documentId?: string; + severity: string; + projectId?: string; +} + +const parseArgs = (): LogOptions => { + const defaults: LogOptions = { + service: process.env.CLOUD_RUN_SERVICE, + functionName: process.env.FUNCTION_NAME || 'api', + region: process.env.FUNCTION_REGION || 'us-central1', + limit: Number(process.env.LOG_LIMIT) || 100, + minutes: Number(process.env.LOG_MINUTES || 120), + documentId: process.env.DOCUMENT_ID, + severity: process.env.LOG_MIN_SEVERITY || 'INFO', + projectId: process.env.GCLOUD_PROJECT || process.env.GOOGLE_CLOUD_PROJECT + }; + + const args = process.argv.slice(2); + for (const arg of args) { + if (!arg.startsWith('--')) continue; + const [flag, value] = arg.split('='); + if (!value) continue; + + switch (flag) { + case '--service': { + defaults.service = value; + break; + } + case '--function': { + defaults.functionName = value; + break; + } + case '--region': { + defaults.region = value; + break; + } + case '--limit': { + defaults.limit = Number(value) || defaults.limit; + break; + } + case '--minutes': { + defaults.minutes = Number(value) || defaults.minutes; + break; + } + case '--documentId': { + defaults.documentId = value; + break; + } + case '--severity': { + defaults.severity = value.toUpperCase(); + break; + } + case '--project': { + defaults.projectId = value; + break; + } + default: + break; + } + } + + return defaults; +}; + +const resolveServiceName = (options: LogOptions, projectId: string): string | undefined => { + if (options.service) { + return options.service; + } + + if (!options.functionName) { + return undefined; + } + + console.log(`Resolving Cloud Run service for function "${options.functionName}"...`); + const describeArgs = [ + 'functions', + 'describe', + options.functionName, + `--project=${projectId}`, + `--region=${options.region}`, + '--gen2', + '--format=value(serviceConfig.service)' + ]; + + const describeResult = spawnSync('gcloud', describeArgs, { encoding: 'utf-8' }); + if (describeResult.status === 0 && describeResult.stdout.trim()) { + const resolved = describeResult.stdout.trim(); + console.log(` → Cloud Run service: ${resolved}`); + return resolved; + } + + console.warn( + 'Unable to resolve Cloud Run service automatically. Falling back to function name.' + ); + return options.functionName; +}; + +const run = async (): Promise => { + const options = parseArgs(); + const projectId = + options.projectId || + config.googleCloud.projectId || + process.env.GCLOUD_PROJECT || + process.env.GOOGLE_CLOUD_PROJECT; + + if (!projectId) { + console.error('Unable to determine project ID. Set GCLOUD_PROJECT or GOOGLE_CLOUD_PROJECT.'); + process.exit(1); + } + + const serviceName = resolveServiceName(options, projectId); + const sinceIso = new Date(Date.now() - options.minutes * 60_000).toISOString(); + const filterParts = [ + 'resource.type="cloud_run_revision"', + `severity>="${options.severity}"`, + `timestamp>="${sinceIso}"` + ]; + + if (serviceName) { + filterParts.push(`resource.labels.service_name="${serviceName}"`); + } + + if (options.documentId) { + filterParts.push(`jsonPayload.documentId="${options.documentId}"`); + } + + const filter = filterParts.join(' AND '); + const args = [ + 'logging', + 'read', + filter, + `--project=${projectId}`, + '--limit', + options.limit.toString(), + '--format=json' + ]; + + console.log('Fetching logs with filter:\n', filter, '\n'); + + const result = spawnSync('gcloud', args, { + encoding: 'utf-8' + }); + + if (result.error) { + console.error('Failed to execute gcloud:', result.error.message); + process.exit(1); + } + + if (result.status !== 0) { + console.error('gcloud logging read failed:\n', result.stderr || result.stdout); + process.exit(result.status ?? 1); + } + + const stdout = result.stdout?.trim(); + if (!stdout) { + console.log('No log entries returned.'); + return; + } + + let entries: any[] = []; + try { + entries = JSON.parse(stdout); + } catch (error) { + console.error('Unable to parse gcloud output as JSON:\n', stdout); + process.exit(1); + } + + if (!entries.length) { + console.log('No log entries matched the filter.'); + return; + } + + entries.forEach((entry, index) => { + const timestamp = entry.timestamp || entry.receiveTimestamp || 'unknown-time'; + const severity = entry.severity || entry.jsonPayload?.severity || 'INFO'; + const payload = entry.textPayload || entry.jsonPayload || entry.protoPayload; + const documentId = entry.jsonPayload?.documentId; + const message = + entry.jsonPayload?.message || + entry.textPayload || + JSON.stringify(payload, null, 2); + + console.log( + `#${index + 1} [${timestamp}] [${severity}]${ + documentId ? ` [documentId=${documentId}]` : '' + }` + ); + console.log(message); + console.log('='.repeat(80)); + }); +}; + +run().catch(error => { + console.error('Unexpected error while fetching logs:', error); + process.exit(1); +}); diff --git a/backend/src/services/documentAiProcessor.ts b/backend/src/services/documentAiProcessor.ts index 59fba13..20ba01e 100644 --- a/backend/src/services/documentAiProcessor.ts +++ b/backend/src/services/documentAiProcessor.ts @@ -1,9 +1,10 @@ import { logger } from '../utils/logger'; import { DocumentProcessorServiceClient } from '@google-cloud/documentai'; -import { Storage } from '@google-cloud/storage'; +import { Storage, StorageOptions } from '@google-cloud/storage'; import { config } from '../config/env'; import pdf from 'pdf-parse'; import { PDFDocument } from 'pdf-lib'; +import { ensureApplicationDefaultCredentials, getGoogleClientOptions } from '../utils/googleServiceAccount'; interface ProcessingResult { success: boolean; @@ -45,8 +46,20 @@ export class DocumentAiProcessor { constructor() { this.gcsBucketName = config.googleCloud.gcsBucketName; - this.documentAiClient = new DocumentProcessorServiceClient(); - this.storageClient = new Storage(); + ensureApplicationDefaultCredentials(); + const clientOptions = getGoogleClientOptions(); + + this.documentAiClient = new DocumentProcessorServiceClient(clientOptions as any); + + const storageOptions: StorageOptions = { + projectId: clientOptions.projectId, + }; + + if (clientOptions.credentials) { + storageOptions.credentials = clientOptions.credentials; + } + + this.storageClient = new Storage(storageOptions); // Construct the processor name this.processorName = `projects/${config.googleCloud.projectId}/locations/${config.googleCloud.documentAiLocation}/processors/${config.googleCloud.documentAiProcessorId}`; diff --git a/backend/src/services/fileStorageService.ts b/backend/src/services/fileStorageService.ts index 955ea34..34c21c6 100644 --- a/backend/src/services/fileStorageService.ts +++ b/backend/src/services/fileStorageService.ts @@ -4,6 +4,7 @@ import { Storage } from '@google-cloud/storage'; import { config } from '../config/env'; import { logger, StructuredLogger } from '../utils/logger'; import { uploadMonitoringService } from './uploadMonitoringService'; +import { ensureApplicationDefaultCredentials, getGoogleClientOptions } from '../utils/googleServiceAccount'; export interface FileInfo { originalName: string; @@ -40,8 +41,9 @@ class FileStorageService { constructor() { this.bucketName = config.googleCloud.gcsBucketName; + ensureApplicationDefaultCredentials(); + // Check if we're in Firebase Functions/Cloud Run environment - // In these environments, Application Default Credentials are used automatically const isCloudEnvironment = process.env.FUNCTION_TARGET || process.env.FUNCTION_NAME || process.env.K_SERVICE || @@ -49,88 +51,61 @@ class FileStorageService { !!process.env.GCLOUD_PROJECT || process.env.X_GOOGLE_GCLOUD_PROJECT; - // Initialize Google Cloud Storage + const clientOptions = getGoogleClientOptions(); const storageConfig: any = { - projectId: config.googleCloud.projectId, + projectId: clientOptions.projectId || config.googleCloud.projectId, }; - // Only use keyFilename in local development - // In Firebase Functions/Cloud Run, use Application Default Credentials - if (isCloudEnvironment) { - // In cloud, ALWAYS clear GOOGLE_APPLICATION_CREDENTIALS to force use of ADC - // Firebase Functions automatically provides credentials via metadata service - // These credentials have signing capabilities for generating signed URLs + if (clientOptions.credentials) { + storageConfig.credentials = clientOptions.credentials; + logger.info('Using inline service account credentials for GCS'); + } else if (isCloudEnvironment) { + // Firebase Functions provides ADC automatically. Ensure any stale env var is cleared. const originalCreds = process.env.GOOGLE_APPLICATION_CREDENTIALS; if (originalCreds) { delete process.env.GOOGLE_APPLICATION_CREDENTIALS; - logger.info('Using Application Default Credentials for GCS (cloud environment)', { - clearedEnvVar: 'GOOGLE_APPLICATION_CREDENTIALS', - originalValue: originalCreds, - projectId: config.googleCloud.projectId + logger.info('Cleared GOOGLE_APPLICATION_CREDENTIALS to use ADC (cloud environment)', { + originalValue: originalCreds }); } else { - logger.info('Using Application Default Credentials for GCS (cloud environment)', { - projectId: config.googleCloud.projectId - }); + logger.info('Using Application Default Credentials for GCS (cloud environment)'); } - - // Explicitly set project ID and let Storage use ADC (metadata service) - // Don't set keyFilename - this forces use of ADC which has signing capabilities - storageConfig.projectId = config.googleCloud.projectId; } else if (config.googleCloud.applicationCredentials) { - // Local development: check if the service account file exists try { const credsPath = config.googleCloud.applicationCredentials; - // Handle relative paths const absolutePath = path.isAbsolute(credsPath) ? credsPath : path.resolve(process.cwd(), credsPath); - + if (fs.existsSync(absolutePath)) { storageConfig.keyFilename = absolutePath; - logger.info('Using service account key file for GCS', { - keyFile: absolutePath - }); + logger.info('Using service account key file for GCS', { keyFile: absolutePath }); } else { - // File doesn't exist - clear GOOGLE_APPLICATION_CREDENTIALS if it points to this file - // and let Storage use Application Default Credentials (gcloud auth) if (process.env.GOOGLE_APPLICATION_CREDENTIALS === credsPath) { delete process.env.GOOGLE_APPLICATION_CREDENTIALS; - logger.warn('Service account key file not found, cleared GOOGLE_APPLICATION_CREDENTIALS, using Application Default Credentials', { - keyFile: credsPath - }); - } else { - logger.warn('Service account key file not found, using Application Default Credentials', { - keyFile: credsPath - }); } + logger.warn('Service account key file not found, falling back to ADC', { keyFile: credsPath }); } } catch (error) { - // If we can't check the file, clear the env var to avoid errors if (process.env.GOOGLE_APPLICATION_CREDENTIALS === config.googleCloud.applicationCredentials) { delete process.env.GOOGLE_APPLICATION_CREDENTIALS; } - logger.warn('Could not check service account key file, cleared GOOGLE_APPLICATION_CREDENTIALS, using Application Default Credentials', { + logger.warn('Could not check service account key file, using Application Default Credentials', { error: error instanceof Error ? error.message : String(error), keyFile: config.googleCloud.applicationCredentials }); } - } else { - // No applicationCredentials config - ensure GOOGLE_APPLICATION_CREDENTIALS is not set to invalid path - if (process.env.GOOGLE_APPLICATION_CREDENTIALS) { - const credsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS; - const absolutePath = path.isAbsolute(credsPath) - ? credsPath - : path.resolve(process.cwd(), credsPath); - - // If the file doesn't exist, clear the env var to avoid Storage initialization errors - if (!fs.existsSync(absolutePath)) { - delete process.env.GOOGLE_APPLICATION_CREDENTIALS; - logger.warn('GOOGLE_APPLICATION_CREDENTIALS pointed to non-existent file, cleared it, using Application Default Credentials', { - clearedPath: credsPath, - absolutePath - }); - } + } else if (process.env.GOOGLE_APPLICATION_CREDENTIALS) { + const credsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS; + const absolutePath = path.isAbsolute(credsPath) + ? credsPath + : path.resolve(process.cwd(), credsPath); + + if (!fs.existsSync(absolutePath)) { + delete process.env.GOOGLE_APPLICATION_CREDENTIALS; + logger.warn('GOOGLE_APPLICATION_CREDENTIALS pointed to non-existent file, cleared it to use ADC', { + clearedPath: credsPath + }); } } @@ -873,4 +848,4 @@ class FileStorageService { } export const fileStorageService = new FileStorageService(); -export default fileStorageService; \ No newline at end of file +export default fileStorageService; diff --git a/backend/src/services/llmService.ts b/backend/src/services/llmService.ts index 216a62e..ef05dd9 100644 --- a/backend/src/services/llmService.ts +++ b/backend/src/services/llmService.ts @@ -95,9 +95,8 @@ class LLMService { this.apiKey = config.llm.anthropicApiKey!; } - // Use configured model instead of hardcoded value - // This ensures we use the latest models (e.g., claude-sonnet-4-5-20250929) - this.defaultModel = config.llm.model; + // Use configured model instead of hardcoded value (normalize to supported identifiers) + this.defaultModel = this.normalizeModelName(config.llm.model) || config.llm.model; this.maxTokens = config.llm.maxTokens; this.temperature = config.llm.temperature; @@ -363,6 +362,11 @@ class LLMService { // Increased from 3 minutes to handle complex CIM analysis even with RAG reduction const timeoutMs = config.llm.timeoutMs || 360000; const timeoutMinutes = Math.round(timeoutMs / 60000); + const normalizedModel = this.normalizeModelName(request.model || this.defaultModel); + const normalizedRequest: LLMRequest = { + ...request, + model: normalizedModel + }; // Add a timeout wrapper to prevent hanging const timeoutPromise = new Promise((_, reject) => { @@ -373,20 +377,20 @@ class LLMService { // CRITICAL DEBUG: Log which provider method we're calling logger.info('Calling LLM provider method', { provider: this.provider, - model: request.model || this.defaultModel, + model: normalizedRequest.model || this.defaultModel, willCallOpenRouter: this.provider === 'openrouter', willCallAnthropic: this.provider === 'anthropic', willCallOpenAI: this.provider === 'openai' }); if (this.provider === 'openai') { - return await this.callOpenAI(request); + return await this.callOpenAI(normalizedRequest); } else if (this.provider === 'openrouter') { logger.info('Routing to callOpenRouter method'); - return await this.callOpenRouter(request); + return await this.callOpenRouter(normalizedRequest); } else if (this.provider === 'anthropic') { logger.info('Routing to callAnthropic method'); - return await this.callAnthropic(request); + return await this.callAnthropic(normalizedRequest); } else { logger.error('Unsupported LLM provider', { provider: this.provider }); throw new Error(`Unsupported LLM provider: ${this.provider}`); @@ -2524,7 +2528,60 @@ CRITICAL REQUIREMENTS: 8. **BPCP PREFERENCES**: BPCP prefers companies which are founder/family-owned and within driving distance of Cleveland and Charlotte. `; } + + private normalizeModelName(model?: string): string | undefined { + if (!model) { + return model; + } + + const trimmed = model.trim(); + const lower = trimmed.toLowerCase(); + const canonicalSonnet46 = 'claude-sonnet-4-6'; + const canonicalHaiku45 = 'claude-haiku-4-5'; + const legacySonnet35 = 'claude-3-5-sonnet-latest'; + const legacyHaiku35 = 'claude-3-5-haiku-latest'; + + // Keep modern 4.6/4.5 identifiers as-is + if (lower.includes('sonnet-4-6')) { + return trimmed; + } + if (lower.includes('haiku-4-5')) { + return trimmed; + } + + // Map older Claude 4.x labels (4.0/4.5) to the active 4.6/4.5 SKUs + if (lower.includes('sonnet-4')) { + if (trimmed !== canonicalSonnet46) { + logger.warn('Normalizing Claude Sonnet 4.x model to 4.6', { + requestedModel: trimmed, + normalizedModel: canonicalSonnet46 + }); + } + return canonicalSonnet46; + } + + if (lower.includes('haiku-4')) { + if (trimmed !== canonicalHaiku45) { + logger.warn('Normalizing Claude Haiku 4.x model to 4.5', { + requestedModel: trimmed, + normalizedModel: canonicalHaiku45 + }); + } + return canonicalHaiku45; + } + + // Keep legacy 3.5 identifiers stable, but normalize ambiguous "sonnet" or "haiku" labels + if (lower.includes('sonnet-3-5') || lower.includes('sonnet3.5')) { + return legacySonnet35; + } + + if (lower.includes('haiku-3-5') || lower.includes('haiku3.5')) { + return legacyHaiku35; + } + + return trimmed; + } } export const llmService = new LLMService(); -export default llmService; \ No newline at end of file +export default llmService; diff --git a/backend/src/services/optimizedAgenticRAGProcessor.ts b/backend/src/services/optimizedAgenticRAGProcessor.ts index f875a6b..a2ea987 100644 --- a/backend/src/services/optimizedAgenticRAGProcessor.ts +++ b/backend/src/services/optimizedAgenticRAGProcessor.ts @@ -3,6 +3,7 @@ import { vectorDatabaseService } from './vectorDatabaseService'; import { VectorDatabaseModel } from '../models/VectorDatabaseModel'; import { llmService } from './llmService'; import { CIMReview } from './llmSchemas'; +import { config } from '../config/env'; import type { ParsedFinancials } from './financialTableParser'; import type { StructuredTable } from './documentAiProcessor'; @@ -1833,9 +1834,24 @@ IMPORTANT EXTRACTION RULES: if (selectedChunks.length === 0) { // Fallback: select chunks based on keywords from the query const keywords = ragQuery.toLowerCase().split(' ').filter(w => w.length > 4); - selectedChunks = chunks + const keywordMatched = chunks .filter(chunk => keywords.some(kw => chunk.content.toLowerCase().includes(kw))) .slice(0, maxChunks); + + // If keyword matching finds very few chunks, use all chunks (spread evenly) + // to avoid sending only a tiny fragment to the LLM + if (keywordMatched.length < 5 && chunks.length > 5) { + logger.warn('Keyword fallback found too few chunks, using evenly-spaced selection', { + keywordMatched: keywordMatched.length, + totalChunks: chunks.length, + maxChunks + }); + // Select evenly-spaced chunks to cover the full document + const step = Math.max(1, Math.floor(chunks.length / maxChunks)); + selectedChunks = chunks.filter((_, i) => i % step === 0).slice(0, maxChunks); + } else { + selectedChunks = keywordMatched; + } } // Limit to maxChunks @@ -2338,7 +2354,7 @@ Do not include any preamble or explanation.`; { maxTokens: 3000, temperature: 0.3, // Lower temperature for more consistent output - model: 'claude-3-7-sonnet-latest' + model: config.llm.model } ); diff --git a/backend/src/utils/googleServiceAccount.ts b/backend/src/utils/googleServiceAccount.ts new file mode 100644 index 0000000..f724830 --- /dev/null +++ b/backend/src/utils/googleServiceAccount.ts @@ -0,0 +1,119 @@ +import fs from 'fs'; +import path from 'path'; +import { config } from '../config/env'; +import { logger } from './logger'; + +export interface GoogleServiceAccountCredentials { + client_email?: string; + private_key?: string; + project_id?: string; +} + +let cachedCredentials: GoogleServiceAccountCredentials | null | undefined; + +const normalizePrivateKey = (key?: string): string | undefined => { + if (!key) { + return key; + } + // Firebase secrets often escape newlines. Normalize so Google SDKs accept the key. + return key.includes('\\n') ? key.replace(/\\n/g, '\n') : key; +}; + +const readServiceAccountFile = (filePath: string): GoogleServiceAccountCredentials | undefined => { + const absolutePath = path.isAbsolute(filePath) ? filePath : path.resolve(process.cwd(), filePath); + if (!fs.existsSync(absolutePath)) { + logger.warn('Service account file not found', { absolutePath }); + return undefined; + } + + try { + const contents = fs.readFileSync(absolutePath, 'utf-8'); + const parsed = JSON.parse(contents) as GoogleServiceAccountCredentials; + if (parsed.private_key) { + parsed.private_key = normalizePrivateKey(parsed.private_key); + } + logger.info('Loaded service account credentials from file', { absolutePath }); + return parsed; + } catch (error) { + logger.error('Failed to parse service account file', { + absolutePath, + error: error instanceof Error ? error.message : String(error) + }); + return undefined; + } +}; + +export const resolveGoogleServiceAccount = (): GoogleServiceAccountCredentials | undefined => { + if (cachedCredentials !== undefined) { + return cachedCredentials || undefined; + } + + // Priority 1: Firebase secret injected as JSON + const inlineSecret = process.env.FIREBASE_SERVICE_ACCOUNT; + if (inlineSecret) { + try { + const parsed = JSON.parse(inlineSecret) as GoogleServiceAccountCredentials; + if (parsed.private_key) { + parsed.private_key = normalizePrivateKey(parsed.private_key); + } + cachedCredentials = parsed; + logger.info('Loaded Google service account from FIREBASE_SERVICE_ACCOUNT'); + return cachedCredentials; + } catch (error) { + logger.error('Failed to parse FIREBASE_SERVICE_ACCOUNT', { + error: error instanceof Error ? error.message : String(error) + }); + } + } + + // Priority 2: Explicit path variables + const serviceAccountPath = process.env.FIREBASE_SERVICE_ACCOUNT_PATH || config.googleCloud.applicationCredentials; + if (serviceAccountPath) { + const parsed = readServiceAccountFile(serviceAccountPath); + if (parsed) { + cachedCredentials = parsed; + return cachedCredentials; + } + } + + cachedCredentials = null; + return undefined; +}; + +export const ensureApplicationDefaultCredentials = (): void => { + const credsPath = process.env.GOOGLE_APPLICATION_CREDENTIALS; + if (!credsPath) { + return; + } + + const absolutePath = path.isAbsolute(credsPath) ? credsPath : path.resolve(process.cwd(), credsPath); + if (fs.existsSync(absolutePath)) { + return; + } + + delete process.env.GOOGLE_APPLICATION_CREDENTIALS; + logger.warn('GOOGLE_APPLICATION_CREDENTIALS pointed to a missing file. Falling back to ADC.', { + clearedPath: credsPath + }); +}; + +export const getGoogleClientOptions = (): { + projectId?: string; + credentials?: GoogleServiceAccountCredentials; +} => { + const credentials = resolveGoogleServiceAccount(); + const options: { + projectId?: string; + credentials?: GoogleServiceAccountCredentials; + } = {}; + + if (config.googleCloud.projectId) { + options.projectId = config.googleCloud.projectId; + } + + if (credentials && credentials.client_email && credentials.private_key) { + options.credentials = credentials; + } + + return options; +};