feat: wire Analytics tab to real data, add markdown rendering, fix UI labels

- Analytics endpoints now query document_processing_events table instead of
  returning hardcoded zeros (/documents/analytics, /processing-stats,
  /health/agentic-rag)
- Add react-markdown for rich text rendering in CIM review template
  (readOnly textarea fields now render markdown formatting)
- Fix upload page references from "Firebase Storage" to "Cloud Storage"
- Fix financial year labels from "FY3" to "FY-3" in DocumentViewer

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
admin
2026-02-25 11:27:46 -05:00
parent 4a25e551ce
commit d4b1658929
9 changed files with 1408 additions and 96 deletions

View File

@@ -130,19 +130,9 @@ app.get('/health/config', (_req, res) => {
// Agentic RAG health check endpoint (for analytics dashboard) // Agentic RAG health check endpoint (for analytics dashboard)
app.get('/health/agentic-rag', async (_req, res) => { app.get('/health/agentic-rag', async (_req, res) => {
try { try {
// Return health status (agentic RAG is not fully implemented) const { getHealthFromEvents } = await import('./services/analyticsService');
const healthStatus = { const healthStatus = await getHealthFromEvents();
status: 'healthy' as const,
agents: {},
overall: {
successRate: 1.0,
averageProcessingTime: 0,
activeSessions: 0,
errorRate: 0
},
timestamp: new Date().toISOString()
};
res.json(healthStatus); res.json(healthStatus);
} catch (error) { } catch (error) {
logger.error('Failed to get agentic RAG health', { error }); logger.error('Failed to get agentic RAG health', { error });

View File

@@ -2,6 +2,7 @@ import express from 'express';
import { verifyFirebaseToken } from '../middleware/firebaseAuth'; import { verifyFirebaseToken } from '../middleware/firebaseAuth';
import { documentController } from '../controllers/documentController'; import { documentController } from '../controllers/documentController';
import { unifiedDocumentProcessor } from '../services/unifiedDocumentProcessor'; import { unifiedDocumentProcessor } from '../services/unifiedDocumentProcessor';
import { getSessionAnalytics, getProcessingStatsFromEvents } from '../services/analyticsService';
import { logger } from '../utils/logger'; import { logger } from '../utils/logger';
import { config } from '../config/env'; import { config } from '../config/env';
import { DocumentModel } from '../models/DocumentModel'; import { DocumentModel } from '../models/DocumentModel';
@@ -40,18 +41,7 @@ router.get('/analytics', async (req, res) => {
} }
const days = parseInt(req.query['days'] as string) || 30; const days = parseInt(req.query['days'] as string) || 30;
// Return empty analytics data (agentic RAG analytics not fully implemented) const analytics = await getSessionAnalytics(days);
const analytics = {
totalSessions: 0,
successfulSessions: 0,
failedSessions: 0,
avgQualityScore: 0.8,
avgCompleteness: 0.9,
avgProcessingTime: 0,
sessionsOverTime: [],
agentPerformance: [],
qualityTrends: []
};
return res.json({ return res.json({
...analytics, ...analytics,
correlationId: req.correlationId || undefined correlationId: req.correlationId || undefined
@@ -70,7 +60,7 @@ router.get('/analytics', async (req, res) => {
router.get('/processing-stats', async (req, res) => { router.get('/processing-stats', async (req, res) => {
try { try {
const stats = await unifiedDocumentProcessor.getProcessingStats(); const stats = await getProcessingStatsFromEvents();
return res.json({ return res.json({
...stats, ...stats,
correlationId: req.correlationId || undefined correlationId: req.correlationId || undefined

View File

@@ -146,3 +146,188 @@ export async function getAnalyticsSummary(range: string = '24h'): Promise<Analyt
generatedAt: new Date().toISOString(), generatedAt: new Date().toISOString(),
}; };
} }
// =============================================================================
// getSessionAnalytics — per-day session stats for Analytics tab
// =============================================================================
export interface SessionAnalytics {
sessionStats: Array<{
date: string;
total_sessions: string;
successful_sessions: string;
failed_sessions: string;
avg_processing_time: string;
avg_cost: string;
}>;
agentStats: Array<Record<string, string>>;
qualityStats: Array<Record<string, string>>;
period: {
startDate: string;
endDate: string;
days: number;
};
}
/**
* Returns per-day session statistics from document_processing_events.
* Groups upload_started events by date, then joins completed/failed counts.
*/
export async function getSessionAnalytics(days: number): Promise<SessionAnalytics> {
const pool = getPostgresPool();
const interval = `${days} days`;
const { rows } = await pool.query<{
date: string;
total_sessions: string;
successful_sessions: string;
failed_sessions: string;
avg_processing_time: string;
}>(`
SELECT
DATE(created_at) AS date,
COUNT(*) FILTER (WHERE event_type = 'upload_started') AS total_sessions,
COUNT(*) FILTER (WHERE event_type = 'completed') AS successful_sessions,
COUNT(*) FILTER (WHERE event_type = 'failed') AS failed_sessions,
COALESCE(AVG(duration_ms) FILTER (WHERE event_type = 'completed'), 0) AS avg_processing_time
FROM document_processing_events
WHERE created_at >= NOW() - $1::interval
GROUP BY DATE(created_at)
ORDER BY date DESC
`, [interval]);
const endDate = new Date();
const startDate = new Date(Date.now() - days * 86400000);
return {
sessionStats: rows.map(r => ({
date: r.date,
total_sessions: r.total_sessions,
successful_sessions: r.successful_sessions,
failed_sessions: r.failed_sessions,
avg_processing_time: r.avg_processing_time,
avg_cost: '0', // cost tracking not implemented
})),
agentStats: [], // agent-level tracking not available in current schema
qualityStats: [], // quality scores not available in current schema
period: {
startDate: startDate.toISOString(),
endDate: endDate.toISOString(),
days,
},
};
}
// =============================================================================
// getProcessingStatsFromEvents — processing pipeline stats for Analytics tab
// =============================================================================
export interface ProcessingStatsFromEvents {
totalDocuments: number;
documentAiAgenticRagSuccess: number;
averageProcessingTime: { documentAiAgenticRag: number };
averageApiCalls: { documentAiAgenticRag: number };
}
/**
* Returns processing pipeline statistics from document_processing_events.
*/
export async function getProcessingStatsFromEvents(): Promise<ProcessingStatsFromEvents> {
const pool = getPostgresPool();
const { rows } = await pool.query<{
total_documents: string;
succeeded: string;
avg_processing_ms: string | null;
}>(`
SELECT
COUNT(DISTINCT document_id) AS total_documents,
COUNT(*) FILTER (WHERE event_type = 'completed') AS succeeded,
AVG(duration_ms) FILTER (WHERE event_type = 'completed') AS avg_processing_ms
FROM document_processing_events
`);
const row = rows[0]!;
return {
totalDocuments: parseInt(row.total_documents, 10),
documentAiAgenticRagSuccess: parseInt(row.succeeded, 10),
averageProcessingTime: {
documentAiAgenticRag: row.avg_processing_ms ? parseFloat(row.avg_processing_ms) : 0,
},
averageApiCalls: {
documentAiAgenticRag: 0, // API call counting not tracked in events table
},
};
}
// =============================================================================
// getHealthFromEvents — system health status for Analytics tab
// =============================================================================
export interface HealthFromEvents {
status: 'healthy' | 'degraded' | 'unhealthy';
agents: Record<string, unknown>;
overall: {
successRate: number;
averageProcessingTime: number;
activeSessions: number;
errorRate: number;
};
timestamp: string;
}
/**
* Derives system health status from recent document_processing_events.
* Looks at the last 24 hours to determine health.
*/
export async function getHealthFromEvents(): Promise<HealthFromEvents> {
const pool = getPostgresPool();
const { rows } = await pool.query<{
total: string;
succeeded: string;
failed: string;
avg_processing_ms: string | null;
active: string;
}>(`
SELECT
COUNT(*) AS total,
COUNT(*) FILTER (WHERE event_type = 'completed') AS succeeded,
COUNT(*) FILTER (WHERE event_type = 'failed') AS failed,
AVG(duration_ms) FILTER (WHERE event_type = 'completed') AS avg_processing_ms,
COUNT(*) FILTER (
WHERE event_type = 'processing_started'
AND document_id NOT IN (
SELECT document_id FROM document_processing_events
WHERE event_type IN ('completed', 'failed')
AND created_at >= NOW() - INTERVAL '24 hours'
)
) AS active
FROM document_processing_events
WHERE created_at >= NOW() - INTERVAL '24 hours'
`);
const row = rows[0]!;
const total = parseInt(row.total, 10);
const succeeded = parseInt(row.succeeded, 10);
const failed = parseInt(row.failed, 10);
const successRate = total > 0 ? succeeded / total : 1.0;
const errorRate = total > 0 ? failed / total : 0;
let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy';
if (errorRate > 0.5) status = 'unhealthy';
else if (errorRate > 0.2) status = 'degraded';
return {
status,
agents: {},
overall: {
successRate,
averageProcessingTime: row.avg_processing_ms ? parseFloat(row.avg_processing_ms) : 0,
activeSessions: parseInt(row.active, 10),
errorRate,
},
timestamp: new Date().toISOString(),
};
}

File diff suppressed because it is too large Load Diff

View File

@@ -21,6 +21,7 @@
"react": "^18.2.0", "react": "^18.2.0",
"react-dom": "^18.2.0", "react-dom": "^18.2.0",
"react-dropzone": "^14.3.8", "react-dropzone": "^14.3.8",
"react-markdown": "^10.1.0",
"react-router-dom": "^6.20.1", "react-router-dom": "^6.20.1",
"tailwind-merge": "^2.0.0" "tailwind-merge": "^2.0.0"
}, },

View File

@@ -35,18 +35,12 @@ interface AnalyticsData {
interface ProcessingStats { interface ProcessingStats {
totalDocuments: number; totalDocuments: number;
chunkingSuccess: number; documentAiAgenticRagSuccess: number;
ragSuccess: number;
agenticRagSuccess: number;
averageProcessingTime: { averageProcessingTime: {
chunking: number; documentAiAgenticRag: number;
rag: number;
agenticRag: number;
}; };
averageApiCalls: { averageApiCalls: {
chunking: number; documentAiAgenticRag: number;
rag: number;
agenticRag: number;
}; };
} }
@@ -208,29 +202,21 @@ const Analytics: React.FC = () => {
<h2 className="text-lg font-semibold text-gray-900 mb-4">Processing Statistics</h2> <h2 className="text-lg font-semibold text-gray-900 mb-4">Processing Statistics</h2>
<div className="grid grid-cols-1 md:grid-cols-3 gap-6"> <div className="grid grid-cols-1 md:grid-cols-3 gap-6">
<div className="space-y-4"> <div className="space-y-4">
<h3 className="text-md font-medium text-gray-700">Success Rates</h3> <h3 className="text-md font-medium text-gray-700">Documents Processed</h3>
<div className="space-y-2"> <div className="space-y-2">
<div className="flex justify-between"> <div className="flex justify-between">
<span className="text-sm text-gray-600">Chunking</span> <span className="text-sm text-gray-600">Total Documents</span>
<span className="text-sm font-medium"> <span className="text-sm font-medium">{processingStats.totalDocuments}</span>
{processingStats.totalDocuments > 0
? (((processingStats.chunkingSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
: 0}%
</span>
</div> </div>
<div className="flex justify-between"> <div className="flex justify-between">
<span className="text-sm text-gray-600">RAG</span> <span className="text-sm text-gray-600">Successful</span>
<span className="text-sm font-medium"> <span className="text-sm font-medium text-success-600">{processingStats.documentAiAgenticRagSuccess ?? 0}</span>
{processingStats.totalDocuments > 0
? (((processingStats.ragSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
: 0}%
</span>
</div> </div>
<div className="flex justify-between"> <div className="flex justify-between">
<span className="text-sm text-gray-600">Agentic RAG</span> <span className="text-sm text-gray-600">Success Rate</span>
<span className="text-sm font-medium"> <span className="text-sm font-medium">
{processingStats.totalDocuments > 0 {processingStats.totalDocuments > 0
? (((processingStats.agenticRagSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1) ? (((processingStats.documentAiAgenticRagSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
: 0}% : 0}%
</span> </span>
</div> </div>
@@ -240,33 +226,17 @@ const Analytics: React.FC = () => {
<h3 className="text-md font-medium text-gray-700">Average Processing Time</h3> <h3 className="text-md font-medium text-gray-700">Average Processing Time</h3>
<div className="space-y-2"> <div className="space-y-2">
<div className="flex justify-between"> <div className="flex justify-between">
<span className="text-sm text-gray-600">Chunking</span> <span className="text-sm text-gray-600">Document AI + Agentic RAG</span>
<span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.chunking ?? 0)}</span> <span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.documentAiAgenticRag ?? 0)}</span>
</div>
<div className="flex justify-between">
<span className="text-sm text-gray-600">RAG</span>
<span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.rag ?? 0)}</span>
</div>
<div className="flex justify-between">
<span className="text-sm text-gray-600">Agentic RAG</span>
<span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.agenticRag ?? 0)}</span>
</div> </div>
</div> </div>
</div> </div>
<div className="space-y-4"> <div className="space-y-4">
<h3 className="text-md font-medium text-gray-700">Average API Calls</h3> <h3 className="text-md font-medium text-gray-700">Pipeline</h3>
<div className="space-y-2"> <div className="space-y-2">
<div className="flex justify-between"> <div className="flex justify-between">
<span className="text-sm text-gray-600">Chunking</span> <span className="text-sm text-gray-600">Processing Method</span>
<span className="text-sm font-medium">{(processingStats.averageApiCalls?.chunking ?? 0).toFixed(1)}</span> <span className="text-sm font-medium">Document AI + Agentic RAG</span>
</div>
<div className="flex justify-between">
<span className="text-sm text-gray-600">RAG</span>
<span className="text-sm font-medium">{(processingStats.averageApiCalls?.rag ?? 0).toFixed(1)}</span>
</div>
<div className="flex justify-between">
<span className="text-sm text-gray-600">Agentic RAG</span>
<span className="text-sm font-medium">{(processingStats.averageApiCalls?.agenticRag ?? 0).toFixed(1)}</span>
</div> </div>
</div> </div>
</div> </div>

View File

@@ -1,4 +1,5 @@
import React, { useState, useEffect } from 'react'; import React, { useState, useEffect } from 'react';
import Markdown from 'react-markdown';
import { Save, Download } from 'lucide-react'; import { Save, Download } from 'lucide-react';
import { cn } from '../utils/cn'; import { cn } from '../utils/cn';
@@ -341,17 +342,23 @@ const CIMReviewTemplate: React.FC<CIMReviewTemplateProps> = ({
{label} {label}
</label> </label>
{type === 'textarea' ? ( {type === 'textarea' ? (
<textarea readOnly && value ? (
value={value || ''} <div className="block w-full rounded-md border border-gray-200 bg-gray-50 px-3 py-2 text-sm text-gray-700 prose prose-sm max-w-none prose-p:my-1 prose-ul:my-1 prose-li:my-0">
onChange={(e) => { <Markdown>{value}</Markdown>
updateNestedField(e.target.value); </div>
triggerAutoSave(); ) : (
}} <textarea
placeholder={placeholder} value={value || ''}
rows={rows || 3} onChange={(e) => {
disabled={readOnly} updateNestedField(e.target.value);
className="block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm disabled:bg-gray-50 disabled:text-gray-500" triggerAutoSave();
/> }}
placeholder={placeholder}
rows={rows || 3}
disabled={readOnly}
className="block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm disabled:bg-gray-50 disabled:text-gray-500"
/>
)
) : type === 'date' ? ( ) : type === 'date' ? (
<input <input
type="date" type="date"

View File

@@ -321,16 +321,16 @@ const DocumentUpload: React.FC<DocumentUploadProps> = ({
const getStatusText = (status: UploadedFile['status'], error?: string, storageError?: boolean) => { const getStatusText = (status: UploadedFile['status'], error?: string, storageError?: boolean) => {
switch (status) { switch (status) {
case 'uploading': case 'uploading':
return 'Uploading to Firebase Storage...'; return 'Uploading to Cloud Storage...';
case 'uploaded': case 'uploaded':
return 'Uploaded to Firebase Storage ✓'; return 'Uploaded to Cloud Storage ✓';
case 'processing': case 'processing':
return 'Processing with Document AI + Optimized Agentic RAG...'; return 'Processing with Document AI + Optimized Agentic RAG...';
case 'completed': case 'completed':
return 'Completed ✓ (PDF automatically deleted)'; return 'Completed ✓ (PDF automatically deleted)';
case 'error': case 'error':
if (error === 'Upload cancelled') return 'Cancelled'; if (error === 'Upload cancelled') return 'Cancelled';
if (storageError) return 'Firebase Storage Error'; if (storageError) return 'Cloud Storage Error';
return 'Error'; return 'Error';
default: default:
return ''; return '';
@@ -372,7 +372,7 @@ const DocumentUpload: React.FC<DocumentUploadProps> = ({
Drag and drop PDF files here, or click to browse Drag and drop PDF files here, or click to browse
</p> </p>
<p className="text-xs text-gray-500"> <p className="text-xs text-gray-500">
Maximum file size: 50MB Supported format: PDF Stored securely in Firebase Storage Automatic Document AI + Optimized Agentic RAG Processing PDFs deleted after processing Maximum file size: 50MB Supported format: PDF Stored securely in Cloud Storage Automatic Document AI + Optimized Agentic RAG Processing PDFs deleted after processing
</p> </p>
</div> </div>
@@ -400,7 +400,7 @@ const DocumentUpload: React.FC<DocumentUploadProps> = ({
<div> <div>
<h4 className="text-sm font-medium text-success-800">Upload Complete</h4> <h4 className="text-sm font-medium text-success-800">Upload Complete</h4>
<p className="text-sm text-success-700 mt-1"> <p className="text-sm text-success-700 mt-1">
Files have been uploaded successfully to Firebase Storage! You can now navigate away from this page. Files have been uploaded successfully! You can now navigate away from this page.
Processing will continue in the background using Document AI + Optimized Agentic RAG. This can take several minutes. PDFs will be automatically deleted after processing to save costs. Processing will continue in the background using Document AI + Optimized Agentic RAG. This can take several minutes. PDFs will be automatically deleted after processing to save costs.
</p> </p>
</div> </div>

View File

@@ -296,7 +296,7 @@ ${user?.name || user?.email || 'CIM Document Processor User'}`);
<div className="space-y-1"> <div className="space-y-1">
{extractedData.financials.revenue.map((value, index) => ( {extractedData.financials.revenue.map((value, index) => (
<div key={index} className="flex justify-between text-sm"> <div key={index} className="flex justify-between text-sm">
<span className="text-gray-600">{index === 3 ? 'LTM' : `FY${3-index}`}</span> <span className="text-gray-600">{index === 3 ? 'LTM' : `FY-${3-index}`}</span>
<span className="font-medium">{value}</span> <span className="font-medium">{value}</span>
</div> </div>
))} ))}
@@ -307,7 +307,7 @@ ${user?.name || user?.email || 'CIM Document Processor User'}`);
<div className="space-y-1"> <div className="space-y-1">
{extractedData.financials.ebitda.map((value, index) => ( {extractedData.financials.ebitda.map((value, index) => (
<div key={index} className="flex justify-between text-sm"> <div key={index} className="flex justify-between text-sm">
<span className="text-gray-600">{index === 3 ? 'LTM' : `FY${3-index}`}</span> <span className="text-gray-600">{index === 3 ? 'LTM' : `FY-${3-index}`}</span>
<span className="font-medium">{value}</span> <span className="font-medium">{value}</span>
</div> </div>
))} ))}
@@ -318,7 +318,7 @@ ${user?.name || user?.email || 'CIM Document Processor User'}`);
<div className="space-y-1"> <div className="space-y-1">
{extractedData.financials.margins.map((value, index) => ( {extractedData.financials.margins.map((value, index) => (
<div key={index} className="flex justify-between text-sm"> <div key={index} className="flex justify-between text-sm">
<span className="text-gray-600">{index === 3 ? 'LTM' : `FY${3-index}`}</span> <span className="text-gray-600">{index === 3 ? 'LTM' : `FY-${3-index}`}</span>
<span className="font-medium">{value}</span> <span className="font-medium">{value}</span>
</div> </div>
))} ))}