feat: wire Analytics tab to real data, add markdown rendering, fix UI labels

- Analytics endpoints now query document_processing_events table instead of returning hardcoded zeros (/documents/analytics, /processing-stats, /health/agentic-rag) - Add react-markdown for rich text rendering in CIM review template (readOnly textarea fields now render markdown formatting) - Fix upload page references from "Firebase Storage" to "Cloud Storage" - Fix financial year labels from "FY3" to "FY-3" in DocumentViewer Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-25 11:27:46 -05:00
parent 4a25e551ce
commit d4b1658929
9 changed files with 1408 additions and 96 deletions
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -130,19 +130,9 @@ app.get('/health/config', (_req, res) => {
 // Agentic RAG health check endpoint (for analytics dashboard)
 app.get('/health/agentic-rag', async (_req, res) => {
  try {
-    // Return health status (agentic RAG is not fully implemented)
-    const healthStatus = {
-      status: 'healthy' as const,
-      agents: {},
-      overall: {
-        successRate: 1.0,
-        averageProcessingTime: 0,
-        activeSessions: 0,
-        errorRate: 0
-      },
-      timestamp: new Date().toISOString()
-    };
-    
+    const { getHealthFromEvents } = await import('./services/analyticsService');
+    const healthStatus = await getHealthFromEvents();
+
    res.json(healthStatus);
  } catch (error) {
    logger.error('Failed to get agentic RAG health', { error });
--- a/backend/src/routes/documents.ts
+++ b/backend/src/routes/documents.ts
@@ -2,6 +2,7 @@ import express from 'express';
 import { verifyFirebaseToken } from '../middleware/firebaseAuth';
 import { documentController } from '../controllers/documentController';
 import { unifiedDocumentProcessor } from '../services/unifiedDocumentProcessor';
+import { getSessionAnalytics, getProcessingStatsFromEvents } from '../services/analyticsService';
 import { logger } from '../utils/logger';
 import { config } from '../config/env';
 import { DocumentModel } from '../models/DocumentModel';
@@ -40,18 +41,7 @@ router.get('/analytics', async (req, res) => {
    }

    const days = parseInt(req.query['days'] as string) || 30;
-    // Return empty analytics data (agentic RAG analytics not fully implemented)
-    const analytics = {
-      totalSessions: 0,
-      successfulSessions: 0,
-      failedSessions: 0,
-      avgQualityScore: 0.8,
-      avgCompleteness: 0.9,
-      avgProcessingTime: 0,
-      sessionsOverTime: [],
-      agentPerformance: [],
-      qualityTrends: []
-    };
+    const analytics = await getSessionAnalytics(days);
    return res.json({
      ...analytics,
      correlationId: req.correlationId || undefined
@@ -70,7 +60,7 @@ router.get('/analytics', async (req, res) => {

 router.get('/processing-stats', async (req, res) => {
  try {
-    const stats = await unifiedDocumentProcessor.getProcessingStats();
+    const stats = await getProcessingStatsFromEvents();
    return res.json({
      ...stats,
      correlationId: req.correlationId || undefined
--- a/backend/src/services/analyticsService.ts
+++ b/backend/src/services/analyticsService.ts
@@ -146,3 +146,188 @@ export async function getAnalyticsSummary(range: string = '24h'): Promise<Analyt
    generatedAt: new Date().toISOString(),
  };
 }
+
+// =============================================================================
+// getSessionAnalytics — per-day session stats for Analytics tab
+// =============================================================================
+
+export interface SessionAnalytics {
+  sessionStats: Array<{
+    date: string;
+    total_sessions: string;
+    successful_sessions: string;
+    failed_sessions: string;
+    avg_processing_time: string;
+    avg_cost: string;
+  }>;
+  agentStats: Array<Record<string, string>>;
+  qualityStats: Array<Record<string, string>>;
+  period: {
+    startDate: string;
+    endDate: string;
+    days: number;
+  };
+}
+
+/**
+ * Returns per-day session statistics from document_processing_events.
+ * Groups upload_started events by date, then joins completed/failed counts.
+ */
+export async function getSessionAnalytics(days: number): Promise<SessionAnalytics> {
+  const pool = getPostgresPool();
+  const interval = `${days} days`;
+
+  const { rows } = await pool.query<{
+    date: string;
+    total_sessions: string;
+    successful_sessions: string;
+    failed_sessions: string;
+    avg_processing_time: string;
+  }>(`
+    SELECT
+      DATE(created_at) AS date,
+      COUNT(*) FILTER (WHERE event_type = 'upload_started')    AS total_sessions,
+      COUNT(*) FILTER (WHERE event_type = 'completed')         AS successful_sessions,
+      COUNT(*) FILTER (WHERE event_type = 'failed')            AS failed_sessions,
+      COALESCE(AVG(duration_ms) FILTER (WHERE event_type = 'completed'), 0) AS avg_processing_time
+    FROM document_processing_events
+    WHERE created_at >= NOW() - $1::interval
+    GROUP BY DATE(created_at)
+    ORDER BY date DESC
+  `, [interval]);
+
+  const endDate = new Date();
+  const startDate = new Date(Date.now() - days * 86400000);
+
+  return {
+    sessionStats: rows.map(r => ({
+      date: r.date,
+      total_sessions: r.total_sessions,
+      successful_sessions: r.successful_sessions,
+      failed_sessions: r.failed_sessions,
+      avg_processing_time: r.avg_processing_time,
+      avg_cost: '0', // cost tracking not implemented
+    })),
+    agentStats: [], // agent-level tracking not available in current schema
+    qualityStats: [], // quality scores not available in current schema
+    period: {
+      startDate: startDate.toISOString(),
+      endDate: endDate.toISOString(),
+      days,
+    },
+  };
+}
+
+// =============================================================================
+// getProcessingStatsFromEvents — processing pipeline stats for Analytics tab
+// =============================================================================
+
+export interface ProcessingStatsFromEvents {
+  totalDocuments: number;
+  documentAiAgenticRagSuccess: number;
+  averageProcessingTime: { documentAiAgenticRag: number };
+  averageApiCalls: { documentAiAgenticRag: number };
+}
+
+/**
+ * Returns processing pipeline statistics from document_processing_events.
+ */
+export async function getProcessingStatsFromEvents(): Promise<ProcessingStatsFromEvents> {
+  const pool = getPostgresPool();
+
+  const { rows } = await pool.query<{
+    total_documents: string;
+    succeeded: string;
+    avg_processing_ms: string | null;
+  }>(`
+    SELECT
+      COUNT(DISTINCT document_id) AS total_documents,
+      COUNT(*) FILTER (WHERE event_type = 'completed') AS succeeded,
+      AVG(duration_ms) FILTER (WHERE event_type = 'completed') AS avg_processing_ms
+    FROM document_processing_events
+  `);
+
+  const row = rows[0]!;
+
+  return {
+    totalDocuments: parseInt(row.total_documents, 10),
+    documentAiAgenticRagSuccess: parseInt(row.succeeded, 10),
+    averageProcessingTime: {
+      documentAiAgenticRag: row.avg_processing_ms ? parseFloat(row.avg_processing_ms) : 0,
+    },
+    averageApiCalls: {
+      documentAiAgenticRag: 0, // API call counting not tracked in events table
+    },
+  };
+}
+
+// =============================================================================
+// getHealthFromEvents — system health status for Analytics tab
+// =============================================================================
+
+export interface HealthFromEvents {
+  status: 'healthy' | 'degraded' | 'unhealthy';
+  agents: Record<string, unknown>;
+  overall: {
+    successRate: number;
+    averageProcessingTime: number;
+    activeSessions: number;
+    errorRate: number;
+  };
+  timestamp: string;
+}
+
+/**
+ * Derives system health status from recent document_processing_events.
+ * Looks at the last 24 hours to determine health.
+ */
+export async function getHealthFromEvents(): Promise<HealthFromEvents> {
+  const pool = getPostgresPool();
+
+  const { rows } = await pool.query<{
+    total: string;
+    succeeded: string;
+    failed: string;
+    avg_processing_ms: string | null;
+    active: string;
+  }>(`
+    SELECT
+      COUNT(*) AS total,
+      COUNT(*) FILTER (WHERE event_type = 'completed') AS succeeded,
+      COUNT(*) FILTER (WHERE event_type = 'failed') AS failed,
+      AVG(duration_ms) FILTER (WHERE event_type = 'completed') AS avg_processing_ms,
+      COUNT(*) FILTER (
+        WHERE event_type = 'processing_started'
+        AND document_id NOT IN (
+          SELECT document_id FROM document_processing_events
+          WHERE event_type IN ('completed', 'failed')
+          AND created_at >= NOW() - INTERVAL '24 hours'
+        )
+      ) AS active
+    FROM document_processing_events
+    WHERE created_at >= NOW() - INTERVAL '24 hours'
+  `);
+
+  const row = rows[0]!;
+  const total = parseInt(row.total, 10);
+  const succeeded = parseInt(row.succeeded, 10);
+  const failed = parseInt(row.failed, 10);
+  const successRate = total > 0 ? succeeded / total : 1.0;
+  const errorRate = total > 0 ? failed / total : 0;
+
+  let status: 'healthy' | 'degraded' | 'unhealthy' = 'healthy';
+  if (errorRate > 0.5) status = 'unhealthy';
+  else if (errorRate > 0.2) status = 'degraded';
+
+  return {
+    status,
+    agents: {},
+    overall: {
+      successRate,
+      averageProcessingTime: row.avg_processing_ms ? parseFloat(row.avg_processing_ms) : 0,
+      activeSessions: parseInt(row.active, 10),
+      errorRate,
+    },
+    timestamp: new Date().toISOString(),
+  };
+}
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -21,6 +21,7 @@
    "react": "^18.2.0",
    "react-dom": "^18.2.0",
    "react-dropzone": "^14.3.8",
+    "react-markdown": "^10.1.0",
    "react-router-dom": "^6.20.1",
    "tailwind-merge": "^2.0.0"
  },
--- a/frontend/src/components/Analytics.tsx
+++ b/frontend/src/components/Analytics.tsx
@@ -35,18 +35,12 @@ interface AnalyticsData {

 interface ProcessingStats {
  totalDocuments: number;
-  chunkingSuccess: number;
-  ragSuccess: number;
-  agenticRagSuccess: number;
+  documentAiAgenticRagSuccess: number;
  averageProcessingTime: {
-    chunking: number;
-    rag: number;
-    agenticRag: number;
+    documentAiAgenticRag: number;
  };
  averageApiCalls: {
-    chunking: number;
-    rag: number;
-    agenticRag: number;
+    documentAiAgenticRag: number;
  };
 }

@@ -208,29 +202,21 @@ const Analytics: React.FC = () => {
          <h2 className="text-lg font-semibold text-gray-900 mb-4">Processing Statistics</h2>
          <div className="grid grid-cols-1 md:grid-cols-3 gap-6">
            <div className="space-y-4">
-              <h3 className="text-md font-medium text-gray-700">Success Rates</h3>
+              <h3 className="text-md font-medium text-gray-700">Documents Processed</h3>
              <div className="space-y-2">
                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">Chunking</span>
-                  <span className="text-sm font-medium">
-                    {processingStats.totalDocuments > 0
-                      ? (((processingStats.chunkingSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
-                      : 0}%
-                  </span>
+                  <span className="text-sm text-gray-600">Total Documents</span>
+                  <span className="text-sm font-medium">{processingStats.totalDocuments}</span>
                </div>
                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">RAG</span>
-                  <span className="text-sm font-medium">
-                    {processingStats.totalDocuments > 0
-                      ? (((processingStats.ragSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
-                      : 0}%
-                  </span>
+                  <span className="text-sm text-gray-600">Successful</span>
+                  <span className="text-sm font-medium text-success-600">{processingStats.documentAiAgenticRagSuccess ?? 0}</span>
                </div>
                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">Agentic RAG</span>
+                  <span className="text-sm text-gray-600">Success Rate</span>
                  <span className="text-sm font-medium">
                    {processingStats.totalDocuments > 0
-                      ? (((processingStats.agenticRagSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
+                      ? (((processingStats.documentAiAgenticRagSuccess ?? 0) / processingStats.totalDocuments) * 100).toFixed(1)
                      : 0}%
                  </span>
                </div>
@@ -240,33 +226,17 @@ const Analytics: React.FC = () => {
              <h3 className="text-md font-medium text-gray-700">Average Processing Time</h3>
              <div className="space-y-2">
                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">Chunking</span>
-                  <span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.chunking ?? 0)}</span>
-                </div>
-                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">RAG</span>
-                  <span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.rag ?? 0)}</span>
-                </div>
-                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">Agentic RAG</span>
-                  <span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.agenticRag ?? 0)}</span>
+                  <span className="text-sm text-gray-600">Document AI + Agentic RAG</span>
+                  <span className="text-sm font-medium">{formatTime(processingStats.averageProcessingTime?.documentAiAgenticRag ?? 0)}</span>
                </div>
              </div>
            </div>
            <div className="space-y-4">
-              <h3 className="text-md font-medium text-gray-700">Average API Calls</h3>
+              <h3 className="text-md font-medium text-gray-700">Pipeline</h3>
              <div className="space-y-2">
                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">Chunking</span>
-                  <span className="text-sm font-medium">{(processingStats.averageApiCalls?.chunking ?? 0).toFixed(1)}</span>
-                </div>
-                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">RAG</span>
-                  <span className="text-sm font-medium">{(processingStats.averageApiCalls?.rag ?? 0).toFixed(1)}</span>
-                </div>
-                <div className="flex justify-between">
-                  <span className="text-sm text-gray-600">Agentic RAG</span>
-                  <span className="text-sm font-medium">{(processingStats.averageApiCalls?.agenticRag ?? 0).toFixed(1)}</span>
+                  <span className="text-sm text-gray-600">Processing Method</span>
+                  <span className="text-sm font-medium">Document AI + Agentic RAG</span>
                </div>
              </div>
            </div>
--- a/frontend/src/components/CIMReviewTemplate.tsx
+++ b/frontend/src/components/CIMReviewTemplate.tsx
@@ -1,4 +1,5 @@
 import React, { useState, useEffect } from 'react';
+import Markdown from 'react-markdown';
 import { Save, Download } from 'lucide-react';
 import { cn } from '../utils/cn';

@@ -341,17 +342,23 @@ const CIMReviewTemplate: React.FC<CIMReviewTemplateProps> = ({
          {label}
        </label>
        {type === 'textarea' ? (
-          <textarea
-            value={value || ''}
-            onChange={(e) => {
-              updateNestedField(e.target.value);
-              triggerAutoSave();
-            }}
-            placeholder={placeholder}
-            rows={rows || 3}
-            disabled={readOnly}
-            className="block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm disabled:bg-gray-50 disabled:text-gray-500"
-          />
+          readOnly && value ? (
+            <div className="block w-full rounded-md border border-gray-200 bg-gray-50 px-3 py-2 text-sm text-gray-700 prose prose-sm max-w-none prose-p:my-1 prose-ul:my-1 prose-li:my-0">
+              <Markdown>{value}</Markdown>
+            </div>
+          ) : (
+            <textarea
+              value={value || ''}
+              onChange={(e) => {
+                updateNestedField(e.target.value);
+                triggerAutoSave();
+              }}
+              placeholder={placeholder}
+              rows={rows || 3}
+              disabled={readOnly}
+              className="block w-full rounded-md border-gray-300 shadow-sm focus:border-blue-500 focus:ring-blue-500 sm:text-sm disabled:bg-gray-50 disabled:text-gray-500"
+            />
+          )
        ) : type === 'date' ? (
          <input
            type="date"
--- a/frontend/src/components/DocumentUpload.tsx
+++ b/frontend/src/components/DocumentUpload.tsx
@@ -321,16 +321,16 @@ const DocumentUpload: React.FC<DocumentUploadProps> = ({
  const getStatusText = (status: UploadedFile['status'], error?: string, storageError?: boolean) => {
    switch (status) {
      case 'uploading':
-        return 'Uploading to Firebase Storage...';
+        return 'Uploading to Cloud Storage...';
      case 'uploaded':
-        return 'Uploaded to Firebase Storage ✓';
+        return 'Uploaded to Cloud Storage ✓';
      case 'processing':
        return 'Processing with Document AI + Optimized Agentic RAG...';
      case 'completed':
        return 'Completed ✓ (PDF automatically deleted)';
      case 'error':
        if (error === 'Upload cancelled') return 'Cancelled';
-        if (storageError) return 'Firebase Storage Error';
+        if (storageError) return 'Cloud Storage Error';
        return 'Error';
      default:
        return '';
@@ -372,7 +372,7 @@ const DocumentUpload: React.FC<DocumentUploadProps> = ({
          Drag and drop PDF files here, or click to browse
        </p>
        <p className="text-xs text-gray-500">
-          Maximum file size: 50MB • Supported format: PDF • Stored securely in Firebase Storage • Automatic Document AI + Optimized Agentic RAG Processing • PDFs deleted after processing
+          Maximum file size: 50MB • Supported format: PDF • Stored securely in Cloud Storage • Automatic Document AI + Optimized Agentic RAG Processing • PDFs deleted after processing
        </p>
      </div>

@@ -400,7 +400,7 @@ const DocumentUpload: React.FC<DocumentUploadProps> = ({
            <div>
              <h4 className="text-sm font-medium text-success-800">Upload Complete</h4>
              <p className="text-sm text-success-700 mt-1">
-                Files have been uploaded successfully to Firebase Storage! You can now navigate away from this page. 
+                Files have been uploaded successfully! You can now navigate away from this page.
                Processing will continue in the background using Document AI + Optimized Agentic RAG. This can take several minutes. PDFs will be automatically deleted after processing to save costs.
              </p>
            </div>
--- a/frontend/src/components/DocumentViewer.tsx
+++ b/frontend/src/components/DocumentViewer.tsx
@@ -296,7 +296,7 @@ ${user?.name || user?.email || 'CIM Document Processor User'}`);
              <div className="space-y-1">
                {extractedData.financials.revenue.map((value, index) => (
                  <div key={index} className="flex justify-between text-sm">
-                    <span className="text-gray-600">{index === 3 ? 'LTM' : `FY${3-index}`}</span>
+                    <span className="text-gray-600">{index === 3 ? 'LTM' : `FY-${3-index}`}</span>
                    <span className="font-medium">{value}</span>
                  </div>
                ))}
@@ -307,7 +307,7 @@ ${user?.name || user?.email || 'CIM Document Processor User'}`);
              <div className="space-y-1">
                {extractedData.financials.ebitda.map((value, index) => (
                  <div key={index} className="flex justify-between text-sm">
-                    <span className="text-gray-600">{index === 3 ? 'LTM' : `FY${3-index}`}</span>
+                    <span className="text-gray-600">{index === 3 ? 'LTM' : `FY-${3-index}`}</span>
                    <span className="font-medium">{value}</span>
                  </div>
                ))}
@@ -318,7 +318,7 @@ ${user?.name || user?.email || 'CIM Document Processor User'}`);
              <div className="space-y-1">
                {extractedData.financials.margins.map((value, index) => (
                  <div key={index} className="flex justify-between text-sm">
-                    <span className="text-gray-600">{index === 3 ? 'LTM' : `FY${3-index}`}</span>
+                    <span className="text-gray-600">{index === 3 ? 'LTM' : `FY-${3-index}`}</span>
                    <span className="font-medium">{value}</span>
                  </div>
                ))}