Files
cim_summary/frontend/src/components/DocumentAnalytics.tsx
Jon 57770fd99d feat: Implement hybrid LLM approach with enhanced prompts for CIM analysis
🎯 Major Features:
- Hybrid LLM configuration: Claude 3.7 Sonnet (primary) + GPT-4.5 (fallback)
- Task-specific model selection for optimal performance
- Enhanced prompts for all analysis types with proven results

🔧 Technical Improvements:
- Enhanced financial analysis with fiscal year mapping (100% success rate)
- Business model analysis with scalability assessment
- Market positioning analysis with TAM/SAM extraction
- Management team assessment with succession planning
- Creative content generation with GPT-4.5

📊 Performance & Cost Optimization:
- Claude 3.7 Sonnet: /5 per 1M tokens (82.2% MATH score)
- GPT-4.5: Premium creative content (5/50 per 1M tokens)
- ~80% cost savings using Claude for analytical tasks
- Automatic fallback system for reliability

 Proven Results:
- Successfully extracted 3-year financial data from STAX CIM
- Correctly mapped fiscal years (2023→FY-3, 2024→FY-2, 2025E→FY-1, LTM Mar-25→LTM)
- Identified revenue: 4M→1M→1M→6M (LTM)
- Identified EBITDA: 8.9M→3.9M→1M→7.2M (LTM)

🚀 Files Added/Modified:
- Enhanced LLM service with task-specific model selection
- Updated environment configuration for hybrid approach
- Enhanced prompt builders for all analysis types
- Comprehensive testing scripts and documentation
- Updated frontend components for improved UX

📚 References:
- Eden AI Model Comparison: Claude 3.7 Sonnet vs GPT-4.5
- Artificial Analysis Benchmarks for performance metrics
- Cost optimization based on model strengths and pricing
2025-07-28 16:46:06 -04:00

371 lines
15 KiB
TypeScript

import React, { useState, useEffect } from 'react';
import { documentService } from '../services/documentService';
import AgenticRAGSessionDetails from './AgenticRAGSessionDetails';
import { cn } from '../utils/cn';
interface DocumentAnalyticsProps {
documentId: string;
documentName: string;
}
interface DocumentAnalyticsData {
documentId: string;
summary: {
totalSessions: number;
successfulSessions: number;
successRate: number;
totalProcessingTime: number;
avgProcessingTime: number;
totalCost: number;
avgCost: number;
avgValidationScore: number;
};
sessions: Array<{
id: string;
strategy: string;
status: string;
totalAgents: number;
completedAgents: number;
failedAgents: number;
overallValidationScore: number;
processingTimeMs: number;
apiCallsCount: number;
totalCost: number;
createdAt: string;
completedAt?: string;
}>;
executions: Array<{
id: string;
agentName: string;
stepNumber: number;
status: string;
processingTimeMs: number;
retryCount: number;
errorMessage?: string;
createdAt: string;
updatedAt: string;
sessionId: string;
}>;
qualityMetrics: Array<{
id: string;
metricType: string;
metricValue: number;
metricDetails: any;
createdAt: string;
sessionId: string;
}>;
}
const DocumentAnalytics: React.FC<DocumentAnalyticsProps> = ({
documentId,
documentName
}) => {
const [analyticsData, setAnalyticsData] = useState<DocumentAnalyticsData | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [selectedSessionId, setSelectedSessionId] = useState<string | null>(null);
useEffect(() => {
loadDocumentAnalytics();
}, [documentId]);
const loadDocumentAnalytics = async () => {
try {
setLoading(true);
setError(null);
const data = await documentService.getDocumentAnalytics(documentId);
setAnalyticsData(data);
} catch (err) {
setError('Failed to load document analytics');
console.error('Document analytics loading error:', err);
} finally {
setLoading(false);
}
};
const formatTime = (ms: number): string => {
if (ms < 1000) return `${Math.round(ms)}ms`;
if (ms < 60000) return `${(ms / 1000).toFixed(1)}s`;
return `${(ms / 60000).toFixed(1)}m`;
};
const formatCost = (cost: number): string => {
return `$${cost.toFixed(4)}`;
};
const getStatusColor = (status: string): string => {
switch (status.toLowerCase()) {
case 'completed':
return 'text-green-600 bg-green-100';
case 'failed':
return 'text-red-600 bg-red-100';
case 'processing':
return 'text-yellow-600 bg-yellow-100';
case 'pending':
return 'text-blue-600 bg-blue-100';
default:
return 'text-gray-600 bg-gray-100';
}
};
const getStrategyDisplayName = (strategy: string): string => {
return strategy.replace(/_/g, ' ').toUpperCase();
};
if (loading) {
return (
<div className="flex items-center justify-center min-h-64">
<div className="animate-spin rounded-full h-8 w-8 border-b-2 border-blue-600"></div>
</div>
);
}
if (error) {
return (
<div className="bg-red-50 border border-red-200 rounded-lg p-4">
<div className="flex">
<div className="flex-shrink-0">
<svg className="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
<path fillRule="evenodd" d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z" clipRule="evenodd" />
</svg>
</div>
<div className="ml-3">
<h3 className="text-sm font-medium text-red-800">Error loading document analytics</h3>
<p className="mt-1 text-sm text-red-700">{error}</p>
</div>
</div>
</div>
);
}
if (!analyticsData) {
return (
<div className="bg-yellow-50 border border-yellow-200 rounded-lg p-4">
<p className="text-yellow-800">No analytics data found for this document</p>
</div>
);
}
const { summary, sessions, executions, qualityMetrics } = analyticsData;
// If a session is selected, show session details
if (selectedSessionId) {
return (
<div>
<div className="mb-4">
<button
onClick={() => setSelectedSessionId(null)}
className="text-blue-600 hover:text-blue-800 flex items-center"
>
<svg className="h-4 w-4 mr-1" fill="none" viewBox="0 0 24 24" stroke="currentColor">
<path strokeLinecap="round" strokeLinejoin="round" strokeWidth={2} d="M15 19l-7-7 7-7" />
</svg>
Back to Document Analytics
</button>
</div>
<AgenticRAGSessionDetails
sessionId={selectedSessionId}
onClose={() => setSelectedSessionId(null)}
/>
</div>
);
}
return (
<div className="space-y-6">
{/* Header */}
<div className="flex justify-between items-center">
<div>
<h1 className="text-2xl font-bold text-gray-900">Document Analytics</h1>
<p className="text-sm text-gray-600">{documentName}</p>
</div>
<button
onClick={loadDocumentAnalytics}
className="bg-blue-600 text-white px-4 py-2 rounded-md text-sm hover:bg-blue-700"
>
Refresh
</button>
</div>
{/* Summary Statistics */}
<div className="bg-white rounded-lg shadow p-6">
<h2 className="text-lg font-semibold text-gray-900 mb-4">Summary Statistics</h2>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
<div className="bg-gray-50 rounded-lg p-4">
<p className="text-sm font-medium text-gray-500">Total Sessions</p>
<p className="text-2xl font-bold text-gray-900">{summary.totalSessions}</p>
</div>
<div className="bg-gray-50 rounded-lg p-4">
<p className="text-sm font-medium text-gray-500">Success Rate</p>
<p className="text-2xl font-bold text-gray-900">{summary.successRate.toFixed(1)}%</p>
</div>
<div className="bg-gray-50 rounded-lg p-4">
<p className="text-sm font-medium text-gray-500">Avg Processing Time</p>
<p className="text-2xl font-bold text-gray-900">{formatTime(summary.avgProcessingTime)}</p>
</div>
<div className="bg-gray-50 rounded-lg p-4">
<p className="text-sm font-medium text-gray-500">Avg Cost</p>
<p className="text-2xl font-bold text-gray-900">{formatCost(summary.avgCost)}</p>
</div>
</div>
<div className="grid grid-cols-1 md:grid-cols-2 gap-4 mt-4">
<div className="bg-gray-50 rounded-lg p-4">
<p className="text-sm font-medium text-gray-500">Total Processing Time</p>
<p className="text-lg font-semibold text-gray-900">{formatTime(summary.totalProcessingTime)}</p>
</div>
<div className="bg-gray-50 rounded-lg p-4">
<p className="text-sm font-medium text-gray-500">Avg Validation Score</p>
<p className="text-lg font-semibold text-gray-900">
{summary.avgValidationScore ? `${(summary.avgValidationScore * 100).toFixed(1)}%` : 'N/A'}
</p>
</div>
</div>
</div>
{/* Processing Sessions */}
<div className="bg-white rounded-lg shadow p-6">
<h2 className="text-lg font-semibold text-gray-900 mb-4">Processing Sessions</h2>
{sessions.length === 0 ? (
<p className="text-gray-500 text-center py-8">No processing sessions found</p>
) : (
<div className="overflow-x-auto">
<table className="min-w-full divide-y divide-gray-200">
<thead className="bg-gray-50">
<tr>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Strategy</th>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Status</th>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Agents</th>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Time</th>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Cost</th>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Score</th>
<th className="px-6 py-3 text-left text-xs font-medium text-gray-500 uppercase tracking-wider">Actions</th>
</tr>
</thead>
<tbody className="bg-white divide-y divide-gray-200">
{sessions.map((session) => (
<tr key={session.id} className="hover:bg-gray-50">
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{getStrategyDisplayName(session.strategy)}
</td>
<td className="px-6 py-4 whitespace-nowrap">
<span className={cn(
"inline-flex px-2 py-1 text-xs font-semibold rounded-full",
getStatusColor(session.status)
)}>
{session.status}
</span>
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{session.completedAgents}/{session.totalAgents}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatTime(session.processingTimeMs)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{formatCost(session.totalCost)}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
{session.overallValidationScore ? `${(session.overallValidationScore * 100).toFixed(1)}%` : 'N/A'}
</td>
<td className="px-6 py-4 whitespace-nowrap text-sm text-gray-900">
<button
onClick={() => setSelectedSessionId(session.id)}
className="text-blue-600 hover:text-blue-800 text-sm"
>
View Details
</button>
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
</div>
{/* Agent Executions Summary */}
{executions.length > 0 && (
<div className="bg-white rounded-lg shadow p-6">
<h2 className="text-lg font-semibold text-gray-900 mb-4">Agent Executions Summary</h2>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
{Array.from(new Set(executions.map(e => e.agentName))).map(agentName => {
const agentExecutions = executions.filter(e => e.agentName === agentName);
const successfulExecutions = agentExecutions.filter(e => e.status === 'completed');
const avgProcessingTime = agentExecutions.reduce((sum, e) => sum + e.processingTimeMs, 0) / agentExecutions.length;
const avgRetries = agentExecutions.reduce((sum, e) => sum + e.retryCount, 0) / agentExecutions.length;
return (
<div key={agentName} className="bg-gray-50 rounded-lg p-4">
<h3 className="text-sm font-medium text-gray-900 mb-2">
{agentName.replace(/_/g, ' ').replace(/\b\w/g, l => l.toUpperCase())}
</h3>
<div className="space-y-1">
<div className="flex justify-between text-xs">
<span className="text-gray-600">Total Executions:</span>
<span className="font-medium">{agentExecutions.length}</span>
</div>
<div className="flex justify-between text-xs">
<span className="text-gray-600">Success Rate:</span>
<span className="font-medium text-green-600">
{agentExecutions.length > 0
? ((successfulExecutions.length / agentExecutions.length) * 100).toFixed(1)
: 0}%
</span>
</div>
<div className="flex justify-between text-xs">
<span className="text-gray-600">Avg Time:</span>
<span className="font-medium">{formatTime(avgProcessingTime)}</span>
</div>
<div className="flex justify-between text-xs">
<span className="text-gray-600">Avg Retries:</span>
<span className="font-medium">{avgRetries.toFixed(1)}</span>
</div>
</div>
</div>
);
})}
</div>
</div>
)}
{/* Quality Metrics Summary */}
{qualityMetrics.length > 0 && (
<div className="bg-white rounded-lg shadow p-6">
<h2 className="text-lg font-semibold text-gray-900 mb-4">Quality Metrics Summary</h2>
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-4">
{Array.from(new Set(qualityMetrics.map(m => m.metricType))).map(metricType => {
const typeMetrics = qualityMetrics.filter(m => m.metricType === metricType);
const avgValue = typeMetrics.reduce((sum, m) => sum + m.metricValue, 0) / typeMetrics.length;
const minValue = Math.min(...typeMetrics.map(m => m.metricValue));
const maxValue = Math.max(...typeMetrics.map(m => m.metricValue));
return (
<div key={metricType} className="bg-gray-50 rounded-lg p-4">
<h3 className="text-sm font-medium text-gray-900 mb-2">
{metricType.charAt(0).toUpperCase() + metricType.slice(1)}
</h3>
<div className="space-y-1">
<div className="flex justify-between text-xs">
<span className="text-gray-600">Average:</span>
<span className="font-medium">{(avgValue * 100).toFixed(1)}%</span>
</div>
<div className="flex justify-between text-xs">
<span className="text-gray-600">Min:</span>
<span className="font-medium">{(minValue * 100).toFixed(1)}%</span>
</div>
<div className="flex justify-between text-xs">
<span className="text-gray-600">Max:</span>
<span className="font-medium">{(maxValue * 100).toFixed(1)}%</span>
</div>
</div>
</div>
);
})}
</div>
</div>
)}
</div>
);
};
export default DocumentAnalytics;