Major release with significant performance improvements and new processing strategy. ## Core Changes - Implemented simple_full_document processing strategy (default) - Full document → LLM approach: 1-2 passes, ~5-6 minutes processing time - Achieved 100% completeness with 2 API calls (down from 5+) - Removed redundant Document AI passes for faster processing ## Financial Data Extraction - Enhanced deterministic financial table parser - Improved FY3/FY2/FY1/LTM identification from varying CIM formats - Automatic merging of parser results with LLM extraction ## Code Quality & Infrastructure - Cleaned up debug logging (removed emoji markers from production code) - Fixed Firebase Secrets configuration (using modern defineSecret approach) - Updated OpenAI API key - Resolved deployment conflicts (secrets vs environment variables) - Added .env files to Firebase ignore list ## Deployment - Firebase Functions v2 deployment successful - All 7 required secrets verified and configured - Function URL: https://api-y56ccs6wva-uc.a.run.app ## Performance Improvements - Processing time: ~5-6 minutes (down from 23+ minutes) - API calls: 1-2 (down from 5+) - Completeness: 100% achievable - LLM Model: claude-3-7-sonnet-latest ## Breaking Changes - Default processing strategy changed to 'simple_full_document' - RAG processor available as alternative strategy 'document_ai_agentic_rag' ## Files Changed - 36 files changed, 5642 insertions(+), 4451 deletions(-) - Removed deprecated documentation files - Cleaned up unused services and models This release represents a major refactoring focused on speed, accuracy, and maintainability.
97 lines
4.2 KiB
PL/PgSQL
97 lines
4.2 KiB
PL/PgSQL
-- Complete Database Setup for CIM Summarizer
|
|
-- Run this in Supabase SQL Editor to create all necessary tables
|
|
|
|
-- 1. Create users table
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
firebase_uid VARCHAR(255) UNIQUE NOT NULL,
|
|
email VARCHAR(255) UNIQUE NOT NULL,
|
|
display_name VARCHAR(255),
|
|
photo_url VARCHAR(1000),
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
|
last_login_at TIMESTAMP WITH TIME ZONE
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_users_firebase_uid ON users(firebase_uid);
|
|
CREATE INDEX IF NOT EXISTS idx_users_email ON users(email);
|
|
|
|
-- 2. Create update_updated_at_column function (needed for triggers)
|
|
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
|
RETURNS TRIGGER AS $$
|
|
BEGIN
|
|
NEW.updated_at = CURRENT_TIMESTAMP;
|
|
RETURN NEW;
|
|
END;
|
|
$$ language 'plpgsql';
|
|
|
|
-- 3. Create documents table
|
|
CREATE TABLE IF NOT EXISTS documents (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
user_id VARCHAR(255) NOT NULL, -- Changed from UUID to VARCHAR to match Firebase UID
|
|
original_file_name VARCHAR(500) NOT NULL,
|
|
file_path VARCHAR(1000) NOT NULL,
|
|
file_size BIGINT NOT NULL CHECK (file_size > 0),
|
|
uploaded_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
|
status VARCHAR(50) NOT NULL DEFAULT 'uploaded' CHECK (status IN ('uploading', 'uploaded', 'extracting_text', 'processing_llm', 'generating_pdf', 'completed', 'failed')),
|
|
extracted_text TEXT,
|
|
generated_summary TEXT,
|
|
summary_markdown_path VARCHAR(1000),
|
|
summary_pdf_path VARCHAR(1000),
|
|
processing_started_at TIMESTAMP WITH TIME ZONE,
|
|
processing_completed_at TIMESTAMP WITH TIME ZONE,
|
|
error_message TEXT,
|
|
analysis_data JSONB, -- Added for storing analysis results
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_documents_user_id ON documents(user_id);
|
|
CREATE INDEX IF NOT EXISTS idx_documents_status ON documents(status);
|
|
CREATE INDEX IF NOT EXISTS idx_documents_uploaded_at ON documents(uploaded_at);
|
|
CREATE INDEX IF NOT EXISTS idx_documents_processing_completed_at ON documents(processing_completed_at);
|
|
CREATE INDEX IF NOT EXISTS idx_documents_user_status ON documents(user_id, status);
|
|
|
|
CREATE TRIGGER update_documents_updated_at
|
|
BEFORE UPDATE ON documents
|
|
FOR EACH ROW
|
|
EXECUTE FUNCTION update_updated_at_column();
|
|
|
|
-- 4. Create processing_jobs table
|
|
CREATE TABLE IF NOT EXISTS processing_jobs (
|
|
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
|
|
user_id VARCHAR(255) NOT NULL,
|
|
status VARCHAR(50) NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'processing', 'completed', 'failed', 'retrying')),
|
|
attempts INTEGER NOT NULL DEFAULT 0,
|
|
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
options JSONB,
|
|
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
|
started_at TIMESTAMP WITH TIME ZONE,
|
|
completed_at TIMESTAMP WITH TIME ZONE,
|
|
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
|
error TEXT,
|
|
last_error_at TIMESTAMP WITH TIME ZONE,
|
|
result JSONB
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_status ON processing_jobs(status);
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_created_at ON processing_jobs(created_at);
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_document_id ON processing_jobs(document_id);
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_user_id ON processing_jobs(user_id);
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_pending ON processing_jobs(status, created_at) WHERE status = 'pending';
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_last_error_at ON processing_jobs(last_error_at) WHERE status = 'retrying';
|
|
CREATE INDEX IF NOT EXISTS idx_processing_jobs_attempts ON processing_jobs(attempts);
|
|
|
|
CREATE TRIGGER update_processing_jobs_updated_at
|
|
BEFORE UPDATE ON processing_jobs
|
|
FOR EACH ROW
|
|
EXECUTE FUNCTION update_updated_at_column();
|
|
|
|
-- Verify all tables were created
|
|
SELECT table_name
|
|
FROM information_schema.tables
|
|
WHERE table_schema = 'public'
|
|
AND table_name IN ('users', 'documents', 'processing_jobs')
|
|
ORDER BY table_name;
|