Initial commit: Virtual Board Member AI System foundation

app/core/celery.py

@@ -0,0 +1,140 @@
+"""
+Celery configuration for background task processing.
+"""
+
+from celery import Celery
+from celery.schedules import crontab
+import structlog
+
+from app.core.config import settings
+
+logger = structlog.get_logger()
+
+# Create Celery app
+celery_app = Celery(
+    "virtual_board_member",
+    broker=settings.CELERY_BROKER_URL,
+    backend=settings.CELERY_RESULT_BACKEND,
+    include=[
+        "app.tasks.document_processing",
+        "app.tasks.commitment_extraction",
+        "app.tasks.notifications",
+        "app.tasks.analytics",
+    ]
+)
+
+# Configure Celery
+celery_app.conf.update(
+    task_serializer=settings.CELERY_TASK_SERIALIZER,
+    result_serializer=settings.CELERY_RESULT_SERIALIZER,
+    accept_content=settings.CELERY_ACCEPT_CONTENT,
+    timezone=settings.CELERY_TIMEZONE,
+    enable_utc=settings.CELERY_ENABLE_UTC,
+    task_track_started=True,
+    task_time_limit=30 * 60,  # 30 minutes
+    task_soft_time_limit=25 * 60,  # 25 minutes
+    worker_prefetch_multiplier=1,
+    worker_max_tasks_per_child=1000,
+    result_expires=3600,  # 1 hour
+    task_always_eager=settings.TESTING,  # Run tasks synchronously in tests
+)
+
+# Configure periodic tasks
+celery_app.conf.beat_schedule = {
+    # Daily commitment reminders
+    "daily-commitment-reminders": {
+        "task": "app.tasks.notifications.send_commitment_reminders",
+        "schedule": crontab(hour=9, minute=0),  # 9 AM daily
+    },
+    
+    # Weekly analytics report
+    "weekly-analytics-report": {
+        "task": "app.tasks.analytics.generate_weekly_report",
+        "schedule": crontab(day_of_week=1, hour=8, minute=0),  # Monday 8 AM
+    },
+    
+    # Daily document processing cleanup
+    "daily-document-cleanup": {
+        "task": "app.tasks.document_processing.cleanup_old_documents",
+        "schedule": crontab(hour=2, minute=0),  # 2 AM daily
+    },
+    
+    # Hourly health check
+    "hourly-health-check": {
+        "task": "app.tasks.system.health_check",
+        "schedule": crontab(minute=0),  # Every hour
+    },
+    
+    # Daily audit log cleanup
+    "daily-audit-cleanup": {
+        "task": "app.tasks.system.cleanup_audit_logs",
+        "schedule": crontab(hour=3, minute=0),  # 3 AM daily
+    },
+}
+
+# Task routing
+celery_app.conf.task_routes = {
+    "app.tasks.document_processing.*": {"queue": "document_processing"},
+    "app.tasks.commitment_extraction.*": {"queue": "commitment_extraction"},
+    "app.tasks.notifications.*": {"queue": "notifications"},
+    "app.tasks.analytics.*": {"queue": "analytics"},
+    "app.tasks.system.*": {"queue": "system"},
+}
+
+# Task annotations for specific configurations
+celery_app.conf.task_annotations = {
+    "app.tasks.document_processing.process_large_document": {
+        "rate_limit": "10/m",  # 10 per minute
+        "time_limit": 1800,  # 30 minutes
+    },
+    "app.tasks.commitment_extraction.extract_commitments": {
+        "rate_limit": "50/m",  # 50 per minute
+        "time_limit": 300,  # 5 minutes
+    },
+    "app.tasks.analytics.generate_weekly_report": {
+        "rate_limit": "1/h",  # 1 per hour
+        "time_limit": 600,  # 10 minutes
+    },
+}
+
+# Error handling
+@celery_app.task(bind=True)
+def debug_task(self):
+    """Debug task for testing."""
+    logger.info(f"Request: {self.request!r}")
+
+
+# Task failure handling
+@celery_app.task(bind=True, autoretry_for=(Exception,), retry_kwargs={'max_retries': 3})
+def retry_task(self, *args, **kwargs):
+    """Base task with retry logic."""
+    try:
+        # Task logic here
+        pass
+    except Exception as exc:
+        logger.error(
+            "Task failed",
+            task_name=self.name,
+            task_id=self.request.id,
+            error=str(exc),
+            retry_count=self.request.retries
+        )
+        raise self.retry(exc=exc, countdown=60 * (2 ** self.request.retries))
+
+
+# Health check task
+@celery_app.task
+def health_check():
+    """Health check task for monitoring."""
+    logger.info("Celery health check completed")
+    return {"status": "healthy", "timestamp": "2025-01-01T00:00:00Z"}
+
+
+# Cleanup task
+@celery_app.task
+def cleanup_old_data():
+    """Cleanup old data and temporary files."""
+    logger.info("Starting data cleanup")
+    # TODO: Implement cleanup logic
+    logger.info("Data cleanup completed")
+    return {"status": "completed", "cleaned_items": 0}