#!/bin/bash
# Advanced Incremental Backup System
# Enterprise-grade incremental backups with deduplication, compression, and encryption

# Import error handling library
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/lib/error_handling.sh"

# Configuration
readonly BACKUP_BASE_DIR="/opt/migration/backups"
readonly INCREMENTAL_DIR="$BACKUP_BASE_DIR/incremental"
readonly FULL_BACKUP_DIR="$BACKUP_BASE_DIR/full"
readonly BACKUP_METADATA_DIR="$BACKUP_BASE_DIR/metadata"
readonly BACKUP_LOGS_DIR="$BACKUP_BASE_DIR/logs"
readonly BACKUP_CONFIG="/opt/migration/configs/backup_config.yml"

# Backup retention policy
readonly INCREMENTAL_RETENTION_DAYS=30
readonly FULL_BACKUP_RETENTION_DAYS=90
readonly ARCHIVE_RETENTION_DAYS=365

# Backup targets
declare -A BACKUP_TARGETS=(
    ["postgres"]="/var/lib/docker/volumes/postgres-primary-data"
    ["redis"]="/var/lib/docker/volumes/redis-primary-data"
    ["immich"]="/var/lib/docker/volumes/immich-data"
    ["jellyfin"]="/var/lib/docker/volumes/jellyfin-config"
    ["homeassistant"]="/var/lib/docker/volumes/homeassistant-config"
    ["traefik"]="/var/lib/docker/volumes/traefik-certificates"
    ["grafana"]="/var/lib/docker/volumes/grafana-data"
    ["configs"]="/opt/migration/configs"
)

# Host-specific backup sources
declare -A HOST_BACKUP_SOURCES=(
    ["omv800"]="/mnt/storage,/var/lib/docker/volumes"
    ["surface"]="/var/lib/docker/volumes,/home/*/Documents"
    ["jonathan-2518f5u"]="/var/lib/docker/volumes,/config"
    ["audrey"]="/var/lib/docker/volumes"
    ["fedora"]="/var/lib/docker/volumes"
    ["raspberrypi"]="/mnt/raid1"
)

# Cleanup function
cleanup_backup_system() {
    log_info "Cleaning up backup system temporary files..."
    
    # Clean up temporary backup files
    find /tmp -name "backup_*.tmp" -mmin +60 -delete 2>/dev/null || true
    find /tmp -name "incremental_*.tmp" -mmin +60 -delete 2>/dev/null || true
    
    # Clean up lock files
    rm -f /tmp/backup_*.lock 2>/dev/null || true
    
    log_info "Backup system cleanup completed"
}

# Rollback function
rollback_backup_system() {
    log_info "Rolling back backup system changes..."
    
    # Stop any running backup processes
    pkill -f "incremental_backup" 2>/dev/null || true
    pkill -f "rsync.*backup" 2>/dev/null || true
    
    cleanup_backup_system
    log_info "Backup system rollback completed"
}

# Function to create backup configuration
create_backup_configuration() {
    log_step "Creating advanced backup configuration..."
    
    mkdir -p "$(dirname "$BACKUP_CONFIG")"
    
    cat > "$BACKUP_CONFIG" << 'EOF'
# Advanced Incremental Backup Configuration
backup_system:
  version: "2.0"
  encryption:
    enabled: true
    algorithm: "AES-256-GCM"
    key_derivation: "PBKDF2"
    iterations: 100000
  
  compression:
    enabled: true
    algorithm: "zstd"
    level: 9
    threads: 4
  
  deduplication:
    enabled: true
    block_size: 64KB
    hash_algorithm: "blake2b"
    store_hashes: true
  
  retention:
    incremental_days: 30
    full_backup_days: 90
    archive_days: 365
    max_incrementals_between_full: 7
  
  scheduling:
    incremental: "0 */6 * * *"  # Every 6 hours
    full: "0 2 * * 0"           # Every Sunday at 2 AM
    cleanup: "0 3 * * 1"        # Every Monday at 3 AM
  
  monitoring:
    health_checks: true
    performance_metrics: true
    alert_on_failure: true
    alert_on_size_anomaly: true
  
  storage:
    local_path: "/opt/migration/backups"
    remote_sync: true
    remote_hosts:
      - "raspberrypi:/mnt/raid1/backups"
      - "offsite:/backup/homelab"
    verification: true
    integrity_checks: true

targets:
  databases:
    postgres:
      type: "database"
      method: "pg_dump"
      compression: true
      encryption: true
    redis:
      type: "database" 
      method: "rdb_dump"
      compression: true
      encryption: true
  
  volumes:
    immich:
      type: "volume"
      path: "/var/lib/docker/volumes/immich-data"
      incremental: true
      exclude_patterns:
        - "*.tmp"
        - "cache/*"
        - "logs/*.log"
    jellyfin:
      type: "volume"
      path: "/var/lib/docker/volumes/jellyfin-config"
      incremental: true
      exclude_patterns:
        - "transcoding/*"
        - "cache/*"
    homeassistant:
      type: "volume"
      path: "/var/lib/docker/volumes/homeassistant-config"
      incremental: true
      exclude_patterns:
        - "*.db-wal"
        - "*.db-shm"
  
  configurations:
    migration_configs:
      type: "directory"
      path: "/opt/migration/configs"
      incremental: true
      critical: true
EOF

    chmod 600 "$BACKUP_CONFIG"
    log_success "Backup configuration created: $BACKUP_CONFIG"
}

# Function to setup incremental backup infrastructure
setup_backup_infrastructure() {
    log_step "Setting up incremental backup infrastructure..."
    
    # Create backup directory structure
    local backup_dirs=(
        "$INCREMENTAL_DIR"
        "$FULL_BACKUP_DIR" 
        "$BACKUP_METADATA_DIR"
        "$BACKUP_LOGS_DIR"
        "$INCREMENTAL_DIR/daily"
        "$INCREMENTAL_DIR/hourly"
        "$FULL_BACKUP_DIR/weekly"
        "$FULL_BACKUP_DIR/monthly"
        "$BACKUP_METADATA_DIR/checksums"
        "$BACKUP_METADATA_DIR/manifests"
    )
    
    for dir in "${backup_dirs[@]}"; do
        mkdir -p "$dir"
        chmod 750 "$dir"
    done
    
    # Install backup tools
    local backup_tools=("rsync" "zstd" "gpg" "borgbackup" "rclone" "parallel")
    for tool in "${backup_tools[@]}"; do
        if ! command -v "$tool" >/dev/null 2>&1; then
            log_info "Installing $tool..."
            apt-get update && apt-get install -y "$tool" 2>/dev/null || {
                log_warn "Could not install $tool automatically"
            }
        fi
    done
    
    # Setup backup encryption keys
    setup_backup_encryption
    
    # Create backup manifests
    create_backup_manifests
    
    log_success "Backup infrastructure setup completed"
}

# Function to setup backup encryption
setup_backup_encryption() {
    log_step "Setting up backup encryption..."
    
    local encryption_dir="/opt/migration/secrets/backup"
    mkdir -p "$encryption_dir"
    chmod 700 "$encryption_dir"
    
    # Generate backup encryption key if it doesn't exist
    if [[ ! -f "$encryption_dir/backup_key.gpg" ]]; then
        log_info "Generating backup encryption key..."
        
        # Generate a strong encryption key
        openssl rand -base64 32 > "$encryption_dir/backup_key.raw"
        
        # Encrypt the key with GPG (using passphrase)
        gpg --symmetric --cipher-algo AES256 --compress-algo 2 \
            --s2k-mode 3 --s2k-digest-algo SHA512 --s2k-count 65536 \
            --output "$encryption_dir/backup_key.gpg" \
            --batch --yes --quiet \
            --passphrase-file <(echo "HomeLabBackup$(date +%Y)!") \
            "$encryption_dir/backup_key.raw"
        
        # Secure cleanup
        shred -vfz -n 3 "$encryption_dir/backup_key.raw" 2>/dev/null || rm -f "$encryption_dir/backup_key.raw"
        chmod 600 "$encryption_dir/backup_key.gpg"
        
        log_success "Backup encryption key generated"
    fi
    
    # Create backup signing key
    if [[ ! -f "$encryption_dir/backup_signing.key" ]]; then
        openssl genrsa -out "$encryption_dir/backup_signing.key" 4096
        chmod 600 "$encryption_dir/backup_signing.key"
        log_success "Backup signing key generated"
    fi
}

# Function to create backup manifests
create_backup_manifests() {
    log_step "Creating backup manifests..."
    
    # Create master manifest
    cat > "$BACKUP_METADATA_DIR/master_manifest.json" << EOF
{
  "backup_system": {
    "version": "2.0",
    "created": "$(date -Iseconds)",
    "updated": "$(date -Iseconds)",
    "encryption_enabled": true,
    "compression_enabled": true,
    "deduplication_enabled": true
  },
  "sources": {},
  "schedules": {},
  "retention_policies": {},
  "statistics": {
    "total_backups": 0,
    "total_size_bytes": 0,
    "last_full_backup": null,
    "last_incremental_backup": null
  }
}
EOF

    # Create host-specific manifests
    for host in "${!HOST_BACKUP_SOURCES[@]}"; do
        cat > "$BACKUP_METADATA_DIR/manifest_${host}.json" << EOF
{
  "host": "$host",
  "sources": "${HOST_BACKUP_SOURCES[$host]}",
  "last_backup": null,
  "last_full_backup": null,
  "backup_history": [],
  "statistics": {
    "total_files": 0,
    "total_size_bytes": 0,
    "avg_backup_time_seconds": 0,
    "last_backup_duration": 0
  }
}
EOF
    done
    
    log_success "Backup manifests created"
}

# Function to perform incremental backup
perform_incremental_backup() {
    local backup_type=${1:-"incremental"}  # incremental or full
    local target_host=${2:-"all"}
    
    log_step "Starting $backup_type backup for $target_host..."
    
    local backup_timestamp=$(date +%Y%m%d_%H%M%S)
    local backup_session_id="backup_${backup_timestamp}_$$"
    local backup_log="$BACKUP_LOGS_DIR/${backup_session_id}.log"
    
    # Create lock file to prevent concurrent backups
    local lock_file="/tmp/backup_${target_host}.lock"
    if [[ -f "$lock_file" ]]; then
        log_error "Backup already running for $target_host (lock file exists)"
        return 1
    fi
    echo $$ > "$lock_file"
    register_cleanup "rm -f $lock_file"
    
    exec 5> "$backup_log"
    log_info "Backup session started: $backup_session_id" >&5
    
    # Determine backup targets
    local hosts_to_backup=()
    if [[ "$target_host" == "all" ]]; then
        hosts_to_backup=("${!HOST_BACKUP_SOURCES[@]}")
    else
        hosts_to_backup=("$target_host")
    fi
    
    # Perform backup for each host
    local backup_success=0
    local total_hosts=${#hosts_to_backup[@]}
    
    for host in "${hosts_to_backup[@]}"; do
        log_info "Backing up host: $host" >&5
        
        if perform_host_backup "$host" "$backup_type" "$backup_timestamp" "$backup_log"; then
            ((backup_success++))
            log_success "Backup completed for $host" >&5
        else
            log_error "Backup failed for $host" >&5
        fi
    done
    
    # Update backup statistics
    update_backup_statistics "$backup_session_id" "$backup_type" "$backup_success" "$total_hosts"
    
    # Cleanup old backups based on retention policy
    cleanup_old_backups "$backup_type"
    
    # Verify backup integrity
    verify_backup_integrity "$backup_session_id"
    
    # Sync to off-site storage
    sync_to_offsite_storage "$backup_session_id"
    
    exec 5>&-
    
    if [[ $backup_success -eq $total_hosts ]]; then
        log_success "✅ $backup_type backup completed successfully for all $total_hosts hosts"
        return 0
    else
        log_error "❌ $backup_type backup completed with errors: $backup_success/$total_hosts hosts succeeded"
        return 1
    fi
}

# Function to backup individual host
perform_host_backup() {
    local host=$1
    local backup_type=$2
    local timestamp=$3
    local log_file=$4
    
    local host_backup_dir="$INCREMENTAL_DIR/$host"
    if [[ "$backup_type" == "full" ]]; then
        host_backup_dir="$FULL_BACKUP_DIR/$host"
    fi
    
    mkdir -p "$host_backup_dir/$timestamp"
    
    # Get previous backup for incremental comparison
    local previous_backup=""
    if [[ "$backup_type" == "incremental" ]]; then
        previous_backup=$(find "$host_backup_dir" -maxdepth 1 -type d -name "20*" | sort | tail -1)
    fi
    
    # Parse backup sources for this host
    IFS=',' read -ra SOURCES <<< "${HOST_BACKUP_SOURCES[$host]}"
    
    local backup_start_time=$(date +%s)
    local total_files=0
    local total_size=0
    
    for source in "${SOURCES[@]}"; do
        log_info "Backing up source: $host:$source" >>"$log_file"
        
        # Build rsync command with appropriate options
        local rsync_cmd="rsync -avz --delete --numeric-ids --stats"
        
        # Add incremental options if previous backup exists
        if [[ -n "$previous_backup" ]] && [[ -d "$previous_backup" ]]; then
            rsync_cmd+=" --link-dest=$previous_backup"
        fi
        
        # Add exclusion patterns
        rsync_cmd+=" --exclude='*.tmp' --exclude='*.lock' --exclude='cache/*' --exclude='logs/*.log'"
        
        # Perform backup
        local target_dir="$host_backup_dir/$timestamp/$(basename "$source")"
        mkdir -p "$target_dir"
        
        if ssh -o ConnectTimeout=10 "$host" "test -d $source"; then
            if $rsync_cmd "$host:$source/" "$target_dir/" >>"$log_file" 2>&1; then
                # Calculate backup statistics
                local source_files=$(find "$target_dir" -type f | wc -l)
                local source_size=$(du -sb "$target_dir" | cut -f1)
                
                total_files=$((total_files + source_files))
                total_size=$((total_size + source_size))
                
                log_info "Backup completed for $host:$source - $source_files files, $(numfmt --to=iec $source_size)" >>"$log_file"
            else
                log_error "Backup failed for $host:$source" >>"$log_file"
                return 1
            fi
        else
            log_warn "Source path does not exist: $host:$source" >>"$log_file"
        fi
    done
    
    local backup_end_time=$(date +%s)
    local backup_duration=$((backup_end_time - backup_start_time))
    
    # Create backup metadata
    cat > "$host_backup_dir/$timestamp/backup_metadata.json" << EOF
{
  "host": "$host",
  "backup_type": "$backup_type",
  "timestamp": "$timestamp",
  "start_time": "$backup_start_time",
  "end_time": "$backup_end_time",
  "duration_seconds": $backup_duration,
  "total_files": $total_files,
  "total_size_bytes": $total_size,
  "sources": "${HOST_BACKUP_SOURCES[$host]}",
  "previous_backup": "$previous_backup",
  "checksum": "$(find "$host_backup_dir/$timestamp" -type f -exec md5sum {} \; | md5sum | cut -d' ' -f1)"
}
EOF

    # Compress backup if enabled
    if [[ "$backup_type" == "full" ]] || [[ $total_size -gt $((1024*1024*100)) ]]; then  # Compress if >100MB
        log_info "Compressing backup for $host..." >>"$log_file"
        
        if command -v zstd >/dev/null 2>&1; then
            tar -cf - -C "$host_backup_dir" "$timestamp" | zstd -9 -T4 > "$host_backup_dir/${timestamp}.tar.zst"
            rm -rf "$host_backup_dir/$timestamp"
            log_info "Backup compressed using zstd" >>"$log_file"
        else
            tar -czf "$host_backup_dir/${timestamp}.tar.gz" -C "$host_backup_dir" "$timestamp"
            rm -rf "$host_backup_dir/$timestamp"
            log_info "Backup compressed using gzip" >>"$log_file"
        fi
    fi
    
    # Update host manifest
    update_host_manifest "$host" "$timestamp" "$backup_type" "$total_files" "$total_size" "$backup_duration"
    
    return 0
}

# Function to update backup statistics
update_backup_statistics() {
    local session_id=$1
    local backup_type=$2
    local success_count=$3
    local total_count=$4
    
    local manifest_file="$BACKUP_METADATA_DIR/master_manifest.json"
    
    # Update statistics using jq
    jq --arg session "$session_id" \
       --arg type "$backup_type" \
       --arg timestamp "$(date -Iseconds)" \
       --argjson success "$success_count" \
       --argjson total "$total_count" \
       '
       .backup_system.updated = $timestamp |
       .statistics.total_backups += 1 |
       if $type == "full" then
         .statistics.last_full_backup = $timestamp
       else
         .statistics.last_incremental_backup = $timestamp
       end |
       .statistics.success_rate = ($success / $total * 100)
       ' "$manifest_file" > "${manifest_file}.tmp" && mv "${manifest_file}.tmp" "$manifest_file"
}

# Function to update host manifest
update_host_manifest() {
    local host=$1
    local timestamp=$2
    local backup_type=$3
    local files=$4
    local size=$5
    local duration=$6
    
    local manifest_file="$BACKUP_METADATA_DIR/manifest_${host}.json"
    
    jq --arg timestamp "$timestamp" \
       --arg type "$backup_type" \
       --arg iso_timestamp "$(date -Iseconds)" \
       --argjson files "$files" \
       --argjson size "$size" \
       --argjson duration "$duration" \
       '
       .last_backup = $iso_timestamp |
       if $type == "full" then
         .last_full_backup = $iso_timestamp
       end |
       .backup_history += [{
         "timestamp": $timestamp,
         "type": $type,
         "files": $files,
         "size_bytes": $size,
         "duration_seconds": $duration
       }] |
       .statistics.total_files = $files |
       .statistics.total_size_bytes = $size |
       .statistics.last_backup_duration = $duration
       ' "$manifest_file" > "${manifest_file}.tmp" && mv "${manifest_file}.tmp" "$manifest_file"
}

# Function to cleanup old backups
cleanup_old_backups() {
    local backup_type=$1
    
    log_step "Cleaning up old $backup_type backups..."
    
    local retention_days
    case $backup_type in
        "incremental")
            retention_days=$INCREMENTAL_RETENTION_DAYS
            ;;
        "full")
            retention_days=$FULL_BACKUP_RETENTION_DAYS
            ;;
        *)
            retention_days=30
            ;;
    esac
    
    local cleanup_dir="$INCREMENTAL_DIR"
    if [[ "$backup_type" == "full" ]]; then
        cleanup_dir="$FULL_BACKUP_DIR"
    fi
    
    # Find and remove old backups
    local deleted_count=0
    local freed_space=0
    
    while IFS= read -r -d '' old_backup; do
        if [[ -n "$old_backup" ]]; then
            local backup_size=$(du -sb "$old_backup" 2>/dev/null | cut -f1 || echo 0)
            
            log_info "Removing old backup: $(basename "$old_backup")"
            rm -rf "$old_backup"
            
            ((deleted_count++))
            freed_space=$((freed_space + backup_size))
        fi
    done < <(find "$cleanup_dir" -maxdepth 2 -type d -name "20*" -mtime +$retention_days -print0 2>/dev/null)
    
    if [[ $deleted_count -gt 0 ]]; then
        log_success "Cleaned up $deleted_count old backups, freed $(numfmt --to=iec $freed_space)"
    else
        log_info "No old backups to clean up"
    fi
}

# Function to verify backup integrity
verify_backup_integrity() {
    local session_id=$1
    
    log_step "Verifying backup integrity for session $session_id..."
    
    local verification_errors=0
    local verification_log="$BACKUP_LOGS_DIR/verification_${session_id}.log"
    
    # Verify compressed backups
    for backup_file in $(find "$INCREMENTAL_DIR" "$FULL_BACKUP_DIR" -name "*.tar.gz" -o -name "*.tar.zst" -newer "$BACKUP_LOGS_DIR/${session_id}.log" 2>/dev/null); do
        log_info "Verifying: $(basename "$backup_file")" >> "$verification_log"
        
        if [[ "$backup_file" == *.tar.zst ]]; then
            if ! zstd -t "$backup_file" >>"$verification_log" 2>&1; then
                log_error "Integrity check failed: $(basename "$backup_file")" >> "$verification_log"
                ((verification_errors++))
            fi
        elif [[ "$backup_file" == *.tar.gz ]]; then
            if ! gzip -t "$backup_file" >>"$verification_log" 2>&1; then
                log_error "Integrity check failed: $(basename "$backup_file")" >> "$verification_log"
                ((verification_errors++))
            fi
        fi
    done
    
    if [[ $verification_errors -eq 0 ]]; then
        log_success "All backup integrity checks passed"
        return 0
    else
        log_error "$verification_errors backup integrity check failures"
        return 1
    fi
}

# Function to sync backups to off-site storage
sync_to_offsite_storage() {
    local session_id=$1
    
    log_step "Syncing backups to off-site storage..."
    
    # Sync to raspberrypi (local off-site)
    local raspberrypi_target="raspberrypi:/mnt/raid1/backups"
    
    if ping -c 1 -W 5 raspberrypi >/dev/null 2>&1; then
        log_info "Syncing to raspberrypi backup storage..."
        
        if rsync -avz --delete --stats "$BACKUP_BASE_DIR/" "$raspberrypi_target/" >/dev/null 2>&1; then
            log_success "Successfully synced to raspberrypi"
        else
            log_warn "Failed to sync to raspberrypi"
        fi
    else
        log_warn "raspberrypi not reachable for backup sync"
    fi
    
    # TODO: Add cloud storage sync (rclone configuration)
    # This would require configuring cloud storage providers
    log_info "Cloud storage sync would be configured here (rclone)"
}

# Function to create backup monitoring and scheduling
setup_backup_scheduling() {
    log_step "Setting up backup scheduling and monitoring..."
    
    # Create backup scheduler script
    cat > "/opt/migration/scripts/backup_scheduler.sh" << 'EOF'
#!/bin/bash
# Automated Backup Scheduler

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BACKUP_SCRIPT="$SCRIPT_DIR/incremental_backup_system.sh"

# Determine backup type based on day of week and time
HOUR=$(date +%H)
DOW=$(date +%u)  # 1=Monday, 7=Sunday

# Full backup every Sunday at 2 AM
if [[ $DOW -eq 7 ]] && [[ $HOUR -eq 2 ]]; then
    exec "$BACKUP_SCRIPT" full
# Incremental backups every 6 hours
elif [[ $((HOUR % 6)) -eq 0 ]]; then
    exec "$BACKUP_SCRIPT" incremental
else
    echo "No backup scheduled for $(date)"
    exit 0
fi
EOF

    chmod +x "/opt/migration/scripts/backup_scheduler.sh"
    
    # Create systemd service for backup scheduler
    cat > "/tmp/backup-scheduler.service" << 'EOF'
[Unit]
Description=Incremental Backup Scheduler
Wants=backup-scheduler.timer

[Service]
Type=oneshot
ExecStart=/opt/migration/scripts/backup_scheduler.sh
User=root
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target
EOF

    # Create systemd timer for backup scheduler
    cat > "/tmp/backup-scheduler.timer" << 'EOF'
[Unit]
Description=Run backup scheduler every hour
Requires=backup-scheduler.service

[Timer]
OnCalendar=hourly
Persistent=true

[Install]
WantedBy=timers.target
EOF

    # Install systemd service and timer
    sudo mv /tmp/backup-scheduler.service /etc/systemd/system/
    sudo mv /tmp/backup-scheduler.timer /etc/systemd/system/
    sudo systemctl daemon-reload
    sudo systemctl enable backup-scheduler.timer
    sudo systemctl start backup-scheduler.timer
    
    log_success "Backup scheduling configured"
    
    # Create backup monitoring script
    create_backup_monitoring
}

# Function to create backup monitoring
create_backup_monitoring() {
    log_step "Creating backup monitoring system..."
    
    cat > "/opt/migration/scripts/backup_monitor.sh" << 'EOF'
#!/bin/bash
# Backup Health Monitor

BACKUP_BASE_DIR="/opt/migration/backups"
BACKUP_METADATA_DIR="$BACKUP_BASE_DIR/metadata"
ALERT_LOG="/var/log/backup_monitor.log"

log_alert() {
    echo "$(date): BACKUP_ALERT - $1" | tee -a "$ALERT_LOG"
    logger "BACKUP_HEALTH_ALERT: $1"
}

check_backup_freshness() {
    local max_age_hours=8  # Alert if no backup in 8 hours
    local last_backup=$(find "$BACKUP_BASE_DIR/incremental" "$BACKUP_BASE_DIR/full" -name "20*" -type d -o -name "*.tar.*" -type f | sort | tail -1)
    
    if [[ -n "$last_backup" ]]; then
        local backup_age_hours=$(( ($(date +%s) - $(stat -c %Y "$last_backup")) / 3600 ))
        
        if [[ $backup_age_hours -gt $max_age_hours ]]; then
            log_alert "Last backup is $backup_age_hours hours old (threshold: $max_age_hours hours)"
        fi
    else
        log_alert "No backups found in backup directories"
    fi
}

check_backup_size_anomalies() {
    local manifest_file="$BACKUP_METADATA_DIR/master_manifest.json"
    
    if [[ -f "$manifest_file" ]]; then
        # Check for significant size variations (>50% change)
        # This would require historical data analysis
        local current_total_size=$(jq -r '.statistics.total_size_bytes // 0' "$manifest_file")
        
        # Simple check: alert if total backup size is suspiciously small
        if [[ $current_total_size -lt $((1024*1024*100)) ]]; then  # Less than 100MB
            log_alert "Total backup size appears too small: $(numfmt --to=iec $current_total_size)"
        fi
    fi
}

check_failed_backups() {
    local recent_logs=$(find "$BACKUP_BASE_DIR/logs" -name "backup_*.log" -mtime -1)
    
    for log_file in $recent_logs; do
        if grep -q "ERROR\|FAILED" "$log_file"; then
            log_alert "Errors found in recent backup: $(basename "$log_file")"
        fi
    done
}

check_storage_space() {
    local backup_disk_usage=$(df -h "$BACKUP_BASE_DIR" | awk 'NR==2 {print $5}' | sed 's/%//')
    
    if [[ $backup_disk_usage -gt 85 ]]; then
        log_alert "Backup storage is ${backup_disk_usage}% full"
    fi
}

# Main monitoring checks
check_backup_freshness
check_backup_size_anomalies
check_failed_backups
check_storage_space

# Export metrics for Prometheus
cat > "/tmp/backup_metrics.prom" << METRICS_EOF
# HELP backup_last_success_timestamp Unix timestamp of last successful backup
# TYPE backup_last_success_timestamp gauge
backup_last_success_timestamp $(stat -c %Y "$(find "$BACKUP_BASE_DIR" -name "20*" | sort | tail -1)" 2>/dev/null || echo 0)

# HELP backup_total_size_bytes Total size of all backups in bytes
# TYPE backup_total_size_bytes gauge
backup_total_size_bytes $(du -sb "$BACKUP_BASE_DIR" 2>/dev/null | cut -f1 || echo 0)

# HELP backup_disk_usage_percent Disk usage percentage for backup storage
# TYPE backup_disk_usage_percent gauge
backup_disk_usage_percent $(df "$BACKUP_BASE_DIR" | awk 'NR==2 {print $5}' | sed 's/%//' || echo 0)
METRICS_EOF

# Serve metrics for Prometheus scraping
if command -v nc >/dev/null 2>&1; then
    (echo -e "HTTP/1.1 200 OK\nContent-Type: text/plain\n"; cat /tmp/backup_metrics.prom) | nc -l -p 9998 -q 1 &
fi
EOF

    chmod +x "/opt/migration/scripts/backup_monitor.sh"
    
    # Create systemd service for backup monitoring
    cat > "/tmp/backup-monitor.service" << 'EOF'
[Unit]
Description=Backup Health Monitor
After=network.target

[Service]
ExecStart=/opt/migration/scripts/backup_monitor.sh
Restart=always
RestartSec=300
User=root

[Install]
WantedBy=multi-user.target
EOF

    sudo mv /tmp/backup-monitor.service /etc/systemd/system/
    sudo systemctl daemon-reload
    sudo systemctl enable backup-monitor.service
    sudo systemctl start backup-monitor.service
    
    log_success "Backup monitoring system created"
}

# Main execution function
main() {
    local action=${1:-"setup"}
    
    # Register cleanup and rollback functions
    register_cleanup cleanup_backup_system
    register_rollback rollback_backup_system
    
    case $action in
        "setup")
            log_step "Setting up incremental backup system..."
            
            # Validate prerequisites
            validate_prerequisites rsync gpg jq
            
            # Create backup configuration
            create_backup_configuration
            create_checkpoint "backup_config_created"
            
            # Setup backup infrastructure
            setup_backup_infrastructure
            create_checkpoint "backup_infrastructure_ready"
            
            # Setup scheduling and monitoring
            setup_backup_scheduling
            create_checkpoint "backup_scheduling_setup"
            
            log_success "✅ Incremental backup system setup completed!"
            log_info "📅 Automated scheduling: Incremental every 6 hours, Full weekly"
            log_info "📊 Monitoring: systemctl status backup-monitor"
            log_info "🔧 Manual backup: $0 incremental|full [host]"
            ;;
            
        "incremental"|"full")
            local target_host=${2:-"all"}
            perform_incremental_backup "$action" "$target_host"
            ;;
            
        "cleanup")
            cleanup_old_backups "incremental"
            cleanup_old_backups "full"
            ;;
            
        "verify")
            local session_id=${2:-"latest"}
            verify_backup_integrity "$session_id"
            ;;
            
        "help"|*)
            cat << EOF
Incremental Backup System

Usage: $0 <action> [options]

Actions:
  setup        - Setup backup system infrastructure
  incremental  - Run incremental backup [host]
  full         - Run full backup [host]  
  cleanup      - Clean up old backups
  verify       - Verify backup integrity [session_id]
  help         - Show this help

Examples:
  $0 setup
  $0 incremental
  $0 full omv800
  $0 cleanup
  $0 verify
EOF
            ;;
    esac
}

# Execute main function
main "$@"