HomeAudit/scripts/comprehensive_pre_migration_backup_automated.sh

#!/bin/bash

# Comprehensive Pre-Migration Backup Script (Automated with Resume Support)
# Uses discovery results and password file to backup 100% of infrastructure
# Based on comprehensive discovery of all services, databases, and data
# Supports resuming from where it left off and shows progress

set -uo pipefail

# Configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
BACKUP_TIMESTAMP=$(date +%Y%m%d_%H%M%S)
BACKUP_DIR="/export/omv800_backup/pre_migration_${BACKUP_TIMESTAMP}"
LOG_FILE="$PROJECT_ROOT/logs/comprehensive_backup_${BACKUP_TIMESTAMP}.log"
PASSWORD_FILE="$PROJECT_ROOT/secrets/ssh_passwords.env"
DISCOVERY_DIR="$PROJECT_ROOT/comprehensive_discovery_results"
PROGRESS_FILE="$PROJECT_ROOT/logs/backup_progress_${BACKUP_TIMESTAMP}.json"

# Resume support - check if we're resuming an existing backup
RESUME_BACKUP_DIR=""
if [[ -n "${1:-}" ]]; then
    # Check if the directory exists on raspberrypi
    if ssh jon@raspberrypi "test -d '$1'" 2>/dev/null; then
        RESUME_BACKUP_DIR="$1"
        BACKUP_DIR="$RESUME_BACKUP_DIR"
        BACKUP_TIMESTAMP=$(basename "$BACKUP_DIR" | sed 's/pre_migration_//')
        LOG_FILE="$PROJECT_ROOT/logs/comprehensive_backup_${BACKUP_TIMESTAMP}.log"
        PROGRESS_FILE="$PROJECT_ROOT/logs/backup_progress_${BACKUP_TIMESTAMP}.json"
        echo "🔄 RESUMING backup from: $BACKUP_DIR"
    fi
fi

# Create directories
mkdir -p "$(dirname "$LOG_FILE")"
mkdir -p "$(dirname "$PROGRESS_FILE")"

# Load passwords
if [[ -f "$PASSWORD_FILE" ]]; then
    source "$PASSWORD_FILE"
else
    echo "Error: Password file not found: $PASSWORD_FILE"
    exit 1
fi

# Progress tracking
declare -A COMPLETED_TASKS
declare -A TOTAL_TASKS

# Initialize progress tracking
init_progress() {
    TOTAL_TASKS["databases"]=0
    TOTAL_TASKS["volumes"]=0
    TOTAL_TASKS["configs"]=0
    TOTAL_TASKS["secrets"]=0
    TOTAL_TASKS["user_data"]=0
    TOTAL_TASKS["system_configs"]=0
    
    COMPLETED_TASKS["databases"]=0
    COMPLETED_TASKS["volumes"]=0
    COMPLETED_TASKS["configs"]=0
    COMPLETED_TASKS["secrets"]=0
    COMPLETED_TASKS["user_data"]=0
    COMPLETED_TASKS["system_configs"]=0
    
    # Count total tasks from discovery results
    if [[ -d "$DISCOVERY_DIR" ]]; then
        # Count databases
        if [[ -f "$DISCOVERY_DIR/databases_fedora.txt" ]]; then
            TOTAL_TASKS["databases"]=$(wc -l < "$DISCOVERY_DIR/databases_fedora.txt")
        fi
        
        # Count volumes (skip header line)
        if [[ -f "$DISCOVERY_DIR/volumes_fedora.txt" ]]; then
            TOTAL_TASKS["volumes"]=$(($(wc -l < "$DISCOVERY_DIR/volumes_fedora.txt") - 1))
        fi
        
        # Estimate other tasks
        TOTAL_TASKS["configs"]=5  # Local + fedora + other hosts
        TOTAL_TASKS["secrets"]=5
        TOTAL_TASKS["user_data"]=5
        TOTAL_TASKS["system_configs"]=5
    fi
    
    save_progress
}

# Save progress to file
save_progress() {
    cat > "$PROGRESS_FILE" << EOF
{
    "timestamp": "$(date -Iseconds)",
    "backup_dir": "$BACKUP_DIR",
    "completed_tasks": $(printf '%s\n' "${COMPLETED_TASKS[@]}" | jq -R . | jq -s .),
    "total_tasks": $(printf '%s\n' "${TOTAL_TASKS[@]}" | jq -R . | jq -s .),
    "task_names": ["databases", "volumes", "configs", "secrets", "user_data", "system_configs"]
}
EOF
}

# Load existing progress if resuming
load_progress() {
    if [[ -n "$RESUME_BACKUP_DIR" ]]; then
        echo "📋 Loading existing progress from backup directory: $BACKUP_DIR"
        # Initialize arrays if not already done
        COMPLETED_TASKS["databases"]="${COMPLETED_TASKS["databases"]:-0}"
        COMPLETED_TASKS["volumes"]="${COMPLETED_TASKS["volumes"]:-0}"
        COMPLETED_TASKS["configs"]="${COMPLETED_TASKS["configs"]:-0}"
        COMPLETED_TASKS["secrets"]="${COMPLETED_TASKS["secrets"]:-0}"
        COMPLETED_TASKS["user_data"]="${COMPLETED_TASKS["user_data"]:-0}"
        COMPLETED_TASKS["system_configs"]="${COMPLETED_TASKS["system_configs"]:-0}"
        
        # Parse completed tasks from existing backup files on raspberrypi
        if ssh jon@raspberrypi "test -d '$BACKUP_DIR/database_dumps'" 2>/dev/null; then
            COMPLETED_TASKS["databases"]=$(ssh jon@raspberrypi "find '$BACKUP_DIR/database_dumps' -name '*.sql' | wc -l" 2>/dev/null || echo "0")
        fi
        if ssh jon@raspberrypi "test -d '$BACKUP_DIR/docker_volumes'" 2>/dev/null; then
            COMPLETED_TASKS["volumes"]=$(ssh jon@raspberrypi "find '$BACKUP_DIR/docker_volumes' -name '*.tar.gz' | wc -l" 2>/dev/null || echo "0")
        fi
        if ssh jon@raspberrypi "test -d '$BACKUP_DIR/configurations'" 2>/dev/null; then
            COMPLETED_TASKS["configs"]=$(ssh jon@raspberrypi "find '$BACKUP_DIR/configurations' -name '*.tar.gz' | wc -l" 2>/dev/null || echo "0")
        fi
        if ssh jon@raspberrypi "test -d '$BACKUP_DIR/secrets'" 2>/dev/null; then
            COMPLETED_TASKS["secrets"]=$(ssh jon@raspberrypi "find '$BACKUP_DIR/secrets' -name '*.tar.gz' | wc -l" 2>/dev/null || echo "0")
        fi
        if ssh jon@raspberrypi "test -d '$BACKUP_DIR/user_data'" 2>/dev/null; then
            COMPLETED_TASKS["user_data"]=$(ssh jon@raspberrypi "find '$BACKUP_DIR/user_data' -name '*.tar.gz' | wc -l" 2>/dev/null || echo "0")
        fi
        if ssh jon@raspberrypi "test -d '$BACKUP_DIR/system_configs'" 2>/dev/null; then
            COMPLETED_TASKS["system_configs"]=$(ssh jon@raspberrypi "find '$BACKUP_DIR/system_configs' -name '*.tar.gz' | wc -l" 2>/dev/null || echo "0")
        fi
        
        echo "📊 Loaded progress:"
        echo "   Databases: ${COMPLETED_TASKS["databases"]}"
        echo "   Volumes: ${COMPLETED_TASKS["volumes"]}"
        echo "   Configs: ${COMPLETED_TASKS["configs"]}"
        echo "   Secrets: ${COMPLETED_TASKS["secrets"]}"
        echo "   User Data: ${COMPLETED_TASKS["user_data"]}"
        echo "   System Configs: ${COMPLETED_TASKS["system_configs"]}"
    fi
}

# Show progress
show_progress() {
    local task_type="$1"
    local current="${COMPLETED_TASKS[$task_type]:-0}"
    local total="${TOTAL_TASKS[$task_type]:-1}"
    local percentage=0
    
    if [[ $total -gt 0 ]]; then
        percentage=$((current * 100 / total))
    fi
    
    echo "📊 Progress [$task_type]: $current/$total ($percentage%)"
    save_progress
}

# Logging function with progress
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

# Error handling
cleanup() {
    log "Cleaning up temporary files..."
    rm -f /tmp/backup_*.sql /tmp/docker_*.txt /tmp/network_*.txt 2>/dev/null || true
}

trap cleanup EXIT

# SSH function with password support
ssh_with_password() {
    local host="$1"
    local user="$2"
    local command="$3"
    local password=""

    # Get password for specific host
    case "$host" in
        "fedora")
            password="$FEDORA_PASSWORD"
            ;;
        "lenovo")
            password="$LENOVO_PASSWORD"
            ;;
        "lenovo420")
            password="$LENOVO420_PASSWORD"
            ;;
        "omv800")
            password="$OMV800_PASSWORD"
            ;;
        "surface")
            password="$SURFACE_PASSWORD"
            ;;
        "audrey")
            password="$AUDREY_PASSWORD"
            ;;
        "raspberrypi")
            password="$RASPBERRYPI_PASSWORD"
            ;;
        *)
            password=""
            ;;
    esac

    if [[ -n "$password" ]]; then
        # Use sshpass for password authentication
        if command -v sshpass >/dev/null 2>&1; then
            sshpass -p "$password" ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$user@$host" "$command"
        else
            log "Warning: sshpass not available, trying SSH key authentication for $host"
            ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$user@$host" "$command"
        fi
    else
        # Try SSH key authentication
        ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$user@$host" "$command"
    fi
}

# Check if backup already exists (for resume)
backup_exists() {
    local backup_path="$1"
    ssh jon@raspberrypi "test -f '$backup_path'" 2>/dev/null
}

# Backup databases with progress
backup_all_databases() {
    log "=== BACKING UP ALL DATABASES ==="
    
    if [[ -f "$DISCOVERY_DIR/databases_fedora.txt" ]]; then
        log "Backing up databases on fedora (user: jonathan)..."
        
        while IFS= read -r line; do
            if [[ -z "$line" || "$line" == *"CONTAINER"* ]]; then
                continue
            fi
            
            local container_name=$(echo "$line" | awk '{print $1}')
            local db_type=$(echo "$line" | awk '{print $2}')
            local backup_name="fedora_${container_name}_${db_type}_${BACKUP_TIMESTAMP}.sql"
            local backup_path="$BACKUP_DIR/database_dumps/$backup_name"
            
            # Check if already backed up
            if backup_exists "$backup_path"; then
                log "⏭️  Skipping existing database backup: $backup_name"
                ((COMPLETED_TASKS["databases"]++))
                show_progress "databases"
                continue
            fi
            
            log "🔄 Backing up database: $container_name ($db_type)"
            
            case "$db_type" in
                "postgres")
                    ssh_with_password "fedora" "jonathan" "docker exec $container_name pg_dumpall -U postgres" > "/tmp/${backup_name}" 2>/dev/null || true
                    ;;
                "mysql"|"mariadb")
                    ssh_with_password "fedora" "jonathan" "docker exec $container_name mysqldump -u root -p --all-databases" > "/tmp/${backup_name}" 2>/dev/null || true
                    ;;
                "redis")
                    ssh_with_password "fedora" "jonathan" "docker exec $container_name redis-cli BGSAVE" > "/tmp/${backup_name}" 2>/dev/null || true
                    ;;
                "mongodb")
                    ssh_with_password "fedora" "jonathan" "docker exec $container_name mongodump --out /tmp/mongo_dump" > "/tmp/${backup_name}" 2>/dev/null || true
                    ;;
            esac
            
            if [[ -s "/tmp/${backup_name}" ]]; then
                rsync -avz "/tmp/${backup_name}" "jon@raspberrypi:$backup_path"
                log "✅ Database backup created: $backup_path ($(stat -c%s "/tmp/${backup_name}") bytes)"
                ((COMPLETED_TASKS["databases"]++))
                show_progress "databases"
            else
                log "⚠️  Empty database backup for: $container_name"
            fi
        done < "$DISCOVERY_DIR/databases_fedora.txt"
    fi
}

# Backup Docker volumes with progress
backup_all_docker_volumes() {
    log "=== BACKING UP ALL DOCKER VOLUMES ==="
    
    if [[ -f "$DISCOVERY_DIR/volumes_fedora.txt" ]]; then
        log "Backing up Docker volumes on fedora (user: jonathan)..."
        
        # Skip header line and process volumes
        local volume_count=0
        while IFS= read -r line; do
            if [[ -z "$line" ]]; then
                continue
            fi
            
            local volume_name=$(echo "$line" | awk '{print $2}')
            local backup_name="fedora_${volume_name}_${BACKUP_TIMESTAMP}.tar.gz"
            local backup_path="$BACKUP_DIR/docker_volumes/$backup_name"
            
            # Check if already backed up
            if backup_exists "$backup_path"; then
                log "⏭️  Skipping existing volume backup: $backup_name"
                ((volume_count++))
                ((COMPLETED_TASKS["volumes"]++))
                show_progress "volumes"
                continue
            fi
            
            log "🔄 Backing up Docker volume: $volume_name on fedora"
            
            # Create volume backup
            ssh_with_password "fedora" "jonathan" "docker run --rm -v $volume_name:/data -v /tmp:/backup alpine tar czf /backup/volume_backup.tar.gz -C /data ." 2>/dev/null || true
            
            if ssh_with_password "fedora" "jonathan" "test -f /tmp/volume_backup.tar.gz" 2>/dev/null; then
                # Copy from fedora to raspberrypi via local machine
                scp "jonathan@fedora:/tmp/volume_backup.tar.gz" "/tmp/volume_backup_temp.tar.gz" 2>/dev/null || true
                if [[ -f "/tmp/volume_backup_temp.tar.gz" ]]; then
                    rsync -avz "/tmp/volume_backup_temp.tar.gz" "jon@raspberrypi:$backup_path"
                    local size=$(stat -c%s "/tmp/volume_backup_temp.tar.gz" 2>/dev/null || echo "0")
                    log "✅ Volume backup created: $backup_path ($size bytes)"
                    rm -f "/tmp/volume_backup_temp.tar.gz" 2>/dev/null || true
                    ssh_with_password "fedora" "jonathan" "rm -f /tmp/volume_backup.tar.gz" 2>/dev/null || true
                    ((COMPLETED_TASKS["volumes"]++))
                    show_progress "volumes"
                else
                    log "⚠️  Failed to copy volume backup from fedora"
                fi
            else
                log "⚠️  Failed to create volume backup for: $volume_name"
            fi
        done < <(tail -n +2 "$DISCOVERY_DIR/volumes_fedora.txt")
    fi
}

# Backup configurations with progress
backup_all_configurations() {
    log "=== BACKING UP ALL CONFIGURATIONS ==="
    
    # Create configurations directory
    ssh jon@raspberrypi "mkdir -p $BACKUP_DIR/configurations"
    
    # Local configs
    if ! backup_exists "$BACKUP_DIR/configurations/local_configs_${BACKUP_TIMESTAMP}.tar.gz"; then
        log "Backing up local configurations..."
        tar czf "/tmp/local_configs_${BACKUP_TIMESTAMP}.tar.gz" -C "$PROJECT_ROOT" . 2>/dev/null || true
        if [[ -f "/tmp/local_configs_${BACKUP_TIMESTAMP}.tar.gz" ]]; then
            rsync -avz "/tmp/local_configs_${BACKUP_TIMESTAMP}.tar.gz" "jon@raspberrypi:$BACKUP_DIR/configurations/"
            local size=$(stat -c%s "/tmp/local_configs_${BACKUP_TIMESTAMP}.tar.gz")
            log "✅ Local config backup created: $BACKUP_DIR/configurations/local_configs_${BACKUP_TIMESTAMP}.tar.gz ($size bytes)"
            ((COMPLETED_TASKS["configs"]++))
            show_progress "configs"
        fi
    else
        log "⏭️  Skipping existing local config backup"
        ((COMPLETED_TASKS["configs"]++))
        show_progress "configs"
    fi
    
    # Remote configs
    if [[ -f "$DISCOVERY_DIR/all_hosts.txt" ]]; then
        while IFS=: read -r host user; do
            if [[ -z "$host" || "$host" == "localhost" ]]; then
                continue
            fi
            
            local backup_name="${host}_configs_${BACKUP_TIMESTAMP}.tar.gz"
            local backup_path="$BACKUP_DIR/configurations/$backup_name"
            
            # Check if already backed up
            if backup_exists "$backup_path"; then
                log "⏭️  Skipping existing config backup for $host"
                ((COMPLETED_TASKS["configs"]++))
                show_progress "configs"
                continue
            fi
            
            log "Backing up configurations on $host (user: $user)..."
            log "Backing up configurations for $host"
            
            # Create config backup
            if ssh_with_password "$host" "$user" "tar czf /tmp/config_backup.tar.gz -C /etc . -C /home . 2>/dev/null || true" 2>/dev/null; then
                if ssh_with_password "$host" "$user" "test -f /tmp/config_backup.tar.gz" 2>/dev/null; then
                    # Copy from host to raspberrypi via local machine
                    scp "$user@$host:/tmp/config_backup.tar.gz" "/tmp/config_backup_temp.tar.gz" 2>/dev/null || true
                    if [[ -f "/tmp/config_backup_temp.tar.gz" ]]; then
                        rsync -avz "/tmp/config_backup_temp.tar.gz" "jon@raspberrypi:$backup_path"
                        local size=$(stat -c%s "/tmp/config_backup_temp.tar.gz" 2>/dev/null || echo "0")
                        log "✅ Config backup created: $backup_path ($size bytes)"
                        rm -f "/tmp/config_backup_temp.tar.gz" 2>/dev/null || true
                        ssh_with_password "$host" "$user" "rm -f /tmp/config_backup.tar.gz" 2>/dev/null || true
                        ((COMPLETED_TASKS["configs"]++))
                        show_progress "configs"
                    else
                        log "⚠️  Failed to copy config backup from $host"
                    fi
                else
                    log "⚠️  Failed to create config backup for $host"
                fi
            else
                log "⚠️  Failed to connect to $host for config backup"
            fi
        done < "$DISCOVERY_DIR/all_hosts.txt"
    fi
}

# Backup secrets with progress
backup_all_secrets() {
    log "=== BACKING UP ALL SECRETS AND SSL CERTIFICATES ==="
    
    # Create secrets directory
    ssh jon@raspberrypi "mkdir -p $BACKUP_DIR/secrets"
    
    # Local secrets
    if ! backup_exists "$BACKUP_DIR/secrets/local_secrets_${BACKUP_TIMESTAMP}.tar.gz"; then
        log "Backing up local secrets..."
        tar czf "/tmp/local_secrets_${BACKUP_TIMESTAMP}.tar.gz" -C "$PROJECT_ROOT/secrets" . 2>/dev/null || true
        if [[ -f "/tmp/local_secrets_${BACKUP_TIMESTAMP}.tar.gz" ]]; then
            rsync -avz "/tmp/local_secrets_${BACKUP_TIMESTAMP}.tar.gz" "jon@raspberrypi:$BACKUP_DIR/secrets/"
            local size=$(stat -c%s "/tmp/local_secrets_${BACKUP_TIMESTAMP}.tar.gz")
            log "✅ Local secrets backup created: $BACKUP_DIR/secrets/local_secrets_${BACKUP_TIMESTAMP}.tar.gz ($size bytes)"
            ((COMPLETED_TASKS["secrets"]++))
            show_progress "secrets"
        fi
    else
        log "⏭️  Skipping existing local secrets backup"
        ((COMPLETED_TASKS["secrets"]++))
        show_progress "secrets"
    fi
    
    # Remote secrets
    if [[ -f "$DISCOVERY_DIR/all_hosts.txt" ]]; then
        while IFS=: read -r host user; do
            if [[ -z "$host" || "$host" == "localhost" ]]; then
                continue
            fi
            
            local backup_name="${host}_secrets_${BACKUP_TIMESTAMP}.tar.gz"
            local backup_path="$BACKUP_DIR/secrets/$backup_name"
            
            # Check if already backed up
            if backup_exists "$backup_path"; then
                log "⏭️  Skipping existing secrets backup for $host"
                ((COMPLETED_TASKS["secrets"]++))
                show_progress "secrets"
                continue
            fi
            
            log "Backing up SSL certificates on $host (user: $user)..."
            log "Backing up secrets for $host"
            
            # Create secrets backup
            if ssh_with_password "$host" "$user" "tar czf /tmp/secrets_backup.tar.gz -C /etc/ssl . -C /etc/letsencrypt . 2>/dev/null || true" 2>/dev/null; then
                if ssh_with_password "$host" "$user" "test -f /tmp/secrets_backup.tar.gz" 2>/dev/null; then
                    # Copy from host to raspberrypi via local machine
                    scp "$user@$host:/tmp/secrets_backup.tar.gz" "/tmp/secrets_backup_temp.tar.gz" 2>/dev/null || true
                    if [[ -f "/tmp/secrets_backup_temp.tar.gz" ]]; then
                        rsync -avz "/tmp/secrets_backup_temp.tar.gz" "jon@raspberrypi:$backup_path"
                        local size=$(stat -c%s "/tmp/secrets_backup_temp.tar.gz" 2>/dev/null || echo "0")
                        log "✅ Secrets backup created: $backup_path ($size bytes)"
                        rm -f "/tmp/secrets_backup_temp.tar.gz" 2>/dev/null || true
                        ssh_with_password "$host" "$user" "rm -f /tmp/secrets_backup.tar.gz" 2>/dev/null || true
                        ((COMPLETED_TASKS["secrets"]++))
                        show_progress "secrets"
                    else
                        log "⚠️  Failed to copy secrets backup from $host"
                    fi
                else
                    log "⚠️  Failed to create secrets backup for $host"
                fi
            else
                log "⚠️  Failed to connect to $host for secrets backup"
            fi
        done < "$DISCOVERY_DIR/all_hosts.txt"
    fi
}

# Backup user data with progress
backup_all_user_data() {
    log "=== BACKING UP ALL USER DATA AND APPLICATIONS ==="
    
    # Create user_data directory
    ssh jon@raspberrypi "mkdir -p $BACKUP_DIR/user_data"
    
    if [[ -f "$DISCOVERY_DIR/all_hosts.txt" ]]; then
        while IFS=: read -r host user; do
            if [[ -z "$host" || "$host" == "localhost" ]]; then
                continue
            fi
            
            local backup_name="${host}_user_data_${BACKUP_TIMESTAMP}.tar.gz"
            local backup_path="$BACKUP_DIR/user_data/$backup_name"
            
            # Check if already backed up
            if backup_exists "$backup_path"; then
                log "⏭️  Skipping existing user data backup for $host"
                ((COMPLETED_TASKS["user_data"]++))
                show_progress "user_data"
                continue
            fi
            
            log "Backing up user data on $host (user: $user)..."
            log "Backing up user data for $host"
            
            # Create user data backup (excluding large directories that might cause issues)
            if ssh_with_password "$host" "$user" "tar czf /tmp/user_data_backup.tar.gz --exclude='*/node_modules' --exclude='*/.git' --exclude='*/Downloads' --exclude='*/Videos' --exclude='*/Music' -C /home . -C /srv . 2>/dev/null || true" 2>/dev/null; then
                if ssh_with_password "$host" "$user" "test -f /tmp/user_data_backup.tar.gz" 2>/dev/null; then
                    # Copy from host to raspberrypi via local machine
                    scp "$user@$host:/tmp/user_data_backup.tar.gz" "/tmp/user_data_backup_temp.tar.gz" 2>/dev/null || true
                    if [[ -f "/tmp/user_data_backup_temp.tar.gz" ]]; then
                        rsync -avz "/tmp/user_data_backup_temp.tar.gz" "jon@raspberrypi:$backup_path"
                        local size=$(stat -c%s "/tmp/user_data_backup_temp.tar.gz" 2>/dev/null || echo "0")
                        log "✅ User data backup created: $backup_path ($size bytes)"
                        rm -f "/tmp/user_data_backup_temp.tar.gz" 2>/dev/null || true
                        ssh_with_password "$host" "$user" "rm -f /tmp/user_data_backup.tar.gz" 2>/dev/null || true
                        ((COMPLETED_TASKS["user_data"]++))
                        show_progress "user_data"
                    else
                        log "⚠️  Failed to copy user data backup from $host"
                    fi
                else
                    log "⚠️  Failed to create user data backup for $host"
                fi
            else
                log "⚠️  Failed to connect to $host for user data backup"
            fi
        done < "$DISCOVERY_DIR/all_hosts.txt"
    fi
}

# Backup system configurations with progress
backup_all_system_configs() {
    log "=== BACKING UP ALL SYSTEM CONFIGURATIONS ==="
    
    # Create system_configs directory
    ssh jon@raspberrypi "mkdir -p $BACKUP_DIR/system_configs"
    
    if [[ -f "$DISCOVERY_DIR/all_hosts.txt" ]]; then
        while IFS=: read -r host user; do
            if [[ -z "$host" || "$host" == "localhost" ]]; then
                continue
            fi
            
            local backup_name="${host}_system_configs_${BACKUP_TIMESTAMP}.tar.gz"
            local backup_path="$BACKUP_DIR/system_configs/$backup_name"
            
            # Check if already backed up
            if backup_exists "$backup_path"; then
                log "⏭️  Skipping existing system config backup for $host"
                ((COMPLETED_TASKS["system_configs"]++))
                show_progress "system_configs"
                continue
            fi
            
            log "Backing up system configurations on $host (user: $user)..."
            
            # Create system config backup
            if ssh_with_password "$host" "$user" "tar czf /tmp/system_configs_backup.tar.gz -C /etc/systemd . -C /etc/network . -C /etc/docker . 2>/dev/null || true" 2>/dev/null; then
                if ssh_with_password "$host" "$user" "test -f /tmp/system_configs_backup.tar.gz" 2>/dev/null; then
                    # Copy from host to raspberrypi via local machine
                    scp "$user@$host:/tmp/system_configs_backup.tar.gz" "/tmp/system_configs_backup_temp.tar.gz" 2>/dev/null || true
                    if [[ -f "/tmp/system_configs_backup_temp.tar.gz" ]]; then
                        rsync -avz "/tmp/system_configs_backup_temp.tar.gz" "jon@raspberrypi:$backup_path"
                        local size=$(stat -c%s "/tmp/system_configs_backup_temp.tar.gz" 2>/dev/null || echo "0")
                        log "✅ System config backup created: $backup_path ($size bytes)"
                        rm -f "/tmp/system_configs_backup_temp.tar.gz" 2>/dev/null || true
                        ssh_with_password "$host" "$user" "rm -f /tmp/system_configs_backup.tar.gz" 2>/dev/null || true
                        ((COMPLETED_TASKS["system_configs"]++))
                        show_progress "system_configs"
                    else
                        log "⚠️  Failed to copy system config backup from $host"
                    fi
                else
                    log "⚠️  Failed to create system config backup for $host"
                fi
            else
                log "⚠️  Failed to connect to $host for system config backup"
            fi
        done < "$DISCOVERY_DIR/all_hosts.txt"
    fi
}

# Create backup manifest
create_backup_manifest() {
    log "=== CREATING BACKUP MANIFEST ==="
    
    local manifest_file="$BACKUP_DIR/backup_manifest_${BACKUP_TIMESTAMP}.json"
    
    cat > "/tmp/manifest.json" << EOF
{
    "backup_timestamp": "$BACKUP_TIMESTAMP",
    "backup_directory": "$BACKUP_DIR",
    "total_size_bytes": $(ssh jon@raspberrypi "du -sb $BACKUP_DIR" | awk '{print $1}'),
    "files": [
EOF
    
    # Add all backup files to manifest
    ssh jon@raspberrypi "find $BACKUP_DIR -name '*.tar.gz' -o -name '*.sql' | sort" | while read -r file; do
        local filename=$(basename "$file")
        local size=$(ssh jon@raspberrypi "stat -c%s '$file'" 2>/dev/null || echo "0")
        local checksum=$(ssh jon@raspberrypi "sha256sum '$file'" 2>/dev/null | awk '{print $1}' || echo "")
        
        cat >> "/tmp/manifest.json" << EOF
        {
            "filename": "$filename",
            "path": "$file",
            "size_bytes": $size,
            "checksum": "$checksum"
        },
EOF
    done
    
    # Remove trailing comma and close JSON
    sed -i '$ s/,$//' "/tmp/manifest.json"
    
    cat >> "/tmp/manifest.json" << EOF
    ],
    "completion_time": "$(date -Iseconds)",
    "total_tasks_completed": $((COMPLETED_TASKS["databases"] + COMPLETED_TASKS["volumes"] + COMPLETED_TASKS["configs"] + COMPLETED_TASKS["secrets"] + COMPLETED_TASKS["user_data"] + COMPLETED_TASKS["system_configs"]))
}
EOF
    
    rsync -avz "/tmp/manifest.json" "jon@raspberrypi:$manifest_file"
    log "✅ Backup manifest created: $manifest_file"
}

# Verify backup completeness
verify_backup_completeness() {
    log "=== VERIFYING BACKUP COMPLETENESS ==="
    
    local total_files=$(ssh jon@raspberrypi "find $BACKUP_DIR -name '*.tar.gz' -o -name '*.sql' | wc -l")
    local total_size=$(ssh jon@raspberrypi "du -sh $BACKUP_DIR" | awk '{print $1}')
    
    log "📊 Backup Summary:"
    log "   Total files: $total_files"
    log "   Total size: $total_size"
    log "   Backup location: $BACKUP_DIR"
    
    # Check for critical components
    local critical_missing=0
    
    if ! ssh jon@raspberrypi "test -d '$BACKUP_DIR/configurations'" 2>/dev/null; then
        log "❌ Missing: configurations"
        ((critical_missing++))
    fi
    
    if ! ssh jon@raspberrypi "test -d '$BACKUP_DIR/secrets'" 2>/dev/null; then
        log "❌ Missing: secrets"
        ((critical_missing++))
    fi
    
    if ! ssh jon@raspberrypi "test -d '$BACKUP_DIR/user_data'" 2>/dev/null; then
        log "❌ Missing: user_data"
        ((critical_missing++))
    fi
    
    if [[ $critical_missing -eq 0 ]]; then
        log "✅ Backup verification passed - all critical components present"
    else
        log "⚠️  Backup verification warning - $critical_missing critical components missing"
    fi
}

# Main backup function
main() {
    if [[ -n "$RESUME_BACKUP_DIR" ]]; then
        log "🔄 RESUMING COMPREHENSIVE PRE-MIGRATION BACKUP"
        log "Resume directory: $RESUME_BACKUP_DIR"
    else
        log "=== COMPREHENSIVE PRE-MIGRATION BACKUP STARTED ==="
        log "Timestamp: $BACKUP_TIMESTAMP"
        log "Backup directory: $BACKUP_DIR"
    fi
    
    log "Password file: $PASSWORD_FILE"
    log "Discovery directory: $DISCOVERY_DIR"
    log "Progress file: $PROGRESS_FILE"

    # Verify discovery results exist
    if [[ ! -d "$DISCOVERY_DIR" ]]; then
        log "ERROR: Discovery results not found. Run discovery script first."
        exit 1
    fi

    # Initialize or load progress
    if [[ -n "$RESUME_BACKUP_DIR" ]]; then
        load_progress
    else
        # Create backup directory on raspberrypi
        log "Creating backup directory on raspberrypi..."
        ssh jon@raspberrypi "mkdir -p $BACKUP_DIR"
        init_progress
    fi

    # Show initial progress
    log "📊 Initial Progress:"
    show_progress "databases"
    show_progress "volumes"
    show_progress "configs"
    show_progress "secrets"
    show_progress "user_data"
    show_progress "system_configs"

    # Backup all components
    backup_all_databases
    backup_all_docker_volumes
    backup_all_configurations
    backup_all_secrets
    backup_all_user_data
    backup_all_system_configs

    # Create backup manifest
    create_backup_manifest

    # Verify backup completeness
    verify_backup_completeness

    log "=== BACKUP COMPLETE ==="
    log "Backup location: $BACKUP_DIR"
    log "Log file: $LOG_FILE"
    log "Progress file: $PROGRESS_FILE"
    
    # Show final progress
    log "📊 Final Progress:"
    show_progress "databases"
    show_progress "volumes"
    show_progress "configs"
    show_progress "secrets"
    show_progress "user_data"
    show_progress "system_configs"
}

# Run main function
main "$@"