#!/bin/bash # Automated Backup Validation Script # Validates backup integrity and recovery procedures set -euo pipefail # Configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" BACKUP_DIR="/backup" LOG_FILE="$PROJECT_ROOT/logs/backup-validation-$(date +%Y%m%d-%H%M%S).log" VALIDATION_RESULTS="$PROJECT_ROOT/logs/backup-validation-results.yaml" # Create directories mkdir -p "$(dirname "$LOG_FILE")" "$PROJECT_ROOT/logs" # Logging function log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE" } # Initialize validation results init_results() { cat > "$VALIDATION_RESULTS" << EOF validation_run: timestamp: "$(date -Iseconds)" script_version: "1.0" results: EOF } # Add result to validation file add_result() { local backup_type="$1" local status="$2" local details="$3" cat >> "$VALIDATION_RESULTS" << EOF - backup_type: "$backup_type" status: "$status" details: "$details" validated_at: "$(date -Iseconds)" EOF } # Validate PostgreSQL backup validate_postgresql_backup() { log "Validating PostgreSQL backups..." local latest_backup latest_backup=$(find "$BACKUP_DIR" -name "postgresql_full_*.sql" -type f -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-) if [[ -z "$latest_backup" ]]; then log "❌ No PostgreSQL backup files found" add_result "postgresql" "FAILED" "No backup files found" return 1 fi log "Testing PostgreSQL backup: $latest_backup" # Test backup file integrity if [[ ! -s "$latest_backup" ]]; then log "❌ PostgreSQL backup file is empty" add_result "postgresql" "FAILED" "Backup file is empty" return 1 fi # Test SQL syntax and structure if ! grep -q "CREATE DATABASE\|CREATE TABLE\|INSERT INTO" "$latest_backup"; then log "❌ PostgreSQL backup appears to be incomplete" add_result "postgresql" "FAILED" "Backup appears incomplete" return 1 fi # Test restore capability (dry run) local temp_container="backup-validation-pg-$$" if docker run --rm --name "$temp_container" \ -e POSTGRES_PASSWORD=testpass \ -v "$latest_backup:/backup.sql:ro" \ postgres:16 \ sh -c " postgres & sleep 10 psql -U postgres -c 'SELECT 1' > /dev/null 2>&1 psql -U postgres -f /backup.sql --single-transaction --set ON_ERROR_STOP=on > /dev/null 2>&1 echo 'Backup restoration test successful' " > /dev/null 2>&1; then log "✅ PostgreSQL backup validation successful" add_result "postgresql" "PASSED" "Backup file integrity and restore test successful" else log "❌ PostgreSQL backup restore test failed" add_result "postgresql" "FAILED" "Restore test failed" return 1 fi } # Validate MariaDB backup validate_mariadb_backup() { log "Validating MariaDB backups..." local latest_backup latest_backup=$(find "$BACKUP_DIR" -name "mariadb_full_*.sql" -type f -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-) if [[ -z "$latest_backup" ]]; then log "❌ No MariaDB backup files found" add_result "mariadb" "FAILED" "No backup files found" return 1 fi log "Testing MariaDB backup: $latest_backup" # Test backup file integrity if [[ ! -s "$latest_backup" ]]; then log "❌ MariaDB backup file is empty" add_result "mariadb" "FAILED" "Backup file is empty" return 1 fi # Test SQL syntax and structure if ! grep -q "CREATE DATABASE\|CREATE TABLE\|INSERT INTO" "$latest_backup"; then log "❌ MariaDB backup appears to be incomplete" add_result "mariadb" "FAILED" "Backup appears incomplete" return 1 fi # Test restore capability (dry run) local temp_container="backup-validation-mariadb-$$" if docker run --rm --name "$temp_container" \ -e MYSQL_ROOT_PASSWORD=testpass \ -v "$latest_backup:/backup.sql:ro" \ mariadb:11 \ sh -c " mysqld & sleep 15 mysql -u root -ptestpass -e 'SELECT 1' > /dev/null 2>&1 mysql -u root -ptestpass < /backup.sql echo 'Backup restoration test successful' " > /dev/null 2>&1; then log "✅ MariaDB backup validation successful" add_result "mariadb" "PASSED" "Backup file integrity and restore test successful" else log "❌ MariaDB backup restore test failed" add_result "mariadb" "FAILED" "Restore test failed" return 1 fi } # Validate file backups (tar.gz archives) validate_file_backups() { log "Validating file backups..." local backup_patterns=("docker_volumes_*.tar.gz" "immich_data_*.tar.gz" "nextcloud_data_*.tar.gz" "homeassistant_data_*.tar.gz") local validation_passed=0 local validation_failed=0 for pattern in "${backup_patterns[@]}"; do local latest_backup latest_backup=$(find "$BACKUP_DIR" -name "$pattern" -type f -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -1 | cut -d' ' -f2- || true) if [[ -z "$latest_backup" ]]; then log "⚠️ No backup found for pattern: $pattern" add_result "file_backup_$pattern" "WARNING" "No backup files found" continue fi log "Testing file backup: $latest_backup" # Test archive integrity if tar -tzf "$latest_backup" >/dev/null 2>&1; then log "✅ Archive integrity test passed for $latest_backup" add_result "file_backup_$pattern" "PASSED" "Archive integrity verified" ((validation_passed++)) else log "❌ Archive integrity test failed for $latest_backup" add_result "file_backup_$pattern" "FAILED" "Archive corruption detected" ((validation_failed++)) fi # Test extraction (sample files only) local temp_dir="/tmp/backup-validation-$$" mkdir -p "$temp_dir" if tar -xzf "$latest_backup" -C "$temp_dir" --strip-components=1 --wildcards "*/[^/]*" -O >/dev/null 2>&1; then log "✅ Sample extraction test passed for $latest_backup" else log "⚠️ Sample extraction test warning for $latest_backup" fi rm -rf "$temp_dir" done log "File backup validation summary: $validation_passed passed, $validation_failed failed" } # Validate container configuration backups validate_container_configs() { log "Validating container configuration backups..." local config_dir="$BACKUP_DIR/container_configs" if [[ ! -d "$config_dir" ]]; then log "❌ Container configuration backup directory not found" add_result "container_configs" "FAILED" "Backup directory missing" return 1 fi local config_files config_files=$(find "$config_dir" -name "*_config.json" -type f | wc -l) if [[ $config_files -eq 0 ]]; then log "❌ No container configuration files found" add_result "container_configs" "FAILED" "No configuration files found" return 1 fi local valid_configs=0 local invalid_configs=0 # Test JSON validity for config_file in "$config_dir"/*_config.json; do if python3 -c "import json; json.load(open('$config_file'))" >/dev/null 2>&1; then ((valid_configs++)) else ((invalid_configs++)) log "❌ Invalid JSON in $config_file" fi done if [[ $invalid_configs -eq 0 ]]; then log "✅ All container configuration files are valid ($valid_configs total)" add_result "container_configs" "PASSED" "$valid_configs valid configuration files" else log "❌ Container configuration validation failed: $invalid_configs invalid files" add_result "container_configs" "FAILED" "$invalid_configs invalid configuration files" return 1 fi } # Validate Docker Compose backups validate_compose_backups() { log "Validating Docker Compose file backups..." local compose_dir="$BACKUP_DIR/compose_files" if [[ ! -d "$compose_dir" ]]; then log "❌ Docker Compose backup directory not found" add_result "compose_files" "FAILED" "Backup directory missing" return 1 fi local compose_files compose_files=$(find "$compose_dir" -name "docker-compose.y*" -type f | wc -l) if [[ $compose_files -eq 0 ]]; then log "❌ No Docker Compose files found" add_result "compose_files" "FAILED" "No compose files found" return 1 fi local valid_compose=0 local invalid_compose=0 # Test YAML validity for compose_file in "$compose_dir"/docker-compose.y*; do if python3 -c "import yaml; yaml.safe_load(open('$compose_file'))" >/dev/null 2>&1; then ((valid_compose++)) else ((invalid_compose++)) log "❌ Invalid YAML in $compose_file" fi done if [[ $invalid_compose -eq 0 ]]; then log "✅ All Docker Compose files are valid ($valid_compose total)" add_result "compose_files" "PASSED" "$valid_compose valid compose files" else log "❌ Docker Compose validation failed: $invalid_compose invalid files" add_result "compose_files" "FAILED" "$invalid_compose invalid compose files" return 1 fi } # Generate validation report generate_report() { log "Generating validation report..." # Add summary to results cat >> "$VALIDATION_RESULTS" << EOF summary: total_tests: $(grep -c "backup_type:" "$VALIDATION_RESULTS") passed_tests: $(grep -c "status: \"PASSED\"" "$VALIDATION_RESULTS") failed_tests: $(grep -c "status: \"FAILED\"" "$VALIDATION_RESULTS") warning_tests: $(grep -c "status: \"WARNING\"" "$VALIDATION_RESULTS") EOF log "✅ Validation report generated: $VALIDATION_RESULTS" # Send notification if configured if command -v mail >/dev/null 2>&1 && [[ -n "${BACKUP_NOTIFICATION_EMAIL:-}" ]]; then local subject="Backup Validation Report - $(date '+%Y-%m-%d')" mail -s "$subject" "$BACKUP_NOTIFICATION_EMAIL" < "$VALIDATION_RESULTS" log "📧 Validation report emailed to $BACKUP_NOTIFICATION_EMAIL" fi } # Setup automated validation setup_automation() { local cron_schedule="0 4 * * 1" # Weekly on Monday at 4 AM local cron_command="$SCRIPT_DIR/automated-backup-validation.sh --validate-all" if crontab -l 2>/dev/null | grep -q "automated-backup-validation.sh"; then log "Cron job already exists for automated backup validation" else (crontab -l 2>/dev/null; echo "$cron_schedule $cron_command") | crontab - log "✅ Automated weekly backup validation scheduled" fi } # Main execution main() { log "Starting automated backup validation" init_results case "${1:-validate-all}" in "--postgresql") validate_postgresql_backup ;; "--mariadb") validate_mariadb_backup ;; "--files") validate_file_backups ;; "--configs") validate_container_configs validate_compose_backups ;; "--validate-all"|"") validate_postgresql_backup || true validate_mariadb_backup || true validate_file_backups || true validate_container_configs || true validate_compose_backups || true ;; "--setup-automation") setup_automation ;; "--help"|"-h") cat << 'EOF' Automated Backup Validation Script USAGE: automated-backup-validation.sh [OPTIONS] OPTIONS: --postgresql Validate PostgreSQL backups only --mariadb Validate MariaDB backups only --files Validate file archive backups only --configs Validate configuration backups only --validate-all Validate all backup types (default) --setup-automation Set up weekly cron job for automated validation --help, -h Show this help message ENVIRONMENT VARIABLES: BACKUP_NOTIFICATION_EMAIL Email address for validation reports EXAMPLES: # Validate all backups ./automated-backup-validation.sh # Validate only database backups ./automated-backup-validation.sh --postgresql ./automated-backup-validation.sh --mariadb # Set up weekly automation ./automated-backup-validation.sh --setup-automation NOTES: - Requires Docker for database restore testing - Creates detailed validation reports in YAML format - Safe to run multiple times (non-destructive testing) - Logs all operations for auditability EOF ;; *) log "❌ Unknown option: $1" log "Use --help for usage information" exit 1 ;; esac generate_report log "🎉 Backup validation completed" } # Execute main function main "$@"