9ea31368f5
Major accomplishments: - ✅ SELinux policy installed and working - ✅ Core Traefik v2.10 deployment running - ✅ Production configuration ready (v3.1) - ✅ Monitoring stack configured - ✅ Comprehensive documentation created - ✅ Security hardening implemented Current status: - 🟡 Partially deployed (60% complete) - ⚠️ Docker socket access needs resolution - ❌ Monitoring stack not deployed yet - ⚠️ Production migration pending Next steps: 1. Fix Docker socket permissions 2. Deploy monitoring stack 3. Migrate to production config 4. Validate full functionality Files added: - Complete Traefik deployment documentation - Production and test configurations - Monitoring stack configurations - SELinux policy module - Security checklists and guides - Current status documentation
393 lines
13 KiB
Bash
Executable File
393 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Automated Backup Validation Script
|
|
# Validates backup integrity and recovery procedures
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
|
BACKUP_DIR="/backup"
|
|
LOG_FILE="$PROJECT_ROOT/logs/backup-validation-$(date +%Y%m%d-%H%M%S).log"
|
|
VALIDATION_RESULTS="$PROJECT_ROOT/logs/backup-validation-results.yaml"
|
|
|
|
# Create directories
|
|
mkdir -p "$(dirname "$LOG_FILE")" "$PROJECT_ROOT/logs"
|
|
|
|
# Logging function
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Initialize validation results
|
|
init_results() {
|
|
cat > "$VALIDATION_RESULTS" << EOF
|
|
validation_run:
|
|
timestamp: "$(date -Iseconds)"
|
|
script_version: "1.0"
|
|
results:
|
|
EOF
|
|
}
|
|
|
|
# Add result to validation file
|
|
add_result() {
|
|
local backup_type="$1"
|
|
local status="$2"
|
|
local details="$3"
|
|
|
|
cat >> "$VALIDATION_RESULTS" << EOF
|
|
- backup_type: "$backup_type"
|
|
status: "$status"
|
|
details: "$details"
|
|
validated_at: "$(date -Iseconds)"
|
|
EOF
|
|
}
|
|
|
|
# Validate PostgreSQL backup
|
|
validate_postgresql_backup() {
|
|
log "Validating PostgreSQL backups..."
|
|
local latest_backup
|
|
latest_backup=$(find "$BACKUP_DIR" -name "postgresql_full_*.sql" -type f -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
|
|
|
if [[ -z "$latest_backup" ]]; then
|
|
log "❌ No PostgreSQL backup files found"
|
|
add_result "postgresql" "FAILED" "No backup files found"
|
|
return 1
|
|
fi
|
|
|
|
log "Testing PostgreSQL backup: $latest_backup"
|
|
|
|
# Test backup file integrity
|
|
if [[ ! -s "$latest_backup" ]]; then
|
|
log "❌ PostgreSQL backup file is empty"
|
|
add_result "postgresql" "FAILED" "Backup file is empty"
|
|
return 1
|
|
fi
|
|
|
|
# Test SQL syntax and structure
|
|
if ! grep -q "CREATE DATABASE\|CREATE TABLE\|INSERT INTO" "$latest_backup"; then
|
|
log "❌ PostgreSQL backup appears to be incomplete"
|
|
add_result "postgresql" "FAILED" "Backup appears incomplete"
|
|
return 1
|
|
fi
|
|
|
|
# Test restore capability (dry run)
|
|
local temp_container="backup-validation-pg-$$"
|
|
if docker run --rm --name "$temp_container" \
|
|
-e POSTGRES_PASSWORD=testpass \
|
|
-v "$latest_backup:/backup.sql:ro" \
|
|
postgres:16 \
|
|
sh -c "
|
|
postgres &
|
|
sleep 10
|
|
psql -U postgres -c 'SELECT 1' > /dev/null 2>&1
|
|
psql -U postgres -f /backup.sql --single-transaction --set ON_ERROR_STOP=on > /dev/null 2>&1
|
|
echo 'Backup restoration test successful'
|
|
" > /dev/null 2>&1; then
|
|
log "✅ PostgreSQL backup validation successful"
|
|
add_result "postgresql" "PASSED" "Backup file integrity and restore test successful"
|
|
else
|
|
log "❌ PostgreSQL backup restore test failed"
|
|
add_result "postgresql" "FAILED" "Restore test failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Validate MariaDB backup
|
|
validate_mariadb_backup() {
|
|
log "Validating MariaDB backups..."
|
|
local latest_backup
|
|
latest_backup=$(find "$BACKUP_DIR" -name "mariadb_full_*.sql" -type f -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
|
|
|
if [[ -z "$latest_backup" ]]; then
|
|
log "❌ No MariaDB backup files found"
|
|
add_result "mariadb" "FAILED" "No backup files found"
|
|
return 1
|
|
fi
|
|
|
|
log "Testing MariaDB backup: $latest_backup"
|
|
|
|
# Test backup file integrity
|
|
if [[ ! -s "$latest_backup" ]]; then
|
|
log "❌ MariaDB backup file is empty"
|
|
add_result "mariadb" "FAILED" "Backup file is empty"
|
|
return 1
|
|
fi
|
|
|
|
# Test SQL syntax and structure
|
|
if ! grep -q "CREATE DATABASE\|CREATE TABLE\|INSERT INTO" "$latest_backup"; then
|
|
log "❌ MariaDB backup appears to be incomplete"
|
|
add_result "mariadb" "FAILED" "Backup appears incomplete"
|
|
return 1
|
|
fi
|
|
|
|
# Test restore capability (dry run)
|
|
local temp_container="backup-validation-mariadb-$$"
|
|
if docker run --rm --name "$temp_container" \
|
|
-e MYSQL_ROOT_PASSWORD=testpass \
|
|
-v "$latest_backup:/backup.sql:ro" \
|
|
mariadb:11 \
|
|
sh -c "
|
|
mysqld &
|
|
sleep 15
|
|
mysql -u root -ptestpass -e 'SELECT 1' > /dev/null 2>&1
|
|
mysql -u root -ptestpass < /backup.sql
|
|
echo 'Backup restoration test successful'
|
|
" > /dev/null 2>&1; then
|
|
log "✅ MariaDB backup validation successful"
|
|
add_result "mariadb" "PASSED" "Backup file integrity and restore test successful"
|
|
else
|
|
log "❌ MariaDB backup restore test failed"
|
|
add_result "mariadb" "FAILED" "Restore test failed"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Validate file backups (tar.gz archives)
|
|
validate_file_backups() {
|
|
log "Validating file backups..."
|
|
local backup_patterns=("docker_volumes_*.tar.gz" "immich_data_*.tar.gz" "nextcloud_data_*.tar.gz" "homeassistant_data_*.tar.gz")
|
|
local validation_passed=0
|
|
local validation_failed=0
|
|
|
|
for pattern in "${backup_patterns[@]}"; do
|
|
local latest_backup
|
|
latest_backup=$(find "$BACKUP_DIR" -name "$pattern" -type f -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -1 | cut -d' ' -f2- || true)
|
|
|
|
if [[ -z "$latest_backup" ]]; then
|
|
log "⚠️ No backup found for pattern: $pattern"
|
|
add_result "file_backup_$pattern" "WARNING" "No backup files found"
|
|
continue
|
|
fi
|
|
|
|
log "Testing file backup: $latest_backup"
|
|
|
|
# Test archive integrity
|
|
if tar -tzf "$latest_backup" >/dev/null 2>&1; then
|
|
log "✅ Archive integrity test passed for $latest_backup"
|
|
add_result "file_backup_$pattern" "PASSED" "Archive integrity verified"
|
|
((validation_passed++))
|
|
else
|
|
log "❌ Archive integrity test failed for $latest_backup"
|
|
add_result "file_backup_$pattern" "FAILED" "Archive corruption detected"
|
|
((validation_failed++))
|
|
fi
|
|
|
|
# Test extraction (sample files only)
|
|
local temp_dir="/tmp/backup-validation-$$"
|
|
mkdir -p "$temp_dir"
|
|
|
|
if tar -xzf "$latest_backup" -C "$temp_dir" --strip-components=1 --wildcards "*/[^/]*" -O >/dev/null 2>&1; then
|
|
log "✅ Sample extraction test passed for $latest_backup"
|
|
else
|
|
log "⚠️ Sample extraction test warning for $latest_backup"
|
|
fi
|
|
|
|
rm -rf "$temp_dir"
|
|
done
|
|
|
|
log "File backup validation summary: $validation_passed passed, $validation_failed failed"
|
|
}
|
|
|
|
# Validate container configuration backups
|
|
validate_container_configs() {
|
|
log "Validating container configuration backups..."
|
|
local config_dir="$BACKUP_DIR/container_configs"
|
|
|
|
if [[ ! -d "$config_dir" ]]; then
|
|
log "❌ Container configuration backup directory not found"
|
|
add_result "container_configs" "FAILED" "Backup directory missing"
|
|
return 1
|
|
fi
|
|
|
|
local config_files
|
|
config_files=$(find "$config_dir" -name "*_config.json" -type f | wc -l)
|
|
|
|
if [[ $config_files -eq 0 ]]; then
|
|
log "❌ No container configuration files found"
|
|
add_result "container_configs" "FAILED" "No configuration files found"
|
|
return 1
|
|
fi
|
|
|
|
local valid_configs=0
|
|
local invalid_configs=0
|
|
|
|
# Test JSON validity
|
|
for config_file in "$config_dir"/*_config.json; do
|
|
if python3 -c "import json; json.load(open('$config_file'))" >/dev/null 2>&1; then
|
|
((valid_configs++))
|
|
else
|
|
((invalid_configs++))
|
|
log "❌ Invalid JSON in $config_file"
|
|
fi
|
|
done
|
|
|
|
if [[ $invalid_configs -eq 0 ]]; then
|
|
log "✅ All container configuration files are valid ($valid_configs total)"
|
|
add_result "container_configs" "PASSED" "$valid_configs valid configuration files"
|
|
else
|
|
log "❌ Container configuration validation failed: $invalid_configs invalid files"
|
|
add_result "container_configs" "FAILED" "$invalid_configs invalid configuration files"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Validate Docker Compose backups
|
|
validate_compose_backups() {
|
|
log "Validating Docker Compose file backups..."
|
|
local compose_dir="$BACKUP_DIR/compose_files"
|
|
|
|
if [[ ! -d "$compose_dir" ]]; then
|
|
log "❌ Docker Compose backup directory not found"
|
|
add_result "compose_files" "FAILED" "Backup directory missing"
|
|
return 1
|
|
fi
|
|
|
|
local compose_files
|
|
compose_files=$(find "$compose_dir" -name "docker-compose.y*" -type f | wc -l)
|
|
|
|
if [[ $compose_files -eq 0 ]]; then
|
|
log "❌ No Docker Compose files found"
|
|
add_result "compose_files" "FAILED" "No compose files found"
|
|
return 1
|
|
fi
|
|
|
|
local valid_compose=0
|
|
local invalid_compose=0
|
|
|
|
# Test YAML validity
|
|
for compose_file in "$compose_dir"/docker-compose.y*; do
|
|
if python3 -c "import yaml; yaml.safe_load(open('$compose_file'))" >/dev/null 2>&1; then
|
|
((valid_compose++))
|
|
else
|
|
((invalid_compose++))
|
|
log "❌ Invalid YAML in $compose_file"
|
|
fi
|
|
done
|
|
|
|
if [[ $invalid_compose -eq 0 ]]; then
|
|
log "✅ All Docker Compose files are valid ($valid_compose total)"
|
|
add_result "compose_files" "PASSED" "$valid_compose valid compose files"
|
|
else
|
|
log "❌ Docker Compose validation failed: $invalid_compose invalid files"
|
|
add_result "compose_files" "FAILED" "$invalid_compose invalid compose files"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Generate validation report
|
|
generate_report() {
|
|
log "Generating validation report..."
|
|
|
|
# Add summary to results
|
|
cat >> "$VALIDATION_RESULTS" << EOF
|
|
summary:
|
|
total_tests: $(grep -c "backup_type:" "$VALIDATION_RESULTS")
|
|
passed_tests: $(grep -c "status: \"PASSED\"" "$VALIDATION_RESULTS")
|
|
failed_tests: $(grep -c "status: \"FAILED\"" "$VALIDATION_RESULTS")
|
|
warning_tests: $(grep -c "status: \"WARNING\"" "$VALIDATION_RESULTS")
|
|
EOF
|
|
|
|
log "✅ Validation report generated: $VALIDATION_RESULTS"
|
|
|
|
# Send notification if configured
|
|
if command -v mail >/dev/null 2>&1 && [[ -n "${BACKUP_NOTIFICATION_EMAIL:-}" ]]; then
|
|
local subject="Backup Validation Report - $(date '+%Y-%m-%d')"
|
|
mail -s "$subject" "$BACKUP_NOTIFICATION_EMAIL" < "$VALIDATION_RESULTS"
|
|
log "📧 Validation report emailed to $BACKUP_NOTIFICATION_EMAIL"
|
|
fi
|
|
}
|
|
|
|
# Setup automated validation
|
|
setup_automation() {
|
|
local cron_schedule="0 4 * * 1" # Weekly on Monday at 4 AM
|
|
local cron_command="$SCRIPT_DIR/automated-backup-validation.sh --validate-all"
|
|
|
|
if crontab -l 2>/dev/null | grep -q "automated-backup-validation.sh"; then
|
|
log "Cron job already exists for automated backup validation"
|
|
else
|
|
(crontab -l 2>/dev/null; echo "$cron_schedule $cron_command") | crontab -
|
|
log "✅ Automated weekly backup validation scheduled"
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
log "Starting automated backup validation"
|
|
init_results
|
|
|
|
case "${1:-validate-all}" in
|
|
"--postgresql")
|
|
validate_postgresql_backup
|
|
;;
|
|
"--mariadb")
|
|
validate_mariadb_backup
|
|
;;
|
|
"--files")
|
|
validate_file_backups
|
|
;;
|
|
"--configs")
|
|
validate_container_configs
|
|
validate_compose_backups
|
|
;;
|
|
"--validate-all"|"")
|
|
validate_postgresql_backup || true
|
|
validate_mariadb_backup || true
|
|
validate_file_backups || true
|
|
validate_container_configs || true
|
|
validate_compose_backups || true
|
|
;;
|
|
"--setup-automation")
|
|
setup_automation
|
|
;;
|
|
"--help"|"-h")
|
|
cat << 'EOF'
|
|
Automated Backup Validation Script
|
|
|
|
USAGE:
|
|
automated-backup-validation.sh [OPTIONS]
|
|
|
|
OPTIONS:
|
|
--postgresql Validate PostgreSQL backups only
|
|
--mariadb Validate MariaDB backups only
|
|
--files Validate file archive backups only
|
|
--configs Validate configuration backups only
|
|
--validate-all Validate all backup types (default)
|
|
--setup-automation Set up weekly cron job for automated validation
|
|
--help, -h Show this help message
|
|
|
|
ENVIRONMENT VARIABLES:
|
|
BACKUP_NOTIFICATION_EMAIL Email address for validation reports
|
|
|
|
EXAMPLES:
|
|
# Validate all backups
|
|
./automated-backup-validation.sh
|
|
|
|
# Validate only database backups
|
|
./automated-backup-validation.sh --postgresql
|
|
./automated-backup-validation.sh --mariadb
|
|
|
|
# Set up weekly automation
|
|
./automated-backup-validation.sh --setup-automation
|
|
|
|
NOTES:
|
|
- Requires Docker for database restore testing
|
|
- Creates detailed validation reports in YAML format
|
|
- Safe to run multiple times (non-destructive testing)
|
|
- Logs all operations for auditability
|
|
EOF
|
|
;;
|
|
*)
|
|
log "❌ Unknown option: $1"
|
|
log "Use --help for usage information"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
generate_report
|
|
log "🎉 Backup validation completed"
|
|
}
|
|
|
|
# Execute main function
|
|
main "$@" |