Complete Traefik infrastructure deployment - 60% complete
Major accomplishments: - ✅ SELinux policy installed and working - ✅ Core Traefik v2.10 deployment running - ✅ Production configuration ready (v3.1) - ✅ Monitoring stack configured - ✅ Comprehensive documentation created - ✅ Security hardening implemented Current status: - 🟡 Partially deployed (60% complete) - ⚠️ Docker socket access needs resolution - ❌ Monitoring stack not deployed yet - ⚠️ Production migration pending Next steps: 1. Fix Docker socket permissions 2. Deploy monitoring stack 3. Migrate to production config 4. Validate full functionality Files added: - Complete Traefik deployment documentation - Production and test configurations - Monitoring stack configurations - SELinux policy module - Security checklists and guides - Current status documentation
This commit is contained in:
admin
393
scripts/automated-backup-validation.sh
Executable file
393
scripts/automated-backup-validation.sh
Executable file
@@ -0,0 +1,393 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Automated Backup Validation Script
|
||||
# Validates backup integrity and recovery procedures
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$SCRIPT_DIR")"
|
||||
BACKUP_DIR="/backup"
|
||||
LOG_FILE="$PROJECT_ROOT/logs/backup-validation-$(date +%Y%m%d-%H%M%S).log"
|
||||
VALIDATION_RESULTS="$PROJECT_ROOT/logs/backup-validation-results.yaml"
|
||||
|
||||
# Create directories
|
||||
mkdir -p "$(dirname "$LOG_FILE")" "$PROJECT_ROOT/logs"
|
||||
|
||||
# Logging function
|
||||
log() {
|
||||
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
|
||||
}
|
||||
|
||||
# Initialize validation results
|
||||
init_results() {
|
||||
cat > "$VALIDATION_RESULTS" << EOF
|
||||
validation_run:
|
||||
timestamp: "$(date -Iseconds)"
|
||||
script_version: "1.0"
|
||||
results:
|
||||
EOF
|
||||
}
|
||||
|
||||
# Add result to validation file
|
||||
add_result() {
|
||||
local backup_type="$1"
|
||||
local status="$2"
|
||||
local details="$3"
|
||||
|
||||
cat >> "$VALIDATION_RESULTS" << EOF
|
||||
- backup_type: "$backup_type"
|
||||
status: "$status"
|
||||
details: "$details"
|
||||
validated_at: "$(date -Iseconds)"
|
||||
EOF
|
||||
}
|
||||
|
||||
# Validate PostgreSQL backup
|
||||
validate_postgresql_backup() {
|
||||
log "Validating PostgreSQL backups..."
|
||||
local latest_backup
|
||||
latest_backup=$(find "$BACKUP_DIR" -name "postgresql_full_*.sql" -type f -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
||||
|
||||
if [[ -z "$latest_backup" ]]; then
|
||||
log "❌ No PostgreSQL backup files found"
|
||||
add_result "postgresql" "FAILED" "No backup files found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Testing PostgreSQL backup: $latest_backup"
|
||||
|
||||
# Test backup file integrity
|
||||
if [[ ! -s "$latest_backup" ]]; then
|
||||
log "❌ PostgreSQL backup file is empty"
|
||||
add_result "postgresql" "FAILED" "Backup file is empty"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Test SQL syntax and structure
|
||||
if ! grep -q "CREATE DATABASE\|CREATE TABLE\|INSERT INTO" "$latest_backup"; then
|
||||
log "❌ PostgreSQL backup appears to be incomplete"
|
||||
add_result "postgresql" "FAILED" "Backup appears incomplete"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Test restore capability (dry run)
|
||||
local temp_container="backup-validation-pg-$$"
|
||||
if docker run --rm --name "$temp_container" \
|
||||
-e POSTGRES_PASSWORD=testpass \
|
||||
-v "$latest_backup:/backup.sql:ro" \
|
||||
postgres:16 \
|
||||
sh -c "
|
||||
postgres &
|
||||
sleep 10
|
||||
psql -U postgres -c 'SELECT 1' > /dev/null 2>&1
|
||||
psql -U postgres -f /backup.sql --single-transaction --set ON_ERROR_STOP=on > /dev/null 2>&1
|
||||
echo 'Backup restoration test successful'
|
||||
" > /dev/null 2>&1; then
|
||||
log "✅ PostgreSQL backup validation successful"
|
||||
add_result "postgresql" "PASSED" "Backup file integrity and restore test successful"
|
||||
else
|
||||
log "❌ PostgreSQL backup restore test failed"
|
||||
add_result "postgresql" "FAILED" "Restore test failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Validate MariaDB backup
|
||||
validate_mariadb_backup() {
|
||||
log "Validating MariaDB backups..."
|
||||
local latest_backup
|
||||
latest_backup=$(find "$BACKUP_DIR" -name "mariadb_full_*.sql" -type f -printf '%T@ %p\n' | sort -nr | head -1 | cut -d' ' -f2-)
|
||||
|
||||
if [[ -z "$latest_backup" ]]; then
|
||||
log "❌ No MariaDB backup files found"
|
||||
add_result "mariadb" "FAILED" "No backup files found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "Testing MariaDB backup: $latest_backup"
|
||||
|
||||
# Test backup file integrity
|
||||
if [[ ! -s "$latest_backup" ]]; then
|
||||
log "❌ MariaDB backup file is empty"
|
||||
add_result "mariadb" "FAILED" "Backup file is empty"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Test SQL syntax and structure
|
||||
if ! grep -q "CREATE DATABASE\|CREATE TABLE\|INSERT INTO" "$latest_backup"; then
|
||||
log "❌ MariaDB backup appears to be incomplete"
|
||||
add_result "mariadb" "FAILED" "Backup appears incomplete"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Test restore capability (dry run)
|
||||
local temp_container="backup-validation-mariadb-$$"
|
||||
if docker run --rm --name "$temp_container" \
|
||||
-e MYSQL_ROOT_PASSWORD=testpass \
|
||||
-v "$latest_backup:/backup.sql:ro" \
|
||||
mariadb:11 \
|
||||
sh -c "
|
||||
mysqld &
|
||||
sleep 15
|
||||
mysql -u root -ptestpass -e 'SELECT 1' > /dev/null 2>&1
|
||||
mysql -u root -ptestpass < /backup.sql
|
||||
echo 'Backup restoration test successful'
|
||||
" > /dev/null 2>&1; then
|
||||
log "✅ MariaDB backup validation successful"
|
||||
add_result "mariadb" "PASSED" "Backup file integrity and restore test successful"
|
||||
else
|
||||
log "❌ MariaDB backup restore test failed"
|
||||
add_result "mariadb" "FAILED" "Restore test failed"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Validate file backups (tar.gz archives)
|
||||
validate_file_backups() {
|
||||
log "Validating file backups..."
|
||||
local backup_patterns=("docker_volumes_*.tar.gz" "immich_data_*.tar.gz" "nextcloud_data_*.tar.gz" "homeassistant_data_*.tar.gz")
|
||||
local validation_passed=0
|
||||
local validation_failed=0
|
||||
|
||||
for pattern in "${backup_patterns[@]}"; do
|
||||
local latest_backup
|
||||
latest_backup=$(find "$BACKUP_DIR" -name "$pattern" -type f -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -1 | cut -d' ' -f2- || true)
|
||||
|
||||
if [[ -z "$latest_backup" ]]; then
|
||||
log "⚠️ No backup found for pattern: $pattern"
|
||||
add_result "file_backup_$pattern" "WARNING" "No backup files found"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "Testing file backup: $latest_backup"
|
||||
|
||||
# Test archive integrity
|
||||
if tar -tzf "$latest_backup" >/dev/null 2>&1; then
|
||||
log "✅ Archive integrity test passed for $latest_backup"
|
||||
add_result "file_backup_$pattern" "PASSED" "Archive integrity verified"
|
||||
((validation_passed++))
|
||||
else
|
||||
log "❌ Archive integrity test failed for $latest_backup"
|
||||
add_result "file_backup_$pattern" "FAILED" "Archive corruption detected"
|
||||
((validation_failed++))
|
||||
fi
|
||||
|
||||
# Test extraction (sample files only)
|
||||
local temp_dir="/tmp/backup-validation-$$"
|
||||
mkdir -p "$temp_dir"
|
||||
|
||||
if tar -xzf "$latest_backup" -C "$temp_dir" --strip-components=1 --wildcards "*/[^/]*" -O >/dev/null 2>&1; then
|
||||
log "✅ Sample extraction test passed for $latest_backup"
|
||||
else
|
||||
log "⚠️ Sample extraction test warning for $latest_backup"
|
||||
fi
|
||||
|
||||
rm -rf "$temp_dir"
|
||||
done
|
||||
|
||||
log "File backup validation summary: $validation_passed passed, $validation_failed failed"
|
||||
}
|
||||
|
||||
# Validate container configuration backups
|
||||
validate_container_configs() {
|
||||
log "Validating container configuration backups..."
|
||||
local config_dir="$BACKUP_DIR/container_configs"
|
||||
|
||||
if [[ ! -d "$config_dir" ]]; then
|
||||
log "❌ Container configuration backup directory not found"
|
||||
add_result "container_configs" "FAILED" "Backup directory missing"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local config_files
|
||||
config_files=$(find "$config_dir" -name "*_config.json" -type f | wc -l)
|
||||
|
||||
if [[ $config_files -eq 0 ]]; then
|
||||
log "❌ No container configuration files found"
|
||||
add_result "container_configs" "FAILED" "No configuration files found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local valid_configs=0
|
||||
local invalid_configs=0
|
||||
|
||||
# Test JSON validity
|
||||
for config_file in "$config_dir"/*_config.json; do
|
||||
if python3 -c "import json; json.load(open('$config_file'))" >/dev/null 2>&1; then
|
||||
((valid_configs++))
|
||||
else
|
||||
((invalid_configs++))
|
||||
log "❌ Invalid JSON in $config_file"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $invalid_configs -eq 0 ]]; then
|
||||
log "✅ All container configuration files are valid ($valid_configs total)"
|
||||
add_result "container_configs" "PASSED" "$valid_configs valid configuration files"
|
||||
else
|
||||
log "❌ Container configuration validation failed: $invalid_configs invalid files"
|
||||
add_result "container_configs" "FAILED" "$invalid_configs invalid configuration files"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Validate Docker Compose backups
|
||||
validate_compose_backups() {
|
||||
log "Validating Docker Compose file backups..."
|
||||
local compose_dir="$BACKUP_DIR/compose_files"
|
||||
|
||||
if [[ ! -d "$compose_dir" ]]; then
|
||||
log "❌ Docker Compose backup directory not found"
|
||||
add_result "compose_files" "FAILED" "Backup directory missing"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local compose_files
|
||||
compose_files=$(find "$compose_dir" -name "docker-compose.y*" -type f | wc -l)
|
||||
|
||||
if [[ $compose_files -eq 0 ]]; then
|
||||
log "❌ No Docker Compose files found"
|
||||
add_result "compose_files" "FAILED" "No compose files found"
|
||||
return 1
|
||||
fi
|
||||
|
||||
local valid_compose=0
|
||||
local invalid_compose=0
|
||||
|
||||
# Test YAML validity
|
||||
for compose_file in "$compose_dir"/docker-compose.y*; do
|
||||
if python3 -c "import yaml; yaml.safe_load(open('$compose_file'))" >/dev/null 2>&1; then
|
||||
((valid_compose++))
|
||||
else
|
||||
((invalid_compose++))
|
||||
log "❌ Invalid YAML in $compose_file"
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $invalid_compose -eq 0 ]]; then
|
||||
log "✅ All Docker Compose files are valid ($valid_compose total)"
|
||||
add_result "compose_files" "PASSED" "$valid_compose valid compose files"
|
||||
else
|
||||
log "❌ Docker Compose validation failed: $invalid_compose invalid files"
|
||||
add_result "compose_files" "FAILED" "$invalid_compose invalid compose files"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Generate validation report
|
||||
generate_report() {
|
||||
log "Generating validation report..."
|
||||
|
||||
# Add summary to results
|
||||
cat >> "$VALIDATION_RESULTS" << EOF
|
||||
summary:
|
||||
total_tests: $(grep -c "backup_type:" "$VALIDATION_RESULTS")
|
||||
passed_tests: $(grep -c "status: \"PASSED\"" "$VALIDATION_RESULTS")
|
||||
failed_tests: $(grep -c "status: \"FAILED\"" "$VALIDATION_RESULTS")
|
||||
warning_tests: $(grep -c "status: \"WARNING\"" "$VALIDATION_RESULTS")
|
||||
EOF
|
||||
|
||||
log "✅ Validation report generated: $VALIDATION_RESULTS"
|
||||
|
||||
# Send notification if configured
|
||||
if command -v mail >/dev/null 2>&1 && [[ -n "${BACKUP_NOTIFICATION_EMAIL:-}" ]]; then
|
||||
local subject="Backup Validation Report - $(date '+%Y-%m-%d')"
|
||||
mail -s "$subject" "$BACKUP_NOTIFICATION_EMAIL" < "$VALIDATION_RESULTS"
|
||||
log "📧 Validation report emailed to $BACKUP_NOTIFICATION_EMAIL"
|
||||
fi
|
||||
}
|
||||
|
||||
# Setup automated validation
|
||||
setup_automation() {
|
||||
local cron_schedule="0 4 * * 1" # Weekly on Monday at 4 AM
|
||||
local cron_command="$SCRIPT_DIR/automated-backup-validation.sh --validate-all"
|
||||
|
||||
if crontab -l 2>/dev/null | grep -q "automated-backup-validation.sh"; then
|
||||
log "Cron job already exists for automated backup validation"
|
||||
else
|
||||
(crontab -l 2>/dev/null; echo "$cron_schedule $cron_command") | crontab -
|
||||
log "✅ Automated weekly backup validation scheduled"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main execution
|
||||
main() {
|
||||
log "Starting automated backup validation"
|
||||
init_results
|
||||
|
||||
case "${1:-validate-all}" in
|
||||
"--postgresql")
|
||||
validate_postgresql_backup
|
||||
;;
|
||||
"--mariadb")
|
||||
validate_mariadb_backup
|
||||
;;
|
||||
"--files")
|
||||
validate_file_backups
|
||||
;;
|
||||
"--configs")
|
||||
validate_container_configs
|
||||
validate_compose_backups
|
||||
;;
|
||||
"--validate-all"|"")
|
||||
validate_postgresql_backup || true
|
||||
validate_mariadb_backup || true
|
||||
validate_file_backups || true
|
||||
validate_container_configs || true
|
||||
validate_compose_backups || true
|
||||
;;
|
||||
"--setup-automation")
|
||||
setup_automation
|
||||
;;
|
||||
"--help"|"-h")
|
||||
cat << 'EOF'
|
||||
Automated Backup Validation Script
|
||||
|
||||
USAGE:
|
||||
automated-backup-validation.sh [OPTIONS]
|
||||
|
||||
OPTIONS:
|
||||
--postgresql Validate PostgreSQL backups only
|
||||
--mariadb Validate MariaDB backups only
|
||||
--files Validate file archive backups only
|
||||
--configs Validate configuration backups only
|
||||
--validate-all Validate all backup types (default)
|
||||
--setup-automation Set up weekly cron job for automated validation
|
||||
--help, -h Show this help message
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
BACKUP_NOTIFICATION_EMAIL Email address for validation reports
|
||||
|
||||
EXAMPLES:
|
||||
# Validate all backups
|
||||
./automated-backup-validation.sh
|
||||
|
||||
# Validate only database backups
|
||||
./automated-backup-validation.sh --postgresql
|
||||
./automated-backup-validation.sh --mariadb
|
||||
|
||||
# Set up weekly automation
|
||||
./automated-backup-validation.sh --setup-automation
|
||||
|
||||
NOTES:
|
||||
- Requires Docker for database restore testing
|
||||
- Creates detailed validation reports in YAML format
|
||||
- Safe to run multiple times (non-destructive testing)
|
||||
- Logs all operations for auditability
|
||||
EOF
|
||||
;;
|
||||
*)
|
||||
log "❌ Unknown option: $1"
|
||||
log "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
generate_report
|
||||
log "🎉 Backup validation completed"
|
||||
}
|
||||
|
||||
# Execute main function
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user