526 lines
19 KiB
Bash
Executable File
526 lines
19 KiB
Bash
Executable File
#!/bin/bash
|
|
# Backup Verification and Testing Script
|
|
# Validates backup integrity and tests restoration procedures
|
|
|
|
# Import error handling library
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/lib/error_handling.sh"
|
|
|
|
# Configuration
|
|
readonly BACKUP_BASE_DIR="/opt/migration/backups"
|
|
readonly VERIFICATION_DIR="/opt/migration/verification"
|
|
readonly TEST_RESTORE_DIR="/opt/migration/test_restore"
|
|
readonly VERIFICATION_LOG="$LOG_DIR/backup_verification_$(date +%Y%m%d_%H%M%S).log"
|
|
|
|
# Cleanup function
|
|
cleanup_verification() {
|
|
log_info "Cleaning up verification directories..."
|
|
|
|
if [[ -d "$TEST_RESTORE_DIR" ]]; then
|
|
rm -rf "$TEST_RESTORE_DIR"
|
|
log_info "Removed test restore directory"
|
|
fi
|
|
|
|
# Clean up any temporary Docker containers
|
|
docker ps -a --filter "name=verification_test_*" -q | xargs -r docker rm -f 2>/dev/null || true
|
|
|
|
# Clean up any temporary networks
|
|
docker network ls --filter "name=verification_*" -q | xargs -r docker network rm 2>/dev/null || true
|
|
}
|
|
|
|
# Rollback function
|
|
rollback_verification() {
|
|
log_info "Rolling back verification processes..."
|
|
cleanup_verification
|
|
|
|
# Stop any running verification containers
|
|
docker ps --filter "name=verification_*" -q | xargs -r docker stop 2>/dev/null || true
|
|
}
|
|
|
|
# Function to verify database dumps
|
|
verify_database_dumps() {
|
|
local snapshot_dir=$1
|
|
local dump_dir="$snapshot_dir/database_dumps"
|
|
|
|
log_step "Verifying database dumps in $dump_dir..."
|
|
|
|
if [[ ! -d "$dump_dir" ]]; then
|
|
log_error "Database dump directory not found: $dump_dir"
|
|
return 1
|
|
fi
|
|
|
|
local verification_results="$VERIFICATION_DIR/database_verification.json"
|
|
echo '{"dumps": []}' > "$verification_results"
|
|
|
|
# Verify PostgreSQL dumps
|
|
for dump_file in "$dump_dir"/postgres_dump_*.sql; do
|
|
if [[ -f "$dump_file" ]]; then
|
|
local host=$(basename "$dump_file" .sql | sed 's/postgres_dump_//')
|
|
log_info "Verifying PostgreSQL dump for $host..."
|
|
|
|
# Check file size
|
|
local size=$(stat -f%z "$dump_file" 2>/dev/null || stat -c%s "$dump_file" 2>/dev/null || echo "0")
|
|
|
|
# Check file content structure
|
|
local has_header=$(head -5 "$dump_file" | grep -c "PostgreSQL database dump" || echo "0")
|
|
local has_footer=$(tail -5 "$dump_file" | grep -c "PostgreSQL database dump complete" || echo "0")
|
|
local table_count=$(grep -c "CREATE TABLE" "$dump_file" || echo "0")
|
|
local data_count=$(grep -c "COPY.*FROM stdin" "$dump_file" || echo "0")
|
|
|
|
# Test dump restoration
|
|
local restore_success="false"
|
|
if test_postgres_restore "$dump_file" "$host"; then
|
|
restore_success="true"
|
|
fi
|
|
|
|
# Update verification results
|
|
local dump_result=$(cat << EOF
|
|
{
|
|
"host": "$host",
|
|
"file": "$dump_file",
|
|
"size_bytes": $size,
|
|
"has_header": $has_header,
|
|
"has_footer": $has_footer,
|
|
"table_count": $table_count,
|
|
"data_count": $data_count,
|
|
"restore_test": $restore_success,
|
|
"verification_time": "$(date -Iseconds)"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
# Add to results JSON
|
|
jq ".dumps += [$dump_result]" "$verification_results" > "${verification_results}.tmp" && mv "${verification_results}.tmp" "$verification_results"
|
|
|
|
if [[ $size -gt 1000 ]] && [[ $has_header -gt 0 ]] && [[ $restore_success == "true" ]]; then
|
|
log_success "✅ PostgreSQL dump verified for $host: ${size} bytes, ${table_count} tables"
|
|
else
|
|
log_error "❌ PostgreSQL dump verification failed for $host"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Verify MySQL dumps
|
|
for dump_file in "$dump_dir"/mysql_dump_*.sql; do
|
|
if [[ -f "$dump_file" ]]; then
|
|
local host=$(basename "$dump_file" .sql | sed 's/mysql_dump_//')
|
|
log_info "Verifying MySQL dump for $host..."
|
|
|
|
local size=$(stat -f%z "$dump_file" 2>/dev/null || stat -c%s "$dump_file" 2>/dev/null || echo "0")
|
|
local has_header=$(head -10 "$dump_file" | grep -c "MySQL dump" || echo "0")
|
|
local database_count=$(grep -c "CREATE DATABASE" "$dump_file" || echo "0")
|
|
|
|
if [[ $size -gt 1000 ]] && [[ $has_header -gt 0 ]]; then
|
|
log_success "✅ MySQL dump verified for $host: ${size} bytes, ${database_count} databases"
|
|
else
|
|
log_warn "⚠️ MySQL dump may have issues for $host"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
log_success "Database dump verification completed"
|
|
return 0
|
|
}
|
|
|
|
# Function to test PostgreSQL dump restoration
|
|
test_postgres_restore() {
|
|
local dump_file=$1
|
|
local host=$2
|
|
|
|
log_info "Testing PostgreSQL restoration for $host..."
|
|
|
|
# Create temporary PostgreSQL container for testing
|
|
local test_container="verification_test_postgres_$host"
|
|
local test_network="verification_network"
|
|
|
|
# Create test network
|
|
docker network create "$test_network" 2>/dev/null || true
|
|
|
|
# Start temporary PostgreSQL container
|
|
if docker run -d \
|
|
--name "$test_container" \
|
|
--network "$test_network" \
|
|
-e POSTGRES_PASSWORD=testpass \
|
|
-e POSTGRES_DB=testdb \
|
|
postgres:13 >/dev/null 2>&1; then
|
|
|
|
# Wait for PostgreSQL to be ready
|
|
if wait_for_service "PostgreSQL-$host" "docker exec $test_container pg_isready -U postgres" 60 5; then
|
|
|
|
# Attempt restoration
|
|
if docker exec -i "$test_container" psql -U postgres -d testdb < "$dump_file" >/dev/null 2>&1; then
|
|
|
|
# Verify data was restored
|
|
local table_count=$(docker exec "$test_container" psql -U postgres -d testdb -t -c "SELECT count(*) FROM information_schema.tables WHERE table_schema='public';" 2>/dev/null | xargs || echo "0")
|
|
|
|
if [[ $table_count -gt 0 ]]; then
|
|
log_success "PostgreSQL dump restoration test passed for $host ($table_count tables)"
|
|
docker rm -f "$test_container" >/dev/null 2>&1
|
|
return 0
|
|
else
|
|
log_warn "PostgreSQL dump restored but no tables found for $host"
|
|
fi
|
|
else
|
|
log_error "PostgreSQL dump restoration failed for $host"
|
|
fi
|
|
else
|
|
log_error "PostgreSQL container failed to start for $host test"
|
|
fi
|
|
|
|
# Cleanup
|
|
docker rm -f "$test_container" >/dev/null 2>&1
|
|
else
|
|
log_error "Failed to create PostgreSQL test container for $host"
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
# Function to verify configuration backups
|
|
verify_configuration_backups() {
|
|
local snapshot_dir=$1
|
|
|
|
log_step "Verifying configuration backups in $snapshot_dir..."
|
|
|
|
local verification_results="$VERIFICATION_DIR/config_verification.json"
|
|
echo '{"configs": []}' > "$verification_results"
|
|
|
|
for config_backup in "$snapshot_dir"/config_backup_*.tar.gz; do
|
|
if [[ -f "$config_backup" ]]; then
|
|
local host=$(basename "$config_backup" .tar.gz | sed 's/config_backup_//')
|
|
log_info "Verifying configuration backup for $host..."
|
|
|
|
# Check file integrity
|
|
local size=$(stat -f%z "$config_backup" 2>/dev/null || stat -c%s "$config_backup" 2>/dev/null || echo "0")
|
|
local is_valid_gzip="false"
|
|
|
|
if gzip -t "$config_backup" 2>/dev/null; then
|
|
is_valid_gzip="true"
|
|
log_success "✅ Configuration backup is valid gzip for $host"
|
|
else
|
|
log_error "❌ Configuration backup is corrupted for $host"
|
|
fi
|
|
|
|
# Test extraction
|
|
local extraction_test="false"
|
|
local test_extract_dir="$TEST_RESTORE_DIR/config_$host"
|
|
mkdir -p "$test_extract_dir"
|
|
|
|
if tar -tzf "$config_backup" >/dev/null 2>&1; then
|
|
if tar -xzf "$config_backup" -C "$test_extract_dir" 2>/dev/null; then
|
|
local extracted_files=$(find "$test_extract_dir" -type f | wc -l)
|
|
if [[ $extracted_files -gt 0 ]]; then
|
|
extraction_test="true"
|
|
log_success "Configuration backup extraction test passed for $host ($extracted_files files)"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Update verification results
|
|
local config_result=$(cat << EOF
|
|
{
|
|
"host": "$host",
|
|
"file": "$config_backup",
|
|
"size_bytes": $size,
|
|
"is_valid_gzip": $is_valid_gzip,
|
|
"extraction_test": $extraction_test,
|
|
"verification_time": "$(date -Iseconds)"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
jq ".configs += [$config_result]" "$verification_results" > "${verification_results}.tmp" && mv "${verification_results}.tmp" "$verification_results"
|
|
|
|
# Cleanup test extraction
|
|
rm -rf "$test_extract_dir" 2>/dev/null || true
|
|
fi
|
|
done
|
|
|
|
log_success "Configuration backup verification completed"
|
|
return 0
|
|
}
|
|
|
|
# Function to verify Docker state backups
|
|
verify_docker_state_backups() {
|
|
local snapshot_dir=$1
|
|
|
|
log_step "Verifying Docker state backups..."
|
|
|
|
local verification_results="$VERIFICATION_DIR/docker_verification.json"
|
|
echo '{"hosts": []}' > "$verification_results"
|
|
|
|
for host_dir in "$snapshot_dir"/*; do
|
|
if [[ -d "$host_dir" ]] && [[ $(basename "$host_dir") != "database_dumps" ]]; then
|
|
local host=$(basename "$host_dir")
|
|
log_info "Verifying Docker state for $host..."
|
|
|
|
local containers_file="$host_dir/docker_containers.txt"
|
|
local images_file="$host_dir/docker_images.txt"
|
|
local networks_file="$host_dir/docker_networks.txt"
|
|
local volumes_file="$host_dir/docker_volumes.txt"
|
|
|
|
local container_count=0
|
|
local image_count=0
|
|
local network_count=0
|
|
local volume_count=0
|
|
|
|
# Count containers
|
|
if [[ -f "$containers_file" ]]; then
|
|
container_count=$(grep -c "^[^$]" "$containers_file" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
# Count images
|
|
if [[ -f "$images_file" ]]; then
|
|
image_count=$(grep -c "^[^$]" "$images_file" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
# Count networks
|
|
if [[ -f "$networks_file" ]]; then
|
|
network_count=$(grep -c "^[^$]" "$networks_file" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
# Count volumes
|
|
if [[ -f "$volumes_file" ]]; then
|
|
volume_count=$(grep -c "^[^$]" "$volumes_file" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
# Check for compose files
|
|
local compose_files=0
|
|
if [[ -d "$host_dir/compose_files" ]]; then
|
|
compose_files=$(find "$host_dir/compose_files" -name "*.yml" -o -name "*.yaml" | wc -l)
|
|
fi
|
|
|
|
local docker_result=$(cat << EOF
|
|
{
|
|
"host": "$host",
|
|
"containers": $container_count,
|
|
"images": $image_count,
|
|
"networks": $network_count,
|
|
"volumes": $volume_count,
|
|
"compose_files": $compose_files,
|
|
"verification_time": "$(date -Iseconds)"
|
|
}
|
|
EOF
|
|
)
|
|
|
|
jq ".hosts += [$docker_result]" "$verification_results" > "${verification_results}.tmp" && mv "${verification_results}.tmp" "$verification_results"
|
|
|
|
log_success "✅ Docker state verified for $host: $container_count containers, $image_count images"
|
|
fi
|
|
done
|
|
|
|
log_success "Docker state verification completed"
|
|
return 0
|
|
}
|
|
|
|
# Function to create comprehensive verification report
|
|
create_verification_report() {
|
|
local snapshot_dir=$1
|
|
local report_file="$VERIFICATION_DIR/verification_report_$(date +%Y%m%d_%H%M%S).md"
|
|
|
|
log_step "Creating comprehensive verification report..."
|
|
|
|
cat > "$report_file" << EOF
|
|
# Backup Verification Report
|
|
|
|
**Generated:** $(date)
|
|
**Snapshot Directory:** $snapshot_dir
|
|
**Verification Directory:** $VERIFICATION_DIR
|
|
|
|
## Executive Summary
|
|
EOF
|
|
|
|
# Database verification summary
|
|
if [[ -f "$VERIFICATION_DIR/database_verification.json" ]]; then
|
|
local total_db_dumps=$(jq '.dumps | length' "$VERIFICATION_DIR/database_verification.json")
|
|
local successful_restores=$(jq '.dumps | map(select(.restore_test == true)) | length' "$VERIFICATION_DIR/database_verification.json")
|
|
|
|
echo "- **Database Dumps:** $total_db_dumps total, $successful_restores passed restore tests" >> "$report_file"
|
|
fi
|
|
|
|
# Configuration verification summary
|
|
if [[ -f "$VERIFICATION_DIR/config_verification.json" ]]; then
|
|
local total_configs=$(jq '.configs | length' "$VERIFICATION_DIR/config_verification.json")
|
|
local valid_configs=$(jq '.configs | map(select(.is_valid_gzip == true and .extraction_test == true)) | length' "$VERIFICATION_DIR/config_verification.json")
|
|
|
|
echo "- **Configuration Backups:** $total_configs total, $valid_configs verified" >> "$report_file"
|
|
fi
|
|
|
|
# Docker verification summary
|
|
if [[ -f "$VERIFICATION_DIR/docker_verification.json" ]]; then
|
|
local total_hosts=$(jq '.hosts | length' "$VERIFICATION_DIR/docker_verification.json")
|
|
local total_containers=$(jq '.hosts | map(.containers) | add' "$VERIFICATION_DIR/docker_verification.json")
|
|
|
|
echo "- **Docker States:** $total_hosts hosts, $total_containers total containers" >> "$report_file"
|
|
fi
|
|
|
|
cat >> "$report_file" << EOF
|
|
|
|
## Detailed Results
|
|
|
|
### Database Verification
|
|
EOF
|
|
|
|
# Database details
|
|
if [[ -f "$VERIFICATION_DIR/database_verification.json" ]]; then
|
|
jq -r '.dumps[] | "- **\(.host)**: \(.size_bytes) bytes, \(.table_count) tables, restore test: \(.restore_test)"' "$VERIFICATION_DIR/database_verification.json" >> "$report_file"
|
|
fi
|
|
|
|
cat >> "$report_file" << EOF
|
|
|
|
### Configuration Verification
|
|
EOF
|
|
|
|
# Configuration details
|
|
if [[ -f "$VERIFICATION_DIR/config_verification.json" ]]; then
|
|
jq -r '.configs[] | "- **\(.host)**: \(.size_bytes) bytes, valid: \(.is_valid_gzip), extractable: \(.extraction_test)"' "$VERIFICATION_DIR/config_verification.json" >> "$report_file"
|
|
fi
|
|
|
|
cat >> "$report_file" << EOF
|
|
|
|
### Docker State Verification
|
|
EOF
|
|
|
|
# Docker details
|
|
if [[ -f "$VERIFICATION_DIR/docker_verification.json" ]]; then
|
|
jq -r '.hosts[] | "- **\(.host)**: \(.containers) containers, \(.images) images, \(.compose_files) compose files"' "$VERIFICATION_DIR/docker_verification.json" >> "$report_file"
|
|
fi
|
|
|
|
cat >> "$report_file" << EOF
|
|
|
|
## Recommendations
|
|
|
|
### Critical Issues
|
|
EOF
|
|
|
|
# Identify critical issues
|
|
local critical_issues=0
|
|
|
|
if [[ -f "$VERIFICATION_DIR/database_verification.json" ]]; then
|
|
local failed_restores=$(jq '.dumps | map(select(.restore_test == false)) | length' "$VERIFICATION_DIR/database_verification.json")
|
|
if [[ $failed_restores -gt 0 ]]; then
|
|
echo "- ❌ **$failed_restores database dumps failed restore tests** - Re-create these backups" >> "$report_file"
|
|
((critical_issues++))
|
|
fi
|
|
fi
|
|
|
|
if [[ -f "$VERIFICATION_DIR/config_verification.json" ]]; then
|
|
local invalid_configs=$(jq '.configs | map(select(.is_valid_gzip == false or .extraction_test == false)) | length' "$VERIFICATION_DIR/config_verification.json")
|
|
if [[ $invalid_configs -gt 0 ]]; then
|
|
echo "- ❌ **$invalid_configs configuration backups are corrupted** - Re-create these backups" >> "$report_file"
|
|
((critical_issues++))
|
|
fi
|
|
fi
|
|
|
|
if [[ $critical_issues -eq 0 ]]; then
|
|
echo "- ✅ **No critical issues identified** - All backups are valid and restorable" >> "$report_file"
|
|
fi
|
|
|
|
cat >> "$report_file" << EOF
|
|
|
|
### Next Steps
|
|
1. **Address Critical Issues:** Fix any failed backups before proceeding
|
|
2. **Test Full Restoration:** Perform end-to-end restoration test in staging
|
|
3. **Document Procedures:** Update restoration procedures based on findings
|
|
4. **Schedule Regular Verification:** Implement automated backup verification
|
|
|
|
## Files and Logs
|
|
- **Verification Log:** $VERIFICATION_LOG
|
|
- **Database Results:** $VERIFICATION_DIR/database_verification.json
|
|
- **Config Results:** $VERIFICATION_DIR/config_verification.json
|
|
- **Docker Results:** $VERIFICATION_DIR/docker_verification.json
|
|
EOF
|
|
|
|
log_success "Verification report created: $report_file"
|
|
echo "$report_file"
|
|
}
|
|
|
|
# Function to run full backup verification
|
|
run_full_verification() {
|
|
local snapshot_dir=${1:-"$BACKUP_BASE_DIR/latest"}
|
|
|
|
if [[ ! -d "$snapshot_dir" ]]; then
|
|
log_error "Snapshot directory not found: $snapshot_dir"
|
|
return 1
|
|
fi
|
|
|
|
log_step "Starting full backup verification for: $snapshot_dir"
|
|
|
|
# Create verification directory
|
|
mkdir -p "$VERIFICATION_DIR"
|
|
mkdir -p "$TEST_RESTORE_DIR"
|
|
|
|
# Register cleanup and rollback
|
|
register_cleanup cleanup_verification
|
|
register_rollback rollback_verification
|
|
|
|
# Validate prerequisites
|
|
validate_prerequisites docker jq gzip tar
|
|
|
|
# Create checkpoint
|
|
create_checkpoint "verification_start"
|
|
|
|
# Verify database dumps
|
|
if verify_database_dumps "$snapshot_dir"; then
|
|
create_checkpoint "database_verification_complete"
|
|
else
|
|
log_error "Database verification failed"
|
|
return 1
|
|
fi
|
|
|
|
# Verify configuration backups
|
|
if verify_configuration_backups "$snapshot_dir"; then
|
|
create_checkpoint "config_verification_complete"
|
|
else
|
|
log_error "Configuration verification failed"
|
|
return 1
|
|
fi
|
|
|
|
# Verify Docker state backups
|
|
if verify_docker_state_backups "$snapshot_dir"; then
|
|
create_checkpoint "docker_verification_complete"
|
|
else
|
|
log_error "Docker verification failed"
|
|
return 1
|
|
fi
|
|
|
|
# Create comprehensive report
|
|
local report_file=$(create_verification_report "$snapshot_dir")
|
|
|
|
# Final summary
|
|
log_success "✅ Backup verification completed successfully!"
|
|
log_info "📊 Verification report: $report_file"
|
|
|
|
# Display summary
|
|
if [[ -f "$report_file" ]]; then
|
|
echo ""
|
|
echo "=== VERIFICATION SUMMARY ==="
|
|
head -20 "$report_file"
|
|
echo ""
|
|
echo "Full report available at: $report_file"
|
|
fi
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
local snapshot_dir=${1:-""}
|
|
|
|
if [[ -z "$snapshot_dir" ]]; then
|
|
# Use latest snapshot if no directory specified
|
|
if [[ -L "$BACKUP_BASE_DIR/latest" ]]; then
|
|
snapshot_dir=$(readlink -f "$BACKUP_BASE_DIR/latest")
|
|
log_info "Using latest snapshot: $snapshot_dir"
|
|
else
|
|
log_error "No snapshot directory specified and no 'latest' link found"
|
|
log_info "Usage: $0 [snapshot_directory]"
|
|
log_info "Available snapshots:"
|
|
ls -la "$BACKUP_BASE_DIR"/snapshot_* 2>/dev/null || echo "No snapshots found"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
run_full_verification "$snapshot_dir"
|
|
}
|
|
|
|
# Execute main function
|
|
main "$@" |