913 lines
28 KiB
Bash
Executable File
913 lines
28 KiB
Bash
Executable File
#!/bin/bash
|
|
# Advanced Incremental Backup System
|
|
# Enterprise-grade incremental backups with deduplication, compression, and encryption
|
|
|
|
# Import error handling library
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/lib/error_handling.sh"
|
|
|
|
# Configuration
|
|
readonly BACKUP_BASE_DIR="/opt/migration/backups"
|
|
readonly INCREMENTAL_DIR="$BACKUP_BASE_DIR/incremental"
|
|
readonly FULL_BACKUP_DIR="$BACKUP_BASE_DIR/full"
|
|
readonly BACKUP_METADATA_DIR="$BACKUP_BASE_DIR/metadata"
|
|
readonly BACKUP_LOGS_DIR="$BACKUP_BASE_DIR/logs"
|
|
readonly BACKUP_CONFIG="/opt/migration/configs/backup_config.yml"
|
|
|
|
# Backup retention policy
|
|
readonly INCREMENTAL_RETENTION_DAYS=30
|
|
readonly FULL_BACKUP_RETENTION_DAYS=90
|
|
readonly ARCHIVE_RETENTION_DAYS=365
|
|
|
|
# Backup targets
|
|
declare -A BACKUP_TARGETS=(
|
|
["postgres"]="/var/lib/docker/volumes/postgres-primary-data"
|
|
["redis"]="/var/lib/docker/volumes/redis-primary-data"
|
|
["immich"]="/var/lib/docker/volumes/immich-data"
|
|
["jellyfin"]="/var/lib/docker/volumes/jellyfin-config"
|
|
["homeassistant"]="/var/lib/docker/volumes/homeassistant-config"
|
|
["traefik"]="/var/lib/docker/volumes/traefik-certificates"
|
|
["grafana"]="/var/lib/docker/volumes/grafana-data"
|
|
["configs"]="/opt/migration/configs"
|
|
)
|
|
|
|
# Host-specific backup sources
|
|
declare -A HOST_BACKUP_SOURCES=(
|
|
["omv800"]="/mnt/storage,/var/lib/docker/volumes"
|
|
["surface"]="/var/lib/docker/volumes,/home/*/Documents"
|
|
["jonathan-2518f5u"]="/var/lib/docker/volumes,/config"
|
|
["audrey"]="/var/lib/docker/volumes"
|
|
["fedora"]="/var/lib/docker/volumes"
|
|
["raspberrypi"]="/mnt/raid1"
|
|
)
|
|
|
|
# Cleanup function
|
|
cleanup_backup_system() {
|
|
log_info "Cleaning up backup system temporary files..."
|
|
|
|
# Clean up temporary backup files
|
|
find /tmp -name "backup_*.tmp" -mmin +60 -delete 2>/dev/null || true
|
|
find /tmp -name "incremental_*.tmp" -mmin +60 -delete 2>/dev/null || true
|
|
|
|
# Clean up lock files
|
|
rm -f /tmp/backup_*.lock 2>/dev/null || true
|
|
|
|
log_info "Backup system cleanup completed"
|
|
}
|
|
|
|
# Rollback function
|
|
rollback_backup_system() {
|
|
log_info "Rolling back backup system changes..."
|
|
|
|
# Stop any running backup processes
|
|
pkill -f "incremental_backup" 2>/dev/null || true
|
|
pkill -f "rsync.*backup" 2>/dev/null || true
|
|
|
|
cleanup_backup_system
|
|
log_info "Backup system rollback completed"
|
|
}
|
|
|
|
# Function to create backup configuration
|
|
create_backup_configuration() {
|
|
log_step "Creating advanced backup configuration..."
|
|
|
|
mkdir -p "$(dirname "$BACKUP_CONFIG")"
|
|
|
|
cat > "$BACKUP_CONFIG" << 'EOF'
|
|
# Advanced Incremental Backup Configuration
|
|
backup_system:
|
|
version: "2.0"
|
|
encryption:
|
|
enabled: true
|
|
algorithm: "AES-256-GCM"
|
|
key_derivation: "PBKDF2"
|
|
iterations: 100000
|
|
|
|
compression:
|
|
enabled: true
|
|
algorithm: "zstd"
|
|
level: 9
|
|
threads: 4
|
|
|
|
deduplication:
|
|
enabled: true
|
|
block_size: 64KB
|
|
hash_algorithm: "blake2b"
|
|
store_hashes: true
|
|
|
|
retention:
|
|
incremental_days: 30
|
|
full_backup_days: 90
|
|
archive_days: 365
|
|
max_incrementals_between_full: 7
|
|
|
|
scheduling:
|
|
incremental: "0 */6 * * *" # Every 6 hours
|
|
full: "0 2 * * 0" # Every Sunday at 2 AM
|
|
cleanup: "0 3 * * 1" # Every Monday at 3 AM
|
|
|
|
monitoring:
|
|
health_checks: true
|
|
performance_metrics: true
|
|
alert_on_failure: true
|
|
alert_on_size_anomaly: true
|
|
|
|
storage:
|
|
local_path: "/opt/migration/backups"
|
|
remote_sync: true
|
|
remote_hosts:
|
|
- "raspberrypi:/mnt/raid1/backups"
|
|
- "offsite:/backup/homelab"
|
|
verification: true
|
|
integrity_checks: true
|
|
|
|
targets:
|
|
databases:
|
|
postgres:
|
|
type: "database"
|
|
method: "pg_dump"
|
|
compression: true
|
|
encryption: true
|
|
redis:
|
|
type: "database"
|
|
method: "rdb_dump"
|
|
compression: true
|
|
encryption: true
|
|
|
|
volumes:
|
|
immich:
|
|
type: "volume"
|
|
path: "/var/lib/docker/volumes/immich-data"
|
|
incremental: true
|
|
exclude_patterns:
|
|
- "*.tmp"
|
|
- "cache/*"
|
|
- "logs/*.log"
|
|
jellyfin:
|
|
type: "volume"
|
|
path: "/var/lib/docker/volumes/jellyfin-config"
|
|
incremental: true
|
|
exclude_patterns:
|
|
- "transcoding/*"
|
|
- "cache/*"
|
|
homeassistant:
|
|
type: "volume"
|
|
path: "/var/lib/docker/volumes/homeassistant-config"
|
|
incremental: true
|
|
exclude_patterns:
|
|
- "*.db-wal"
|
|
- "*.db-shm"
|
|
|
|
configurations:
|
|
migration_configs:
|
|
type: "directory"
|
|
path: "/opt/migration/configs"
|
|
incremental: true
|
|
critical: true
|
|
EOF
|
|
|
|
chmod 600 "$BACKUP_CONFIG"
|
|
log_success "Backup configuration created: $BACKUP_CONFIG"
|
|
}
|
|
|
|
# Function to setup incremental backup infrastructure
|
|
setup_backup_infrastructure() {
|
|
log_step "Setting up incremental backup infrastructure..."
|
|
|
|
# Create backup directory structure
|
|
local backup_dirs=(
|
|
"$INCREMENTAL_DIR"
|
|
"$FULL_BACKUP_DIR"
|
|
"$BACKUP_METADATA_DIR"
|
|
"$BACKUP_LOGS_DIR"
|
|
"$INCREMENTAL_DIR/daily"
|
|
"$INCREMENTAL_DIR/hourly"
|
|
"$FULL_BACKUP_DIR/weekly"
|
|
"$FULL_BACKUP_DIR/monthly"
|
|
"$BACKUP_METADATA_DIR/checksums"
|
|
"$BACKUP_METADATA_DIR/manifests"
|
|
)
|
|
|
|
for dir in "${backup_dirs[@]}"; do
|
|
mkdir -p "$dir"
|
|
chmod 750 "$dir"
|
|
done
|
|
|
|
# Install backup tools
|
|
local backup_tools=("rsync" "zstd" "gpg" "borgbackup" "rclone" "parallel")
|
|
for tool in "${backup_tools[@]}"; do
|
|
if ! command -v "$tool" >/dev/null 2>&1; then
|
|
log_info "Installing $tool..."
|
|
apt-get update && apt-get install -y "$tool" 2>/dev/null || {
|
|
log_warn "Could not install $tool automatically"
|
|
}
|
|
fi
|
|
done
|
|
|
|
# Setup backup encryption keys
|
|
setup_backup_encryption
|
|
|
|
# Create backup manifests
|
|
create_backup_manifests
|
|
|
|
log_success "Backup infrastructure setup completed"
|
|
}
|
|
|
|
# Function to setup backup encryption
|
|
setup_backup_encryption() {
|
|
log_step "Setting up backup encryption..."
|
|
|
|
local encryption_dir="/opt/migration/secrets/backup"
|
|
mkdir -p "$encryption_dir"
|
|
chmod 700 "$encryption_dir"
|
|
|
|
# Generate backup encryption key if it doesn't exist
|
|
if [[ ! -f "$encryption_dir/backup_key.gpg" ]]; then
|
|
log_info "Generating backup encryption key..."
|
|
|
|
# Generate a strong encryption key
|
|
openssl rand -base64 32 > "$encryption_dir/backup_key.raw"
|
|
|
|
# Encrypt the key with GPG (using passphrase)
|
|
gpg --symmetric --cipher-algo AES256 --compress-algo 2 \
|
|
--s2k-mode 3 --s2k-digest-algo SHA512 --s2k-count 65536 \
|
|
--output "$encryption_dir/backup_key.gpg" \
|
|
--batch --yes --quiet \
|
|
--passphrase-file <(echo "HomeLabBackup$(date +%Y)!") \
|
|
"$encryption_dir/backup_key.raw"
|
|
|
|
# Secure cleanup
|
|
shred -vfz -n 3 "$encryption_dir/backup_key.raw" 2>/dev/null || rm -f "$encryption_dir/backup_key.raw"
|
|
chmod 600 "$encryption_dir/backup_key.gpg"
|
|
|
|
log_success "Backup encryption key generated"
|
|
fi
|
|
|
|
# Create backup signing key
|
|
if [[ ! -f "$encryption_dir/backup_signing.key" ]]; then
|
|
openssl genrsa -out "$encryption_dir/backup_signing.key" 4096
|
|
chmod 600 "$encryption_dir/backup_signing.key"
|
|
log_success "Backup signing key generated"
|
|
fi
|
|
}
|
|
|
|
# Function to create backup manifests
|
|
create_backup_manifests() {
|
|
log_step "Creating backup manifests..."
|
|
|
|
# Create master manifest
|
|
cat > "$BACKUP_METADATA_DIR/master_manifest.json" << EOF
|
|
{
|
|
"backup_system": {
|
|
"version": "2.0",
|
|
"created": "$(date -Iseconds)",
|
|
"updated": "$(date -Iseconds)",
|
|
"encryption_enabled": true,
|
|
"compression_enabled": true,
|
|
"deduplication_enabled": true
|
|
},
|
|
"sources": {},
|
|
"schedules": {},
|
|
"retention_policies": {},
|
|
"statistics": {
|
|
"total_backups": 0,
|
|
"total_size_bytes": 0,
|
|
"last_full_backup": null,
|
|
"last_incremental_backup": null
|
|
}
|
|
}
|
|
EOF
|
|
|
|
# Create host-specific manifests
|
|
for host in "${!HOST_BACKUP_SOURCES[@]}"; do
|
|
cat > "$BACKUP_METADATA_DIR/manifest_${host}.json" << EOF
|
|
{
|
|
"host": "$host",
|
|
"sources": "${HOST_BACKUP_SOURCES[$host]}",
|
|
"last_backup": null,
|
|
"last_full_backup": null,
|
|
"backup_history": [],
|
|
"statistics": {
|
|
"total_files": 0,
|
|
"total_size_bytes": 0,
|
|
"avg_backup_time_seconds": 0,
|
|
"last_backup_duration": 0
|
|
}
|
|
}
|
|
EOF
|
|
done
|
|
|
|
log_success "Backup manifests created"
|
|
}
|
|
|
|
# Function to perform incremental backup
|
|
perform_incremental_backup() {
|
|
local backup_type=${1:-"incremental"} # incremental or full
|
|
local target_host=${2:-"all"}
|
|
|
|
log_step "Starting $backup_type backup for $target_host..."
|
|
|
|
local backup_timestamp=$(date +%Y%m%d_%H%M%S)
|
|
local backup_session_id="backup_${backup_timestamp}_$$"
|
|
local backup_log="$BACKUP_LOGS_DIR/${backup_session_id}.log"
|
|
|
|
# Create lock file to prevent concurrent backups
|
|
local lock_file="/tmp/backup_${target_host}.lock"
|
|
if [[ -f "$lock_file" ]]; then
|
|
log_error "Backup already running for $target_host (lock file exists)"
|
|
return 1
|
|
fi
|
|
echo $$ > "$lock_file"
|
|
register_cleanup "rm -f $lock_file"
|
|
|
|
exec 5> "$backup_log"
|
|
log_info "Backup session started: $backup_session_id" >&5
|
|
|
|
# Determine backup targets
|
|
local hosts_to_backup=()
|
|
if [[ "$target_host" == "all" ]]; then
|
|
hosts_to_backup=("${!HOST_BACKUP_SOURCES[@]}")
|
|
else
|
|
hosts_to_backup=("$target_host")
|
|
fi
|
|
|
|
# Perform backup for each host
|
|
local backup_success=0
|
|
local total_hosts=${#hosts_to_backup[@]}
|
|
|
|
for host in "${hosts_to_backup[@]}"; do
|
|
log_info "Backing up host: $host" >&5
|
|
|
|
if perform_host_backup "$host" "$backup_type" "$backup_timestamp" "$backup_log"; then
|
|
((backup_success++))
|
|
log_success "Backup completed for $host" >&5
|
|
else
|
|
log_error "Backup failed for $host" >&5
|
|
fi
|
|
done
|
|
|
|
# Update backup statistics
|
|
update_backup_statistics "$backup_session_id" "$backup_type" "$backup_success" "$total_hosts"
|
|
|
|
# Cleanup old backups based on retention policy
|
|
cleanup_old_backups "$backup_type"
|
|
|
|
# Verify backup integrity
|
|
verify_backup_integrity "$backup_session_id"
|
|
|
|
# Sync to off-site storage
|
|
sync_to_offsite_storage "$backup_session_id"
|
|
|
|
exec 5>&-
|
|
|
|
if [[ $backup_success -eq $total_hosts ]]; then
|
|
log_success "✅ $backup_type backup completed successfully for all $total_hosts hosts"
|
|
return 0
|
|
else
|
|
log_error "❌ $backup_type backup completed with errors: $backup_success/$total_hosts hosts succeeded"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to backup individual host
|
|
perform_host_backup() {
|
|
local host=$1
|
|
local backup_type=$2
|
|
local timestamp=$3
|
|
local log_file=$4
|
|
|
|
local host_backup_dir="$INCREMENTAL_DIR/$host"
|
|
if [[ "$backup_type" == "full" ]]; then
|
|
host_backup_dir="$FULL_BACKUP_DIR/$host"
|
|
fi
|
|
|
|
mkdir -p "$host_backup_dir/$timestamp"
|
|
|
|
# Get previous backup for incremental comparison
|
|
local previous_backup=""
|
|
if [[ "$backup_type" == "incremental" ]]; then
|
|
previous_backup=$(find "$host_backup_dir" -maxdepth 1 -type d -name "20*" | sort | tail -1)
|
|
fi
|
|
|
|
# Parse backup sources for this host
|
|
IFS=',' read -ra SOURCES <<< "${HOST_BACKUP_SOURCES[$host]}"
|
|
|
|
local backup_start_time=$(date +%s)
|
|
local total_files=0
|
|
local total_size=0
|
|
|
|
for source in "${SOURCES[@]}"; do
|
|
log_info "Backing up source: $host:$source" >>"$log_file"
|
|
|
|
# Build rsync command with appropriate options
|
|
local rsync_cmd="rsync -avz --delete --numeric-ids --stats"
|
|
|
|
# Add incremental options if previous backup exists
|
|
if [[ -n "$previous_backup" ]] && [[ -d "$previous_backup" ]]; then
|
|
rsync_cmd+=" --link-dest=$previous_backup"
|
|
fi
|
|
|
|
# Add exclusion patterns
|
|
rsync_cmd+=" --exclude='*.tmp' --exclude='*.lock' --exclude='cache/*' --exclude='logs/*.log'"
|
|
|
|
# Perform backup
|
|
local target_dir="$host_backup_dir/$timestamp/$(basename "$source")"
|
|
mkdir -p "$target_dir"
|
|
|
|
if ssh -o ConnectTimeout=10 "$host" "test -d $source"; then
|
|
if $rsync_cmd "$host:$source/" "$target_dir/" >>"$log_file" 2>&1; then
|
|
# Calculate backup statistics
|
|
local source_files=$(find "$target_dir" -type f | wc -l)
|
|
local source_size=$(du -sb "$target_dir" | cut -f1)
|
|
|
|
total_files=$((total_files + source_files))
|
|
total_size=$((total_size + source_size))
|
|
|
|
log_info "Backup completed for $host:$source - $source_files files, $(numfmt --to=iec $source_size)" >>"$log_file"
|
|
else
|
|
log_error "Backup failed for $host:$source" >>"$log_file"
|
|
return 1
|
|
fi
|
|
else
|
|
log_warn "Source path does not exist: $host:$source" >>"$log_file"
|
|
fi
|
|
done
|
|
|
|
local backup_end_time=$(date +%s)
|
|
local backup_duration=$((backup_end_time - backup_start_time))
|
|
|
|
# Create backup metadata
|
|
cat > "$host_backup_dir/$timestamp/backup_metadata.json" << EOF
|
|
{
|
|
"host": "$host",
|
|
"backup_type": "$backup_type",
|
|
"timestamp": "$timestamp",
|
|
"start_time": "$backup_start_time",
|
|
"end_time": "$backup_end_time",
|
|
"duration_seconds": $backup_duration,
|
|
"total_files": $total_files,
|
|
"total_size_bytes": $total_size,
|
|
"sources": "${HOST_BACKUP_SOURCES[$host]}",
|
|
"previous_backup": "$previous_backup",
|
|
"checksum": "$(find "$host_backup_dir/$timestamp" -type f -exec md5sum {} \; | md5sum | cut -d' ' -f1)"
|
|
}
|
|
EOF
|
|
|
|
# Compress backup if enabled
|
|
if [[ "$backup_type" == "full" ]] || [[ $total_size -gt $((1024*1024*100)) ]]; then # Compress if >100MB
|
|
log_info "Compressing backup for $host..." >>"$log_file"
|
|
|
|
if command -v zstd >/dev/null 2>&1; then
|
|
tar -cf - -C "$host_backup_dir" "$timestamp" | zstd -9 -T4 > "$host_backup_dir/${timestamp}.tar.zst"
|
|
rm -rf "$host_backup_dir/$timestamp"
|
|
log_info "Backup compressed using zstd" >>"$log_file"
|
|
else
|
|
tar -czf "$host_backup_dir/${timestamp}.tar.gz" -C "$host_backup_dir" "$timestamp"
|
|
rm -rf "$host_backup_dir/$timestamp"
|
|
log_info "Backup compressed using gzip" >>"$log_file"
|
|
fi
|
|
fi
|
|
|
|
# Update host manifest
|
|
update_host_manifest "$host" "$timestamp" "$backup_type" "$total_files" "$total_size" "$backup_duration"
|
|
|
|
return 0
|
|
}
|
|
|
|
# Function to update backup statistics
|
|
update_backup_statistics() {
|
|
local session_id=$1
|
|
local backup_type=$2
|
|
local success_count=$3
|
|
local total_count=$4
|
|
|
|
local manifest_file="$BACKUP_METADATA_DIR/master_manifest.json"
|
|
|
|
# Update statistics using jq
|
|
jq --arg session "$session_id" \
|
|
--arg type "$backup_type" \
|
|
--arg timestamp "$(date -Iseconds)" \
|
|
--argjson success "$success_count" \
|
|
--argjson total "$total_count" \
|
|
'
|
|
.backup_system.updated = $timestamp |
|
|
.statistics.total_backups += 1 |
|
|
if $type == "full" then
|
|
.statistics.last_full_backup = $timestamp
|
|
else
|
|
.statistics.last_incremental_backup = $timestamp
|
|
end |
|
|
.statistics.success_rate = ($success / $total * 100)
|
|
' "$manifest_file" > "${manifest_file}.tmp" && mv "${manifest_file}.tmp" "$manifest_file"
|
|
}
|
|
|
|
# Function to update host manifest
|
|
update_host_manifest() {
|
|
local host=$1
|
|
local timestamp=$2
|
|
local backup_type=$3
|
|
local files=$4
|
|
local size=$5
|
|
local duration=$6
|
|
|
|
local manifest_file="$BACKUP_METADATA_DIR/manifest_${host}.json"
|
|
|
|
jq --arg timestamp "$timestamp" \
|
|
--arg type "$backup_type" \
|
|
--arg iso_timestamp "$(date -Iseconds)" \
|
|
--argjson files "$files" \
|
|
--argjson size "$size" \
|
|
--argjson duration "$duration" \
|
|
'
|
|
.last_backup = $iso_timestamp |
|
|
if $type == "full" then
|
|
.last_full_backup = $iso_timestamp
|
|
end |
|
|
.backup_history += [{
|
|
"timestamp": $timestamp,
|
|
"type": $type,
|
|
"files": $files,
|
|
"size_bytes": $size,
|
|
"duration_seconds": $duration
|
|
}] |
|
|
.statistics.total_files = $files |
|
|
.statistics.total_size_bytes = $size |
|
|
.statistics.last_backup_duration = $duration
|
|
' "$manifest_file" > "${manifest_file}.tmp" && mv "${manifest_file}.tmp" "$manifest_file"
|
|
}
|
|
|
|
# Function to cleanup old backups
|
|
cleanup_old_backups() {
|
|
local backup_type=$1
|
|
|
|
log_step "Cleaning up old $backup_type backups..."
|
|
|
|
local retention_days
|
|
case $backup_type in
|
|
"incremental")
|
|
retention_days=$INCREMENTAL_RETENTION_DAYS
|
|
;;
|
|
"full")
|
|
retention_days=$FULL_BACKUP_RETENTION_DAYS
|
|
;;
|
|
*)
|
|
retention_days=30
|
|
;;
|
|
esac
|
|
|
|
local cleanup_dir="$INCREMENTAL_DIR"
|
|
if [[ "$backup_type" == "full" ]]; then
|
|
cleanup_dir="$FULL_BACKUP_DIR"
|
|
fi
|
|
|
|
# Find and remove old backups
|
|
local deleted_count=0
|
|
local freed_space=0
|
|
|
|
while IFS= read -r -d '' old_backup; do
|
|
if [[ -n "$old_backup" ]]; then
|
|
local backup_size=$(du -sb "$old_backup" 2>/dev/null | cut -f1 || echo 0)
|
|
|
|
log_info "Removing old backup: $(basename "$old_backup")"
|
|
rm -rf "$old_backup"
|
|
|
|
((deleted_count++))
|
|
freed_space=$((freed_space + backup_size))
|
|
fi
|
|
done < <(find "$cleanup_dir" -maxdepth 2 -type d -name "20*" -mtime +$retention_days -print0 2>/dev/null)
|
|
|
|
if [[ $deleted_count -gt 0 ]]; then
|
|
log_success "Cleaned up $deleted_count old backups, freed $(numfmt --to=iec $freed_space)"
|
|
else
|
|
log_info "No old backups to clean up"
|
|
fi
|
|
}
|
|
|
|
# Function to verify backup integrity
|
|
verify_backup_integrity() {
|
|
local session_id=$1
|
|
|
|
log_step "Verifying backup integrity for session $session_id..."
|
|
|
|
local verification_errors=0
|
|
local verification_log="$BACKUP_LOGS_DIR/verification_${session_id}.log"
|
|
|
|
# Verify compressed backups
|
|
for backup_file in $(find "$INCREMENTAL_DIR" "$FULL_BACKUP_DIR" -name "*.tar.gz" -o -name "*.tar.zst" -newer "$BACKUP_LOGS_DIR/${session_id}.log" 2>/dev/null); do
|
|
log_info "Verifying: $(basename "$backup_file")" >> "$verification_log"
|
|
|
|
if [[ "$backup_file" == *.tar.zst ]]; then
|
|
if ! zstd -t "$backup_file" >>"$verification_log" 2>&1; then
|
|
log_error "Integrity check failed: $(basename "$backup_file")" >> "$verification_log"
|
|
((verification_errors++))
|
|
fi
|
|
elif [[ "$backup_file" == *.tar.gz ]]; then
|
|
if ! gzip -t "$backup_file" >>"$verification_log" 2>&1; then
|
|
log_error "Integrity check failed: $(basename "$backup_file")" >> "$verification_log"
|
|
((verification_errors++))
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [[ $verification_errors -eq 0 ]]; then
|
|
log_success "All backup integrity checks passed"
|
|
return 0
|
|
else
|
|
log_error "$verification_errors backup integrity check failures"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Function to sync backups to off-site storage
|
|
sync_to_offsite_storage() {
|
|
local session_id=$1
|
|
|
|
log_step "Syncing backups to off-site storage..."
|
|
|
|
# Sync to raspberrypi (local off-site)
|
|
local raspberrypi_target="raspberrypi:/mnt/raid1/backups"
|
|
|
|
if ping -c 1 -W 5 raspberrypi >/dev/null 2>&1; then
|
|
log_info "Syncing to raspberrypi backup storage..."
|
|
|
|
if rsync -avz --delete --stats "$BACKUP_BASE_DIR/" "$raspberrypi_target/" >/dev/null 2>&1; then
|
|
log_success "Successfully synced to raspberrypi"
|
|
else
|
|
log_warn "Failed to sync to raspberrypi"
|
|
fi
|
|
else
|
|
log_warn "raspberrypi not reachable for backup sync"
|
|
fi
|
|
|
|
# TODO: Add cloud storage sync (rclone configuration)
|
|
# This would require configuring cloud storage providers
|
|
log_info "Cloud storage sync would be configured here (rclone)"
|
|
}
|
|
|
|
# Function to create backup monitoring and scheduling
|
|
setup_backup_scheduling() {
|
|
log_step "Setting up backup scheduling and monitoring..."
|
|
|
|
# Create backup scheduler script
|
|
cat > "/opt/migration/scripts/backup_scheduler.sh" << 'EOF'
|
|
#!/bin/bash
|
|
# Automated Backup Scheduler
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
BACKUP_SCRIPT="$SCRIPT_DIR/incremental_backup_system.sh"
|
|
|
|
# Determine backup type based on day of week and time
|
|
HOUR=$(date +%H)
|
|
DOW=$(date +%u) # 1=Monday, 7=Sunday
|
|
|
|
# Full backup every Sunday at 2 AM
|
|
if [[ $DOW -eq 7 ]] && [[ $HOUR -eq 2 ]]; then
|
|
exec "$BACKUP_SCRIPT" full
|
|
# Incremental backups every 6 hours
|
|
elif [[ $((HOUR % 6)) -eq 0 ]]; then
|
|
exec "$BACKUP_SCRIPT" incremental
|
|
else
|
|
echo "No backup scheduled for $(date)"
|
|
exit 0
|
|
fi
|
|
EOF
|
|
|
|
chmod +x "/opt/migration/scripts/backup_scheduler.sh"
|
|
|
|
# Create systemd service for backup scheduler
|
|
cat > "/tmp/backup-scheduler.service" << 'EOF'
|
|
[Unit]
|
|
Description=Incremental Backup Scheduler
|
|
Wants=backup-scheduler.timer
|
|
|
|
[Service]
|
|
Type=oneshot
|
|
ExecStart=/opt/migration/scripts/backup_scheduler.sh
|
|
User=root
|
|
StandardOutput=journal
|
|
StandardError=journal
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
|
|
# Create systemd timer for backup scheduler
|
|
cat > "/tmp/backup-scheduler.timer" << 'EOF'
|
|
[Unit]
|
|
Description=Run backup scheduler every hour
|
|
Requires=backup-scheduler.service
|
|
|
|
[Timer]
|
|
OnCalendar=hourly
|
|
Persistent=true
|
|
|
|
[Install]
|
|
WantedBy=timers.target
|
|
EOF
|
|
|
|
# Install systemd service and timer
|
|
sudo mv /tmp/backup-scheduler.service /etc/systemd/system/
|
|
sudo mv /tmp/backup-scheduler.timer /etc/systemd/system/
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable backup-scheduler.timer
|
|
sudo systemctl start backup-scheduler.timer
|
|
|
|
log_success "Backup scheduling configured"
|
|
|
|
# Create backup monitoring script
|
|
create_backup_monitoring
|
|
}
|
|
|
|
# Function to create backup monitoring
|
|
create_backup_monitoring() {
|
|
log_step "Creating backup monitoring system..."
|
|
|
|
cat > "/opt/migration/scripts/backup_monitor.sh" << 'EOF'
|
|
#!/bin/bash
|
|
# Backup Health Monitor
|
|
|
|
BACKUP_BASE_DIR="/opt/migration/backups"
|
|
BACKUP_METADATA_DIR="$BACKUP_BASE_DIR/metadata"
|
|
ALERT_LOG="/var/log/backup_monitor.log"
|
|
|
|
log_alert() {
|
|
echo "$(date): BACKUP_ALERT - $1" | tee -a "$ALERT_LOG"
|
|
logger "BACKUP_HEALTH_ALERT: $1"
|
|
}
|
|
|
|
check_backup_freshness() {
|
|
local max_age_hours=8 # Alert if no backup in 8 hours
|
|
local last_backup=$(find "$BACKUP_BASE_DIR/incremental" "$BACKUP_BASE_DIR/full" -name "20*" -type d -o -name "*.tar.*" -type f | sort | tail -1)
|
|
|
|
if [[ -n "$last_backup" ]]; then
|
|
local backup_age_hours=$(( ($(date +%s) - $(stat -c %Y "$last_backup")) / 3600 ))
|
|
|
|
if [[ $backup_age_hours -gt $max_age_hours ]]; then
|
|
log_alert "Last backup is $backup_age_hours hours old (threshold: $max_age_hours hours)"
|
|
fi
|
|
else
|
|
log_alert "No backups found in backup directories"
|
|
fi
|
|
}
|
|
|
|
check_backup_size_anomalies() {
|
|
local manifest_file="$BACKUP_METADATA_DIR/master_manifest.json"
|
|
|
|
if [[ -f "$manifest_file" ]]; then
|
|
# Check for significant size variations (>50% change)
|
|
# This would require historical data analysis
|
|
local current_total_size=$(jq -r '.statistics.total_size_bytes // 0' "$manifest_file")
|
|
|
|
# Simple check: alert if total backup size is suspiciously small
|
|
if [[ $current_total_size -lt $((1024*1024*100)) ]]; then # Less than 100MB
|
|
log_alert "Total backup size appears too small: $(numfmt --to=iec $current_total_size)"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
check_failed_backups() {
|
|
local recent_logs=$(find "$BACKUP_BASE_DIR/logs" -name "backup_*.log" -mtime -1)
|
|
|
|
for log_file in $recent_logs; do
|
|
if grep -q "ERROR\|FAILED" "$log_file"; then
|
|
log_alert "Errors found in recent backup: $(basename "$log_file")"
|
|
fi
|
|
done
|
|
}
|
|
|
|
check_storage_space() {
|
|
local backup_disk_usage=$(df -h "$BACKUP_BASE_DIR" | awk 'NR==2 {print $5}' | sed 's/%//')
|
|
|
|
if [[ $backup_disk_usage -gt 85 ]]; then
|
|
log_alert "Backup storage is ${backup_disk_usage}% full"
|
|
fi
|
|
}
|
|
|
|
# Main monitoring checks
|
|
check_backup_freshness
|
|
check_backup_size_anomalies
|
|
check_failed_backups
|
|
check_storage_space
|
|
|
|
# Export metrics for Prometheus
|
|
cat > "/tmp/backup_metrics.prom" << METRICS_EOF
|
|
# HELP backup_last_success_timestamp Unix timestamp of last successful backup
|
|
# TYPE backup_last_success_timestamp gauge
|
|
backup_last_success_timestamp $(stat -c %Y "$(find "$BACKUP_BASE_DIR" -name "20*" | sort | tail -1)" 2>/dev/null || echo 0)
|
|
|
|
# HELP backup_total_size_bytes Total size of all backups in bytes
|
|
# TYPE backup_total_size_bytes gauge
|
|
backup_total_size_bytes $(du -sb "$BACKUP_BASE_DIR" 2>/dev/null | cut -f1 || echo 0)
|
|
|
|
# HELP backup_disk_usage_percent Disk usage percentage for backup storage
|
|
# TYPE backup_disk_usage_percent gauge
|
|
backup_disk_usage_percent $(df "$BACKUP_BASE_DIR" | awk 'NR==2 {print $5}' | sed 's/%//' || echo 0)
|
|
METRICS_EOF
|
|
|
|
# Serve metrics for Prometheus scraping
|
|
if command -v nc >/dev/null 2>&1; then
|
|
(echo -e "HTTP/1.1 200 OK\nContent-Type: text/plain\n"; cat /tmp/backup_metrics.prom) | nc -l -p 9998 -q 1 &
|
|
fi
|
|
EOF
|
|
|
|
chmod +x "/opt/migration/scripts/backup_monitor.sh"
|
|
|
|
# Create systemd service for backup monitoring
|
|
cat > "/tmp/backup-monitor.service" << 'EOF'
|
|
[Unit]
|
|
Description=Backup Health Monitor
|
|
After=network.target
|
|
|
|
[Service]
|
|
ExecStart=/opt/migration/scripts/backup_monitor.sh
|
|
Restart=always
|
|
RestartSec=300
|
|
User=root
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
|
|
sudo mv /tmp/backup-monitor.service /etc/systemd/system/
|
|
sudo systemctl daemon-reload
|
|
sudo systemctl enable backup-monitor.service
|
|
sudo systemctl start backup-monitor.service
|
|
|
|
log_success "Backup monitoring system created"
|
|
}
|
|
|
|
# Main execution function
|
|
main() {
|
|
local action=${1:-"setup"}
|
|
|
|
# Register cleanup and rollback functions
|
|
register_cleanup cleanup_backup_system
|
|
register_rollback rollback_backup_system
|
|
|
|
case $action in
|
|
"setup")
|
|
log_step "Setting up incremental backup system..."
|
|
|
|
# Validate prerequisites
|
|
validate_prerequisites rsync gpg jq
|
|
|
|
# Create backup configuration
|
|
create_backup_configuration
|
|
create_checkpoint "backup_config_created"
|
|
|
|
# Setup backup infrastructure
|
|
setup_backup_infrastructure
|
|
create_checkpoint "backup_infrastructure_ready"
|
|
|
|
# Setup scheduling and monitoring
|
|
setup_backup_scheduling
|
|
create_checkpoint "backup_scheduling_setup"
|
|
|
|
log_success "✅ Incremental backup system setup completed!"
|
|
log_info "📅 Automated scheduling: Incremental every 6 hours, Full weekly"
|
|
log_info "📊 Monitoring: systemctl status backup-monitor"
|
|
log_info "🔧 Manual backup: $0 incremental|full [host]"
|
|
;;
|
|
|
|
"incremental"|"full")
|
|
local target_host=${2:-"all"}
|
|
perform_incremental_backup "$action" "$target_host"
|
|
;;
|
|
|
|
"cleanup")
|
|
cleanup_old_backups "incremental"
|
|
cleanup_old_backups "full"
|
|
;;
|
|
|
|
"verify")
|
|
local session_id=${2:-"latest"}
|
|
verify_backup_integrity "$session_id"
|
|
;;
|
|
|
|
"help"|*)
|
|
cat << EOF
|
|
Incremental Backup System
|
|
|
|
Usage: $0 <action> [options]
|
|
|
|
Actions:
|
|
setup - Setup backup system infrastructure
|
|
incremental - Run incremental backup [host]
|
|
full - Run full backup [host]
|
|
cleanup - Clean up old backups
|
|
verify - Verify backup integrity [session_id]
|
|
help - Show this help
|
|
|
|
Examples:
|
|
$0 setup
|
|
$0 incremental
|
|
$0 full omv800
|
|
$0 cleanup
|
|
$0 verify
|
|
EOF
|
|
;;
|
|
esac
|
|
}
|
|
|
|
# Execute main function
|
|
main "$@" |