COMPREHENSIVE CHANGES: INFRASTRUCTURE MIGRATION: - Migrated services to Docker Swarm on OMV800 (192.168.50.229) - Deployed PostgreSQL database for Vaultwarden migration - Updated all stack configurations for Docker Swarm compatibility - Added comprehensive monitoring stack (Prometheus, Grafana, Blackbox) - Implemented proper secret management for all services VAULTWARDEN POSTGRESQL MIGRATION: - Attempted migration from SQLite to PostgreSQL for NFS compatibility - Created PostgreSQL stack with proper user/password configuration - Built custom Vaultwarden image with PostgreSQL support - Troubleshot persistent SQLite fallback issue despite PostgreSQL config - Identified known issue where Vaultwarden silently falls back to SQLite - Added ENABLE_DB_WAL=false to prevent filesystem compatibility issues - Current status: Old Vaultwarden on lenovo410 still working, new one has config issues PAPERLESS SERVICES: - Successfully deployed Paperless-NGX and Paperless-AI on OMV800 - Both services running on ports 8000 and 3000 respectively - Caddy configuration updated for external access - Services accessible via paperless.pressmess.duckdns.org and paperless-ai.pressmess.duckdns.org CADDY CONFIGURATION: - Updated Caddyfile on Surface (192.168.50.254) for new service locations - Fixed Vaultwarden reverse proxy to point to new Docker Swarm service - Removed old notification hub reference that was causing conflicts - All services properly configured for external access via DuckDNS BACKUP AND DISCOVERY: - Created comprehensive backup system for all hosts - Generated detailed discovery reports for infrastructure analysis - Implemented automated backup validation scripts - Created migration progress tracking and verification reports MONITORING STACK: - Deployed Prometheus, Grafana, and Blackbox monitoring - Created infrastructure and system overview dashboards - Added proper service discovery and alerting configuration - Implemented performance monitoring for all critical services DOCUMENTATION: - Reorganized documentation into logical structure - Created comprehensive migration playbook and troubleshooting guides - Added hardware specifications and optimization recommendations - Documented all configuration changes and service dependencies CURRENT STATUS: - Paperless services: ✅ Working and accessible externally - Vaultwarden: ❌ PostgreSQL configuration issues, old instance still working - Monitoring: ✅ Deployed and operational - Caddy: ✅ Updated and working for external access - PostgreSQL: ✅ Database running, connection issues with Vaultwarden NEXT STEPS: - Continue troubleshooting Vaultwarden PostgreSQL configuration - Consider alternative approaches for Vaultwarden migration - Validate all external service access - Complete final migration validation TECHNICAL NOTES: - Used Docker Swarm for orchestration on OMV800 - Implemented proper secret management for sensitive data - Added comprehensive logging and monitoring - Created automated backup and validation scripts
143 lines
5.9 KiB
Bash
Executable File
143 lines
5.9 KiB
Bash
Executable File
#!/bin/bash
|
|
# Document Current Infrastructure State
|
|
# This script creates a complete snapshot of the current infrastructure
|
|
|
|
set -euo pipefail
|
|
|
|
echo "🔍 Documenting current infrastructure state..."
|
|
|
|
# Create timestamp for this snapshot
|
|
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
SNAPSHOT_DIR="/opt/migration/backups/snapshot_${TIMESTAMP}"
|
|
mkdir -p "$SNAPSHOT_DIR"
|
|
|
|
# Define hosts
|
|
HOSTS=("omv800" "fedora" "surface" "jonathan-2518f5u" "audrey" "raspberrypi")
|
|
HOST_IPS=("192.168.50.229" "192.168.50.225" "192.168.50.254" "192.168.50.181" "192.168.50.145" "192.168.50.107")
|
|
|
|
echo "📋 Creating snapshot in: $SNAPSHOT_DIR"
|
|
|
|
# 1. Docker state documentation
|
|
echo "📦 Documenting Docker state..."
|
|
for i in "${!HOSTS[@]}"; do
|
|
host="${HOSTS[$i]}"
|
|
ip="${HOST_IPS[$i]}"
|
|
|
|
echo " Processing $host ($ip)..."
|
|
|
|
# Create host-specific directory
|
|
host_dir="$SNAPSHOT_DIR/$host"
|
|
mkdir -p "$host_dir"
|
|
|
|
# Docker containers
|
|
ssh -o ConnectTimeout=10 "$host" "docker ps -a --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'" > "$host_dir/docker_containers.txt" 2>/dev/null || echo "No Docker on $host" > "$host_dir/docker_containers.txt"
|
|
|
|
# Docker images
|
|
ssh -o ConnectTimeout=10 "$host" "docker images" > "$host_dir/docker_images.txt" 2>/dev/null || echo "No Docker images on $host" > "$host_dir/docker_images.txt"
|
|
|
|
# Docker networks
|
|
ssh -o ConnectTimeout=10 "$host" "docker network ls" > "$host_dir/docker_networks.txt" 2>/dev/null || echo "No Docker networks on $host" > "$host_dir/docker_networks.txt"
|
|
|
|
# Docker volumes
|
|
ssh -o ConnectTimeout=10 "$host" "docker volume ls" > "$host_dir/docker_volumes.txt" 2>/dev/null || echo "No Docker volumes on $host" > "$host_dir/docker_volumes.txt"
|
|
|
|
# Docker compose files
|
|
ssh -o ConnectTimeout=10 "$host" "find /opt /home -name 'docker-compose*.yml' -exec cat {} \;" > "$host_dir/docker_compose_files.txt" 2>/dev/null || echo "No docker-compose files found on $host" > "$host_dir/docker_compose_files.txt"
|
|
done
|
|
|
|
# 2. Database dumps
|
|
echo "🗄️ Creating database dumps..."
|
|
DUMP_DIR="$SNAPSHOT_DIR/database_dumps"
|
|
mkdir -p "$DUMP_DIR"
|
|
|
|
# PostgreSQL dumps
|
|
for host in "omv800" "surface" "jonathan-2518f5u"; do
|
|
echo " Dumping PostgreSQL from $host..."
|
|
ssh -o ConnectTimeout=10 "$host" "docker ps | grep postgres" > /dev/null 2>&1 && {
|
|
ssh "$host" "docker exec \$(docker ps -q --filter 'ancestor=postgres') pg_dumpall > /tmp/postgres_dump_${host}.sql"
|
|
scp "$host:/tmp/postgres_dump_${host}.sql" "$DUMP_DIR/"
|
|
} || echo "No PostgreSQL found on $host" > "$DUMP_DIR/postgres_dump_${host}.sql"
|
|
done
|
|
|
|
# 3. Configuration backups
|
|
echo "⚙️ Backing up configurations..."
|
|
for i in "${!HOSTS[@]}"; do
|
|
host="${HOSTS[$i]}"
|
|
echo " Backing up configs from $host..."
|
|
|
|
ssh -o ConnectTimeout=10 "$host" "tar czf /tmp/config_backup_${host}.tar.gz /etc/docker /opt /home/*/.config 2>/dev/null || echo 'No configs to backup'" > /dev/null 2>&1
|
|
scp "$host:/tmp/config_backup_${host}.tar.gz" "$SNAPSHOT_DIR/" 2>/dev/null || echo "No config backup available for $host" > "$SNAPSHOT_DIR/config_backup_${host}.txt"
|
|
done
|
|
|
|
# 4. File system snapshots
|
|
echo "💾 Creating file system snapshots..."
|
|
for host in "omv800" "surface" "jonathan-2518f5u"; do
|
|
echo " Creating FS snapshot for $host..."
|
|
|
|
ssh -o ConnectTimeout=10 "$host" "sudo tar czf /tmp/fs_snapshot_${host}.tar.gz /mnt /var/lib/docker 2>/dev/null || echo 'No files to snapshot'" > /dev/null 2>&1
|
|
scp "$host:/tmp/fs_snapshot_${host}.tar.gz" "$SNAPSHOT_DIR/" 2>/dev/null || echo "No FS snapshot available for $host" > "$SNAPSHOT_DIR/fs_snapshot_${host}.txt"
|
|
done
|
|
|
|
# 5. Network configuration
|
|
echo "🌐 Documenting network configuration..."
|
|
for i in "${!HOSTS[@]}"; do
|
|
host="${HOSTS[$i]}"
|
|
echo " Documenting network for $host..."
|
|
|
|
ssh -o ConnectTimeout=10 "$host" "ip addr show" > "$SNAPSHOT_DIR/network_${host}.txt" 2>/dev/null || echo "Cannot get network info for $host" > "$SNAPSHOT_DIR/network_${host}.txt"
|
|
ssh -o ConnectTimeout=10 "$host" "ip route show" > "$SNAPSHOT_DIR/routing_${host}.txt" 2>/dev/null || echo "Cannot get routing info for $host" > "$SNAPSHOT_DIR/routing_${host}.txt"
|
|
done
|
|
|
|
# 6. Service health status
|
|
echo "🏥 Documenting service health..."
|
|
for i in "${!HOSTS[@]}"; do
|
|
host="${HOSTS[$i]}"
|
|
echo " Checking health for $host..."
|
|
|
|
ssh -o ConnectTimeout=10 "$host" "docker ps --format 'table {{.Names}}\t{{.Status}}\t{{.Ports}}'" > "$SNAPSHOT_DIR/health_${host}.txt" 2>/dev/null || echo "No Docker health info for $host" > "$SNAPSHOT_DIR/health_${host}.txt"
|
|
done
|
|
|
|
# 7. System information
|
|
echo "💻 Collecting system information..."
|
|
for i in "${!HOSTS[@]}"; do
|
|
host="${HOSTS[$i]}"
|
|
echo " Getting system info for $host..."
|
|
|
|
ssh -o ConnectTimeout=10 "$host" "uname -a && df -h && free -h && uptime" > "$SNAPSHOT_DIR/system_${host}.txt" 2>/dev/null || echo "Cannot get system info for $host" > "$SNAPSHOT_DIR/system_${host}.txt"
|
|
done
|
|
|
|
# 8. Create summary report
|
|
echo "📋 Creating summary report..."
|
|
cat > "$SNAPSHOT_DIR/summary.txt" << EOF
|
|
Infrastructure Snapshot Summary
|
|
Generated: $(date)
|
|
Snapshot Directory: $SNAPSHOT_DIR
|
|
|
|
Hosts Documented:
|
|
$(for i in "${!HOSTS[@]}"; do echo " - ${HOSTS[$i]}: ${HOST_IPS[$i]}"; done)
|
|
|
|
Files Created:
|
|
$(find "$SNAPSHOT_DIR" -type f | wc -l) total files
|
|
$(du -sh "$SNAPSHOT_DIR" | cut -f1) total size
|
|
|
|
Critical Services Found:
|
|
$(grep -r "immich\|jellyfin\|homeassistant\|appflowy\|paperless" "$SNAPSHOT_DIR" | head -10)
|
|
|
|
Database Dumps:
|
|
$(ls -la "$DUMP_DIR"/*.sql 2>/dev/null | wc -l) PostgreSQL dumps
|
|
|
|
Next Steps:
|
|
1. Verify all critical data is captured
|
|
2. Test backup restoration procedures
|
|
3. Proceed with migration planning
|
|
EOF
|
|
|
|
echo "✅ Current state documented in $SNAPSHOT_DIR"
|
|
echo "📋 Snapshot summary:"
|
|
cat "$SNAPSHOT_DIR/summary.txt"
|
|
|
|
# Create symbolic link to latest
|
|
ln -sfn "$SNAPSHOT_DIR" "/opt/migration/backups/latest"
|
|
|
|
echo "🔗 Latest snapshot linked to: /opt/migration/backups/latest"
|