COMPREHENSIVE CHANGES: INFRASTRUCTURE MIGRATION: - Migrated services to Docker Swarm on OMV800 (192.168.50.229) - Deployed PostgreSQL database for Vaultwarden migration - Updated all stack configurations for Docker Swarm compatibility - Added comprehensive monitoring stack (Prometheus, Grafana, Blackbox) - Implemented proper secret management for all services VAULTWARDEN POSTGRESQL MIGRATION: - Attempted migration from SQLite to PostgreSQL for NFS compatibility - Created PostgreSQL stack with proper user/password configuration - Built custom Vaultwarden image with PostgreSQL support - Troubleshot persistent SQLite fallback issue despite PostgreSQL config - Identified known issue where Vaultwarden silently falls back to SQLite - Added ENABLE_DB_WAL=false to prevent filesystem compatibility issues - Current status: Old Vaultwarden on lenovo410 still working, new one has config issues PAPERLESS SERVICES: - Successfully deployed Paperless-NGX and Paperless-AI on OMV800 - Both services running on ports 8000 and 3000 respectively - Caddy configuration updated for external access - Services accessible via paperless.pressmess.duckdns.org and paperless-ai.pressmess.duckdns.org CADDY CONFIGURATION: - Updated Caddyfile on Surface (192.168.50.254) for new service locations - Fixed Vaultwarden reverse proxy to point to new Docker Swarm service - Removed old notification hub reference that was causing conflicts - All services properly configured for external access via DuckDNS BACKUP AND DISCOVERY: - Created comprehensive backup system for all hosts - Generated detailed discovery reports for infrastructure analysis - Implemented automated backup validation scripts - Created migration progress tracking and verification reports MONITORING STACK: - Deployed Prometheus, Grafana, and Blackbox monitoring - Created infrastructure and system overview dashboards - Added proper service discovery and alerting configuration - Implemented performance monitoring for all critical services DOCUMENTATION: - Reorganized documentation into logical structure - Created comprehensive migration playbook and troubleshooting guides - Added hardware specifications and optimization recommendations - Documented all configuration changes and service dependencies CURRENT STATUS: - Paperless services: ✅ Working and accessible externally - Vaultwarden: ❌ PostgreSQL configuration issues, old instance still working - Monitoring: ✅ Deployed and operational - Caddy: ✅ Updated and working for external access - PostgreSQL: ✅ Database running, connection issues with Vaultwarden NEXT STEPS: - Continue troubleshooting Vaultwarden PostgreSQL configuration - Consider alternative approaches for Vaultwarden migration - Validate all external service access - Complete final migration validation TECHNICAL NOTES: - Used Docker Swarm for orchestration on OMV800 - Implemented proper secret management for sensitive data - Added comprehensive logging and monitoring - Created automated backup and validation scripts
203 lines
5.5 KiB
Bash
Executable File
203 lines
5.5 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Safe Vaultwarden Backup Script
|
|
# Creates comprehensive backups without requiring NFS write access
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SOURCE_HOST="jonathan@192.168.50.181"
|
|
SOURCE_PATH="/home/jonathan/vaultwarden/data"
|
|
BACKUP_DIR="./backups/vaultwarden"
|
|
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Logging function
|
|
log() {
|
|
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] SUCCESS:${NC} $1"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1"
|
|
}
|
|
|
|
# Create backup directory
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
log "Starting comprehensive Vaultwarden backup"
|
|
|
|
# Step 1: Verify source Vaultwarden is running
|
|
log "Step 1: Verifying source Vaultwarden container status"
|
|
if ! ssh "$SOURCE_HOST" "docker ps | grep -q vaultwarden"; then
|
|
log_error "Vaultwarden container is not running on $SOURCE_HOST"
|
|
exit 1
|
|
fi
|
|
|
|
# Get container ID
|
|
CONTAINER_ID=$(ssh "$SOURCE_HOST" "docker ps | grep vaultwarden | awk '{print \$1}'")
|
|
log "Found Vaultwarden container: $CONTAINER_ID"
|
|
|
|
# Step 2: Create comprehensive backup
|
|
log "Step 2: Creating comprehensive backup"
|
|
BACKUP_FILE="$BACKUP_DIR/vaultwarden_complete_backup_${TIMESTAMP}.tar.gz"
|
|
|
|
# Stop Vaultwarden temporarily for consistent backup
|
|
log "Stopping Vaultwarden container for consistent backup"
|
|
ssh "$SOURCE_HOST" "docker stop $CONTAINER_ID"
|
|
|
|
# Wait a moment for graceful shutdown
|
|
sleep 5
|
|
|
|
# Create backup
|
|
log "Creating backup archive"
|
|
ssh "$SOURCE_HOST" "tar czf - -C $SOURCE_PATH ." > "$BACKUP_FILE"
|
|
|
|
# Verify backup was created
|
|
if [ -f "$BACKUP_FILE" ]; then
|
|
log_success "Backup file created successfully"
|
|
else
|
|
log_error "Failed to create backup file"
|
|
exit 1
|
|
fi
|
|
|
|
# Verify backup size
|
|
BACKUP_SIZE=$(stat -c%s "$BACKUP_FILE")
|
|
log "Backup size: ${BACKUP_SIZE} bytes"
|
|
|
|
if [ "$BACKUP_SIZE" -gt 1000000 ]; then
|
|
log_success "Backup size is reasonable (${BACKUP_SIZE} bytes)"
|
|
else
|
|
log_warning "Backup seems small (${BACKUP_SIZE} bytes)"
|
|
fi
|
|
|
|
# Verify backup contents
|
|
log "Verifying backup contents"
|
|
BACKUP_CONTENTS=$(tar tzf "$BACKUP_FILE" | wc -l)
|
|
log "Backup contains $BACKUP_CONTENTS files"
|
|
|
|
if [ "$BACKUP_CONTENTS" -gt 5 ]; then
|
|
log_success "Backup contains expected number of files"
|
|
else
|
|
log_warning "Backup contains fewer files than expected"
|
|
fi
|
|
|
|
# Check for critical files in backup
|
|
if tar tzf "$BACKUP_FILE" | grep -q "db.sqlite3"; then
|
|
log_success "SQLite database included in backup"
|
|
else
|
|
log_error "SQLite database not found in backup"
|
|
exit 1
|
|
fi
|
|
|
|
if tar tzf "$BACKUP_FILE" | grep -q "rsa_key.pem"; then
|
|
log_success "RSA key included in backup"
|
|
else
|
|
log_error "RSA key not found in backup"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 3: Restart source Vaultwarden
|
|
log "Step 3: Restarting source Vaultwarden container"
|
|
ssh "$SOURCE_HOST" "docker start $CONTAINER_ID"
|
|
|
|
# Wait for container to be healthy
|
|
log "Waiting for Vaultwarden to be healthy"
|
|
for i in {1..30}; do
|
|
if ssh "$SOURCE_HOST" "docker ps | grep -q vaultwarden.*healthy"; then
|
|
log_success "Vaultwarden container is healthy"
|
|
break
|
|
fi
|
|
if [ $i -eq 30 ]; then
|
|
log_error "Vaultwarden container failed to become healthy"
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Step 4: Create secondary backup
|
|
log "Step 4: Creating secondary backup"
|
|
SECONDARY_BACKUP="/tmp/vaultwarden_emergency_backup_${TIMESTAMP}.tar.gz"
|
|
cp "$BACKUP_FILE" "$SECONDARY_BACKUP"
|
|
|
|
if [ -f "$SECONDARY_BACKUP" ]; then
|
|
log_success "Secondary backup created at $SECONDARY_BACKUP"
|
|
else
|
|
log_error "Failed to create secondary backup"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 5: Create backup manifest
|
|
log "Step 5: Creating backup manifest"
|
|
MANIFEST_FILE="$BACKUP_DIR/vaultwarden_backup_manifest_${TIMESTAMP}.txt"
|
|
|
|
cat > "$MANIFEST_FILE" << EOF
|
|
Vaultwarden Backup Manifest
|
|
===========================
|
|
Created: $(date)
|
|
Source Host: $SOURCE_HOST
|
|
Source Path: $SOURCE_PATH
|
|
Container ID: $CONTAINER_ID
|
|
|
|
Backup Files:
|
|
- Primary: $BACKUP_FILE (${BACKUP_SIZE} bytes)
|
|
- Secondary: $SECONDARY_BACKUP
|
|
|
|
Backup Contents:
|
|
$(tar tzf "$BACKUP_FILE" | head -20)
|
|
|
|
Total Files: $BACKUP_CONTENTS
|
|
|
|
Critical Files Verified:
|
|
- db.sqlite3: $(tar tzf "$BACKUP_FILE" | grep -c "db.sqlite3" || echo "0")
|
|
- rsa_key.pem: $(tar tzf "$BACKUP_FILE" | grep -c "rsa_key.pem" || echo "0")
|
|
- attachments/: $(tar tzf "$BACKUP_FILE" | grep -c "attachments/" || echo "0")
|
|
- icon_cache/: $(tar tzf "$BACKUP_FILE" | grep -c "icon_cache/" || echo "0")
|
|
- sends/: $(tar tzf "$BACKUP_FILE" | grep -c "sends/" || echo "0")
|
|
|
|
Restore Instructions:
|
|
1. Stop Vaultwarden container
|
|
2. Extract backup: tar xzf $BACKUP_FILE -C /target/path
|
|
3. Set permissions: chown -R 1000:1000 /target/path
|
|
4. Start Vaultwarden container
|
|
|
|
EOF
|
|
|
|
log_success "Backup manifest created: $MANIFEST_FILE"
|
|
|
|
# Step 6: Final summary
|
|
log "Step 6: Backup summary"
|
|
log ""
|
|
log "=== BACKUP COMPLETED SUCCESSFULLY ==="
|
|
log "Primary backup: $BACKUP_FILE"
|
|
log "Secondary backup: $SECONDARY_BACKUP"
|
|
log "Manifest: $MANIFEST_FILE"
|
|
log "Backup size: ${BACKUP_SIZE} bytes"
|
|
log "Files backed up: $BACKUP_CONTENTS"
|
|
log ""
|
|
log "=== NEXT STEPS ==="
|
|
log "1. Verify backup integrity: tar tzf $BACKUP_FILE"
|
|
log "2. Test restore in a safe environment"
|
|
log "3. Proceed with migration when ready"
|
|
log ""
|
|
log "⚠️ IMPORTANT: Keep these backup files safe!"
|
|
log " - Primary: $BACKUP_FILE"
|
|
log " - Secondary: $SECONDARY_BACKUP"
|
|
log " - Manifest: $MANIFEST_FILE"
|
|
log ""
|
|
|
|
log_success "Vaultwarden backup completed successfully!"
|