COMPREHENSIVE CHANGES: INFRASTRUCTURE MIGRATION: - Migrated services to Docker Swarm on OMV800 (192.168.50.229) - Deployed PostgreSQL database for Vaultwarden migration - Updated all stack configurations for Docker Swarm compatibility - Added comprehensive monitoring stack (Prometheus, Grafana, Blackbox) - Implemented proper secret management for all services VAULTWARDEN POSTGRESQL MIGRATION: - Attempted migration from SQLite to PostgreSQL for NFS compatibility - Created PostgreSQL stack with proper user/password configuration - Built custom Vaultwarden image with PostgreSQL support - Troubleshot persistent SQLite fallback issue despite PostgreSQL config - Identified known issue where Vaultwarden silently falls back to SQLite - Added ENABLE_DB_WAL=false to prevent filesystem compatibility issues - Current status: Old Vaultwarden on lenovo410 still working, new one has config issues PAPERLESS SERVICES: - Successfully deployed Paperless-NGX and Paperless-AI on OMV800 - Both services running on ports 8000 and 3000 respectively - Caddy configuration updated for external access - Services accessible via paperless.pressmess.duckdns.org and paperless-ai.pressmess.duckdns.org CADDY CONFIGURATION: - Updated Caddyfile on Surface (192.168.50.254) for new service locations - Fixed Vaultwarden reverse proxy to point to new Docker Swarm service - Removed old notification hub reference that was causing conflicts - All services properly configured for external access via DuckDNS BACKUP AND DISCOVERY: - Created comprehensive backup system for all hosts - Generated detailed discovery reports for infrastructure analysis - Implemented automated backup validation scripts - Created migration progress tracking and verification reports MONITORING STACK: - Deployed Prometheus, Grafana, and Blackbox monitoring - Created infrastructure and system overview dashboards - Added proper service discovery and alerting configuration - Implemented performance monitoring for all critical services DOCUMENTATION: - Reorganized documentation into logical structure - Created comprehensive migration playbook and troubleshooting guides - Added hardware specifications and optimization recommendations - Documented all configuration changes and service dependencies CURRENT STATUS: - Paperless services: ✅ Working and accessible externally - Vaultwarden: ❌ PostgreSQL configuration issues, old instance still working - Monitoring: ✅ Deployed and operational - Caddy: ✅ Updated and working for external access - PostgreSQL: ✅ Database running, connection issues with Vaultwarden NEXT STEPS: - Continue troubleshooting Vaultwarden PostgreSQL configuration - Consider alternative approaches for Vaultwarden migration - Validate all external service access - Complete final migration validation TECHNICAL NOTES: - Used Docker Swarm for orchestration on OMV800 - Implemented proper secret management for sensitive data - Added comprehensive logging and monitoring - Created automated backup and validation scripts
215 lines
6.6 KiB
Bash
Executable File
215 lines
6.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Vaultwarden SQLite Database Migration Script
|
|
# Safely migrates Vaultwarden data from lenovo410 to new Docker Swarm infrastructure
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
SOURCE_HOST="jonathan@192.168.50.181"
|
|
SOURCE_PATH="/home/jonathan/vaultwarden/data"
|
|
BACKUP_DIR="./backups/vaultwarden"
|
|
TARGET_PATH="/export/vaultwarden"
|
|
LOG_FILE="./logs/vaultwarden_migration.log"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Logging function
|
|
log() {
|
|
echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] SUCCESS:${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1" | tee -a "$LOG_FILE"
|
|
}
|
|
|
|
# Create necessary directories
|
|
mkdir -p "$BACKUP_DIR"
|
|
mkdir -p "$(dirname "$LOG_FILE")"
|
|
|
|
log "Starting Vaultwarden SQLite database migration"
|
|
|
|
# Step 1: Verify source Vaultwarden is running and healthy
|
|
log "Step 1: Verifying source Vaultwarden container status"
|
|
if ! ssh "$SOURCE_HOST" "docker ps | grep -q vaultwarden"; then
|
|
log_error "Vaultwarden container is not running on $SOURCE_HOST"
|
|
exit 1
|
|
fi
|
|
|
|
# Get container ID
|
|
CONTAINER_ID=$(ssh "$SOURCE_HOST" "docker ps | grep vaultwarden | awk '{print \$1}'")
|
|
log "Found Vaultwarden container: $CONTAINER_ID"
|
|
|
|
# Step 2: Create comprehensive backup
|
|
log "Step 2: Creating comprehensive backup of current Vaultwarden data"
|
|
BACKUP_FILE="$BACKUP_DIR/vaultwarden_backup_$(date +%Y%m%d_%H%M%S).tar.gz"
|
|
|
|
# Stop Vaultwarden temporarily for consistent backup
|
|
log "Stopping Vaultwarden container for consistent backup"
|
|
ssh "$SOURCE_HOST" "docker stop $CONTAINER_ID"
|
|
|
|
# Wait a moment for graceful shutdown
|
|
sleep 5
|
|
|
|
# Create backup
|
|
log "Creating backup archive"
|
|
ssh "$SOURCE_HOST" "tar czf - -C $SOURCE_PATH ." > "$BACKUP_FILE"
|
|
|
|
# Verify backup size
|
|
BACKUP_SIZE=$(stat -c%s "$BACKUP_FILE")
|
|
log "Backup created: $BACKUP_FILE (${BACKUP_SIZE} bytes)"
|
|
|
|
if [ "$BACKUP_SIZE" -lt 1000000 ]; then
|
|
log_warning "Backup seems small, verifying contents"
|
|
tar tzf "$BACKUP_FILE" | head -10
|
|
fi
|
|
|
|
# Step 3: Restart source Vaultwarden
|
|
log "Step 3: Restarting source Vaultwarden container"
|
|
ssh "$SOURCE_HOST" "docker start $CONTAINER_ID"
|
|
|
|
# Wait for container to be healthy
|
|
log "Waiting for Vaultwarden to be healthy"
|
|
for i in {1..30}; do
|
|
if ssh "$SOURCE_HOST" "docker ps | grep -q vaultwarden.*healthy"; then
|
|
log_success "Vaultwarden container is healthy"
|
|
break
|
|
fi
|
|
if [ $i -eq 30 ]; then
|
|
log_error "Vaultwarden container failed to become healthy"
|
|
exit 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Step 4: Verify NFS export exists and is accessible
|
|
log "Step 4: Verifying NFS export accessibility"
|
|
if [ ! -d "$TARGET_PATH" ]; then
|
|
log_error "Target NFS path $TARGET_PATH does not exist"
|
|
log "Please ensure the NFS export is properly configured on OMV800"
|
|
exit 1
|
|
fi
|
|
|
|
# Test write access
|
|
if ! touch "$TARGET_PATH/test_write_access" 2>/dev/null; then
|
|
log_error "Cannot write to target NFS path $TARGET_PATH"
|
|
exit 1
|
|
fi
|
|
rm -f "$TARGET_PATH/test_write_access"
|
|
|
|
# Step 5: Extract backup to target location
|
|
log "Step 5: Extracting backup to target location"
|
|
cd "$TARGET_PATH"
|
|
tar xzf "$BACKUP_FILE"
|
|
|
|
# Verify extraction
|
|
if [ ! -f "$TARGET_PATH/db.sqlite3" ]; then
|
|
log_error "SQLite database not found in target location"
|
|
exit 1
|
|
fi
|
|
|
|
log_success "Database extracted to target location"
|
|
|
|
# Step 6: Set proper permissions
|
|
log "Step 6: Setting proper permissions"
|
|
chmod 644 "$TARGET_PATH/db.sqlite3"
|
|
chmod 644 "$TARGET_PATH/rsa_key.pem"
|
|
chmod -R 755 "$TARGET_PATH/attachments"
|
|
chmod -R 755 "$TARGET_PATH/icon_cache"
|
|
chmod -R 755 "$TARGET_PATH/sends"
|
|
chmod -R 755 "$TARGET_PATH/tmp"
|
|
|
|
# Step 7: Verify database integrity
|
|
log "Step 7: Verifying database integrity"
|
|
if [ -f "$TARGET_PATH/db.sqlite3" ]; then
|
|
# Check if database is readable
|
|
if file "$TARGET_PATH/db.sqlite3" | grep -q "SQLite"; then
|
|
log_success "SQLite database format verified"
|
|
else
|
|
log_error "Database file does not appear to be a valid SQLite database"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_error "Database file not found in target location"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 8: Create Docker secret for admin token if it doesn't exist
|
|
log "Step 8: Creating admin token secret"
|
|
if ! docker secret ls | grep -q vaultwarden_admin_token; then
|
|
# Generate a secure admin token
|
|
ADMIN_TOKEN=$(openssl rand -base64 32)
|
|
echo "$ADMIN_TOKEN" | docker secret create vaultwarden_admin_token -
|
|
log_success "Created vaultwarden_admin_token secret"
|
|
log "Admin token generated. You can access the admin interface at:"
|
|
log "https://vaultwarden.pressmess.duckdns.org/admin"
|
|
log "Token: $ADMIN_TOKEN"
|
|
else
|
|
log "Admin token secret already exists"
|
|
fi
|
|
|
|
# Step 9: Deploy new Vaultwarden stack
|
|
log "Step 9: Deploying new Vaultwarden stack"
|
|
if docker stack deploy -c stacks/apps/vaultwarden.yml vaultwarden; then
|
|
log_success "Vaultwarden stack deployed successfully"
|
|
else
|
|
log_error "Failed to deploy Vaultwarden stack"
|
|
exit 1
|
|
fi
|
|
|
|
# Step 10: Wait for new service to be ready
|
|
log "Step 10: Waiting for new Vaultwarden service to be ready"
|
|
for i in {1..60}; do
|
|
if docker service ls | grep -q vaultwarden.*1/1; then
|
|
log_success "New Vaultwarden service is running"
|
|
break
|
|
fi
|
|
if [ $i -eq 60 ]; then
|
|
log_error "New Vaultwarden service failed to start"
|
|
exit 1
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
# Step 11: Verify new service is accessible
|
|
log "Step 11: Verifying new service accessibility"
|
|
sleep 10 # Give service time to fully initialize
|
|
|
|
if curl -s -f "https://vaultwarden.pressmess.duckdns.org" > /dev/null; then
|
|
log_success "New Vaultwarden service is accessible"
|
|
else
|
|
log_warning "New Vaultwarden service may not be accessible yet (this is normal during startup)"
|
|
fi
|
|
|
|
# Step 12: Validation period
|
|
log "Step 12: Starting validation period"
|
|
log "Vaultwarden migration completed successfully!"
|
|
log ""
|
|
log "IMPORTANT: Please test the new Vaultwarden service for the next 24 hours:"
|
|
log "1. Access https://vaultwarden.pressmess.duckdns.org"
|
|
log "2. Verify all your passwords and data are present"
|
|
log "3. Test login/logout functionality"
|
|
log "4. Test password creation and editing"
|
|
log "5. Test browser extensions if you use them"
|
|
log ""
|
|
log "If everything works correctly after 24 hours, you can stop the old service:"
|
|
log "ssh $SOURCE_HOST 'docker stop $CONTAINER_ID'"
|
|
log ""
|
|
log "Backup location: $BACKUP_FILE"
|
|
log "Migration log: $LOG_FILE"
|
|
|
|
log_success "Vaultwarden SQLite migration completed successfully!"
|