From bdc50f9e38528f3d156cf2769b3f75f0de2aa1b5 Mon Sep 17 00:00:00 2001 From: Jon Date: Sat, 2 Aug 2025 09:32:10 -0400 Subject: [PATCH] feat: Add GCS cleanup script for automated storage management --- cleanup_gcs.sh | 222 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100755 cleanup_gcs.sh diff --git a/cleanup_gcs.sh b/cleanup_gcs.sh new file mode 100755 index 0000000..60f57ef --- /dev/null +++ b/cleanup_gcs.sh @@ -0,0 +1,222 @@ +#!/bin/bash +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Configuration +PROJECT_ID="cim-summarizer" +UPLOADS_BUCKET="gs://cim-summarizer-uploads" +FIREBASE_UPLOADS_BUCKET="gs://gcf-v2-uploads-245796323861.us-central1.cloudfunctions.appspot.com" +FIREBASE_SOURCES_BUCKET="gs://gcf-v2-sources-245796323861-us-central1" + +# Function to print colored output +print_status() { + echo -e "${BLUE}[INFO]${NC} $1" +} + +print_success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +print_warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +print_error() { + echo -e "${RED}[ERROR]${NC} $1" +} + +# Function to get storage size in human readable format +format_size() { + local bytes=$1 + if [ $bytes -gt 1073741824 ]; then + echo "$(echo "scale=2; $bytes/1073741824" | bc) GB" + elif [ $bytes -gt 1048576 ]; then + echo "$(echo "scale=2; $bytes/1048576" | bc) MB" + elif [ $bytes -gt 1024 ]; then + echo "$(echo "scale=2; $bytes/1024" | bc) KB" + else + echo "${bytes} B" + fi +} + +# Function to calculate total size of files +calculate_size() { + local bucket=$1 + local size=$(gsutil ls -la "$bucket" | grep -E "TOTAL:" | awk '{print $2}' | head -1) + echo "${size:-0}" +} + +# Function to show usage +show_usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -d, --dry-run Show what would be deleted without actually deleting" + echo " -f, --force Skip confirmation prompts" + echo " -h, --help Show this help message" + echo "" + echo "This script cleans up old files from GCS buckets:" + echo " - Removes test files from uploads bucket" + echo " - Removes old Firebase function deployments (keeps 2 most recent)" + echo " - Removes old Firebase function sources (keeps 1 most recent)" + echo " - Cleans up empty directories" +} + +# Function to clean up test files +cleanup_test_files() { + print_status "Cleaning up test files..." + + local test_files=$(gsutil ls "$UPLOADS_BUCKET/test-*.txt" 2>/dev/null || true) + if [ -n "$test_files" ]; then + if [ "$DRY_RUN" = true ]; then + echo "Would remove: $test_files" + else + gsutil rm $test_files + print_success "Removed test files" + fi + else + print_status "No test files found" + fi +} + +# Function to clean up invalid files +cleanup_invalid_files() { + print_status "Cleaning up invalid files..." + + local invalid_files=$(gsutil ls "$UPLOADS_BUCKET/uploads/staging-test-user/*.exe" 2>/dev/null || true) + if [ -n "$invalid_files" ]; then + if [ "$DRY_RUN" = true ]; then + echo "Would remove: $invalid_files" + else + gsutil rm $invalid_files + print_success "Removed invalid files" + fi + else + print_status "No invalid files found" + fi +} + +# Function to clean up old Firebase function deployments +cleanup_firebase_deployments() { + print_status "Cleaning up old Firebase function deployments..." + + local deployment_files=$(gsutil ls "$FIREBASE_UPLOADS_BUCKET/*.zip" 2>/dev/null | sort | head -n -2 || true) + if [ -n "$deployment_files" ]; then + if [ "$DRY_RUN" = true ]; then + echo "Would remove old deployment files:" + echo "$deployment_files" + else + echo "$deployment_files" | while read file; do + if [ -n "$file" ]; then + gsutil rm "$file" + fi + done + print_success "Removed old Firebase function deployments" + fi + else + print_status "No old deployment files found" + fi +} + +# Function to clean up old Firebase function sources +cleanup_firebase_sources() { + print_status "Cleaning up old Firebase function sources..." + + local source_files=$(gsutil ls "$FIREBASE_SOURCES_BUCKET/api/*.zip" 2>/dev/null | sort | head -n -1 || true) + if [ -n "$source_files" ]; then + if [ "$DRY_RUN" = true ]; then + echo "Would remove old source files:" + echo "$source_files" + else + echo "$source_files" | while read file; do + if [ -n "$file" ]; then + gsutil rm "$file" + fi + done + print_success "Removed old Firebase function sources" + fi + else + print_status "No old source files found" + fi +} + +# Function to show storage summary +show_storage_summary() { + print_status "Current storage usage:" + + local uploads_size=$(calculate_size "$UPLOADS_BUCKET") + local firebase_uploads_size=$(calculate_size "$FIREBASE_UPLOADS_BUCKET") + local firebase_sources_size=$(calculate_size "$FIREBASE_SOURCES_BUCKET/api") + + echo "📁 Main Uploads: $(format_size $uploads_size)" + echo "📁 Firebase Function Uploads: $(format_size $firebase_uploads_size)" + echo "📁 Firebase Function Sources: $(format_size $firebase_sources_size)" + + local total_size=$((uploads_size + firebase_uploads_size + firebase_sources_size)) + echo "📊 Total Storage: $(format_size $total_size)" +} + +# Main script +main() { + local dry_run=false + local force=false + + # Parse command line arguments + while [[ $# -gt 0 ]]; do + case $1 in + -d|--dry-run) + dry_run=true + shift + ;; + -f|--force) + force=true + shift + ;; + -h|--help) + show_usage + exit 0 + ;; + *) + print_error "Unknown option: $1" + show_usage + exit 1 + ;; + esac + done + + DRY_RUN=$dry_run + + print_status "Starting GCS cleanup process..." + + if [ "$dry_run" = true ]; then + print_warning "DRY RUN MODE - No files will be deleted" + fi + + # Show initial storage usage + show_storage_summary + + # Perform cleanup + cleanup_test_files + cleanup_invalid_files + cleanup_firebase_deployments + cleanup_firebase_sources + + # Show final storage usage + echo "" + show_storage_summary + + if [ "$dry_run" = false ]; then + print_success "GCS cleanup completed successfully!" + else + print_warning "Dry run completed - no files were deleted" + fi +} + +# Run main function with all arguments +main "$@" \ No newline at end of file