#!/bin/bash # ============================================================================= # LetsBe Backup Script # ============================================================================= # Backs up databases, env files, nginx configs, and tool configs. # Uploads to rclone remote if configured. # Rotates: 7 daily + 4 weekly backups. # # Usage: # /opt/letsbe/scripts/backups.sh # # Cron (installed by setup.sh): # 0 2 * * * /bin/bash /opt/letsbe/scripts/backups.sh >> /opt/letsbe/logs/backup.log 2>&1 # ============================================================================= set -uo pipefail # ============================================================================= # CONFIGURATION # ============================================================================= LETSBE_BASE="/opt/letsbe" BACKUP_DIR="/tmp/letsbe-backups" DATE=$(date +%Y%m%d_%H%M%S) DAY_OF_WEEK=$(date +%u) # 1=Monday, 7=Sunday RCLONE_REMOTE="backup" LOG_FILE="${LETSBE_BASE}/logs/backup.log" STATUS_FILE="${LETSBE_BASE}/config/backup-status.json" # Ensure directories exist mkdir -p "$BACKUP_DIR" mkdir -p "${LETSBE_BASE}/logs" mkdir -p "${LETSBE_BASE}/config" # Tracking variables ERRORS=() FILES_BACKED_UP=0 log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" } log_error() { local msg="$*" echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $msg" >&2 ERRORS+=("$msg") } # ============================================================================= # BACKUP FUNCTIONS # ============================================================================= # Backup a PostgreSQL database from a running container backup_postgres() { local container=$1 local db_name=$2 local db_user=${3:-postgres} local label=$4 # Find container by pattern (supports both prefixed and exact names) local actual_container actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1) if [[ -z "$actual_container" ]]; then return 0 # Container not running, skip silently fi log "Backing up PostgreSQL: $label ($actual_container -> $db_name)..." local output_file="${BACKUP_DIR}/pg_${label}_${DATE}.sql.gz" if docker exec "$actual_container" pg_dump -U "$db_user" "$db_name" 2>/dev/null | gzip > "$output_file"; then # Verify the file is not empty (just gzip header) if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) log " OK: $output_file" else rm -f "$output_file" log_error "PostgreSQL dump empty for $label ($actual_container)" fi else rm -f "$output_file" log_error "PostgreSQL dump failed for $label ($actual_container)" fi } # Backup a MySQL/MariaDB database from a running container backup_mysql() { local container=$1 local db_name=$2 local db_user=${3:-root} local db_pass=$4 local label=$5 local actual_container actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1) if [[ -z "$actual_container" ]]; then return 0 fi log "Backing up MySQL: $label ($actual_container -> $db_name)..." local output_file="${BACKUP_DIR}/mysql_${label}_${DATE}.sql.gz" if docker exec "$actual_container" mysqldump -u"$db_user" -p"$db_pass" --single-transaction "$db_name" 2>/dev/null | gzip > "$output_file"; then if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) log " OK: $output_file" else rm -f "$output_file" log_error "MySQL dump empty for $label ($actual_container)" fi else rm -f "$output_file" log_error "MySQL dump failed for $label ($actual_container)" fi } # Backup a MongoDB database from a running container backup_mongo() { local container=$1 local db_name=$2 local label=$3 local actual_container actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1) if [[ -z "$actual_container" ]]; then return 0 fi log "Backing up MongoDB: $label ($actual_container -> $db_name)..." local output_file="${BACKUP_DIR}/mongo_${label}_${DATE}.archive.gz" if docker exec "$actual_container" mongodump --db "$db_name" --archive 2>/dev/null | gzip > "$output_file"; then if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) log " OK: $output_file" else rm -f "$output_file" log_error "MongoDB dump empty for $label ($actual_container)" fi else rm -f "$output_file" log_error "MongoDB dump failed for $label ($actual_container)" fi } # Backup a directory as a tarball backup_directory() { local src_dir=$1 local label=$2 if [[ ! -d "$src_dir" ]]; then return 0 fi log "Backing up directory: $label ($src_dir)..." local output_file="${BACKUP_DIR}/dir_${label}_${DATE}.tar.gz" if tar czf "$output_file" -C "$(dirname "$src_dir")" "$(basename "$src_dir")" 2>/dev/null; then FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) log " OK: $output_file" else rm -f "$output_file" log_error "Directory backup failed for $label ($src_dir)" fi } # ============================================================================= # HELPER: Read credentials from env files # ============================================================================= # Read a variable from an env file read_env_var() { local file=$1 local var_name=$2 local default=${3:-} if [[ -f "$file" ]]; then local value value=$(grep -E "^${var_name}=" "$file" 2>/dev/null | head -1 | cut -d'=' -f2- | sed 's/^["'"'"']//;s/["'"'"']$//') if [[ -n "$value" ]]; then echo "$value" return fi fi echo "$default" } # ============================================================================= # START BACKUP # ============================================================================= log "=== LetsBe Backup Started - $DATE ===" # ============================================================================= # 1. POSTGRESQL DATABASE BACKUPS # ============================================================================= log "--- PostgreSQL Databases ---" # Read credentials from env files where needed CREDS_FILE="${LETSBE_BASE}/env/credentials.env" # Chatwoot (user from credentials or default) CHATWOOT_USER=$(read_env_var "$CREDS_FILE" "CHATWOOT_POSTGRES_USERNAME" "chatwoot") backup_postgres "chatwoot-postgres" "chatwoot_production" "$CHATWOOT_USER" "chatwoot" # Nextcloud NC_USER=$(read_env_var "$CREDS_FILE" "NEXTCLOUD_POSTGRES_USER" "nextcloud") backup_postgres "nextcloud-postgres" "nextcloud" "$NC_USER" "nextcloud" # Keycloak backup_postgres "keycloak-db" "keycloak" "keycloak" "keycloak" # n8n N8N_USER=$(read_env_var "${LETSBE_BASE}/env/n8n.env" "POSTGRES_USER" "postgres") backup_postgres "n8n-postgres" "n8n" "$N8N_USER" "n8n" # Cal.com CALCOM_USER=$(read_env_var "${LETSBE_BASE}/env/calcom.env" "POSTGRES_USER" "postgres") backup_postgres "calcom-postgres" "calcom" "$CALCOM_USER" "calcom" # Umami UMAMI_USER=$(read_env_var "$CREDS_FILE" "UMAMI_POSTGRES_USER" "postgres") backup_postgres "umami-db" "umami" "$UMAMI_USER" "umami" # NocoDB backup_postgres "nocodb-postgres" "nocodb" "postgres" "nocodb" # Typebot backup_postgres "typebot-db" "typebot" "postgres" "typebot" # Windmill backup_postgres "windmill-db" "windmill" "postgres" "windmill" # GlitchTip backup_postgres "glitchtip-postgres" "postgres" "postgres" "glitchtip" # Penpot PENPOT_USER=$(read_env_var "$CREDS_FILE" "PENPOT_DB_USER" "postgres") backup_postgres "penpot-postgres" "penpot" "$PENPOT_USER" "penpot" # Gitea GITEA_USER=$(read_env_var "$CREDS_FILE" "GITEA_POSTGRES_USER" "postgres") backup_postgres "gitea-db" "gitea" "$GITEA_USER" "gitea" # Odoo ODOO_USER=$(read_env_var "$CREDS_FILE" "ODOO_POSTGRES_USER" "postgres") backup_postgres "odoo-postgres" "postgres" "$ODOO_USER" "odoo" # Listmonk LISTMONK_USER=$(read_env_var "$CREDS_FILE" "LISTMONK_DB_USER" "postgres") backup_postgres "listmonk-db" "listmonk" "$LISTMONK_USER" "listmonk" # Documenso DOCUMENSO_USER=$(read_env_var "$CREDS_FILE" "DOCUMENSO_POSTGRES_USER" "postgres") backup_postgres "documenso-db" "documenso_db" "$DOCUMENSO_USER" "documenso" # Redash (container name may not have customer prefix) REDASH_USER=$(read_env_var "${LETSBE_BASE}/env/redash.env" "POSTGRES_USER" "postgres") backup_postgres "redash-postgres" "postgres" "$REDASH_USER" "redash" # Activepieces (container name may not have customer prefix) ACTIVEPIECES_USER=$(read_env_var "${LETSBE_BASE}/env/activepieces.env" "AP_POSTGRES_USERNAME" "postgres") ACTIVEPIECES_DB=$(read_env_var "${LETSBE_BASE}/env/activepieces.env" "AP_POSTGRES_DATABASE" "activepieces") backup_postgres "activepieces-postgres" "$ACTIVEPIECES_DB" "$ACTIVEPIECES_USER" "activepieces" # LibreChat vectordb (pgvector) LIBRECHAT_PG_USER=$(read_env_var "$CREDS_FILE" "LIBRECHAT_POSTGRES_USER" "postgres") backup_postgres "librechat-vectordb" "librechat" "$LIBRECHAT_PG_USER" "librechat-vectordb" # Also try the generic volume-based container name backup_postgres "vectordb" "librechat" "$LIBRECHAT_PG_USER" "librechat-vectordb" # Orchestrator backup_postgres "orchestrator-db" "orchestrator" "orchestrator" "orchestrator" # ============================================================================= # 2. MYSQL / MARIADB DATABASE BACKUPS # ============================================================================= log "--- MySQL/MariaDB Databases ---" # WordPress (MariaDB) WP_USER=$(read_env_var "$CREDS_FILE" "WORDPRESS_DB_USER" "root") WP_PASS=$(read_env_var "$CREDS_FILE" "WORDPRESS_DB_PASSWORD" "") WP_ROOT_PASS=$(read_env_var "$CREDS_FILE" "WORDPRESS_MARIADB_ROOT_PASSWORD" "$WP_PASS") if [[ -n "$WP_ROOT_PASS" ]]; then backup_mysql "wordpress-mysql" "wordpress" "root" "$WP_ROOT_PASS" "wordpress" fi # Ghost (MySQL) GHOST_PASS=$(read_env_var "$CREDS_FILE" "GHOST_MYSQL_PASSWORD" "") if [[ -n "$GHOST_PASS" ]]; then backup_mysql "ghost-db" "ghost" "root" "$GHOST_PASS" "ghost" fi # ============================================================================= # 3. MONGODB BACKUPS # ============================================================================= log "--- MongoDB Databases ---" # LibreChat MongoDB backup_mongo "librechat-mongodb" "LibreChat" "librechat" # ============================================================================= # 4. ENV FILES BACKUP # ============================================================================= log "--- Configuration Backups ---" backup_directory "${LETSBE_BASE}/env" "env-files" # ============================================================================= # 5. NGINX CONFIGS BACKUP # ============================================================================= backup_directory "${LETSBE_BASE}/nginx" "nginx-configs" # Also backup active nginx sites if [[ -d "/etc/nginx/sites-enabled" ]]; then backup_directory "/etc/nginx/sites-enabled" "nginx-sites-enabled" fi # ============================================================================= # 6. TOOL CONFIGS BACKUP # ============================================================================= backup_directory "${LETSBE_BASE}/config" "letsbe-config" # Backup rclone config if it exists if [[ -f "/root/.config/rclone/rclone.conf" ]]; then log "Backing up rclone config..." cp "/root/.config/rclone/rclone.conf" "${BACKUP_DIR}/rclone_conf_${DATE}.conf" FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) fi # Backup crontab log "Backing up crontab..." crontab -l > "${BACKUP_DIR}/crontab_${DATE}.txt" 2>/dev/null && FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) || true # ============================================================================= # 7. UPLOAD TO RCLONE REMOTE # ============================================================================= log "--- Remote Upload ---" if command -v rclone &> /dev/null; then if rclone listremotes 2>/dev/null | grep -q "^${RCLONE_REMOTE}:"; then log "Uploading backups to ${RCLONE_REMOTE}:letsbe-backups/${DATE}/..." if rclone copy "$BACKUP_DIR" "${RCLONE_REMOTE}:letsbe-backups/${DATE}/" --quiet 2>&1; then log "Upload complete." else log_error "rclone upload failed" fi else log "WARNING: rclone remote '${RCLONE_REMOTE}' not configured. Backups stored locally only." fi else log "WARNING: rclone not installed. Backups stored locally only." fi # ============================================================================= # 8. ROTATION: Keep 7 daily + 4 weekly # ============================================================================= log "--- Backup Rotation ---" # Daily cleanup: remove files older than 7 days find "$BACKUP_DIR" -maxdepth 1 -type f -mtime +7 -delete 2>/dev/null || true log "Local daily rotation applied (7 days)." # Weekly rotation on remote (keep 4 weeks) if command -v rclone &> /dev/null && rclone listremotes 2>/dev/null | grep -q "^${RCLONE_REMOTE}:"; then # If today is Sunday (day 7), copy today's backup as a weekly backup if [[ "$DAY_OF_WEEK" -eq 7 ]]; then WEEK_NUM=$(date +%Y-W%V) log "Creating weekly backup: ${WEEK_NUM}" rclone copy "${RCLONE_REMOTE}:letsbe-backups/${DATE}/" "${RCLONE_REMOTE}:letsbe-backups/weekly/${WEEK_NUM}/" --quiet 2>/dev/null || true fi # Remove daily remote backups older than 7 days # List remote directories and delete old ones rclone lsd "${RCLONE_REMOTE}:letsbe-backups/" 2>/dev/null | while read -r _ _ _ dirname; do # Skip 'weekly' directory [[ "$dirname" == "weekly" ]] && continue # Parse date from directory name (format: YYYYMMDD_HHMMSS) dir_date=$(echo "$dirname" | cut -c1-8) if [[ "$dir_date" =~ ^[0-9]{8}$ ]]; then dir_epoch=$(date -d "${dir_date:0:4}-${dir_date:4:2}-${dir_date:6:2}" +%s 2>/dev/null || echo "0") cutoff_epoch=$(date -d "7 days ago" +%s 2>/dev/null || echo "0") if [[ "$dir_epoch" -gt 0 && "$cutoff_epoch" -gt 0 && "$dir_epoch" -lt "$cutoff_epoch" ]]; then log "Removing old remote daily: $dirname" rclone purge "${RCLONE_REMOTE}:letsbe-backups/${dirname}/" --quiet 2>/dev/null || true fi fi done # Remove weekly backups older than 4 weeks rclone lsd "${RCLONE_REMOTE}:letsbe-backups/weekly/" 2>/dev/null | while read -r _ _ _ dirname; do # Parse week from directory name (format: YYYY-WNN) week_year=$(echo "$dirname" | cut -d'-' -f1) week_num=$(echo "$dirname" | sed 's/.*W//') if [[ "$week_year" =~ ^[0-9]{4}$ && "$week_num" =~ ^[0-9]+$ ]]; then current_year=$(date +%Y) current_week=$(date +%V) # Calculate approximate age in weeks age_weeks=$(( (current_year - week_year) * 52 + (current_week - week_num) )) if [[ "$age_weeks" -gt 4 ]]; then log "Removing old remote weekly: $dirname" rclone purge "${RCLONE_REMOTE}:letsbe-backups/weekly/${dirname}/" --quiet 2>/dev/null || true fi fi done fi # ============================================================================= # 9. STATUS FILE # ============================================================================= # Calculate total backup size TOTAL_SIZE_BYTES=$(du -sb "$BACKUP_DIR" 2>/dev/null | cut -f1 || echo "0") TOTAL_SIZE_MB=$(( TOTAL_SIZE_BYTES / 1048576 )) # Determine status if [[ ${#ERRORS[@]} -eq 0 ]]; then STATUS="success" else STATUS="partial" fi # Build errors JSON array ERRORS_JSON="[]" if [[ ${#ERRORS[@]} -gt 0 ]]; then ERRORS_JSON="[" for i in "${!ERRORS[@]}"; do # Escape quotes in error messages escaped=$(echo "${ERRORS[$i]}" | sed 's/"/\\"/g') if [[ $i -gt 0 ]]; then ERRORS_JSON+="," fi ERRORS_JSON+="\"${escaped}\"" done ERRORS_JSON+="]" fi cat > "$STATUS_FILE" <