letsbe
/
LetsBeBiz-Redesign
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
LetsBeBiz-Redesign/letsbe-ansible-runner/scripts/backups.sh

473 lines
17 KiB
Bash
Raw Permalink Blame History

#!/bin/bash
# =============================================================================
# LetsBe Backup Script
# =============================================================================
# Backs up databases, env files, nginx configs, and tool configs.
# Uploads to rclone remote if configured.
# Rotates: 7 daily + 4 weekly backups.
#
# Usage:
# /opt/letsbe/scripts/backups.sh
#
# Cron (installed by setup.sh):
# 0 2 * * * /bin/bash /opt/letsbe/scripts/backups.sh >> /opt/letsbe/logs/backup.log 2>&1
# =============================================================================
set -uo pipefail
# =============================================================================
# CONFIGURATION
# =============================================================================
LETSBE_BASE="/opt/letsbe"
BACKUP_DIR="/tmp/letsbe-backups"
DATE=$(date +%Y%m%d_%H%M%S)
DAY_OF_WEEK=$(date +%u) # 1=Monday, 7=Sunday
RCLONE_REMOTE="backup"
LOG_FILE="${LETSBE_BASE}/logs/backup.log"
STATUS_FILE="${LETSBE_BASE}/config/backup-status.json"
# Ensure directories exist
mkdir -p "$BACKUP_DIR"
mkdir -p "${LETSBE_BASE}/logs"
mkdir -p "${LETSBE_BASE}/config"
# Tracking variables
ERRORS=()
FILES_BACKED_UP=0
log() {
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
}
log_error() {
local msg="$*"
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $msg" >&2
ERRORS+=("$msg")
}
# =============================================================================
# BACKUP FUNCTIONS
# =============================================================================
# Backup a PostgreSQL database from a running container
backup_postgres() {
local container=$1
local db_name=$2
local db_user=${3:-postgres}
local label=$4
# Find container by pattern (supports both prefixed and exact names)
local actual_container
actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1)
if [[ -z "$actual_container" ]]; then
return 0 # Container not running, skip silently
fi
log "Backing up PostgreSQL: $label ($actual_container -> $db_name)..."
local output_file="${BACKUP_DIR}/pg_${label}_${DATE}.sql.gz"
if docker exec "$actual_container" pg_dump -U "$db_user" "$db_name" 2>/dev/null | gzip > "$output_file"; then
# Verify the file is not empty (just gzip header)
if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
log " OK: $output_file"
else
rm -f "$output_file"
log_error "PostgreSQL dump empty for $label ($actual_container)"
fi
else
rm -f "$output_file"
log_error "PostgreSQL dump failed for $label ($actual_container)"
fi
}
# Backup a MySQL/MariaDB database from a running container
backup_mysql() {
local container=$1
local db_name=$2
local db_user=${3:-root}
local db_pass=$4
local label=$5
local actual_container
actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1)
if [[ -z "$actual_container" ]]; then
return 0
fi
log "Backing up MySQL: $label ($actual_container -> $db_name)..."
local output_file="${BACKUP_DIR}/mysql_${label}_${DATE}.sql.gz"
if docker exec "$actual_container" mysqldump -u"$db_user" -p"$db_pass" --single-transaction "$db_name" 2>/dev/null | gzip > "$output_file"; then
if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
log " OK: $output_file"
else
rm -f "$output_file"
log_error "MySQL dump empty for $label ($actual_container)"
fi
else
rm -f "$output_file"
log_error "MySQL dump failed for $label ($actual_container)"
fi
}
# Backup a MongoDB database from a running container
backup_mongo() {
local container=$1
local db_name=$2
local label=$3
local actual_container
actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1)
if [[ -z "$actual_container" ]]; then
return 0
fi
log "Backing up MongoDB: $label ($actual_container -> $db_name)..."
local output_file="${BACKUP_DIR}/mongo_${label}_${DATE}.archive.gz"
if docker exec "$actual_container" mongodump --db "$db_name" --archive 2>/dev/null | gzip > "$output_file"; then
if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
log " OK: $output_file"
else
rm -f "$output_file"
log_error "MongoDB dump empty for $label ($actual_container)"
fi
else
rm -f "$output_file"
log_error "MongoDB dump failed for $label ($actual_container)"
fi
}
# Backup a directory as a tarball
backup_directory() {
local src_dir=$1
local label=$2
if [[ ! -d "$src_dir" ]]; then
return 0
fi
log "Backing up directory: $label ($src_dir)..."
local output_file="${BACKUP_DIR}/dir_${label}_${DATE}.tar.gz"
if tar czf "$output_file" -C "$(dirname "$src_dir")" "$(basename "$src_dir")" 2>/dev/null; then
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
log " OK: $output_file"
else
rm -f "$output_file"
log_error "Directory backup failed for $label ($src_dir)"
fi
}
# =============================================================================
# HELPER: Read credentials from env files
# =============================================================================
# Read a variable from an env file
read_env_var() {
local file=$1
local var_name=$2
local default=${3:-}
if [[ -f "$file" ]]; then
local value
value=$(grep -E "^${var_name}=" "$file" 2>/dev/null | head -1 | cut -d'=' -f2- | sed 's/^["'"'"']//;s/["'"'"']$//')
if [[ -n "$value" ]]; then
echo "$value"
return
fi
fi
echo "$default"
}
# =============================================================================
# START BACKUP
# =============================================================================
log "=== LetsBe Backup Started - $DATE ==="
# =============================================================================
# 1. POSTGRESQL DATABASE BACKUPS
# =============================================================================
log "--- PostgreSQL Databases ---"
# Read credentials from env files where needed
CREDS_FILE="${LETSBE_BASE}/env/credentials.env"
# Chatwoot (user from credentials or default)
CHATWOOT_USER=$(read_env_var "$CREDS_FILE" "CHATWOOT_POSTGRES_USERNAME" "chatwoot")
backup_postgres "chatwoot-postgres" "chatwoot_production" "$CHATWOOT_USER" "chatwoot"
# Nextcloud
NC_USER=$(read_env_var "$CREDS_FILE" "NEXTCLOUD_POSTGRES_USER" "nextcloud")
backup_postgres "nextcloud-postgres" "nextcloud" "$NC_USER" "nextcloud"
# Keycloak
backup_postgres "keycloak-db" "keycloak" "keycloak" "keycloak"
# n8n
N8N_USER=$(read_env_var "${LETSBE_BASE}/env/n8n.env" "POSTGRES_USER" "postgres")
backup_postgres "n8n-postgres" "n8n" "$N8N_USER" "n8n"
# Cal.com
CALCOM_USER=$(read_env_var "${LETSBE_BASE}/env/calcom.env" "POSTGRES_USER" "postgres")
backup_postgres "calcom-postgres" "calcom" "$CALCOM_USER" "calcom"
# Umami
UMAMI_USER=$(read_env_var "$CREDS_FILE" "UMAMI_POSTGRES_USER" "postgres")
backup_postgres "umami-db" "umami" "$UMAMI_USER" "umami"
# NocoDB
backup_postgres "nocodb-postgres" "nocodb" "postgres" "nocodb"
# Typebot
backup_postgres "typebot-db" "typebot" "postgres" "typebot"
# Windmill
backup_postgres "windmill-db" "windmill" "postgres" "windmill"
# GlitchTip
backup_postgres "glitchtip-postgres" "postgres" "postgres" "glitchtip"
# Penpot
PENPOT_USER=$(read_env_var "$CREDS_FILE" "PENPOT_DB_USER" "postgres")
backup_postgres "penpot-postgres" "penpot" "$PENPOT_USER" "penpot"
# Gitea
GITEA_USER=$(read_env_var "$CREDS_FILE" "GITEA_POSTGRES_USER" "postgres")
backup_postgres "gitea-db" "gitea" "$GITEA_USER" "gitea"
# Odoo
ODOO_USER=$(read_env_var "$CREDS_FILE" "ODOO_POSTGRES_USER" "postgres")
backup_postgres "odoo-postgres" "postgres" "$ODOO_USER" "odoo"
# Listmonk
LISTMONK_USER=$(read_env_var "$CREDS_FILE" "LISTMONK_DB_USER" "postgres")
backup_postgres "listmonk-db" "listmonk" "$LISTMONK_USER" "listmonk"
# Documenso
DOCUMENSO_USER=$(read_env_var "$CREDS_FILE" "DOCUMENSO_POSTGRES_USER" "postgres")
backup_postgres "documenso-db" "documenso_db" "$DOCUMENSO_USER" "documenso"
# Redash (container name may not have customer prefix)
REDASH_USER=$(read_env_var "${LETSBE_BASE}/env/redash.env" "POSTGRES_USER" "postgres")
backup_postgres "redash-postgres" "postgres" "$REDASH_USER" "redash"
# Activepieces (container name may not have customer prefix)
ACTIVEPIECES_USER=$(read_env_var "${LETSBE_BASE}/env/activepieces.env" "AP_POSTGRES_USERNAME" "postgres")
ACTIVEPIECES_DB=$(read_env_var "${LETSBE_BASE}/env/activepieces.env" "AP_POSTGRES_DATABASE" "activepieces")
backup_postgres "activepieces-postgres" "$ACTIVEPIECES_DB" "$ACTIVEPIECES_USER" "activepieces"
# LibreChat vectordb (pgvector)
LIBRECHAT_PG_USER=$(read_env_var "$CREDS_FILE" "LIBRECHAT_POSTGRES_USER" "postgres")
backup_postgres "librechat-vectordb" "librechat" "$LIBRECHAT_PG_USER" "librechat-vectordb"
# Also try the generic volume-based container name
backup_postgres "vectordb" "librechat" "$LIBRECHAT_PG_USER" "librechat-vectordb"
# Orchestrator
backup_postgres "orchestrator-db" "orchestrator" "orchestrator" "orchestrator"
# =============================================================================
# 2. MYSQL / MARIADB DATABASE BACKUPS
# =============================================================================
log "--- MySQL/MariaDB Databases ---"
# WordPress (MariaDB)
WP_USER=$(read_env_var "$CREDS_FILE" "WORDPRESS_DB_USER" "root")
WP_PASS=$(read_env_var "$CREDS_FILE" "WORDPRESS_DB_PASSWORD" "")
WP_ROOT_PASS=$(read_env_var "$CREDS_FILE" "WORDPRESS_MARIADB_ROOT_PASSWORD" "$WP_PASS")
if [[ -n "$WP_ROOT_PASS" ]]; then
backup_mysql "wordpress-mysql" "wordpress" "root" "$WP_ROOT_PASS" "wordpress"
fi
# Ghost (MySQL)
GHOST_PASS=$(read_env_var "$CREDS_FILE" "GHOST_MYSQL_PASSWORD" "")
if [[ -n "$GHOST_PASS" ]]; then
backup_mysql "ghost-db" "ghost" "root" "$GHOST_PASS" "ghost"
fi
# =============================================================================
# 3. MONGODB BACKUPS
# =============================================================================
log "--- MongoDB Databases ---"
# LibreChat MongoDB
backup_mongo "librechat-mongodb" "LibreChat" "librechat"
# =============================================================================
# 4. ENV FILES BACKUP
# =============================================================================
log "--- Configuration Backups ---"
backup_directory "${LETSBE_BASE}/env" "env-files"
# =============================================================================
# 5. NGINX CONFIGS BACKUP
# =============================================================================
backup_directory "${LETSBE_BASE}/nginx" "nginx-configs"
# Also backup active nginx sites
if [[ -d "/etc/nginx/sites-enabled" ]]; then
backup_directory "/etc/nginx/sites-enabled" "nginx-sites-enabled"
fi
# =============================================================================
# 6. TOOL CONFIGS BACKUP
# =============================================================================
backup_directory "${LETSBE_BASE}/config" "letsbe-config"
# Backup rclone config if it exists
if [[ -f "/root/.config/rclone/rclone.conf" ]]; then
log "Backing up rclone config..."
cp "/root/.config/rclone/rclone.conf" "${BACKUP_DIR}/rclone_conf_${DATE}.conf"
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
fi
# Backup crontab
log "Backing up crontab..."
crontab -l > "${BACKUP_DIR}/crontab_${DATE}.txt" 2>/dev/null && FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) || true
# =============================================================================
# 7. UPLOAD TO RCLONE REMOTE
# =============================================================================
log "--- Remote Upload ---"
if command -v rclone &> /dev/null; then
if rclone listremotes 2>/dev/null | grep -q "^${RCLONE_REMOTE}:"; then
log "Uploading backups to ${RCLONE_REMOTE}:letsbe-backups/${DATE}/..."
if rclone copy "$BACKUP_DIR" "${RCLONE_REMOTE}:letsbe-backups/${DATE}/" --quiet 2>&1; then
log "Upload complete."
else
log_error "rclone upload failed"
fi
else
log "WARNING: rclone remote '${RCLONE_REMOTE}' not configured. Backups stored locally only."
fi
else
log "WARNING: rclone not installed. Backups stored locally only."
fi
# =============================================================================
# 8. ROTATION: Keep 7 daily + 4 weekly
# =============================================================================
log "--- Backup Rotation ---"
# Daily cleanup: remove files older than 7 days
find "$BACKUP_DIR" -maxdepth 1 -type f -mtime +7 -delete 2>/dev/null || true
log "Local daily rotation applied (7 days)."
# Weekly rotation on remote (keep 4 weeks)
if command -v rclone &> /dev/null && rclone listremotes 2>/dev/null | grep -q "^${RCLONE_REMOTE}:"; then
# If today is Sunday (day 7), copy today's backup as a weekly backup
if [[ "$DAY_OF_WEEK" -eq 7 ]]; then
WEEK_NUM=$(date +%Y-W%V)
log "Creating weekly backup: ${WEEK_NUM}"
rclone copy "${RCLONE_REMOTE}:letsbe-backups/${DATE}/" "${RCLONE_REMOTE}:letsbe-backups/weekly/${WEEK_NUM}/" --quiet 2>/dev/null || true
fi
# Remove daily remote backups older than 7 days
# List remote directories and delete old ones
rclone lsd "${RCLONE_REMOTE}:letsbe-backups/" 2>/dev/null | while read -r _ _ _ dirname; do
# Skip 'weekly' directory
[[ "$dirname" == "weekly" ]] && continue
# Parse date from directory name (format: YYYYMMDD_HHMMSS)
dir_date=$(echo "$dirname" | cut -c1-8)
if [[ "$dir_date" =~ ^[0-9]{8}$ ]]; then
dir_epoch=$(date -d "${dir_date:0:4}-${dir_date:4:2}-${dir_date:6:2}" +%s 2>/dev/null || echo "0")
cutoff_epoch=$(date -d "7 days ago" +%s 2>/dev/null || echo "0")
if [[ "$dir_epoch" -gt 0 && "$cutoff_epoch" -gt 0 && "$dir_epoch" -lt "$cutoff_epoch" ]]; then
log "Removing old remote daily: $dirname"
rclone purge "${RCLONE_REMOTE}:letsbe-backups/${dirname}/" --quiet 2>/dev/null || true
fi
fi
done
# Remove weekly backups older than 4 weeks
rclone lsd "${RCLONE_REMOTE}:letsbe-backups/weekly/" 2>/dev/null | while read -r _ _ _ dirname; do
# Parse week from directory name (format: YYYY-WNN)
week_year=$(echo "$dirname" | cut -d'-' -f1)
week_num=$(echo "$dirname" | sed 's/.*W//')
if [[ "$week_year" =~ ^[0-9]{4}$ && "$week_num" =~ ^[0-9]+$ ]]; then
current_year=$(date +%Y)
current_week=$(date +%V)
# Calculate approximate age in weeks
age_weeks=$(( (current_year - week_year) * 52 + (current_week - week_num) ))
if [[ "$age_weeks" -gt 4 ]]; then
log "Removing old remote weekly: $dirname"
rclone purge "${RCLONE_REMOTE}:letsbe-backups/weekly/${dirname}/" --quiet 2>/dev/null || true
fi
fi
done
fi
# =============================================================================
# 9. STATUS FILE
# =============================================================================
# Calculate total backup size
TOTAL_SIZE_BYTES=$(du -sb "$BACKUP_DIR" 2>/dev/null | cut -f1 || echo "0")
TOTAL_SIZE_MB=$(( TOTAL_SIZE_BYTES / 1048576 ))
# Determine status
if [[ ${#ERRORS[@]} -eq 0 ]]; then
STATUS="success"
else
STATUS="partial"
fi
# Build errors JSON array
ERRORS_JSON="[]"
if [[ ${#ERRORS[@]} -gt 0 ]]; then
ERRORS_JSON="["
for i in "${!ERRORS[@]}"; do
# Escape quotes in error messages
escaped=$(echo "${ERRORS[$i]}" | sed 's/"/\\"/g')
if [[ $i -gt 0 ]]; then
ERRORS_JSON+=","
fi
ERRORS_JSON+="\"${escaped}\""
done
ERRORS_JSON+="]"
fi
cat > "$STATUS_FILE" <<EOF
{
"last_run": "$(date -u '+%Y-%m-%dT%H:%M:%SZ')",
"status": "${STATUS}",
"size_mb": ${TOTAL_SIZE_MB},
"files_backed_up": ${FILES_BACKED_UP},
"errors": ${ERRORS_JSON}
}
EOF
# =============================================================================
# DONE
# =============================================================================
log "=== Backup Complete ==="
log "Status: ${STATUS}"
log "Files backed up: ${FILES_BACKED_UP}"
log "Total size: ${TOTAL_SIZE_MB} MB"
log "Local backups: ${BACKUP_DIR}"
if [[ ${#ERRORS[@]} -gt 0 ]]; then
log "Errors (${#ERRORS[@]}):"
for err in "${ERRORS[@]}"; do
log " - $err"
done
fi
Reference in New Issue View Git Blame Copy Permalink