473 lines
17 KiB
Bash
473 lines
17 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
# LetsBe Backup Script
|
|
# =============================================================================
|
|
# Backs up databases, env files, nginx configs, and tool configs.
|
|
# Uploads to rclone remote if configured.
|
|
# Rotates: 7 daily + 4 weekly backups.
|
|
#
|
|
# Usage:
|
|
# /opt/letsbe/scripts/backups.sh
|
|
#
|
|
# Cron (installed by setup.sh):
|
|
# 0 2 * * * /bin/bash /opt/letsbe/scripts/backups.sh >> /opt/letsbe/logs/backup.log 2>&1
|
|
# =============================================================================
|
|
|
|
set -uo pipefail
|
|
|
|
# =============================================================================
|
|
# CONFIGURATION
|
|
# =============================================================================
|
|
|
|
LETSBE_BASE="/opt/letsbe"
|
|
BACKUP_DIR="/tmp/letsbe-backups"
|
|
DATE=$(date +%Y%m%d_%H%M%S)
|
|
DAY_OF_WEEK=$(date +%u) # 1=Monday, 7=Sunday
|
|
RCLONE_REMOTE="backup"
|
|
LOG_FILE="${LETSBE_BASE}/logs/backup.log"
|
|
STATUS_FILE="${LETSBE_BASE}/config/backup-status.json"
|
|
|
|
# Ensure directories exist
|
|
mkdir -p "$BACKUP_DIR"
|
|
mkdir -p "${LETSBE_BASE}/logs"
|
|
mkdir -p "${LETSBE_BASE}/config"
|
|
|
|
# Tracking variables
|
|
ERRORS=()
|
|
FILES_BACKED_UP=0
|
|
|
|
log() {
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"
|
|
}
|
|
|
|
log_error() {
|
|
local msg="$*"
|
|
echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $msg" >&2
|
|
ERRORS+=("$msg")
|
|
}
|
|
|
|
# =============================================================================
|
|
# BACKUP FUNCTIONS
|
|
# =============================================================================
|
|
|
|
# Backup a PostgreSQL database from a running container
|
|
backup_postgres() {
|
|
local container=$1
|
|
local db_name=$2
|
|
local db_user=${3:-postgres}
|
|
local label=$4
|
|
|
|
# Find container by pattern (supports both prefixed and exact names)
|
|
local actual_container
|
|
actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1)
|
|
|
|
if [[ -z "$actual_container" ]]; then
|
|
return 0 # Container not running, skip silently
|
|
fi
|
|
|
|
log "Backing up PostgreSQL: $label ($actual_container -> $db_name)..."
|
|
local output_file="${BACKUP_DIR}/pg_${label}_${DATE}.sql.gz"
|
|
|
|
if docker exec "$actual_container" pg_dump -U "$db_user" "$db_name" 2>/dev/null | gzip > "$output_file"; then
|
|
# Verify the file is not empty (just gzip header)
|
|
if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then
|
|
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
|
|
log " OK: $output_file"
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "PostgreSQL dump empty for $label ($actual_container)"
|
|
fi
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "PostgreSQL dump failed for $label ($actual_container)"
|
|
fi
|
|
}
|
|
|
|
# Backup a MySQL/MariaDB database from a running container
|
|
backup_mysql() {
|
|
local container=$1
|
|
local db_name=$2
|
|
local db_user=${3:-root}
|
|
local db_pass=$4
|
|
local label=$5
|
|
|
|
local actual_container
|
|
actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1)
|
|
|
|
if [[ -z "$actual_container" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
log "Backing up MySQL: $label ($actual_container -> $db_name)..."
|
|
local output_file="${BACKUP_DIR}/mysql_${label}_${DATE}.sql.gz"
|
|
|
|
if docker exec "$actual_container" mysqldump -u"$db_user" -p"$db_pass" --single-transaction "$db_name" 2>/dev/null | gzip > "$output_file"; then
|
|
if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then
|
|
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
|
|
log " OK: $output_file"
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "MySQL dump empty for $label ($actual_container)"
|
|
fi
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "MySQL dump failed for $label ($actual_container)"
|
|
fi
|
|
}
|
|
|
|
# Backup a MongoDB database from a running container
|
|
backup_mongo() {
|
|
local container=$1
|
|
local db_name=$2
|
|
local label=$3
|
|
|
|
local actual_container
|
|
actual_container=$(docker ps --format '{{.Names}}' | grep -E "(^|-)${container}$" | head -1)
|
|
|
|
if [[ -z "$actual_container" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
log "Backing up MongoDB: $label ($actual_container -> $db_name)..."
|
|
local output_file="${BACKUP_DIR}/mongo_${label}_${DATE}.archive.gz"
|
|
|
|
if docker exec "$actual_container" mongodump --db "$db_name" --archive 2>/dev/null | gzip > "$output_file"; then
|
|
if [[ $(stat -c%s "$output_file" 2>/dev/null || stat -f%z "$output_file" 2>/dev/null) -gt 100 ]]; then
|
|
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
|
|
log " OK: $output_file"
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "MongoDB dump empty for $label ($actual_container)"
|
|
fi
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "MongoDB dump failed for $label ($actual_container)"
|
|
fi
|
|
}
|
|
|
|
# Backup a directory as a tarball
|
|
backup_directory() {
|
|
local src_dir=$1
|
|
local label=$2
|
|
|
|
if [[ ! -d "$src_dir" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
log "Backing up directory: $label ($src_dir)..."
|
|
local output_file="${BACKUP_DIR}/dir_${label}_${DATE}.tar.gz"
|
|
|
|
if tar czf "$output_file" -C "$(dirname "$src_dir")" "$(basename "$src_dir")" 2>/dev/null; then
|
|
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
|
|
log " OK: $output_file"
|
|
else
|
|
rm -f "$output_file"
|
|
log_error "Directory backup failed for $label ($src_dir)"
|
|
fi
|
|
}
|
|
|
|
# =============================================================================
|
|
# HELPER: Read credentials from env files
|
|
# =============================================================================
|
|
|
|
# Read a variable from an env file
|
|
read_env_var() {
|
|
local file=$1
|
|
local var_name=$2
|
|
local default=${3:-}
|
|
|
|
if [[ -f "$file" ]]; then
|
|
local value
|
|
value=$(grep -E "^${var_name}=" "$file" 2>/dev/null | head -1 | cut -d'=' -f2- | sed 's/^["'"'"']//;s/["'"'"']$//')
|
|
if [[ -n "$value" ]]; then
|
|
echo "$value"
|
|
return
|
|
fi
|
|
fi
|
|
echo "$default"
|
|
}
|
|
|
|
# =============================================================================
|
|
# START BACKUP
|
|
# =============================================================================
|
|
|
|
log "=== LetsBe Backup Started - $DATE ==="
|
|
|
|
# =============================================================================
|
|
# 1. POSTGRESQL DATABASE BACKUPS
|
|
# =============================================================================
|
|
|
|
log "--- PostgreSQL Databases ---"
|
|
|
|
# Read credentials from env files where needed
|
|
CREDS_FILE="${LETSBE_BASE}/env/credentials.env"
|
|
|
|
# Chatwoot (user from credentials or default)
|
|
CHATWOOT_USER=$(read_env_var "$CREDS_FILE" "CHATWOOT_POSTGRES_USERNAME" "chatwoot")
|
|
backup_postgres "chatwoot-postgres" "chatwoot_production" "$CHATWOOT_USER" "chatwoot"
|
|
|
|
# Nextcloud
|
|
NC_USER=$(read_env_var "$CREDS_FILE" "NEXTCLOUD_POSTGRES_USER" "nextcloud")
|
|
backup_postgres "nextcloud-postgres" "nextcloud" "$NC_USER" "nextcloud"
|
|
|
|
# Keycloak
|
|
backup_postgres "keycloak-db" "keycloak" "keycloak" "keycloak"
|
|
|
|
# n8n
|
|
N8N_USER=$(read_env_var "${LETSBE_BASE}/env/n8n.env" "POSTGRES_USER" "postgres")
|
|
backup_postgres "n8n-postgres" "n8n" "$N8N_USER" "n8n"
|
|
|
|
# Cal.com
|
|
CALCOM_USER=$(read_env_var "${LETSBE_BASE}/env/calcom.env" "POSTGRES_USER" "postgres")
|
|
backup_postgres "calcom-postgres" "calcom" "$CALCOM_USER" "calcom"
|
|
|
|
# Umami
|
|
UMAMI_USER=$(read_env_var "$CREDS_FILE" "UMAMI_POSTGRES_USER" "postgres")
|
|
backup_postgres "umami-db" "umami" "$UMAMI_USER" "umami"
|
|
|
|
# NocoDB
|
|
backup_postgres "nocodb-postgres" "nocodb" "postgres" "nocodb"
|
|
|
|
# Typebot
|
|
backup_postgres "typebot-db" "typebot" "postgres" "typebot"
|
|
|
|
# Windmill
|
|
backup_postgres "windmill-db" "windmill" "postgres" "windmill"
|
|
|
|
# GlitchTip
|
|
backup_postgres "glitchtip-postgres" "postgres" "postgres" "glitchtip"
|
|
|
|
# Penpot
|
|
PENPOT_USER=$(read_env_var "$CREDS_FILE" "PENPOT_DB_USER" "postgres")
|
|
backup_postgres "penpot-postgres" "penpot" "$PENPOT_USER" "penpot"
|
|
|
|
# Gitea
|
|
GITEA_USER=$(read_env_var "$CREDS_FILE" "GITEA_POSTGRES_USER" "postgres")
|
|
backup_postgres "gitea-db" "gitea" "$GITEA_USER" "gitea"
|
|
|
|
# Odoo
|
|
ODOO_USER=$(read_env_var "$CREDS_FILE" "ODOO_POSTGRES_USER" "postgres")
|
|
backup_postgres "odoo-postgres" "postgres" "$ODOO_USER" "odoo"
|
|
|
|
# Listmonk
|
|
LISTMONK_USER=$(read_env_var "$CREDS_FILE" "LISTMONK_DB_USER" "postgres")
|
|
backup_postgres "listmonk-db" "listmonk" "$LISTMONK_USER" "listmonk"
|
|
|
|
# Documenso
|
|
DOCUMENSO_USER=$(read_env_var "$CREDS_FILE" "DOCUMENSO_POSTGRES_USER" "postgres")
|
|
backup_postgres "documenso-db" "documenso_db" "$DOCUMENSO_USER" "documenso"
|
|
|
|
# Redash (container name may not have customer prefix)
|
|
REDASH_USER=$(read_env_var "${LETSBE_BASE}/env/redash.env" "POSTGRES_USER" "postgres")
|
|
backup_postgres "redash-postgres" "postgres" "$REDASH_USER" "redash"
|
|
|
|
# Activepieces (container name may not have customer prefix)
|
|
ACTIVEPIECES_USER=$(read_env_var "${LETSBE_BASE}/env/activepieces.env" "AP_POSTGRES_USERNAME" "postgres")
|
|
ACTIVEPIECES_DB=$(read_env_var "${LETSBE_BASE}/env/activepieces.env" "AP_POSTGRES_DATABASE" "activepieces")
|
|
backup_postgres "activepieces-postgres" "$ACTIVEPIECES_DB" "$ACTIVEPIECES_USER" "activepieces"
|
|
|
|
# LibreChat vectordb (pgvector)
|
|
LIBRECHAT_PG_USER=$(read_env_var "$CREDS_FILE" "LIBRECHAT_POSTGRES_USER" "postgres")
|
|
backup_postgres "librechat-vectordb" "librechat" "$LIBRECHAT_PG_USER" "librechat-vectordb"
|
|
# Also try the generic volume-based container name
|
|
backup_postgres "vectordb" "librechat" "$LIBRECHAT_PG_USER" "librechat-vectordb"
|
|
|
|
# Orchestrator
|
|
backup_postgres "orchestrator-db" "orchestrator" "orchestrator" "orchestrator"
|
|
|
|
# =============================================================================
|
|
# 2. MYSQL / MARIADB DATABASE BACKUPS
|
|
# =============================================================================
|
|
|
|
log "--- MySQL/MariaDB Databases ---"
|
|
|
|
# WordPress (MariaDB)
|
|
WP_USER=$(read_env_var "$CREDS_FILE" "WORDPRESS_DB_USER" "root")
|
|
WP_PASS=$(read_env_var "$CREDS_FILE" "WORDPRESS_DB_PASSWORD" "")
|
|
WP_ROOT_PASS=$(read_env_var "$CREDS_FILE" "WORDPRESS_MARIADB_ROOT_PASSWORD" "$WP_PASS")
|
|
if [[ -n "$WP_ROOT_PASS" ]]; then
|
|
backup_mysql "wordpress-mysql" "wordpress" "root" "$WP_ROOT_PASS" "wordpress"
|
|
fi
|
|
|
|
# Ghost (MySQL)
|
|
GHOST_PASS=$(read_env_var "$CREDS_FILE" "GHOST_MYSQL_PASSWORD" "")
|
|
if [[ -n "$GHOST_PASS" ]]; then
|
|
backup_mysql "ghost-db" "ghost" "root" "$GHOST_PASS" "ghost"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# 3. MONGODB BACKUPS
|
|
# =============================================================================
|
|
|
|
log "--- MongoDB Databases ---"
|
|
|
|
# LibreChat MongoDB
|
|
backup_mongo "librechat-mongodb" "LibreChat" "librechat"
|
|
|
|
# =============================================================================
|
|
# 4. ENV FILES BACKUP
|
|
# =============================================================================
|
|
|
|
log "--- Configuration Backups ---"
|
|
|
|
backup_directory "${LETSBE_BASE}/env" "env-files"
|
|
|
|
# =============================================================================
|
|
# 5. NGINX CONFIGS BACKUP
|
|
# =============================================================================
|
|
|
|
backup_directory "${LETSBE_BASE}/nginx" "nginx-configs"
|
|
|
|
# Also backup active nginx sites
|
|
if [[ -d "/etc/nginx/sites-enabled" ]]; then
|
|
backup_directory "/etc/nginx/sites-enabled" "nginx-sites-enabled"
|
|
fi
|
|
|
|
# =============================================================================
|
|
# 6. TOOL CONFIGS BACKUP
|
|
# =============================================================================
|
|
|
|
backup_directory "${LETSBE_BASE}/config" "letsbe-config"
|
|
|
|
# Backup rclone config if it exists
|
|
if [[ -f "/root/.config/rclone/rclone.conf" ]]; then
|
|
log "Backing up rclone config..."
|
|
cp "/root/.config/rclone/rclone.conf" "${BACKUP_DIR}/rclone_conf_${DATE}.conf"
|
|
FILES_BACKED_UP=$((FILES_BACKED_UP + 1))
|
|
fi
|
|
|
|
# Backup crontab
|
|
log "Backing up crontab..."
|
|
crontab -l > "${BACKUP_DIR}/crontab_${DATE}.txt" 2>/dev/null && FILES_BACKED_UP=$((FILES_BACKED_UP + 1)) || true
|
|
|
|
# =============================================================================
|
|
# 7. UPLOAD TO RCLONE REMOTE
|
|
# =============================================================================
|
|
|
|
log "--- Remote Upload ---"
|
|
|
|
if command -v rclone &> /dev/null; then
|
|
if rclone listremotes 2>/dev/null | grep -q "^${RCLONE_REMOTE}:"; then
|
|
log "Uploading backups to ${RCLONE_REMOTE}:letsbe-backups/${DATE}/..."
|
|
if rclone copy "$BACKUP_DIR" "${RCLONE_REMOTE}:letsbe-backups/${DATE}/" --quiet 2>&1; then
|
|
log "Upload complete."
|
|
else
|
|
log_error "rclone upload failed"
|
|
fi
|
|
else
|
|
log "WARNING: rclone remote '${RCLONE_REMOTE}' not configured. Backups stored locally only."
|
|
fi
|
|
else
|
|
log "WARNING: rclone not installed. Backups stored locally only."
|
|
fi
|
|
|
|
# =============================================================================
|
|
# 8. ROTATION: Keep 7 daily + 4 weekly
|
|
# =============================================================================
|
|
|
|
log "--- Backup Rotation ---"
|
|
|
|
# Daily cleanup: remove files older than 7 days
|
|
find "$BACKUP_DIR" -maxdepth 1 -type f -mtime +7 -delete 2>/dev/null || true
|
|
log "Local daily rotation applied (7 days)."
|
|
|
|
# Weekly rotation on remote (keep 4 weeks)
|
|
if command -v rclone &> /dev/null && rclone listremotes 2>/dev/null | grep -q "^${RCLONE_REMOTE}:"; then
|
|
# If today is Sunday (day 7), copy today's backup as a weekly backup
|
|
if [[ "$DAY_OF_WEEK" -eq 7 ]]; then
|
|
WEEK_NUM=$(date +%Y-W%V)
|
|
log "Creating weekly backup: ${WEEK_NUM}"
|
|
rclone copy "${RCLONE_REMOTE}:letsbe-backups/${DATE}/" "${RCLONE_REMOTE}:letsbe-backups/weekly/${WEEK_NUM}/" --quiet 2>/dev/null || true
|
|
fi
|
|
|
|
# Remove daily remote backups older than 7 days
|
|
# List remote directories and delete old ones
|
|
rclone lsd "${RCLONE_REMOTE}:letsbe-backups/" 2>/dev/null | while read -r _ _ _ dirname; do
|
|
# Skip 'weekly' directory
|
|
[[ "$dirname" == "weekly" ]] && continue
|
|
# Parse date from directory name (format: YYYYMMDD_HHMMSS)
|
|
dir_date=$(echo "$dirname" | cut -c1-8)
|
|
if [[ "$dir_date" =~ ^[0-9]{8}$ ]]; then
|
|
dir_epoch=$(date -d "${dir_date:0:4}-${dir_date:4:2}-${dir_date:6:2}" +%s 2>/dev/null || echo "0")
|
|
cutoff_epoch=$(date -d "7 days ago" +%s 2>/dev/null || echo "0")
|
|
if [[ "$dir_epoch" -gt 0 && "$cutoff_epoch" -gt 0 && "$dir_epoch" -lt "$cutoff_epoch" ]]; then
|
|
log "Removing old remote daily: $dirname"
|
|
rclone purge "${RCLONE_REMOTE}:letsbe-backups/${dirname}/" --quiet 2>/dev/null || true
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Remove weekly backups older than 4 weeks
|
|
rclone lsd "${RCLONE_REMOTE}:letsbe-backups/weekly/" 2>/dev/null | while read -r _ _ _ dirname; do
|
|
# Parse week from directory name (format: YYYY-WNN)
|
|
week_year=$(echo "$dirname" | cut -d'-' -f1)
|
|
week_num=$(echo "$dirname" | sed 's/.*W//')
|
|
if [[ "$week_year" =~ ^[0-9]{4}$ && "$week_num" =~ ^[0-9]+$ ]]; then
|
|
current_year=$(date +%Y)
|
|
current_week=$(date +%V)
|
|
# Calculate approximate age in weeks
|
|
age_weeks=$(( (current_year - week_year) * 52 + (current_week - week_num) ))
|
|
if [[ "$age_weeks" -gt 4 ]]; then
|
|
log "Removing old remote weekly: $dirname"
|
|
rclone purge "${RCLONE_REMOTE}:letsbe-backups/weekly/${dirname}/" --quiet 2>/dev/null || true
|
|
fi
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# =============================================================================
|
|
# 9. STATUS FILE
|
|
# =============================================================================
|
|
|
|
# Calculate total backup size
|
|
TOTAL_SIZE_BYTES=$(du -sb "$BACKUP_DIR" 2>/dev/null | cut -f1 || echo "0")
|
|
TOTAL_SIZE_MB=$(( TOTAL_SIZE_BYTES / 1048576 ))
|
|
|
|
# Determine status
|
|
if [[ ${#ERRORS[@]} -eq 0 ]]; then
|
|
STATUS="success"
|
|
else
|
|
STATUS="partial"
|
|
fi
|
|
|
|
# Build errors JSON array
|
|
ERRORS_JSON="[]"
|
|
if [[ ${#ERRORS[@]} -gt 0 ]]; then
|
|
ERRORS_JSON="["
|
|
for i in "${!ERRORS[@]}"; do
|
|
# Escape quotes in error messages
|
|
escaped=$(echo "${ERRORS[$i]}" | sed 's/"/\\"/g')
|
|
if [[ $i -gt 0 ]]; then
|
|
ERRORS_JSON+=","
|
|
fi
|
|
ERRORS_JSON+="\"${escaped}\""
|
|
done
|
|
ERRORS_JSON+="]"
|
|
fi
|
|
|
|
cat > "$STATUS_FILE" <<EOF
|
|
{
|
|
"last_run": "$(date -u '+%Y-%m-%dT%H:%M:%SZ')",
|
|
"status": "${STATUS}",
|
|
"size_mb": ${TOTAL_SIZE_MB},
|
|
"files_backed_up": ${FILES_BACKED_UP},
|
|
"errors": ${ERRORS_JSON}
|
|
}
|
|
EOF
|
|
|
|
# =============================================================================
|
|
# DONE
|
|
# =============================================================================
|
|
|
|
log "=== Backup Complete ==="
|
|
log "Status: ${STATUS}"
|
|
log "Files backed up: ${FILES_BACKED_UP}"
|
|
log "Total size: ${TOTAL_SIZE_MB} MB"
|
|
log "Local backups: ${BACKUP_DIR}"
|
|
if [[ ${#ERRORS[@]} -gt 0 ]]; then
|
|
log "Errors (${#ERRORS[@]}):"
|
|
for err in "${ERRORS[@]}"; do
|
|
log " - $err"
|
|
done
|
|
fi
|