From 5a25eca92991c2b31bb191ce9f3203f46e897841 Mon Sep 17 00:00:00 2001 From: Timo Behrendt Date: Thu, 4 Sep 2025 20:46:33 +0200 Subject: [PATCH] sync --- Dockerfile | 4 ++ README.md | 133 +++++++++++++++++++++++++++++++++++++++++++++++++- src/backup.sh | 105 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 239 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6cad040..f5c0f2a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,10 @@ RUN apk update && apk add --no-cache \ postgresql-client \ jq +RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc \ + && chmod +x mc \ + && mv mc /usr/local/bin/ + WORKDIR /app COPY src/backup.sh /app/backup.sh diff --git a/README.md b/README.md index 5358bc8..4e85cd3 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ BackupSidecar is configured through environment variables. Below is a breakdown These variables apply to both backup and restore operations. - **`OPERATION_MODE`** _(optional)_ - Defines the operation type (`backup` or `restore`). Defaults to `backup`. -- **`BACKUP_MODE`** _(optional)_ - Defines the backup type (`directory` or `postgres`). Defaults to `directory`. +- **`BACKUP_MODE`** _(optional)_ - Defines the backup type (`directory`, `postgres`, or `s3`). Defaults to `directory`. - **`RESTIC_PASSWORD`** _(required)_ - The encryption password for Restic. - **`RESTIC_REPOSITORY`** _(required)_ - The URI of the Restic repository (e.g., `rest:http://your-rest-server:8000/backup`). - **`RESTIC_REST_USERNAME`** _(optional)_ - The username for REST server authentication. @@ -55,6 +55,22 @@ For `postgres` mode, the following database-related variables are required: - **`RESTORE_SNAPSHOT_ID`** _(optional)_ - The specific snapshot ID to restore (defaults to `latest`). - **`PSQL_ARGS`** _(optional)_ - Additional flags for `psql` (e.g., `--single-transaction`). +### S3 Operations + +For `s3` mode, the following S3-related variables are required: + +**Common Variables:** + +- **`S3_BUCKET`** _(required)_ - The name of the S3 bucket to backup/restore. +- **`S3_ENDPOINT`** _(required)_ - The S3 endpoint URL (e.g., `http://minio:9000` for MinIO). +- **`MINIO_ACCESS_KEY`** _(required)_ - The MinIO access key for S3 authentication. +- **`MINIO_SECRET_KEY`** _(required)_ - The MinIO secret key for S3 authentication. +- **`S3_PREFIX`** _(optional)_ - Optional path prefix within the bucket to backup/restore. + +**Restore-Specific Variables:** + +- **`RESTORE_SNAPSHOT_ID`** _(optional)_ - The specific snapshot ID to restore (defaults to `latest`). + ## Dependencies Ensure the following commands are available in the container: @@ -64,6 +80,7 @@ Ensure the following commands are available in the container: - `jq` - `pg_dump` _(only required for PostgreSQL backup operations)_ - `psql` _(only required for PostgreSQL restore operations)_ +- `mc` _(only required for S3 operations)_ ## Usage @@ -236,6 +253,120 @@ spec: value: "Database Restore Notification" ``` +Example Kubernetes Job manifest for running BackupSidecar to backup an S3 bucket: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: backupsidecar-s3-backup + namespace: authentik +spec: + backoffLimit: 3 + activeDeadlineSeconds: 600 + template: + spec: + restartPolicy: OnFailure + containers: + - name: backupsidecar + image: backupsidecar:latest + env: + - name: OPERATION_MODE + value: "backup" + - name: BACKUP_MODE + value: "s3" + - name: S3_BUCKET + value: "my-bucket" + - name: S3_ENDPOINT + value: "http://minio:9000" + - name: S3_PREFIX + value: "data" # optional + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-secret + key: access_key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-secret + key: secret_key + - name: RESTIC_REPOSITORY + value: "rest:http://rest-server:8000/backup" + - name: RESTIC_PASSWORD + valueFrom: + secretKeyRef: + name: backupsidecar-secret + key: restic_password + - name: GOTIFYHOST + value: "http://gotify.example.com" + - name: GOTIFYTOKEN + valueFrom: + secretKeyRef: + name: backupsidecar-secret + key: gotify_token + - name: GOTIFYTOPIC + value: "S3 Backup Notification" +``` + +Example Kubernetes Job manifest for running BackupSidecar to restore an S3 bucket: + +```yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: backupsidecar-s3-restore + namespace: authentik +spec: + backoffLimit: 3 + activeDeadlineSeconds: 600 + template: + spec: + restartPolicy: OnFailure + containers: + - name: backupsidecar + image: backupsidecar:latest + env: + - name: OPERATION_MODE + value: "restore" + - name: BACKUP_MODE + value: "s3" + - name: S3_BUCKET + value: "my-bucket" + - name: S3_ENDPOINT + value: "http://minio:9000" + - name: S3_PREFIX + value: "data" # optional + - name: RESTORE_SNAPSHOT_ID + value: "abc123def456" # optional, defaults to latest + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: minio-secret + key: access_key + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: minio-secret + key: secret_key + - name: RESTIC_REPOSITORY + value: "rest:http://rest-server:8000/backup" + - name: RESTIC_PASSWORD + valueFrom: + secretKeyRef: + name: backupsidecar-secret + key: restic_password + - name: GOTIFYHOST + value: "http://gotify.example.com" + - name: GOTIFYTOKEN + valueFrom: + secretKeyRef: + name: backupsidecar-secret + key: gotify_token + - name: GOTIFYTOPIC + value: "S3 Restore Notification" +``` + ## Notifications The script can send success or failure notifications via Gotify when enabled. To enable notifications, set `ENABLE_GOTIFY=true` and provide the required Gotify configuration variables (`GOTIFYHOST`, `GOTIFYTOKEN`, `GOTIFYTOPIC`). When notifications are disabled, backup status messages are still logged to the console. diff --git a/src/backup.sh b/src/backup.sh index ee1d4bc..8e9411a 100644 --- a/src/backup.sh +++ b/src/backup.sh @@ -24,7 +24,7 @@ OPERATION_MODE="${OPERATION_MODE:-backup}" ####################################### # Determine backup mode from the environment only. -# Valid values: "directory" or "postgres". +# Valid values: "directory", "postgres", or "s3". # Default to "directory" if not provided. ####################################### BACKUP_MODE="${BACKUP_MODE:-directory}" @@ -39,6 +39,8 @@ if [ "$BACKUP_MODE" = "postgres" ]; then elif [ "$OPERATION_MODE" = "restore" ]; then REQUIRED_CMDS+=(psql) fi +elif [ "$BACKUP_MODE" = "s3" ]; then + REQUIRED_CMDS+=(mc) fi for cmd in "${REQUIRED_CMDS[@]}"; do @@ -107,8 +109,16 @@ case "$BACKUP_MODE" in fi fi ;; + s3) + : "${S3_BUCKET:?Environment variable S3_BUCKET is not set (required for S3 mode)}" + : "${S3_ENDPOINT:?Environment variable S3_ENDPOINT is not set (required for S3 mode)}" + : "${MINIO_ACCESS_KEY:?Environment variable MINIO_ACCESS_KEY is not set (required for S3 mode)}" + : "${MINIO_SECRET_KEY:?Environment variable MINIO_SECRET_KEY is not set (required for S3 mode)}" + # Optional: S3 path prefix + : "${S3_PREFIX:=}" + ;; *) - echo "Error: Unknown backup mode '$BACKUP_MODE'. Valid modes are 'directory' and 'postgres'." >&2 + echo "Error: Unknown backup mode '$BACKUP_MODE'. Valid modes are 'directory', 'postgres', and 's3'." >&2 exit 1 ;; esac @@ -287,6 +297,91 @@ restore_postgres() { fi } +####################################### +# Backup an S3 bucket. +# Syncs the S3 bucket to a temporary directory and then backs it up. +####################################### +backup_s3() { + log "Starting S3 backup for bucket '${S3_BUCKET}' at endpoint '${S3_ENDPOINT}'" + + # Create a temporary directory for the S3 sync. + TEMP_BACKUP_DIR=$(mktemp -d) + log "Created temporary directory: ${TEMP_BACKUP_DIR}" + + # Configure MinIO Client alias + local alias_name="backupsidecar" + if ! mc alias set "${alias_name}" "${S3_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"; then + local msg="Failed to configure MinIO client alias" + log "$msg" + send_notification "$msg" + exit 1 + fi + + # Build S3 path + local s3_path="${alias_name}/${S3_BUCKET}" + if [ -n "${S3_PREFIX}" ]; then + s3_path="${s3_path}/${S3_PREFIX}" + fi + + log "Syncing S3 bucket from ${s3_path} to ${TEMP_BACKUP_DIR}..." + if mc mirror "${s3_path}" "${TEMP_BACKUP_DIR}" --remove; then + log "S3 sync completed successfully." + else + local exit_code=$? + local msg="S3 sync failed with error code ${exit_code}" + log "$msg" + send_notification "$msg" + exit "$exit_code" + fi + + # Back up the directory containing the S3 content. + run_restic_backup "${TEMP_BACKUP_DIR}" +} + +####################################### +# Restore an S3 bucket. +# Restores the S3 content from the backup and syncs it back to S3. +####################################### +restore_s3() { + local snapshot_id="${RESTORE_SNAPSHOT_ID:-latest}" + log "Starting S3 restore for bucket '${S3_BUCKET}' at endpoint '${S3_ENDPOINT}'" + + # Create a temporary directory for the restore. + TEMP_RESTORE_DIR=$(mktemp -d) + log "Created temporary directory: ${TEMP_RESTORE_DIR}" + + # Restore the backup to the temporary directory + run_restic_restore "${TEMP_RESTORE_DIR}" "${snapshot_id}" + + # Configure MinIO Client alias + local alias_name="backupsidecar" + if ! mc alias set "${alias_name}" "${S3_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"; then + local msg="Failed to configure MinIO client alias" + log "$msg" + send_notification "$msg" + exit 1 + fi + + # Build S3 path + local s3_path="${alias_name}/${S3_BUCKET}" + if [ -n "${S3_PREFIX}" ]; then + s3_path="${s3_path}/${S3_PREFIX}" + fi + + log "Syncing restored content from ${TEMP_RESTORE_DIR} to ${s3_path}..." + if mc mirror "${TEMP_RESTORE_DIR}" "${s3_path}" --remove; then + local msg="S3 restore completed successfully" + log "$msg" + send_notification "$msg" + else + local exit_code=$? + local msg="S3 restore failed with error code ${exit_code}" + log "$msg" + send_notification "$msg" + exit "$exit_code" + fi +} + ####################################### # Cleanup temporary resources. ####################################### @@ -315,6 +410,9 @@ main() { postgres) backup_postgres ;; + s3) + backup_s3 + ;; esac ;; restore) @@ -325,6 +423,9 @@ main() { postgres) restore_postgres ;; + s3) + restore_s3 + ;; esac ;; esac