sync

Dockerfile

@@ -7,6 +7,10 @@ RUN apk update && apk add --no-cache \
     postgresql-client \
     jq
 
+RUN curl -O https://dl.min.io/client/mc/release/linux-amd64/mc \
+    && chmod +x mc \
+    && mv mc /usr/local/bin/
+
 WORKDIR /app
 
 COPY src/backup.sh /app/backup.sh

README.md

@@ -11,7 +11,7 @@ BackupSidecar is configured through environment variables. Below is a breakdown
 These variables apply to both backup and restore operations.
 
 - **`OPERATION_MODE`** _(optional)_ - Defines the operation type (`backup` or `restore`). Defaults to `backup`.
-- **`BACKUP_MODE`** _(optional)_ - Defines the backup type (`directory` or `postgres`). Defaults to `directory`.
+- **`BACKUP_MODE`** _(optional)_ - Defines the backup type (`directory`, `postgres`, or `s3`). Defaults to `directory`.
 - **`RESTIC_PASSWORD`** _(required)_ - The encryption password for Restic.
 - **`RESTIC_REPOSITORY`** _(required)_ - The URI of the Restic repository (e.g., `rest:http://your-rest-server:8000/backup`).
 - **`RESTIC_REST_USERNAME`** _(optional)_ - The username for REST server authentication.
@@ -55,6 +55,22 @@ For `postgres` mode, the following database-related variables are required:
 - **`RESTORE_SNAPSHOT_ID`** _(optional)_ - The specific snapshot ID to restore (defaults to `latest`).
 - **`PSQL_ARGS`** _(optional)_ - Additional flags for `psql` (e.g., `--single-transaction`).
 
+### S3 Operations
+
+For `s3` mode, the following S3-related variables are required:
+
+**Common Variables:**
+
+- **`S3_BUCKET`** _(required)_ - The name of the S3 bucket to backup/restore.
+- **`S3_ENDPOINT`** _(required)_ - The S3 endpoint URL (e.g., `http://minio:9000` for MinIO).
+- **`MINIO_ACCESS_KEY`** _(required)_ - The MinIO access key for S3 authentication.
+- **`MINIO_SECRET_KEY`** _(required)_ - The MinIO secret key for S3 authentication.
+- **`S3_PREFIX`** _(optional)_ - Optional path prefix within the bucket to backup/restore.
+
+**Restore-Specific Variables:**
+
+- **`RESTORE_SNAPSHOT_ID`** _(optional)_ - The specific snapshot ID to restore (defaults to `latest`).
+
 ## Dependencies
 
 Ensure the following commands are available in the container:
@@ -64,6 +80,7 @@ Ensure the following commands are available in the container:
 - `jq`
 - `pg_dump` _(only required for PostgreSQL backup operations)_
 - `psql` _(only required for PostgreSQL restore operations)_
+- `mc` _(only required for S3 operations)_
 
 ## Usage
 
@@ -236,6 +253,120 @@ spec:
               value: "Database Restore Notification"
 ```
 
+Example Kubernetes Job manifest for running BackupSidecar to backup an S3 bucket:
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: backupsidecar-s3-backup
+  namespace: authentik
+spec:
+  backoffLimit: 3
+  activeDeadlineSeconds: 600
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: backupsidecar
+          image: backupsidecar:latest
+          env:
+            - name: OPERATION_MODE
+              value: "backup"
+            - name: BACKUP_MODE
+              value: "s3"
+            - name: S3_BUCKET
+              value: "my-bucket"
+            - name: S3_ENDPOINT
+              value: "http://minio:9000"
+            - name: S3_PREFIX
+              value: "data" # optional
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: minio-secret
+                  key: access_key
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: minio-secret
+                  key: secret_key
+            - name: RESTIC_REPOSITORY
+              value: "rest:http://rest-server:8000/backup"
+            - name: RESTIC_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: backupsidecar-secret
+                  key: restic_password
+            - name: GOTIFYHOST
+              value: "http://gotify.example.com"
+            - name: GOTIFYTOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: backupsidecar-secret
+                  key: gotify_token
+            - name: GOTIFYTOPIC
+              value: "S3 Backup Notification"
+```
+
+Example Kubernetes Job manifest for running BackupSidecar to restore an S3 bucket:
+
+```yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: backupsidecar-s3-restore
+  namespace: authentik
+spec:
+  backoffLimit: 3
+  activeDeadlineSeconds: 600
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: backupsidecar
+          image: backupsidecar:latest
+          env:
+            - name: OPERATION_MODE
+              value: "restore"
+            - name: BACKUP_MODE
+              value: "s3"
+            - name: S3_BUCKET
+              value: "my-bucket"
+            - name: S3_ENDPOINT
+              value: "http://minio:9000"
+            - name: S3_PREFIX
+              value: "data" # optional
+            - name: RESTORE_SNAPSHOT_ID
+              value: "abc123def456" # optional, defaults to latest
+            - name: MINIO_ACCESS_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: minio-secret
+                  key: access_key
+            - name: MINIO_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: minio-secret
+                  key: secret_key
+            - name: RESTIC_REPOSITORY
+              value: "rest:http://rest-server:8000/backup"
+            - name: RESTIC_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: backupsidecar-secret
+                  key: restic_password
+            - name: GOTIFYHOST
+              value: "http://gotify.example.com"
+            - name: GOTIFYTOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: backupsidecar-secret
+                  key: gotify_token
+            - name: GOTIFYTOPIC
+              value: "S3 Restore Notification"
+```
+
 ## Notifications
 
 The script can send success or failure notifications via Gotify when enabled. To enable notifications, set `ENABLE_GOTIFY=true` and provide the required Gotify configuration variables (`GOTIFYHOST`, `GOTIFYTOKEN`, `GOTIFYTOPIC`). When notifications are disabled, backup status messages are still logged to the console.

src/backup.sh

@@ -24,7 +24,7 @@ OPERATION_MODE="${OPERATION_MODE:-backup}"
 
 #######################################
 # Determine backup mode from the environment only.
-# Valid values: "directory" or "postgres".
+# Valid values: "directory", "postgres", or "s3".
 # Default to "directory" if not provided.
 #######################################
 BACKUP_MODE="${BACKUP_MODE:-directory}"
@@ -39,6 +39,8 @@ if [ "$BACKUP_MODE" = "postgres" ]; then
     elif [ "$OPERATION_MODE" = "restore" ]; then
         REQUIRED_CMDS+=(psql)
     fi
+elif [ "$BACKUP_MODE" = "s3" ]; then
+    REQUIRED_CMDS+=(mc)
 fi
 
 for cmd in "${REQUIRED_CMDS[@]}"; do
@@ -107,8 +109,16 @@ case "$BACKUP_MODE" in
             fi
         fi
         ;;
+    s3)
+        : "${S3_BUCKET:?Environment variable S3_BUCKET is not set (required for S3 mode)}"
+        : "${S3_ENDPOINT:?Environment variable S3_ENDPOINT is not set (required for S3 mode)}"
+        : "${MINIO_ACCESS_KEY:?Environment variable MINIO_ACCESS_KEY is not set (required for S3 mode)}"
+        : "${MINIO_SECRET_KEY:?Environment variable MINIO_SECRET_KEY is not set (required for S3 mode)}"
+        # Optional: S3 path prefix
+        : "${S3_PREFIX:=}"
+        ;;
     *)
-        echo "Error: Unknown backup mode '$BACKUP_MODE'. Valid modes are 'directory' and 'postgres'." >&2
+        echo "Error: Unknown backup mode '$BACKUP_MODE'. Valid modes are 'directory', 'postgres', and 's3'." >&2
         exit 1
         ;;
 esac
@@ -287,6 +297,91 @@ restore_postgres() {
     fi
 }
 
+#######################################
+# Backup an S3 bucket.
+# Syncs the S3 bucket to a temporary directory and then backs it up.
+#######################################
+backup_s3() {
+    log "Starting S3 backup for bucket '${S3_BUCKET}' at endpoint '${S3_ENDPOINT}'"
+    
+    # Create a temporary directory for the S3 sync.
+    TEMP_BACKUP_DIR=$(mktemp -d)
+    log "Created temporary directory: ${TEMP_BACKUP_DIR}"
+
+    # Configure MinIO Client alias
+    local alias_name="backupsidecar"
+    if ! mc alias set "${alias_name}" "${S3_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"; then
+        local msg="Failed to configure MinIO client alias"
+        log "$msg"
+        send_notification "$msg"
+        exit 1
+    fi
+
+    # Build S3 path
+    local s3_path="${alias_name}/${S3_BUCKET}"
+    if [ -n "${S3_PREFIX}" ]; then
+        s3_path="${s3_path}/${S3_PREFIX}"
+    fi
+
+    log "Syncing S3 bucket from ${s3_path} to ${TEMP_BACKUP_DIR}..."
+    if mc mirror "${s3_path}" "${TEMP_BACKUP_DIR}" --remove; then
+        log "S3 sync completed successfully."
+    else
+        local exit_code=$?
+        local msg="S3 sync failed with error code ${exit_code}"
+        log "$msg"
+        send_notification "$msg"
+        exit "$exit_code"
+    fi
+
+    # Back up the directory containing the S3 content.
+    run_restic_backup "${TEMP_BACKUP_DIR}"
+}
+
+#######################################
+# Restore an S3 bucket.
+# Restores the S3 content from the backup and syncs it back to S3.
+#######################################
+restore_s3() {
+    local snapshot_id="${RESTORE_SNAPSHOT_ID:-latest}"
+    log "Starting S3 restore for bucket '${S3_BUCKET}' at endpoint '${S3_ENDPOINT}'"
+    
+    # Create a temporary directory for the restore.
+    TEMP_RESTORE_DIR=$(mktemp -d)
+    log "Created temporary directory: ${TEMP_RESTORE_DIR}"
+
+    # Restore the backup to the temporary directory
+    run_restic_restore "${TEMP_RESTORE_DIR}" "${snapshot_id}"
+
+    # Configure MinIO Client alias
+    local alias_name="backupsidecar"
+    if ! mc alias set "${alias_name}" "${S3_ENDPOINT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"; then
+        local msg="Failed to configure MinIO client alias"
+        log "$msg"
+        send_notification "$msg"
+        exit 1
+    fi
+
+    # Build S3 path
+    local s3_path="${alias_name}/${S3_BUCKET}"
+    if [ -n "${S3_PREFIX}" ]; then
+        s3_path="${s3_path}/${S3_PREFIX}"
+    fi
+
+    log "Syncing restored content from ${TEMP_RESTORE_DIR} to ${s3_path}..."
+    if mc mirror "${TEMP_RESTORE_DIR}" "${s3_path}" --remove; then
+        local msg="S3 restore completed successfully"
+        log "$msg"
+        send_notification "$msg"
+    else
+        local exit_code=$?
+        local msg="S3 restore failed with error code ${exit_code}"
+        log "$msg"
+        send_notification "$msg"
+        exit "$exit_code"
+    fi
+}
+
 #######################################
 # Cleanup temporary resources.
 #######################################
@@ -315,6 +410,9 @@ main() {
                 postgres)
                     backup_postgres
                     ;;
+                s3)
+                    backup_s3
+                    ;;
             esac
             ;;
         restore)
@@ -325,6 +423,9 @@ main() {
                 postgres)
                     restore_postgres
                     ;;
+                s3)
+                    restore_s3
+                    ;;
             esac
             ;;
     esac