diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..f2d460d --- /dev/null +++ b/TESTING.md @@ -0,0 +1,348 @@ +# Backup Script Testing Guide + +This document provides step-by-step instructions for testing the backup script locally using Docker Compose with a Restic server and PostgreSQL database. + +## Prerequisites + +- Docker and Docker Compose installed +- The backup script (`src/backup.sh`) and its dependencies (restic, curl, jq, pg_dump, psql) +- Basic understanding of bash and PostgreSQL + +## Setup + +### 1. Start the Test Environment + +```bash +# Start the services +docker-compose up -d + +# Wait for services to be ready (about 10-15 seconds) +sleep 15 + +# Verify services are running +docker-compose ps +``` + +### 2. Verify Services + +```bash +# Check if restic server is accessible +curl -s http://localhost:8000/api/v1/repos + +# Check if PostgreSQL is accessible +docker exec postgres-test psql -U testuser -d testdb -c "SELECT version();" +``` + +## Directory Backup Testing + +This section tests the directory backup and restore functionality. + +### 1. Prepare Test Data + +```bash +# Create a test directory with sample files +mkdir -p test_data +cd test_data + +# Create various types of files +echo "This is a text file" > sample.txt +echo '{"name": "test", "value": 123}' > data.json +mkdir -p subdir +echo "Nested file content" > subdir/nested.txt +echo "Binary data" | base64 > binary.dat + +# Create some larger files for testing +dd if=/dev/urandom of=large_file.bin bs=1M count=10 2>/dev/null + +# Go back to project root +cd .. +``` + +### 2. Perform Directory Backup + +```bash +# Set environment variables for directory backup +export OPERATION_MODE="backup" +export BACKUP_MODE="directory" +export RESTIC_PASSWORD="testpassword123" +export RESTIC_REPOSITORY="rest:http://localhost:8000/test-backup" +export SOURCEDIR="/home/tbehrendt/dev/backupsidecar/test_data" +export ENABLE_GOTIFY="false" + +# Run the backup +./src/backup.sh +``` + +### 3. Verify Backup + +```bash +# List snapshots +restic -r "rest:http://localhost:8000/test-backup" snapshots --password-file <(echo "testpassword123") + +# Check backup contents +restic -r "rest:http://localhost:8000/test-backup" ls latest --password-file <(echo "testpassword123") +``` + +### 4. Perform Directory Restore + +```bash +# Create restore directory +mkdir -p restored_data + +# Set environment variables for directory restore +export OPERATION_MODE="restore" +export RESTOREDIR="/home/tbehrendt/dev/backupsidecar/restored_data" +export RESTORE_SNAPSHOT_ID="latest" + +# Run the restore +./src/backup.sh +``` + +### 5. Verify Directory Restore + +```bash +# Compare original and restored directories +diff -r test_data restored_data + +# Check file contents +echo "Original file:" +cat test_data/sample.txt +echo "Restored file:" +cat restored_data/sample.txt + +# Verify binary file integrity +md5sum test_data/large_file.bin +md5sum restored_data/large_file.bin + +# Check directory structure +tree test_data +tree restored_data +``` + +### 6. Cleanup Directory Test + +```bash +# Remove test directories +rm -rf test_data restored_data +``` + +## PostgreSQL Backup Testing + +This section tests the PostgreSQL database backup and restore functionality. + +### 1. Generate Test Data + +```bash +# Generate test data in PostgreSQL +./generate_test_data.sh +``` + +### 2. Verify Initial Data + +```bash +# Check that data exists +docker exec postgres-test psql -U testuser -d testdb -c " +SELECT 'customers' as table_name, COUNT(*) as row_count FROM customers +UNION ALL +SELECT 'orders' as table_name, COUNT(*) as row_count FROM orders; +" +``` + +### 3. Perform PostgreSQL Backup + +```bash +# Set environment variables for PostgreSQL backup +export OPERATION_MODE="backup" +export BACKUP_MODE="postgres" +export RESTIC_PASSWORD="testpassword123" +export RESTIC_REPOSITORY="rest:http://localhost:8000/postgres-backup" +export PGHOST="localhost" +export PGPORT="5432" +export PGDATABASE="testdb" +export PGUSER="testuser" +export PGPASSWORD="testpass" +export ENABLE_GOTIFY="false" + +# Run the backup +./src/backup.sh +``` + +### 4. Verify PostgreSQL Backup + +```bash +# List snapshots +restic -r "rest:http://localhost:8000/postgres-backup" snapshots --password-file <(echo "testpassword123") + +# Check backup contents +restic -r "rest:http://localhost:8000/postgres-backup" ls latest --password-file <(echo "testpassword123") +``` + +### 5. Clear the Database + +```bash +# Drop all tables to simulate data loss +docker exec postgres-test psql -U testuser -d testdb -c " +DROP TABLE IF EXISTS orders CASCADE; +DROP TABLE IF EXISTS customers CASCADE; +" + +# Verify database is empty +docker exec postgres-test psql -U testuser -d testdb -c " +SELECT table_name FROM information_schema.tables +WHERE table_schema = 'public'; +" +``` + +### 6. Perform PostgreSQL Restore + +```bash +# Set environment variables for PostgreSQL restore +export OPERATION_MODE="restore" +export RESTORE_SNAPSHOT_ID="latest" + +# Run the restore +./src/backup.sh +``` + +### 7. Verify PostgreSQL Restore + +```bash +# Check that data has been restored +docker exec postgres-test psql -U testuser -d testdb -c " +SELECT 'customers' as table_name, COUNT(*) as row_count FROM customers +UNION ALL +SELECT 'orders' as table_name, COUNT(*) as row_count FROM orders; +" + +# Verify data integrity +docker exec postgres-test psql -U testuser -d testdb -c " +SELECT + c.name as customer_name, + COUNT(o.id) as order_count, + SUM(o.price * o.quantity) as total_spent +FROM customers c +LEFT JOIN orders o ON c.id = o.customer_id +GROUP BY c.id, c.name +ORDER BY total_spent DESC +LIMIT 5; +" + +# Check foreign key constraints +docker exec postgres-test psql -U testuser -d testdb -c " +SELECT + tc.constraint_name, + tc.table_name, + kcu.column_name, + ccu.table_name AS foreign_table_name, + ccu.column_name AS foreign_column_name +FROM information_schema.table_constraints AS tc +JOIN information_schema.key_column_usage AS kcu + ON tc.constraint_name = kcu.constraint_name + AND tc.table_schema = kcu.table_schema +JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = tc.constraint_name + AND ccu.table_schema = tc.table_schema +WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.table_name IN ('customers', 'orders'); +" +``` + +## Advanced Testing Scenarios + +### 1. Test with Different Snapshot IDs + +```bash +# List all snapshots +restic -r "rest:http://localhost:8000/test-backup" snapshots --password-file <(echo "testpassword123") + +# Restore a specific snapshot +export RESTORE_SNAPSHOT_ID="" +./src/backup.sh +``` + +### 2. Test Error Handling + +```bash +# Test with invalid repository +export RESTIC_REPOSITORY="rest:http://localhost:8000/nonexistent" +./src/backup.sh + +# Test with wrong password +export RESTIC_PASSWORD="wrongpassword" +./src/backup.sh +``` + +### 3. Test with Gotify Notifications (Optional) + +```bash +# If you have a Gotify server running +export ENABLE_GOTIFY="true" +export GOTIFYHOST="http://your-gotify-server:80" +export GOTIFYTOKEN="your-token" +export GOTIFYTOPIC="Backup Test" +./src/backup.sh +``` + +## Cleanup + +### 1. Stop Services + +```bash +# Stop and remove containers +docker-compose down + +# Remove volumes (optional - this will delete all data) +docker-compose down -v +``` + +### 2. Clean Up Test Files + +```bash +# Remove any remaining test files +rm -rf test_data restored_data +``` + +## Troubleshooting + +### Common Issues + +1. **Connection refused to restic server**: Wait a bit longer for the container to start up +2. **PostgreSQL connection failed**: Ensure the database container is fully initialized +3. **Permission denied**: Make sure the backup script is executable (`chmod +x src/backup.sh`) +4. **Restic repository not found**: Check that the repository URL is correct and the server is running + +### Debug Commands + +```bash +# Check container logs +docker-compose logs restic-server +docker-compose logs postgres + +# Test restic connectivity +restic -r "rest:http://localhost:8000/test-backup" snapshots --password-file <(echo "testpassword123") + +# Test PostgreSQL connectivity +docker exec postgres-test psql -U testuser -d testdb -c "SELECT 1;" +``` + +## Expected Results + +### Directory Backup Test + +- ✅ Backup completes successfully +- ✅ Files are backed up to restic repository +- ✅ Restore completes successfully +- ✅ Restored files match original files exactly +- ✅ Directory structure is preserved + +### PostgreSQL Backup Test + +- ✅ Database backup completes successfully +- ✅ Database dump is backed up to restic repository +- ✅ Database can be cleared successfully +- ✅ Database restore completes successfully +- ✅ All data is restored correctly +- ✅ Foreign key relationships are maintained +- ✅ Data integrity is preserved + +This testing procedure ensures that both directory and PostgreSQL backup/restore functionality works correctly and can be used as a foundation for automated testing in CI/CD pipelines. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3bf3604 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,21 @@ +version: "3.8" + +services: + restic-server: + image: restic/rest-server:latest + container_name: restic-server + ports: + - "8000:8000" + command: rest-server --no-auth --path /data + restart: unless-stopped + + postgres: + image: postgres:17 + container_name: postgres-test + ports: + - "5432:5432" + environment: + POSTGRES_DB: testdb + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + restart: unless-stopped diff --git a/generate_dir_test_data.sh b/generate_dir_test_data.sh new file mode 100755 index 0000000..c81dbdc --- /dev/null +++ b/generate_dir_test_data.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -euo pipefail + +# Script to generate test data for Directory backup testing +# This script creates a few directories with a few files and directories in each and populates it with test data + +# Create base test directory +TEST_DIR="/tmp/test-data" +echo "Creating test directory structure in $TEST_DIR..." + +# Remove existing test directory if it exists +rm -rf "$TEST_DIR" +mkdir -p "$TEST_DIR" + +# Create various subdirectories +mkdir -p "$TEST_DIR/documents/reports" +mkdir -p "$TEST_DIR/documents/contracts" +mkdir -p "$TEST_DIR/data/logs" +mkdir -p "$TEST_DIR/data/backups" + +# Create text files with content +echo "This is the annual report for 2023" > "$TEST_DIR/documents/reports/annual_2023.txt" +echo "Q4 financial summary" > "$TEST_DIR/documents/reports/q4_summary.txt" +echo "Contract terms and conditions" > "$TEST_DIR/documents/contracts/agreement.txt" + +# Create JSON files +cat << 'EOF' > "$TEST_DIR/data/config.json" +{ + "app_name": "TestApp", + "version": "1.0.0", + "settings": { + "debug": true, + "max_retries": 3, + "timeout": 30 + } +} +EOF + +# Create some log files +for i in {1..3}; do + echo "$(date) - Log entry $i" >> "$TEST_DIR/data/logs/app.log" + echo "$(date) - Error $i: Sample error message" >> "$TEST_DIR/data/logs/error.log" +done + +# Create symbolic links +ln -s "../reports/annual_2023.txt" "$TEST_DIR/documents/contracts/report_link" +ln -s "../../data/config.json" "$TEST_DIR/documents/reports/config_link" + +echo "Test data generation completed successfully!" +echo "Created directory structure:" +tree "$TEST_DIR" diff --git a/generate_sql_test_data.sh b/generate_sql_test_data.sh new file mode 100755 index 0000000..3bdb612 --- /dev/null +++ b/generate_sql_test_data.sh @@ -0,0 +1,124 @@ +#!/bin/bash +set -euo pipefail + +# Script to generate test data for PostgreSQL backup testing +# This script creates two tables with a foreign key relationship and populates them with test data + +# Database connection parameters +PGHOST="${PGHOST:-localhost}" +PGPORT="${PGPORT:-5432}" +PGDATABASE="${PGDATABASE:-testdb}" +PGUSER="${PGUSER:-testuser}" +PGPASSWORD="${PGPASSWORD:-testpass}" + +# Export password for psql +export PGPASSWORD + +echo "Generating test data for PostgreSQL database..." + +# Create tables +echo "Creating tables..." +psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" << 'EOF' +-- Drop tables if they exist +DROP TABLE IF EXISTS orders CASCADE; +DROP TABLE IF EXISTS customers CASCADE; + +-- Create customers table +CREATE TABLE customers ( + id SERIAL PRIMARY KEY, + name VARCHAR(100) NOT NULL, + email VARCHAR(100) UNIQUE NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Create orders table with foreign key to customers +CREATE TABLE orders ( + id SERIAL PRIMARY KEY, + customer_id INTEGER NOT NULL REFERENCES customers(id) ON DELETE CASCADE, + product_name VARCHAR(100) NOT NULL, + quantity INTEGER NOT NULL CHECK (quantity > 0), + price DECIMAL(10,2) NOT NULL CHECK (price >= 0), + order_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +-- Create indexes for better performance +CREATE INDEX idx_orders_customer_id ON orders(customer_id); +CREATE INDEX idx_customers_email ON customers(email); +CREATE INDEX idx_orders_order_date ON orders(order_date); +EOF + +# Insert test data +echo "Inserting test data..." +psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" << 'EOF' +-- Insert customers +INSERT INTO customers (name, email) VALUES + ('John Doe', 'john.doe@example.com'), + ('Jane Smith', 'jane.smith@example.com'), + ('Bob Johnson', 'bob.johnson@example.com'), + ('Alice Brown', 'alice.brown@example.com'), + ('Charlie Wilson', 'charlie.wilson@example.com'), + ('Diana Davis', 'diana.davis@example.com'), + ('Eve Miller', 'eve.miller@example.com'), + ('Frank Garcia', 'frank.garcia@example.com'), + ('Grace Lee', 'grace.lee@example.com'), + ('Henry Taylor', 'henry.taylor@example.com'); + +-- Insert orders +INSERT INTO orders (customer_id, product_name, quantity, price) VALUES + (1, 'Laptop', 1, 999.99), + (1, 'Mouse', 2, 25.50), + (2, 'Keyboard', 1, 75.00), + (2, 'Monitor', 1, 299.99), + (3, 'Headphones', 1, 150.00), + (3, 'Webcam', 1, 89.99), + (4, 'Tablet', 1, 399.99), + (4, 'Stylus', 1, 49.99), + (5, 'Smartphone', 1, 699.99), + (5, 'Case', 1, 19.99), + (6, 'Desktop', 1, 1299.99), + (6, 'RAM', 2, 79.99), + (7, 'SSD', 1, 199.99), + (7, 'Graphics Card', 1, 599.99), + (8, 'Motherboard', 1, 199.99), + (8, 'CPU', 1, 399.99), + (9, 'Power Supply', 1, 149.99), + (9, 'Cooling Fan', 2, 29.99), + (10, 'Cable Set', 1, 39.99), + (10, 'USB Hub', 1, 24.99); +EOF + +# Verify data +echo "Verifying test data..." +psql -h "$PGHOST" -p "$PGPORT" -U "$PGUSER" -d "$PGDATABASE" << 'EOF' +-- Show table counts +SELECT 'customers' as table_name, COUNT(*) as row_count FROM customers +UNION ALL +SELECT 'orders' as table_name, COUNT(*) as row_count FROM orders; + +-- Show sample data +SELECT 'Sample customers:' as info; +SELECT id, name, email FROM customers LIMIT 5; + +SELECT 'Sample orders:' as info; +SELECT o.id, c.name as customer_name, o.product_name, o.quantity, o.price +FROM orders o +JOIN customers c ON o.customer_id = c.id +LIMIT 5; + +-- Show foreign key relationship +SELECT 'Foreign key relationship check:' as info; +SELECT + c.name as customer_name, + COUNT(o.id) as order_count, + SUM(o.price * o.quantity) as total_spent +FROM customers c +LEFT JOIN orders o ON c.id = o.customer_id +GROUP BY c.id, c.name +ORDER BY total_spent DESC; +EOF + +echo "Test data generation completed successfully!" +echo "Database contains:" +echo "- 10 customers" +echo "- 20 orders with foreign key relationships" +echo "- Various data types and constraints" diff --git a/src/backup.sh b/src/backup.sh old mode 100644 new mode 100755