Ana içeriğe geç

🗄️ PHASE 8: BACKUP & DISASTER RECOVERY (Gün 19-20)#

💾 9.1 Velero Backup Setup#

# 9.1.1 AWS S3 bucket oluştur
BACKUP_BUCKET="mycompany-k8s-backups-$(openssl rand -hex 4)"
aws s3 mb s3://$BACKUP_BUCKET --region eu-west-1

# S3 bucket policy
cat > backup-bucket-policy.json << EOF
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Sid": "VeleroBackupAccess",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::$(aws sts get-caller-identity --query Account --output text):role/velero-role"
            },
            "Action": [
                "s3:GetObject",
                "s3:DeleteObject",
                "s3:PutObject",
                "s3:AbortMultipartUpload",
                "s3:ListMultipartUploadParts"
            ],
            "Resource": "arn:aws:s3:::$BACKUP_BUCKET/*"
        },
        {
            "Sid": "VeleroBackupList",
            "Effect": "Allow",
            "Principal": {
                "AWS": "arn:aws:iam::$(aws sts get-caller-identity --query Account --output text):role/velero-role"
            },
            "Action": [
                "s3:ListBucket"
            ],
            "Resource": "arn:aws:s3:::$BACKUP_BUCKET"
        }
    ]
}
EOF

# IAM policy için Velero permissions
cat > velero-policy.json << 'EOF'
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Action": [
                "ec2:DescribeVolumes",
                "ec2:DescribeSnapshots",
                "ec2:CreateTags",
                "ec2:CreateVolume",
                "ec2:CreateSnapshot",
                "ec2:DeleteSnapshot"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:GetObject",
                "s3:DeleteObject",
                "s3:PutObject",
                "s3:AbortMultipartUpload",
                "s3:ListMultipartUploadParts"
            ],
            "Resource": [
                "arn:aws:s3:::BUCKET-NAME/*"
            ]
        },
        {
            "Effect": "Allow",
            "Action": [
                "s3:ListBucket"
            ],
            "Resource": [
                "arn:aws:s3:::BUCKET-NAME"
            ]
        }
    ]
}
EOF

sed -i "s/BUCKET-NAME/$BACKUP_BUCKET/g" velero-policy.json

# IAM policy oluştur
aws iam create-policy \
    --policy-name VeleroBackupPolicy \
    --policy-document file://velero-policy.json

# Service account için trust policy
cat > velero-trust-policy.json << 'EOF'
{
    "Version": "2012-10-17",
    "Statement": [
        {
            "Effect": "Allow",
            "Principal": {
                "Federated": "arn:aws:iam::ACCOUNT-ID:oidc-provider/OIDC-URL"
            },
            "Action": "sts:AssumeRoleWithWebIdentity",
            "Condition": {
                "StringEquals": {
                    "OIDC-URL:sub": "system:serviceaccount:velero:velero",
                    "OIDC-URL:aud": "sts.amazonaws.com"
                }
            }
        }
    ]
}
EOF

# OIDC provider URL'ini al
OIDC_URL=$(aws eks describe-cluster --name mycompany-dev-eks --query "cluster.identity.oidc.issuer" --output text | sed 's|https://||')
ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text)

sed -i "s/ACCOUNT-ID/$ACCOUNT_ID/g" velero-trust-policy.json
sed -i "s/OIDC-URL/$OIDC_URL/g" velero-trust-policy.json

# IAM role oluştur
aws iam create-role \
    --role-name velero-role \
    --assume-role-policy-document file://velero-trust-policy.json

# Policy'yi role'e attach et
aws iam attach-role-policy \
    --role-arn arn:aws:iam::$ACCOUNT_ID:role/velero-role \
    --policy-arn arn:aws:iam::$ACCOUNT_ID:policy/VeleroBackupPolicy

# 9.1.2 Velero CLI kurulumu
wget https://github.com/vmware-tanzu/velero/releases/download/v1.12.0/velero-v1.12.0-linux-amd64.tar.gz
tar -xzf velero-v1.12.0-linux-amd64.tar.gz
sudo mv velero-v1.12.0-linux-amd64/velero /usr/local/bin/
rm -rf velero-v1.12.0-linux-amd64*

# 9.1.3 Velero kurulumu
cat > velero-values.yaml << EOF
configuration:
  backupStorageLocation:
  - name: aws
    provider: aws
    bucket: $BACKUP_BUCKET
    config:
      region: eu-west-1
  volumeSnapshotLocation:
  - name: aws
    provider: aws
    config:
      region: eu-west-1

credentials:
  useSecret: false

serviceAccount:
  server:
    annotations:
      eks.amazonaws.com/role-arn: arn:aws:iam::$ACCOUNT_ID:role/velero-role

initContainers:
- name: velero-plugin-for-aws
  image: velero/velero-plugin-for-aws:v1.8.0
  volumeMounts:
  - mountPath: /target
    name: plugins

resources:
  requests:
    cpu: 500m
    memory: 128Mi
  limits:
    cpu: 1000m
    memory: 512Mi

nodeAgent:
  resources:
    requests:
      cpu: 500m
      memory: 512Mi
    limits:
      cpu: 1000m
      memory: 1024Mi

schedules:
  daily-backup:
    disabled: false
    schedule: "0 2 * * *"
    template:
      includedNamespaces:
      - dev
      - staging
      - production
      - monitoring
      - vault
      excludedResources:
      - events
      - events.events.k8s.io
      storageLocation: aws
      ttl: 720h0m0s
      snapshotVolumes: true

  weekly-backup:
    disabled: false
    schedule: "0 3 * * 0"
    template:
      includedNamespaces:
      - dev
      - staging
      - production
      - monitoring
      - vault
      excludedResources:
      - events
      - events.events.k8s.io
      storageLocation: aws
      ttl: 2160h0m0s
      snapshotVolumes: true
EOF

helm repo add vmware-tanzu https://vmware-tanzu.github.io/helm-charts
helm repo update

kubectl create namespace velero
helm install velero vmware-tanzu/velero \
  --namespace velero \
  --values velero-values.yaml

# 9.1.4 Manual backup test
velero backup create test-backup --include-namespaces dev
velero backup describe test-backup
velero backup logs test-backup

echo "Backup bucket: $BACKUP_BUCKET"

🔄 9.2 Database Backup Strategy#

# 9.2.1 RDS automated backup script
cat > ~/devops-infrastructure/scripts/rds-backup.sh << 'EOF'
#!/bin/bash

# RDS Backup Script
set -e

DB_IDENTIFIER="mycompany-dev-db"
BACKUP_PREFIX="manual-backup"
REGION="eu-west-1"

# Create manual snapshot
SNAPSHOT_ID="${BACKUP_PREFIX}-$(date +%Y%m%d%H%M%S)"

echo "Creating RDS snapshot: $SNAPSHOT_ID"
aws rds create-db-snapshot \
    --db-instance-identifier $DB_IDENTIFIER \
    --db-snapshot-identifier $SNAPSHOT_ID \
    --region $REGION

# Wait for snapshot completion
echo "Waiting for snapshot completion..."
aws rds wait db-snapshot-completed \
    --db-snapshot-identifier $SNAPSHOT_ID \
    --region $REGION

echo "Snapshot created successfully: $SNAPSHOT_ID"

# List recent snapshots
echo "Recent snapshots:"
aws rds describe-db-snapshots \
    --db-instance-identifier $DB_IDENTIFIER \
    --snapshot-type manual \
    --region $REGION \
    --query 'DBSnapshots[0:5].[DBSnapshotIdentifier,Status,SnapshotCreateTime]' \
    --output table

# Cleanup old manual snapshots (keep last 7)
OLD_SNAPSHOTS=$(aws rds describe-db-snapshots \
    --db-instance-identifier $DB_IDENTIFIER \
    --snapshot-type manual \
    --region $REGION \
    --query 'DBSnapshots[7:].DBSnapshotIdentifier' \
    --output text)

if [ ! -z "$OLD_SNAPSHOTS" ]; then
    echo "Cleaning up old snapshots..."
    for snapshot in $OLD_SNAPSHOTS; do
        echo "Deleting snapshot: $snapshot"
        aws rds delete-db-snapshot \
            --db-snapshot-identifier $snapshot \
            --region $REGION
    done
fi

echo "Backup completed successfully!"
EOF

chmod +x ~/devops-infrastructure/scripts/rds-backup.sh

# 9.2.2 PostgreSQL logical backup (for application data)
cat > ~/devops-infrastructure/scripts/postgres-logical-backup.sh << 'EOF'
#!/bin/bash

# PostgreSQL Logical Backup Script
set -e

# Configuration
DB_HOST="your-rds-endpoint"
DB_NAME="mycompanydb"
DB_USER="admin"
BACKUP_DIR="/tmp/pg-backups"
S3_BUCKET="mycompany-db-logical-backups"
DATE=$(date +%Y%m%d_%H%M%S)

# Create backup directory
mkdir -p $BACKUP_DIR

# Get password from Kubernetes secret
DB_PASSWORD=$(kubectl get secret database-secret -n dev -o jsonpath='{.data.password}' | base64 -d)

export PGPASSWORD=$DB_PASSWORD

# Create backup
echo "Creating logical backup..."
pg_dump -h $DB_HOST -U $DB_USER -d $DB_NAME \
    --verbose \
    --no-password \
    --format=custom \
    --compress=9 \
    --file=$BACKUP_DIR/logical-backup-$DATE.dump

# Upload to S3
echo "Uploading to S3..."
aws s3 cp $BACKUP_DIR/logical-backup-$DATE.dump \
    s3://$S3_BUCKET/logical-backups/logical-backup-$DATE.dump

# Cleanup local file
rm $BACKUP_DIR/logical-backup-$DATE.dump

# Cleanup old S3 backups (keep last 30 days)
echo "Cleaning up old backups..."
aws s3 ls s3://$S3_BUCKET/logical-backups/ \
    --recursive \
    --query "Contents[?LastModified<='$(date -d '30 days ago' --iso-8601)'].Key" \
    --output text | \
    xargs -I {} aws s3 rm s3://$S3_BUCKET/{}

echo "Logical backup completed successfully!"
EOF

chmod +x ~/devops-infrastructure/scripts/postgres-logical-backup.sh

# 9.2.3 CronJob for automated database backups
cat > database-backup-cronjob.yaml << 'EOF'
apiVersion: batch/v1
kind: CronJob
metadata:
  name: postgres-logical-backup
  namespace: dev
spec:
  schedule: "0 1 * * *"  # Daily at 1 AM
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: backup-sa
          containers:
          - name: backup
            image: postgres:15-alpine
            env:
            - name: DB_HOST
              value: "your-rds-endpoint"
            - name: DB_NAME
              value: "mycompanydb"
            - name: DB_USER
              value: "admin"
            - name: DB_PASSWORD
              valueFrom:
                secretKeyRef:
                  name: database-secret
                  key: password
            - name: S3_BUCKET
              value: "mycompany-db-logical-backups"
            command:
            - /bin/bash
            - -c
            - |
              set -e
              apk add --no-cache aws-cli

              DATE=$(date +%Y%m%d_%H%M%S)
              BACKUP_FILE="/tmp/logical-backup-$DATE.dump"

              export PGPASSWORD=$DB_PASSWORD

              echo "Creating logical backup..."
              pg_dump -h $DB_HOST -U $DB_USER -d $DB_NAME \
                  --verbose \
                  --no-password \
                  --format=custom \
                  --compress=9 \
                  --file=$BACKUP_FILE

              echo "Uploading to S3..."
              aws s3 cp $BACKUP_FILE s3://$S3_BUCKET/logical-backups/

              echo "Backup completed successfully!"
            resources:
              requests:
                memory: "256Mi"
                cpu: "100m"
              limits:
                memory: "512Mi"
                cpu: "200m"
          restartPolicy: OnFailure
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: backup-sa
  namespace: dev
  annotations:
    eks.amazonaws.com/role-arn: arn:aws:iam::ACCOUNT_ID:role/backup-role
EOF

kubectl apply -f database-backup-cronjob.yaml

📋 9.3 Disaster Recovery Runbook#

# 9.3.1 DR runbook oluştur
cat > ~/devops-infrastructure/docs/disaster-recovery-runbook.md << 'EOF'
# Disaster Recovery Runbook

## Overview
Bu doküman Kubernetes cluster ve RDS veritabanı için disaster recovery prosedürlerini içerir.

## RTO/RPO Targets
- **RTO (Recovery Time Objective)**: 4 saat
- **RPO (Recovery Point Objective)**: 1 saat

## Disaster Scenarios

### 1. Complete Cluster Loss

#### Assessment
```bash
# Cluster durumunu kontrol et
kubectl get nodes
kubectl get pods --all-namespaces

# AWS EKS cluster durumu
aws eks describe-cluster --name mycompany-dev-eks

Recovery Steps#

  1. Yeni cluster oluştur 

    cd ~/devops-infrastructure/terraform/environments/dev
terraform plan -target=module.eks
terraform apply -target=module.eks

  2. Velero restore 

    # En son backup'ı listele
velero backup get

# Restore işlemi
velero restore create restore-$(date +%Y%m%d) \
    --from-backup daily-backup-YYYYMMDD

  3. Database connectivity kontrol 

    kubectl get secrets database-secret -n dev
kubectl run test-db-connection --rm -i --tty \
    --image=postgres:15-alpine -- \
    psql -h RDS_ENDPOINT -U admin -d mycompanydb

2. Database Disaster#

Assessment#

# RDS status kontrol
aws rds describe-db-instances \
    --db-instance-identifier mycompany-dev-db

# Connection test
kubectl run db-test --rm -i --tty \
    --image=postgres:15-alpine -- \
    psql -h RDS_ENDPOINT -U admin -d mycompanydb -c "SELECT 1;"

Recovery Steps#

  1. Point-in-time recovery 

    # Son valid backup time'ı bul
aws rds describe-db-instances \
    --db-instance-identifier mycompany-dev-db \
    --query 'DBInstances[0].LatestRestorableTime'

# Point-in-time restore
aws rds restore-db-instance-to-point-in-time \
    --source-db-instance-identifier mycompany-dev-db \
    --target-db-instance-identifier mycompany-dev-db-recovered \
    --restore-time 2024-XX-XXTXX:XX:XX.000Z

  2. Manual snapshot restore 

    # Available snapshots
aws rds describe-db-snapshots \
    --db-instance-identifier mycompany-dev-db

# Restore from snapshot
aws rds restore-db-instance-from-db-snapshot \
    --db-instance-identifier mycompany-dev-db-recovered \
    --db-snapshot-identifier manual-backup-YYYYMMDDHHMMSS

  3. Application reconnection 

    # Update database endpoint in secrets
kubectl patch secret database-secret -n dev \
    --type='json' \
    -p='[{"op": "replace", "path": "/data/host", "value":"'$(echo NEW_RDS_ENDPOINT | base64 -w 0)'"}]'

# Restart applications
kubectl rollout restart deployment -n dev

3. Data Corruption#

Assessment#

# Check for data inconsistencies
kubectl exec -it deployment/backend -n dev -- \
    python manage.py check_data_integrity

# Check database logs
aws rds describe-db-log-files \
    --db-instance-identifier mycompany-dev-db

Recovery Steps#

  1. Identify corruption scope 

    # Analyze affected data
kubectl exec -it deployment/backend -n dev -- \
    python manage.py analyze_corruption

  2. Restore from logical backup 

    # Download latest logical backup
aws s3 cp s3://mycompany-db-logical-backups/logical-backups/latest.dump /tmp/

# Restore specific tables
pg_restore -h RDS_ENDPOINT -U admin -d mycompanydb \
    --table=affected_table \
    --clean \
    /tmp/latest.dump

Testing Procedures#

Monthly DR Drill#

  1. Create test restore in separate namespace
  2. Verify data integrity
  3. Test application functionality
  4. Document lessons learned

Quarterly Full DR Test#

  1. Complete environment recreation
  2. Full data restore
  3. End-to-end testing
  4. Performance validation

Emergency Contacts#

  • DevOps Team: +90-XXX-XXX-XXXX
  • Database Team: +90-XXX-XXX-XXXX
  • On-call Engineer: +90-XXX-XXX-XXXX

Post-Incident Actions#

  1. Root Cause Analysis
  2. Document incident timeline
  3. Identify failure points

  4. Implement preventive measures

  5. Update Procedures

  6. Update runbooks
  7. Improve monitoring

  8. Enhance alerting

  9. Team Communication

  10. Share lessons learned
  11. Update training materials
  12. Schedule review meeting EOF

9.3.2 DR test script#

cat > ~/devops-infrastructure/scripts/dr-test.sh << 'EOF'

!/bin/bash#

Disaster Recovery Test Script#

set -e

NAMESPACE="dr-test" BACKUP_NAME="$1"

if [ -z "$BACKUP_NAME" ]; then echo "Usage: $0 " echo "Available backups:" velero backup get exit 1 fi

echo "Starting DR test with backup: $BACKUP_NAME"

Create test namespace#

kubectl create namespace $NAMESPACE --dry-run=client -o yaml | kubectl apply -f -

Restore from backup to test namespace#

velero restore create dr-test-$(date +%Y%m%d%H%M%S) \ --from-backup \(BACKUP_NAME \ --namespace-mappings dev:\)NAMESPACE,staging:$NAMESPACE

Wait for restore completion#

echo "Waiting for restore completion..." sleep 60

Check restored resources#

echo "Checking restored resources..." kubectl get all -n $NAMESPACE

Test database connectivity#

echo "Testing database connectivity..." kubectl run db-test -n $NAMESPACE --rm -i --tty \ --image=postgres:15-alpine -- \ psql -h $(kubectl get secret database-secret -n $NAMESPACE -o jsonpath='{.data.host}' | base64 -d) \ -U $(kubectl get secret database-secret -n $NAMESPACE -o jsonpath='{.data.username}' | base64 -d) \ -d mycompanydb \ -c "SELECT COUNT(*) FROM information_schema.tables;"

echo "DR test completed successfully!" echo "Cleanup: kubectl delete namespace $NAMESPACE" EOF

chmod +x ~/devops-infrastructure/scripts/dr-test.sh ```