IMPORTANT NOTE: This site is not official Red Hat documentation and is provided for informational purposes only. These guides may be experimental, proof of concept, or early adoption. Officially supported documentation is available at docs.openshift.com and access.redhat.com.

Deploying OpenShift API for Data Protection on a ROSA cluster


Authors: Paul Czarkowski, Dustin Scott
Last Editor: Dustin Scott
Published Date: 14 September 2022
Modified Date: 25 May 2023


Prerequisites

Getting Started

  1. Create the following environment variables

    Change the cluster name to match your ROSA cluster and ensure you’re logged into the cluster as an Administrator. Ensure all fields are outputted correctly before moving on.

    export CLUSTER_NAME=my-cluster
    export ROSA_CLUSTER_ID=$(rosa describe cluster -c ${CLUSTER_NAME} --output json | jq -r .id)
    export REGION=$(rosa describe cluster -c ${CLUSTER_NAME} --output json | jq -r .region.id)
    export OIDC_ENDPOINT=$(oc get authentication.config.openshift.io cluster -o json | jq -r .spec.serviceAccountIssuer)
    export AWS_ACCOUNT_ID=`aws sts get-caller-identity --query Account --output text`
    export CLUSTER_VERSION=`rosa describe cluster -c ${CLUSTER_NAME} -o json | jq -r .version.raw_id | cut -f -2 -d '.'`
    export ROLE_NAME="${CLUSTER_NAME}-openshift-oadp-aws-cloud-credentials"
    export AWS_PAGER=""
    export SCRATCH="/tmp/${CLUSTER_NAME}/oadp"
    mkdir -p ${SCRATCH}
    echo "Cluster ID: ${ROSA_CLUSTER_ID}, Region: ${REGION}, OIDC Endpoint: ${OIDC_ENDPOINT}, AWS Account ID: ${AWS_ACCOUNT_ID}"
    

Prepare AWS Account

  1. Create an IAM Policy to allow for S3 Access

    POLICY_ARN=$(aws iam list-policies --query "Policies[?PolicyName=='RosaOadp'].{ARN:Arn}" --output text)
    if [[ -z "${POLICY_ARN}" ]]; then
    cat << EOF > ${SCRATCH}/policy.json
    {
    "Version": "2012-10-17",
    "Statement": [
      {
        "Effect": "Allow",
        "Action": [
          "s3:CreateBucket",
          "s3:DeleteBucket",
          "s3:PutBucketTagging",
          "s3:GetBucketTagging",
          "s3:PutEncryptionConfiguration",
          "s3:GetEncryptionConfiguration",
          "s3:PutLifecycleConfiguration",
          "s3:GetLifecycleConfiguration",
          "s3:GetBucketLocation",
          "s3:ListBucket",
          "s3:GetObject",
          "s3:PutObject",
          "s3:DeleteObject",
          "s3:ListBucketMultipartUploads",
          "s3:AbortMultipartUpload",
          "s3:ListMultipartUploadParts",
          "ec2:DescribeSnapshots",
          "ec2:CreateTags",
          "ec2:CreateVolume",
          "ec2:CreateSnapshot",
          "ec2:DeleteSnapshot"
        ],
        "Resource": "*"
      }
     ]}
    EOF
    POLICY_ARN=$(aws iam create-policy --policy-name "RosaOadp" \
    --policy-document file:///${SCRATCH}/policy.json --query Policy.Arn \
    --tags Key=rosa_openshift_version,Value=4.9 Key=rosa_role_prefix,Value=ManagedOpenShift Key=operator_namespace,Value=openshift-oadp Key=operator_name,Value=openshift-oadp \
    --output text)
    fi
    echo ${POLICY_ARN}
    
  2. Create an IAM Role trust policy for the cluster

    cat <<EOF > ${SCRATCH}/trust-policy.json
    {
       "Version": "2012-10-17",
       "Statement": [{
         "Effect": "Allow",
         "Principal": {
           "Federated": "arn:aws:iam::${AWS_ACCOUNT_ID}:oidc-provider/rh-oidc.s3.us-east-1.amazonaws.com/${ROSA_CLUSTER_ID}"
         },
         "Action": "sts:AssumeRoleWithWebIdentity",
         "Condition": {
           "StringEquals": {
              "rh-oidc.s3.us-east-1.amazonaws.com/${ROSA_CLUSTER_ID}:sub": [
                "system:serviceaccount:openshift-adp:openshift-adp-controller-manager",
                "system:serviceaccount:openshift-adp:velero"]
           }
         }
       }]
    }
    EOF
    ROLE_ARN=$(aws iam create-role --role-name \
      "${ROLE_NAME}" \
       --assume-role-policy-document file://${SCRATCH}/trust-policy.json \
       --tags Key=rosa_cluster_id,Value=${ROSA_CLUSTER_ID} Key=rosa_openshift_version,Value=${CLUSTER_VERSION} Key=rosa_role_prefix,Value=ManagedOpenShift Key=operator_namespace,Value=openshift-adp Key=operator_name,Value=openshift-oadp \
       --query Role.Arn --output text)
    
    echo ${ROLE_ARN}
    
  3. Attach the IAM Policy to the IAM Role

    aws iam attach-role-policy --role-name "${ROLE_NAME}" \
      --policy-arn ${POLICY_ARN}
    

Deploy OADP on cluster

  1. Create a namespace for OADP

    oc create namespace openshift-adp
    
  2. Create a credentials secret

    cat <<EOF > ${SCRATCH}/credentials
    [default]
    role_arn = ${ROLE_ARN}
    web_identity_token_file = /var/run/secrets/openshift/serviceaccount/token
    EOF
    oc -n openshift-adp create secret generic cloud-credentials \
      --from-file=${SCRATCH}/credentials
    
  3. Deploy OADP Operator

NOTE: there is currently an issue with 1.1 of the operator with backups that have a PartiallyFailed status. This does not seem to affect the backup and restore process, but it should be noted as there are issues with it.

cat << EOF | oc create -f -
apiVersion: operators.coreos.com/v1
kind: OperatorGroup
metadata:
  generateName: openshift-adp-
  namespace: openshift-adp
  name: oadp
spec:
  targetNamespaces:
  - openshift-adp
---
apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata:
  name: redhat-oadp-operator
  namespace: openshift-adp
spec:
  channel: stable-1.0
  installPlanApproval: Automatic
  name: redhat-oadp-operator
  source: redhat-operators
  sourceNamespace: openshift-marketplace
  startingCSV: oadp-operator.v1.0.8
EOF
  1. Wait for the operator to be ready

    watch oc -n openshift-adp get pods
    
    NAME                                                READY   STATUS    RESTARTS   AGE
    openshift-adp-controller-manager-546684844f-qqjhn   1/1     Running   0          22s
    
  2. Create Cloud Storage

    cat << EOF | oc create -f -
    apiVersion: oadp.openshift.io/v1alpha1
    kind: CloudStorage
    metadata:
      name: ${CLUSTER_NAME}-oadp
      namespace: openshift-adp
    spec:
      creationSecret:
        key: credentials
        name: cloud-credentials
      enableSharedConfig: true
      name: ${CLUSTER_NAME}-oadp
      provider: aws
      region: $REGION
    EOF
    
  3. Deploy a Data Protection Application

    cat << EOF | oc create -f -
    apiVersion: oadp.openshift.io/v1alpha1
    kind: DataProtectionApplication
    metadata:
      name: ${CLUSTER_NAME}-dpa
      namespace: openshift-adp
    spec:
      backupLocations:
      - bucket:
          cloudStorageRef:
            name: ${CLUSTER_NAME}-oadp
          credential:
            key: credentials
            name: cloud-credentials
          default: true
      configuration:
        velero:
          defaultPlugins:
          - openshift
          - aws
        restic:
          enable: false
      volumeSnapshots:
      - velero:
          config:
            credentialsFile: /tmp/credentials/openshift-adp/cloud-credentials-credentials
            enableSharedConfig: "true"
            region: ${REGION}
          provider: aws
    EOF
    

Perform a backup

  1. Create a workload to backup

    oc create namespace hello-world
    oc new-app -n hello-world --docker-image=docker.io/openshift/hello-openshift
    
  2. Backup workload

    cat << EOF | oc create -f -
    apiVersion: velero.io/v1
    kind: Backup
    metadata:
      name: hello-world
      namespace: openshift-adp
    spec:
      includedNamespaces:
      - hello-world
      storageLocation: ${CLUSTER_NAME}-dpa-1
      ttl: 720h0m0s
    EOF
    
  3. Wait until backup is done

    watch "oc -n openshift-adp get backup hello-world -o json | jq .status"
    
    {
      "completionTimestamp": "2022-09-07T22:20:44Z",
      "expiration": "2022-10-07T22:20:22Z",
      "formatVersion": "1.1.0",
      "phase": "Completed",
      "progress": {
        "itemsBackedUp": 58,
        "totalItems": 58
      },
      "startTimestamp": "2022-09-07T22:20:22Z",
      "version": 1
    }
    
  4. Delete the demo workload

    oc delete ns hello-world
    
  5. Restore from the backup

    cat << EOF | oc create -f -
    apiVersion: velero.io/v1
    kind: Restore
    metadata:
      name: hello-world
      namespace: openshift-adp
    spec:
      backupName: hello-world
    EOF
    
  6. Wait for the Restore to finish

    watch "oc -n openshift-adp get restore hello-world -o json | jq .status"
    
    {
      "completionTimestamp": "2022-09-07T22:25:47Z",
      "phase": "Completed",
      "progress": {
        "itemsRestored": 38,
        "totalItems": 38
      },
      "startTimestamp": "2022-09-07T22:25:28Z",
      "warnings": 9
    }
    
  7. Check the workload is restored

    oc -n hello-world get pods
    
    NAME                              READY   STATUS    RESTARTS   AGE
    hello-openshift-9f885f7c6-kdjpj   1/1     Running   0          90s
    

Cleanup

  1. Delete the workload

    oc delete ns hello-world
    
  2. Delete the Data Protection Application

    oc -n openshift-adp delete dpa ${CLUSTER_NAME}-dpa
    
  3. Delete the Cloud Storage

    oc -n openshift-adp delete cloudstorage ${CLUSTER_NAME}-oadp
    

WARNING: if this command hangs, you may need to delete the finalizer:

oc -n openshift-adp patch cloudstorage ${CLUSTER_NAME}-oadp -p '{"metadata":{"finalizers":null}}' --type=merge
  1. Remove the operator if it is no longer required:
oc -n openshift-adp delete subscription oadp-operator
  1. Remove the namespace for the operator:
oc delete ns openshift-adp
  1. Remove the backup and restore resources from the cluster if they are no longer required:
oc delete backup hello-world
oc delete restore hello-world
  1. Remove the Custom Resource Definitinos from the cluster if you no longer wish to have them:
for CRD in `oc get crds | grep velero | awk '{print $1}'`; do oc delete crd $CRD; done
  1. Delete the AWS S3 Bucket

    aws s3 rm s3://${CLUSTER_NAME}-oadp --recursive
    aws s3api delete-bucket --bucket ${CLUSTER_NAME}-oadp
    
  2. Detach the Policy from the role

    aws iam detach-role-policy --role-name "${ROLE_NAME}" \
      --policy-arn "${POLICY_ARN}"
    
  3. Delete the role

    aws iam delete-role --role-name "${ROLE_NAME}"