Migrating an existing managed controller to High Availability (HA)

cat > rwx-jenkins-home-${DOMAIN}-0.yaml <<EOF
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: rwx-jenkins-home-<domain>
spec:
  storageClassName: <your-new-storage-class>
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 2560Gi # Change this to whatever your storage class requires, or to your needs
EOF
▼

cat > allocate-rwx-jenkins-home-${DOMAIN}-0.yaml <<EOF
apiVersion: batch/v1
kind: Job
metadata:
  name: allocate-rwx-jenkins-home-<domain>
spec:
  template:
    spec:
      volumes:
      - name: volume
        persistentVolumeClaim:
          claimName: rwx-jenkins-home-${DOMAIN}-0
      containers:
      - name: busybox
        image: busybox
        command: ["true"]
        volumeMounts:
          - mountPath: /var/volume
            name: volume
        resources:
          limits:
            cpu: 100m
            memory: 100Mi
          requests:
            cpu: 100m
            memory: 100Mi
      restartPolicy: Never
  backoffLimit: 4
EOF
▼

kubectl wait --for=condition=complete job/allocate-rwx-jenkins-home-${DOMAIN}-0
▼

#!/bin/bash
set -euo pipefail

if [ $# -lt 2 ]; then
  echo "Usage: $0 <source_pvc> <dest_pvc>"
  echo "Example: $0 backup-source-pvc new-volume-rwx"
  exit 1
fi
source_pvc="${1:?}"
dest_pvc="${2:?}"

if ! kubectl get "pvc/$source_pvc" -o name > /dev/null 2>&1; then
  echo "PVC $source_pvc does not exist."
  exit 1
fi

if ! kubectl get "pvc/$dest_pvc" -o name > /dev/null 2>&1; then
  echo "PVC $dest_pvc does not exist."
  exit 1
fi

if [ "$source_pvc" == "$dest_pvc" ]; then
  echo "Source and destination PVC must be different."
  exit 1
fi

echo "1. Migration step"
kubectl apply -f - <<JOB
apiVersion: batch/v1
kind: Job
metadata:
  name: migration
spec:
  template:
    metadata:
      annotations:
        cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
    spec:
      volumes:
      - name: volume1
        persistentVolumeClaim:
          claimName: ${source_pvc}
      - name: volume2
        persistentVolumeClaim:
          claimName: ${dest_pvc}
      containers:
      - name: migration
        image: registry.access.redhat.com/ubi8/ubi
        command: [sh]
        args: [-c, "dnf install -y rsync; rsync -avvu --delete /var/volume1/ /var/volume2"]
        volumeMounts:
          - mountPath: /var/volume1
            name: volume1
          - mountPath: /var/volume2
            name: volume2
        resources:
          limits:
            cpu: "2"
            memory: 4G
          requests:
            cpu: "2"
            memory: 4G
      restartPolicy: Never
  backoffLimit: 4
JOB

echo "Waiting for migration to complete"
echo "You can inspect progress using kubectl logs -f job/migration"
kubectl wait --for=condition=complete --timeout=900m job/migration
echo "== Data from $source_pvc has been copied over to $dest_pvc"
kubectl delete job migration
▼

chown 1000:1000 /var/volume
▼

#!/bin/bash
set -euo pipefail

if [ $# -ne 1 ]; then
  echo "Usage: $0 <domain>"
  echo "Example: $0 mc"
  exit 1
fi
domain="${1:?}"
source_pvc="jenkins-home-$domain-0"
dest_pvc="old-jenkins-home-$domain-0"

if ! kubectl get "pvc/$source_pvc" -o name > /dev/null 2>&1; then
  echo "PVC $source_pvc does not exist."
  exit 1
fi

if kubectl get "pvc/$dest_pvc" -o name > /dev/null 2>&1; then
  echo "PVC $dest_pvc already exists. It will be replaced by persistent volume of $source_pvc."
  read -p "Are you sure? " -n 1 -r
  # Delete PVC-1, keep PV as backup
  pv1_name=$(kubectl get "pvc/${dest_pvc}" -o go-template={{.spec.volumeName}})
  echo "$pv1_name" > old_pv
  echo "== ${dest_pvc} points to pv/${pv1_name}"
  kubectl patch pv ${pv1_name} -p '{"spec": {"persistentVolumeReclaimPolicy": "Retain"}}'
  echo "Deleting ${dest_pvc}, we keep PV ${pv1_name} around"
  kubectl delete pvc/${dest_pvc}
  #kubectl patch pv ${pv1_name} -p '{"spec":{"claimRef": null}}'
fi

mkdir generated
# Rename pvc-2 to pvc-1
# Change PV RetainPolicy to "Retain"

pv_name=$(kubectl get "pvc/${source_pvc}" -o go-template={{.spec.volumeName}})
echo "== ${source_pvc} points to pv/${pv_name}"
kubectl get "pvc/${source_pvc}" -o yaml > generated/source_pvc.yaml
kubectl patch pv ${pv_name} -p '{"spec": {"persistentVolumeReclaimPolicy": "Retain"}}'
kubectl delete "pvc/${source_pvc}"
kubectl patch pv ${pv_name} -p '{"spec":{"claimRef": null}}'

# We ideally want to retain any user annotation
cat <<EOF >generated/patch.yaml
- op: replace
  path: /metadata/name
  value: ${dest_pvc}
- op: replace
  path: /spec/volumeName
  value: ${pv_name}
- op: remove
  path: /metadata/finalizers
- op: remove
  path: /metadata/creationTimestamp
- op: remove
  path: /metadata/namespace
- op: remove
  path: /metadata/resourceVersion
- op: remove
  path: /metadata/uid
- op: remove
  path: /status
EOF

cat <<EOF >generated/kustomization.yaml
patches:
 - target:
      version: v1
      kind: PersistentVolumeClaim
      name: ${source_pvc}
   path: patch.yaml
resources:
- source_pvc.yaml
EOF

trap "rm -rf generated" EXIT

kubectl apply -k generated/
pv_name=$(kubectl get pvc/${dest_pvc} -o go-template={{.spec.volumeName}})
echo "== ${dest_pvc} points to pv/${pv_name}"
echo "== Resetting ${pv_name} retain policy to Delete"
kubectl patch pv ${pv_name} -p '{"spec": {"persistentVolumeReclaimPolicy": "Delete"}}'

echo "== You can rename ${dest_pvc} to ${source_pvc} using ./2-rename_pvc.sh ${dest_pvc} ${source_pvc}"
▼

apiVersion: "apps/v1"
kind: Deployment
spec:
  template:
    spec:
      securityContext:
        fsGroupChangePolicy: OnRootMismatch
▼

kubectl delete pvc backup-jenkins-home-$DOMAIN-0 old-jenkins-home-$DOMAIN-0
▼

kubectl delete pv backup-jenkins-home-$DOMAIN-0
▼