percona · hors · Oct 5, 2024 · Oct 8, 2024 · Oct 8, 2024 · Oct 8, 2024
diff --git a/percona-xtradb-cluster-5.7-backup/Dockerfile b/percona-xtradb-cluster-5.7-backup/Dockerfile
@@ -27,13 +27,17 @@ LABEL org.opencontainers.image.version=${PXC_VERSION}
 # check repository package signature in secure way
 RUN set -ex; \
  export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F; \
+ gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F FB5DB77FD5C118B80511ADA8A6310ACC4672475C; \
  gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \
  gpg --batch --export --armor 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F > ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
- rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
+ gpg --batch --export --armor FB5DB77FD5C118B80511ADA8A6310ACC4672475C > ${GNUPGHOME}/RPM-GPG-KEY-aws; \
+ rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-oracle ${GNUPGHOME}/RPM-GPG-KEY-aws; \
  microdnf install -y findutils; \
  curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \
  rpmkeys --checksig /tmp/percona-release.rpm; \
+ curl -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip; \
+ curl -o /tmp/awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig; \
+ gpg --verify /tmp/awscliv2.sig /tmp/awscliv2.zip; \
  rpm -i /tmp/percona-release.rpm; \
  rm -rf "$GNUPGHOME" /tmp/percona-release.rpm; \
  rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY; \
@@ -61,14 +65,18 @@ RUN set -ex; \
  iputils \
  procps-ng \
  util-linux \
+ unzip \
  procps-ng \
  qpress \
  tar \
  cracklib-dicts \
  libatomic \
  libaio; \
  microdnf clean all; \
- rm -rf /var/cache/dnf /var/cache/yum
+ unzip -q /tmp/awscliv2.zip -d /tmp/; \
+ /tmp/aws/install; \
+ aws --version; \
+ rm -rf /var/cache/dnf /var/cache/yum /tmp/awscliv2.zip /tmp/awscliv2.sig /tmp/aws
 
 # create mysql user/group before mysql installation
 RUN groupadd -g 1001 mysql; \
@@ -102,15 +110,6 @@ RUN set -ex; \
 RUN install -d -o 1001 -g 0 -m 0775 /backup; \
  mkdir /usr/lib/pxc
 
-ENV MC_VERSION=RELEASE.2024-07-08T20-59-24Z
-ENV MC_SHA256SUM=e111d2b4bea05aadbffaa3fc8d2436a3fefedf030cd1318568bccb72810024f0
-RUN set -ex; \
- curl -o /usr/bin/mc -O https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.${MC_VERSION} \
- && chmod +x /usr/bin/mc \
- && echo "${MC_SHA256SUM} /usr/bin/mc" | sha256sum -c - \
- && curl -o /licenses/LICENSE.mc \
- https://raw.githubusercontent.com/minio/mc/${MC_VERSION}/LICENSE
-
 COPY lib/pxc /usr/lib/pxc
 COPY recovery-*.sh backup.sh get-pxc-state /usr/bin/
 

diff --git a/percona-xtradb-cluster-5.7-backup/backup.sh b/percona-xtradb-cluster-5.7-backup/backup.sh
@@ -5,6 +5,7 @@ set -o xtrace
 
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/vault.sh
+. ${LIB_PATH}/aws.sh
 
 GARBD_OPTS=""
 SOCAT_OPTS="TCP-LISTEN:4444,reuseaddr,retry=30"
@@ -141,22 +142,12 @@ backup_volume() {
  echo '[INFO] Backup was finished successfully'
 }
 
-is_object_exist() {
- local bucket="$1"
- local object="$2"
-
- if [[ -n "$(mc -C /tmp/mc ${INSECURE_ARG} --json ls "dest/$bucket/$object" | jq '.status')" ]]; then
- return 1
- fi
-}
-
 backup_s3() {
  S3_BUCKET_PATH=${S3_BUCKET_PATH:-$PXC_SERVICE-$(date +%F-%H-%M)-xtrabackup.stream}
 
  echo "[INFO] Backup to s3://$S3_BUCKET/$S3_BUCKET_PATH started"
  { set +x; } 2>/dev/null
- echo "+ mc -C /tmp/mc ${INSECURE_ARG} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" ACCESS_KEY_ID SECRET_ACCESS_KEY"
- mc -C /tmp/mc ${INSECURE_ARG} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
+ s3_add_bucket_dest
  set -x
  is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME" || xbcloud delete ${INSECURE_ARG} $XBCLOUD_EXTRA_ARGS --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME"
  is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH" || xbcloud delete ${INSECURE_ARG} $XBCLOUD_EXTRA_ARGS --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH"
@@ -176,8 +167,8 @@ backup_s3() {
  | xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 ${INSECURE_ARG} $XBCLOUD_EXTRA_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH" 2>&1 \
  | (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
 
- mc -C /tmp/mc ${INSECURE_ARG} stat "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5"
- md5_size=$(mc -C /tmp/mc ${INSECURE_ARG} stat --json "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5" | sed -e 's/.*"size":\([0-9]*\).*/\1/')
+ aws $AWS_S3_NO_VERIFY_SSL s3 ls s3://$S3_BUCKET/$S3_BUCKET_PATH.md5
+ md5_size=$(aws $AWS_S3_NO_VERIFY_SSL --output json s3api list-objects --bucket "$S3_BUCKET" --prefix "$S3_BUCKET_PATH.md5" --query 'Contents[0].Size' | sed -e 's/.*"size":\([0-9]*\).*/\1/')
  if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)); then
  echo '[ERROR] Backup is empty'
  echo '[ERROR] Backup was finished unsuccessfully'

diff --git a/percona-xtradb-cluster-5.7-backup/recovery-cloud.sh b/percona-xtradb-cluster-5.7-backup/recovery-cloud.sh
@@ -6,21 +6,19 @@ set -o xtrace
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/check-version.sh
 . ${LIB_PATH}/vault.sh
+. ${LIB_PATH}/aws.sh
 
-MC_ARGS='-C /tmp/mc'
 XBCLOUD_ARGS="$XBCLOUD_EXTRA_ARGS"
 
 if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
  XBCLOUD_ARGS="--insecure ${XBCLOUD_ARGS}"
- MC_ARGS="${MC_ARGS} --insecure"
 fi
 
 if [ -n "$S3_BUCKET_URL" ]; then
  { set +x; } 2>/dev/null
- echo "+ mc ${MC_ARGS} config host add dest ${ENDPOINT:-https://s3.amazonaws.com} ACCESS_KEY_ID SECRET_ACCESS_KEY"
- mc ${MC_ARGS} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
+ s3_add_bucket_dest
  set -x
- mc ${MC_ARGS} ls "dest/${S3_BUCKET_URL}"
+ aws $AWS_S3_NO_VERIFY_SSL s3 ls "${S3_BUCKET_URL}"
 elif [ -n "${BACKUP_PATH}" ]; then
  XBCLOUD_ARGS="${XBCLOUD_ARGS} --storage=azure"
 fi

diff --git a/percona-xtradb-cluster-8.0-backup/Dockerfile b/percona-xtradb-cluster-8.0-backup/Dockerfile
@@ -32,13 +32,17 @@ LABEL org.opencontainers.image.version=${PXC_VERSION}
 # check repository package signature in secure way
 RUN set -ex; \
  export GNUPGHOME="$(mktemp -d)"; \
- gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F; \
+ gpg --batch --keyserver keyserver.ubuntu.com --recv-keys 4D1BB29D63D98E422B2113B19334A25F8507EFA5 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F FB5DB77FD5C118B80511ADA8A6310ACC4672475C; \
  gpg --batch --export --armor 4D1BB29D63D98E422B2113B19334A25F8507EFA5 > ${GNUPGHOME}/PERCONA-PACKAGING-KEY; \
  gpg --batch --export --armor 3E6D826D3FBAB389C2F38E34BC4D06A08D8B756F > ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
- rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-oracle; \
+ gpg --batch --export --armor FB5DB77FD5C118B80511ADA8A6310ACC4672475C > ${GNUPGHOME}/RPM-GPG-KEY-aws; \
+ rpmkeys --import ${GNUPGHOME}/PERCONA-PACKAGING-KEY ${GNUPGHOME}/RPM-GPG-KEY-oracle ${GNUPGHOME}/RPM-GPG-KEY-aws; \
  microdnf install -y findutils; \
  curl -Lf -o /tmp/percona-release.rpm https://repo.percona.com/yum/percona-release-latest.noarch.rpm; \
  rpmkeys --checksig /tmp/percona-release.rpm; \
+ curl -o /tmp/awscliv2.zip https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip; \
+ curl -o /tmp/awscliv2.sig https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip.sig; \
+ gpg --verify /tmp/awscliv2.sig /tmp/awscliv2.zip; \
  rpm -i /tmp/percona-release.rpm; \
  rm -rf "$GNUPGHOME" /tmp/percona-release.rpm; \
  rpm --import /etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY
@@ -66,9 +70,13 @@ RUN set -ex; \
  iputils \
  procps-ng \
  util-linux \
+ unzip \
  findutils; \
  microdnf clean all; \
- rm -rf /var/cache/dnf /var/cache/yum
+ unzip -q /tmp/awscliv2.zip -d /tmp/; \
+ /tmp/aws/install; \
+ aws --version; \
+ rm -rf /var/cache/dnf /var/cache/yum /tmp/awscliv2.zip /tmp/awscliv2.sig /tmp/aws
 
 # create mysql user/group before mysql installation
 RUN groupadd -g 1001 mysql; \
@@ -90,7 +98,8 @@ RUN set -ex; \
 
 COPY LICENSE /licenses/LICENSE.Dockerfile
 RUN cp /usr/share/doc/percona-xtrabackup-80/LICENSE /licenses/LICENSE.xtrabackup; \
- cp /usr/share/doc/percona-xtradb-cluster-garbd-3/COPYING /licenses/LICENSE.garbd
+ cp /usr/share/doc/percona-xtradb-cluster-garbd-3/COPYING /licenses/LICENSE.garbd; \
+ curl -o /licenses/LICENSE.aws-cli https://raw.githubusercontent.com/aws/aws-cli/refs/heads/master/LICENSE.txt
 
 RUN set -ex; \
  curl -o /usr/bin/kubectl -LO \
@@ -103,15 +112,6 @@ RUN set -ex; \
 RUN install -d -o 1001 -g 0 -m 0775 /backup; \
  mkdir /usr/lib/pxc
 
-ENV MC_VERSION=RELEASE.2024-07-08T20-59-24Z
-ENV MC_SHA256SUM=e111d2b4bea05aadbffaa3fc8d2436a3fefedf030cd1318568bccb72810024f0
-RUN set -ex; \
- curl -o /usr/bin/mc -O https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.${MC_VERSION} \
- && chmod +x /usr/bin/mc \
- && echo "${MC_SHA256SUM} /usr/bin/mc" | sha256sum -c - \
- && curl -o /licenses/LICENSE.mc \
- https://raw.githubusercontent.com/minio/mc/${MC_VERSION}/LICENSE
-
 COPY lib/pxc /usr/lib/pxc
 COPY recovery-*.sh run_backup.sh backup.sh post_backup.sh get-pxc-state /usr/bin/
 

diff --git a/percona-xtradb-cluster-8.0-backup/lib/pxc/aws.sh b/percona-xtradb-cluster-8.0-backup/lib/pxc/aws.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -o errexit
+
+export AWS_SHARED_CREDENTIALS_FILE='/tmp/aws-credfile'
+export AWS_ENDPOINT_URL="${ENDPOINT:-https://s3.amazonaws.com}"
+export AWS_REGION="${DEFAULT_REGION:-us-west-2}"
+
+if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
+ AWS_S3_NO_VERIFY_SSL='--no-verify-ssl'
+fi
+
+is_object_exist() {
+ local bucket="$1"
+ local object="$2"
+
+ aws $AWS_S3_NO_VERIFY_SSL s3api head-object --bucket $bucket --key "$object" || NOT_EXIST=true
+ if [[ -z "$NOT_EXIST" ]]; then
+ return 1
+ fi
+}
+
+s3_add_bucket_dest() {
+ { set +x; } 2>/dev/null
+ aws configure set aws_access_key_id "$ACCESS_KEY_ID"
+ aws configure set aws_secret_access_key "$SECRET_ACCESS_KEY"
+ set -x
+}
+
diff --git a/percona-xtradb-cluster-8.0-backup/lib/pxc/backup.sh b/percona-xtradb-cluster-8.0-backup/lib/pxc/backup.sh
@@ -2,13 +2,15 @@
 
 set -o errexit
 
+LIB_PATH='/usr/lib/pxc'
+. ${LIB_PATH}/aws.sh
+
 SST_INFO_NAME=sst_info
 XBCLOUD_ARGS="--curl-retriable-errors=7 $XBCLOUD_EXTRA_ARGS"
-
 INSECURE_ARG=""
+
 if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
- INSECURE_ARG="--insecure"
- XBCLOUD_ARGS="${INSECURE_ARG} ${XBCLOUD_ARGS}"
+ XBCLOUD_ARGS="--insecure ${XBCLOUD_ARGS}"
 fi
 
 S3_BUCKET_PATH=${S3_BUCKET_PATH:-$PXC_SERVICE-$(date +%F-%H-%M)-xtrabackup.stream}
@@ -24,24 +26,8 @@ log() {
  set -x
 }
 
-is_object_exist() {
- local bucket="$1"
- local object="$2"
-
- if [[ -n "$(mc -C /tmp/mc ${INSECURE_ARG} --json ls "dest/$bucket/$object" | jq '.status')" ]]; then
- return 1
- fi
-}
-
-mc_add_bucket_dest() {
- echo "+ mc -C /tmp/mc ${INSECURE_ARG} config host add dest ${ENDPOINT:-https://s3.amazonaws.com} ACCESS_KEY_ID SECRET_ACCESS_KEY "
- { set +x; } 2>/dev/null
- mc -C /tmp/mc ${INSECURE_ARG} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
- set -x
-}
-
 clean_backup_s3() {
- mc_add_bucket_dest
+ s3_add_bucket_dest
 
  is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME" || xbcloud delete ${XBCLOUD_ARGS} --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH.$SST_INFO_NAME"
  is_object_exist "$S3_BUCKET" "$S3_BUCKET_PATH/" || xbcloud delete ${XBCLOUD_ARGS} --storage=s3 --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH"

diff --git a/percona-xtradb-cluster-8.0-backup/post_backup.sh b/percona-xtradb-cluster-8.0-backup/post_backup.sh
@@ -7,6 +7,7 @@ set -m
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/vault.sh
 . ${LIB_PATH}/backup.sh
+. ${LIB_PATH}/aws.sh
 
 handle_sigterm() {
  log 'INFO' 'Post recv script was finished'
@@ -28,12 +29,12 @@ backup_volume() {
 
 backup_s3() {
  log 'INFO' 'Checking backup in S3'
- mc -C /tmp/mc stat ${INSECURE_ARG} "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5"
- md5_size=$(mc -C /tmp/mc stat ${INSECURE_ARG} --json "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5" | sed -e 's/.*"size":\([0-9]*\).*/\1/')
- if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)); then
-  log 'ERROR' 'Backup is empty'
-  log 'ERROR' 'Backup was finished unsuccessfull'
-  exit 1
+ aws $AWS_S3_NO_VERIFY_SSL s3 ls s3://$S3_BUCKET/$S3_BUCKET_PATH.md5
+ md5_size=$(aws $AWS_S3_NO_VERIFY_SSL --output json s3api list-objects --bucket "$S3_BUCKET" --prefix "$S3_BUCKET_PATH.md5" --query 'Contents[0].Size' | sed -e 's/.*"size":\([0-9]*\).*/\1/')
+ if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)) ; then
+ log 'ERROR' 'Backup is empty'
+ log 'ERROR' 'Backup was finished unsuccessfull'
+ exit 1
  fi
 }
 

diff --git a/percona-xtradb-cluster-8.0-backup/recovery-cloud.sh b/percona-xtradb-cluster-8.0-backup/recovery-cloud.sh
@@ -6,23 +6,20 @@ set -o xtrace
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/check-version.sh
 . ${LIB_PATH}/vault.sh
+. ${LIB_PATH}/aws.sh
 
 # temporary fix for PXB-2784
 XBCLOUD_ARGS="--curl-retriable-errors=7 $XBCLOUD_EXTRA_ARGS"
 
-MC_ARGS='-C /tmp/mc'
-
 if [ -n "$VERIFY_TLS" ] && [[ $VERIFY_TLS == "false" ]]; then
  XBCLOUD_ARGS="--insecure ${XBCLOUD_ARGS}"
- MC_ARGS="${MC_ARGS} --insecure"
 fi
 
 if [ -n "$S3_BUCKET_URL" ]; then
  { set +x; } 2>/dev/null
- echo "+ mc ${MC_ARGS} config host add dest ${ENDPOINT:-https://s3.amazonaws.com} ACCESS_KEY_ID SECRET_ACCESS_KEY"
- mc ${MC_ARGS} config host add dest "${ENDPOINT:-https://s3.amazonaws.com}" "$ACCESS_KEY_ID" "$SECRET_ACCESS_KEY"
+ s3_add_bucket_dest
  set -x
- mc ${MC_ARGS} ls "dest/${S3_BUCKET_URL}"
+ aws $AWS_S3_NO_VERIFY_SSL s3 ls "${S3_BUCKET_URL}"
 elif [ -n "${BACKUP_PATH}" ]; then
  XBCLOUD_ARGS="${XBCLOUD_ARGS} --storage=azure"
 fi

diff --git a/percona-xtradb-cluster-8.0-backup/run_backup.sh b/percona-xtradb-cluster-8.0-backup/run_backup.sh
@@ -7,6 +7,7 @@ set -m
 LIB_PATH='/usr/lib/pxc'
 . ${LIB_PATH}/vault.sh
 . ${LIB_PATH}/backup.sh
+. ${LIB_PATH}/aws.sh
 
 SOCAT_OPTS="TCP-LISTEN:4444,reuseaddr,retry=30"
 
@@ -91,7 +92,7 @@ backup_volume() {
 }
 
 backup_s3() {
- mc_add_bucket_dest
+ s3_add_bucket_dest
 
  socat -u "$SOCAT_OPTS" stdio | xbstream -x -C /tmp $XBSTREAM_EXTRA_ARGS &
  wait $!