diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..fab409a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,33 @@ + +name: build docker image + +on: + push: + tags: + - "**" + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: Checkout the code + uses: actions/checkout@v4 + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Get latest release version number + id: docker-tag + uses: yuya-takeyama/docker-tag-from-github-ref-action@v1 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: fradelg + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Build multiarch image + run: | + docker buildx build --push \ + --tag fradelg/mysql-cron-backup:${{ steps.docker-tag.outputs.tag }} \ + --platform linux/amd64,linux/arm/v7,linux/arm64 . diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml deleted file mode 100644 index 8d51bd3..0000000 --- a/.github/workflows/image.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: build docker image - -on: - workflow_dispatch: - push: - branches: - - "**" - tags: - - "**" - -jobs: - test: - runs-on: ubuntu-20.04 - steps: - - name: Checkout the code - uses: actions/checkout@v3 - - name: Test Bash scripts - run: sudo apt-get -qq update && sudo apt-get install -y devscripts shellcheck && make test - build: - runs-on: ubuntu-20.04 - needs: test - steps: - - name: Checkout the code - uses: actions/checkout@v3 - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Get latest release version number - id: docker-tag - uses: yuya-takeyama/docker-tag-from-github-ref-action@v1 - - name: Login to Docker Hub - run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin - - name: Build multiarch image - run: | - docker buildx build --push \ - --tag fradelg/mysql-cron-backup:${{ steps.docker-tag.outputs.tag }} \ - --platform linux/amd64,linux/arm/v7,linux/arm64 . \ No newline at end of file diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..4460983 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,26 @@ +name: build docker image + +on: + workflow_dispatch: + push: + branches: + - "**" + +jobs: + test: + runs-on: ubuntu-22.04 + steps: + - name: Checkout the code + uses: actions/checkout@v4 + - name: Test Bash scripts + run: sudo apt-get -qq update && sudo apt-get install -y devscripts shellcheck && make test + - name: Test image + env: + VOLUME_PATH: /tmp/mariadb + DATABASE_NAME: foo + MARIADB_ROOT_PASSWORD: abcd + run: | + docker compose up -d mariadb + docker compose run backup /backup.sh + docker compose run backup /restore.sh /backup/latest.foo.sql.gz + docker compose stop diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6320cd2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +data \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fdf3895..73a2588 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,28 @@ -FROM golang:1.15.8-alpine3.12 AS binary +FROM golang:1.20.4-alpine3.18 AS binary RUN apk -U add openssl git -ARG DOCKERIZE_VERSION=v0.6.1 +ARG DOCKERIZE_VERSION=v0.7.0 WORKDIR /go/src/github.com/jwilder RUN git clone https://github.com/jwilder/dockerize.git && \ cd dockerize && \ git checkout ${DOCKERIZE_VERSION} WORKDIR /go/src/github.com/jwilder/dockerize -RUN go get github.com/robfig/glock -RUN glock sync -n < GLOCKFILE -RUN go install +ENV GO111MODULE=on +RUN go mod tidy +RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -a -o /go/bin/dockerize . -FROM alpine:3.16.2 +FROM alpine:3.20.3 LABEL maintainer "Fco. Javier Delgado del Hoyo <frandelhoyo@gmail.com>" RUN apk add --update \ - tzdata \ - bash \ - mysql-client \ - gzip \ - openssl \ - mariadb-connector-c && \ + tzdata \ + bash \ + gzip \ + openssl \ + mysql-client=~10.11 \ + mariadb-connector-c \ + fdupes && \ rm -rf /var/cache/apk/* COPY --from=binary /go/bin/dockerize /usr/local/bin @@ -32,13 +33,16 @@ ENV CRON_TIME="0 3 * * sun" \ TIMEOUT="10s" \ MYSQLDUMP_OPTS="--quick" -COPY ["run.sh", "backup.sh", "restore.sh", "/"] +COPY ["run.sh", "backup.sh", "restore.sh", "/delete.sh", "/"] RUN mkdir /backup && \ chmod 777 /backup && \ - chmod 755 /run.sh /backup.sh /restore.sh && \ + chmod 755 /run.sh /backup.sh /restore.sh /delete.sh && \ touch /mysql_backup.log && \ chmod 666 /mysql_backup.log VOLUME ["/backup"] +HEALTHCHECK --interval=2s --retries=1800 \ + CMD stat /HEALTHY.status || exit 1 + CMD dockerize -wait tcp://${MYSQL_HOST}:${MYSQL_PORT} -timeout ${TIMEOUT} /run.sh diff --git a/Makefile b/Makefile index dc20422..ca5d51c 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ test: # Checking for syntax errors set -e; for SCRIPT in *.sh; \ do \ - sh -n $$SCRIPT; \ + bash -n $$SCRIPT; \ done # Checking for bashisms (currently not failing, but only listing) diff --git a/README.md b/README.md index 923ec5e..16a2d65 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,12 @@ docker container run -d \ fradelg/mysql-cron-backup ``` +### Healthcheck + + +Healthcheck is provided as a basic init control. +Container is **Healthy** after the database init phase, that is after `INIT_BACKUP` or `INIT_RESTORE_LATEST` happends without check if there is an error, **Starting** otherwise. Not other checks are actually provided. + ## Variables @@ -31,10 +37,12 @@ docker container run -d \ - `MAX_BACKUPS`: The number of backups to keep. When reaching the limit, the old backup will be discarded. No limit by default. - `INIT_BACKUP`: If set, create a backup when the container starts. - `INIT_RESTORE_LATEST`: If set, restores latest backup. +- `EXIT_BACKUP`: If set, create a backup when the container stops. - `TIMEOUT`: Wait a given number of seconds for the database to be ready and make the first backup, `10s` by default. After that time, the initial attempt for backup gives up and only the Cron job will try to make a backup. - `GZIP_LEVEL`: Specify the level of gzip compression from 1 (quickest, least compressed) to 9 (slowest, most compressed), default is 6. - `USE_PLAIN_SQL`: If set, back up and restore plain SQL files without gzip. - `TZ`: Specify TIMEZONE in Container. E.g. "Europe/Berlin". Default is UTC. +- `REMOVE_DUPLICATES`: Use [fdupes](https://github.com/adrianlopezroche/fdupes) to remove duplicate database dumps If you want to make this image the perfect companion of your MySQL container, use [docker-compose](https://docs.docker.com/compose/). You can add more services that will be able to connect to the MySQL image using the name `my_mariadb`, note that you only expose the port `3306` internally to the servers and not to the host: @@ -73,6 +81,8 @@ services: - CRON_TIME=0 3 * * * # Make it small - GZIP_LEVEL=9 + # As of MySQL 8.0.21 this is needed + - MYSQLDUMP_OPTS=--no-tablespaces restart: unless-stopped volumes: @@ -184,4 +194,65 @@ mysql-cron-backup: docker container exec <your_mysql_backup_container_name> /restore.sh /backup/<your_sql_backup_gz_file> ``` -if no database name is specified, `restore.sh` will try to find the database name from the backup file. \ No newline at end of file +if no database name is specified, `restore.sh` will try to find the database name from the backup file. + +### Automatic backup and restore on container starts and stops + +Set `INIT_RESTORE_LATEST` to automatic restore the last backup on startup. +Set `EXIT_BACKUP` to automatic create a last backup on shutdown. + +```yaml + mysql-cron-backup: + image: fradelg/mysql-cron-backup + depends_on: + - mariadb + volumes: + - ${VOLUME_PATH}/backup:/backup + environment: + - MYSQL_HOST=my_mariadb + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASS=${MYSQL_PASSWORD} + - MAX_BACKUPS=15 + - INIT_RESTORE_LATEST=1 + - EXIT_BACKUP=1 + # Every day at 03:00 + - CRON_TIME=0 3 * * * + # Make it small + - GZIP_LEVEL=9 + restart: unless-stopped + +volumes: + data: +``` + +Docker database image could expose a directory you could add files as init sql script. + +```yaml + mysql: + image: mysql + expose: + - 3306 + volumes: + - data:/var/lib/mysql + # If there is not scheme, restore using the init script (if exists) + - ./init-script.sql:/docker-entrypoint-initdb.d/database.sql.gz + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${DATABASE_NAME} + restart: unless-stopped +``` + +```yaml + mariadb: + image: mariadb + expose: + - 3306 + volumes: + - data:/var/lib/mysql + # If there is not scheme, restore using the init script (if exists) + - ./init-script.sql:/docker-entrypoint-initdb.d/database.sql.gz + environment: + - MYSQL_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD} + - MYSQL_DATABASE=${DATABASE_NAME} + restart: unless-stopped +``` \ No newline at end of file diff --git a/backup.sh b/backup.sh index 33ff1b1..499575d 100755 --- a/backup.sh +++ b/backup.sh @@ -29,13 +29,18 @@ do echo "==> Dumping database: $db" FILENAME=/backup/$DATE.$db.sql LATEST=/backup/latest.$db.sql - if mysqldump --single-transaction $MYSQLDUMP_OPTS -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASS" $MYSQL_SSL_OPTS "$db" > "$FILENAME" + BASIC_OPTS="--single-transaction" + if [ -n "$REMOVE_DUPLICATES" ] + then + BASIC_OPTS="$BASIC_OPTS --skip-dump-date" + fi + if mysqldump $BASIC_OPTS $MYSQLDUMP_OPTS -h "$MYSQL_HOST" -P "$MYSQL_PORT" -u "$MYSQL_USER" -p"$MYSQL_PASS" $MYSQL_SSL_OPTS "$db" > "$FILENAME" then EXT= if [ -z "${USE_PLAIN_SQL}" ] then echo "==> Compressing $db with LEVEL $GZIP_LEVEL" - gzip "-$GZIP_LEVEL" -f "$FILENAME" + gzip "-$GZIP_LEVEL" -n -f "$FILENAME" EXT=.gz FILENAME=$FILENAME$EXT LATEST=$LATEST$EXT @@ -44,15 +49,15 @@ do echo "==> Creating symlink to latest backup: $BASENAME" rm "$LATEST" 2> /dev/null cd /backup || exit && ln -s "$BASENAME" "$(basename "$LATEST")" + if [ -n "$REMOVE_DUPLICATES" ] + then + echo "==> Removing duplicate database dumps" + fdupes -idN /backup/ + fi if [ -n "$MAX_BACKUPS" ] then - while [ "$(find /backup -maxdepth 1 -name "*.$db.sql$EXT" -type f | wc -l)" -gt "$MAX_BACKUPS" ] - do - TARGET=$(find /backup -maxdepth 1 -name "*.$db.sql$EXT" -type f | sort | head -n 1) - echo "==> Max number of ($MAX_BACKUPS) backups reached. Deleting ${TARGET} ..." - rm -rf "${TARGET}" - echo "==> Backup ${TARGET} deleted" - done + # Execute the delete script, delete older backup or other custom delete script + /delete.sh "$db" $EXT fi else rm -rf "$FILENAME" diff --git a/delete.sh b/delete.sh new file mode 100755 index 0000000..929ed7b --- /dev/null +++ b/delete.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +db=$1 +EXT=$2 + +# This file could be customized to create custom delete strategy + +while [ "$(find /backup -maxdepth 1 -name "*.$db.sql$EXT" -type f | wc -l)" -gt "$MAX_BACKUPS" ] +do + TARGET=$(find /backup -maxdepth 1 -name "*.$db.sql$EXT" -type f | sort | head -n 1) + echo "==> Max number of ($MAX_BACKUPS) backups reached. Deleting ${TARGET} ..." + rm -rf "${TARGET}" + echo "==> Backup ${TARGET} deleted" +done \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index d0b693f..5ac0e9d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,8 +1,9 @@ -version: "2" services: mariadb: - image: mariadb:10 + image: mariadb:10.11 container_name: my_mariadb + security_opt: + - seccomp:unconfined expose: - 3306 volumes: @@ -11,13 +12,19 @@ services: environment: - MYSQL_DATABASE=${DATABASE_NAME} - MYSQL_ROOT_PASSWORD=${MARIADB_ROOT_PASSWORD} + - MYSQL_ALLOW_EMPTY_ROOT_PASSWORD=yes restart: unless-stopped + healthcheck: + test: [ "CMD", "healthcheck.sh", "--su-mysql", "--connect" ] + timeout: 5s + retries: 10 backup: build: . image: fradelg/mysql-cron-backup depends_on: - - mariadb + mariadb: + condition: service_healthy volumes: - ${VOLUME_PATH}/backup:/backup environment: @@ -28,6 +35,6 @@ services: - INIT_BACKUP=1 - CRON_TIME=0 0 * * * restart: unless-stopped - -volumes: - data: \ No newline at end of file + +volumes: + data: diff --git a/restore.sh b/restore.sh index 0607d6f..6c03634 100755 --- a/restore.sh +++ b/restore.sh @@ -13,6 +13,7 @@ if [ "$#" -ne 1 ] then echo "You must pass the path of the backup file to restore" + exit 1 fi set -o pipefail diff --git a/run.sh b/run.sh index 95f8009..17a420c 100755 --- a/run.sh +++ b/run.sh @@ -1,7 +1,7 @@ #!/bin/bash tail -F /mysql_backup.log & -if [ "${INIT_BACKUP}" -gt "0" ]; then +if [ "${INIT_BACKUP:-0}" -gt "0" ]; then echo "=> Create a backup on the startup" /backup.sh elif [ -n "${INIT_RESTORE_LATEST}" ]; then @@ -11,10 +11,32 @@ elif [ -n "${INIT_RESTORE_LATEST}" ]; then echo "waiting database container..." sleep 1 done - find /backup -maxdepth 1 -name '*.sql.gz' | tail -1 | xargs /restore.sh + # Needed to exclude the 'latest.<database>.sql.gz' file, consider only filenames starting with number + # Only data-tagged backups, eg. '202212250457.database.sql.gz', must be trapped by the regex + find /backup -maxdepth 1 -name '[0-9]*.*.sql.gz' | sort | tail -1 | xargs /restore.sh fi +function final_backup { + echo "=> Captured trap for final backup" + echo "=> Requested last backup at $(date "+%Y-%m-%d %H:%M:%S")" + exec /backup.sh + exit 0 +} + +if [ -n "${EXIT_BACKUP}" ]; then + echo "=> Listening on container shutdown gracefully to make last backup before close" + trap final_backup SIGHUP SIGINT SIGTERM +fi + +touch /HEALTHY.status + echo "${CRON_TIME} /backup.sh >> /mysql_backup.log 2>&1" > /tmp/crontab.conf crontab /tmp/crontab.conf echo "=> Running cron task manager in foreground" -exec crond -f -l 8 -L /mysql_backup.log +crond -f -l 8 -L /mysql_backup.log & + +echo "Listening on crond, and wait..." + +tail -f /dev/null & wait $! + +echo "Script is shutted down." \ No newline at end of file