Changelog.md

@@ -2,6 +2,10 @@
 
 ## Version 2.7
 
+* Support multiple `pg_dump` output formats (PostgreSQL):
+  - Add a `PGDUMP_FORMAT` configuration variable to select output format (`plain`, `custom`, `tar` or `directory` are supported)
+  - Add a `PGDUMP_FORMAT_COMP` configuration variable to choose output format compression method (only for `custom` or `directory` output format and `pg_dump` >= 16)
+  - Add a `PGDUMP_FORMAT_COMP_LEVEL` configuration variable to choose output format compression level
 * Add a `DBDEFAULT` configuration variable (currently PostgreSQL only) to override the default database (postgres) used when connecting to the server (Closes: [#51](https://github.com/k0lter/autopostgresqlbackup/pull/51)
 
 ## Version 2.6

Documentation.md

@@ -279,6 +279,37 @@ page).
 
 *Only while using PostgreSQL database engine*
 
+### PGDUMP_FORMAT
+
+Output format:
+
+ - `plain`: plain-text SQL script (default)
+ - `custom`: custom-format archive suitable for use with `pg_restore` (compressed; see `PGDUMP_FORMAT_COMP` and `PGDUMP_FORMAT_COMP_LEVEL`)
+ - `tar`: tar-format archive for use with `pg_restore` (no compression)
+ - `directory`: directory-format archive for use with `pg_restore`, stores each table in a separate file, supports parallel backup/restore, supported compression and encapsulated in a single tar file.
+
+**default**: `plain`
+
+*Only while using PostgreSQL database engine*
+
+### PGDUMP_FORMAT_COMP
+
+Compression method to use when `PGDUMP_FORMAT` is `custom` or `directory`. Supported values are `gzip`, `lz4`, `zstd`, and `none`.
+
+Ignored with `pg_dump` ≤ 15 (not supported). Falling back to the built-in compression method (e.g. gzip).
+
+**default**: `gzip` (built-in)
+
+*Only while using PostgreSQL database engine*
+
+### PGDUMP_FORMAT_COMP_LEVEL
+
+Compression level to use with when `PGDUMP_FORMAT` is `custom` or `directory`: useable values from `0` to `9`.
+
+**default**: `""` (empty)
+
+*Only while using PostgreSQL database engine*
+
 ### PGDUMPALL_OPTS
 
 Options string for use with pg_dumpall (see

autopostgresqlbackup

@@ -111,6 +111,15 @@ PGDUMPALL=
 # Options string for use with all_dump (see pg_dump manual page).
 PGDUMP_OPTS=
 
+# Output format (plain, custom, tar or directory)
+PGDUMP_FORMAT="plain"
+
+# Compression method (gzip, lz4, zstd or none) to use when PGDUMP_FORMAT is custom or directory
+PGDUMP_FORMAT_COMP="gzip"
+
+# Compression level (0 to 9) to use with when PGDUMP_FORMAT is custom or directory
+PGDUMP_FORMAT_COMP_LEVEL=
+
 # Options string for use with pg_dumpall (see pg_dumpall manual page).
 PGDUMPALL_OPTS=
 
@@ -156,11 +165,8 @@ GPG_HOMEDIR=
 # Database connection arguments
 CONN_ARGS=()
 
-# Hostname
-HOSTNAME="$(uname -n)"
-if [[ "${HOSTNAME}" != *.* ]]; then
-    HOSTNAME="$(hostname --fqdn)"
-fi
+# pg_dump major version
+PGDUMP_VERSION_MAJOR=
 
 # Return Code
 RC=0
@@ -314,6 +320,23 @@ postgresqldb_init () {
     if [ -z "${PGDUMPALL}" ]; then
         PGDUMPALL="pg_dumpall"
     fi
+
+    PGDUMP_VERSION_MAJOR=$("${PGDUMP}" --version 2>/dev/null | cut -d' ' -f3 | cut -d'.' -f1 || true)
+
+    case "${PGDUMP_FORMAT}" in
+        tar)
+            EXT="tar"
+            ;;
+        custom|directory)
+            if [ "${PGDUMP_FORMAT}" = "directory" ]; then
+                EXT="tar"
+            else
+                EXT="pgdump"
+            fi
+            COMP=""
+            COMP_OPTS=""
+            ;;
+    esac
 }
 # }}}
 
@@ -376,6 +399,8 @@ postgresqldb_dump () {
     local db_name cmd_prog cmd_args pg_args
 
     db_name="${1}"
+    db_file="${2}"
+    db_file_dir="${db_file}.dir"
 
     if [ -n "${PGDUMP_OPTS}" ]; then
         IFS=" " read -r -a PGDUMP_ARGS <<< "${PGDUMP_OPTS}"
@@ -390,6 +415,47 @@ postgresqldb_dump () {
         PGDUMPALL_ARGS=()
     fi
 
+    case "${PGDUMP_FORMAT}" in
+        plain)
+            PGDUMP_ARGS+=(-Fp)
+            ;;
+        tar)
+            PGDUMP_ARGS+=(-Ft)
+            ;;
+        custom|directory)
+            if [ "${PGDUMP_FORMAT}" = "directory" ]; then
+                PGDUMP_ARGS+=(-Fd --file "${db_file_dir}")
+                if [ -n "${SU_USERNAME}" ]; then
+                    db_dir="$(dirname "${db_file}")"
+                    db_dir_grp="$(id -gn "${SU_USERNAME}")"
+                    chgrp "${db_dir_grp}" "${db_dir}"
+                    chmod g+w "${db_dir}"
+                fi
+            else
+                PGDUMP_ARGS+=(-Fc)
+            fi
+
+            if [ -n "${PGDUMP_VERSION_MAJOR}" ] && [ "${PGDUMP_VERSION_MAJOR}" -gt 15 ]; then
+                log_info "VERS=${PGDUMP_VERSION_MAJOR}"
+                FORMAT_COMP="${PGDUMP_FORMAT_COMP}"
+                if [ -n "${PGDUMP_FORMAT_COMP_LEVEL}" ] ; then
+                    FORMAT_COMP+=":${PGDUMP_FORMAT_COMP_LEVEL}"
+                fi
+            else
+                log_debug "Falling back to built-in compression (e.g. gzip) for pg_dump <= 15"
+                if [ -n "${PGDUMP_FORMAT_COMP_LEVEL}" ] ; then
+                    FORMAT_COMP="${PGDUMP_FORMAT_COMP_LEVEL}"
+                fi
+            fi
+            if [ -n "${FORMAT_COMP}" ]; then
+                PGDUMP_ARGS+=(--compress "${FORMAT_COMP}")
+            fi
+            ;;
+        *)
+            log_error "Unsupported PGDUMP_FORMAT: ${PGDUMP_FORMAT}"
+            ;;
+    esac
+
     if [ "${db_name}" = "${GLOBALS_OBJECTS}" ]; then
         cmd_prog="${PGDUMPALL}"
         cmd_args=(--globals-only)
@@ -425,6 +491,17 @@ postgresqldb_dump () {
     elif ! "${cmd_prog}" "${cmd_args[@]}" 2> >(logger "err" "error"); then
         log_error "Running command '${cmd_prog} ${cmd_args[*]}' has failed"
     fi
+
+    # when PGDUMP_FORMAT is directory, create tar archive on the fly
+    if [ -d "${db_file_dir}" ] ; then
+        cmd_prog="tar"
+        cmd_args=(--directory "${db_file_dir}")
+        cmd_args+=(--create --remove-files)
+        cmd_args+=(--file - .)
+        if ! "${cmd_prog}" "${cmd_args[@]}" 2> >(logger "err" "error"); then
+            log_error "Running command '${cmd_prog} ${cmd_args[*]}' has failed"
+        fi
+    fi
 }
 # }}}
 
@@ -574,7 +651,7 @@ db_list () {
 db_dump () {
     case ${DBENGINE} in
         postgresql)
-            postgresqldb_dump "${1}"
+            postgresqldb_dump "${1}" "${2}"
             ;;
         mysql)
             mysqldb_dump "${1}"
@@ -607,7 +684,7 @@ db_purge() {
         sort -r | \
         sed -E -n 's/\S+ //p' | \
         tail -n "+${count}" | \
-        xargs -L1 rm -fv | \
+        xargs -L1 rm -rfv | \
         while IFS= read -r line ; do
             log_info "${line}"
         done
@@ -620,9 +697,10 @@ dump() {
 
     db_name="${1}"
     dump_file="${2}"
-
+    backup_file="${dump_file}"
+ 
     if [ -n "${COMP}" ]; then
-        comp_ext=".comp"
+        comp_ext=""
         case "${COMP}" in
             gzip|pigz)
                 comp_ext=".gz"
@@ -646,16 +724,16 @@ dump() {
 
     if [ -n "${COMP}" ] && [ "${ENCRYPTION}" = "yes" ]; then
         log_debug "Dumping (${db_name}) +compress +encrypt to '${dump_file}'"
-        db_dump "${db_name}" | compression | encryption > "${dump_file}"
+        db_dump "${db_name}" "${backup_file}" | compression | encryption > "${dump_file}"
     elif [ -n "${COMP}" ]; then
         log_debug "Dumping (${db_name}) +compress to '${dump_file}'"
-        db_dump "${db_name}" | compression > "${dump_file}"
+        db_dump "${db_name}" "${backup_file}" | compression > "${dump_file}"
     elif [ "${ENCRYPTION}" = "yes" ]; then
         log_debug "Dumping (${db_name}) +encrypt to '${dump_file}'"
-        db_dump "${db_name}" | encryption > "${dump_file}"
+        db_dump "${db_name}" "${backup_file}" | encryption > "${dump_file}"
     else
         log_debug "Dumping (${db_name}) to '${dump_file}'"
-        db_dump "${db_name}" > "${dump_file}"
+        db_dump "${db_name}" "${backup_file}" > "${dump_file}"
     fi
 
     if [ -f "${dump_file}" ]; then
@@ -673,8 +751,8 @@ dump() {
 }
 # }}}
 
-# {{{ setup()
-setup() {
+# {{{ setup_logger()
+setup_logger() {
     # Using a shared memory filesystem (if available) to avoid
     # issues when there is no left space on backup storage
     if [ -w "/dev/shm" ]; then
@@ -684,6 +762,16 @@ setup() {
     LOG_PREFIX="${LOG_DIR}/${NAME}_${DBHOST//\//_}-$(date '+%Y-%m-%d_%Hh%Mm')"
     LOG_FILE="${LOG_PREFIX}.log"
     LOG_REPORT="${LOG_PREFIX}.report"
+}
+# }}}
+
+# {{{ setup()
+setup() {
+    # Hostname
+    HOSTNAME="$(uname -n)"
+    if [[ "${HOSTNAME}" != *.* ]]; then
+        HOSTNAME="$(hostname --fqdn)"
+    fi
 
     HOST="${DBHOST}:${DBPORT}"
     if [ "${DBHOST}" = "localhost" ]; then
@@ -829,6 +917,10 @@ while getopts "hdc:" OPTION ; do
 done
 # }}}
 
+# {{{ Setup logger
+setup_logger
+# }}}
+
 # {{{ I/O redirection(s) for logging
 setup_io
 # }}}
@@ -946,6 +1038,10 @@ if [ "${DBNAMES}" = "all" ]; then
     DBNAMES="$(db_list)"
 fi
 
+if [ -n "${EXT}" ]; then
+    EXT=".${EXT}"
+fi
+
 for db_enc in ${DBNAMES} ; do
     db="$(echo "${db_enc}" | arg_decode)"
 
@@ -959,7 +1055,7 @@ for db_enc in ${DBNAMES} ; do
 
     db_purge "${BACKUPDIR}" "${db_enc}" "${period}" "${rotate}"
 
-    backupfile="${backupdbdir}/${db_enc}_${DATE}.${EXT}"
+    backupfile="${backupdbdir}/${db_enc}_${DATE}${EXT}"
     dump "${db}" "${backupfile}"
 done
 log_info "Backup End: $(date)"