marcusconsulting/docker-skeleton
2
0
Fork 0
Code Activity

Initial commit - forked from the corresponding Smartfront repositories.

Browse Source
This commit is contained in:
Kovács Zoltán
2023-09-11 18:01:37 +02:00
commit 7803630c76
53 changed files with 12259 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
.templates/apache2/apache2.conf Normal file
View File

@ -0,0 +1,95 @@
# Apache2 configuration for a virtualhost proxied to a Docker service.
# Uses https://github.com/acmesh-official/acme.sh to manage SSL certificates.
<VirtualHost *:80 >
ServerAdmin webmaster@$PAR_SERVERNAME
ServerName $PAR_SERVERNAME
# ServerAlias $PAR_SERVERNAMES
DocumentRoot /var/www/html
# Common log settings.
ErrorLog $PAR_SERVICE/logs/web/error.log
CustomLog $PAR_SERVICE/logs/web/access.log combined
# Custom error messages.
<IfModule mod_macro.c>
<Macro try_other $response>
ErrorDocument $response "<span style='font-size: x-large'>Sorry try <a href='http://$PAR_SERVERNAME/$PAR_LOCATION'>http://$PAR_SERVERNAME/$PAR_LOCATION</a> instead.</span>"
</Macro>
<Macro try_later $response>
ErrorDocument $response "<span style='font-size: x-large'>Sorry something went wrong. Try again a bit later.<br>\
You may report this at <a href='mailto:webmaster@$PAR_SERVERNAME'>webmaster@$PAR_SERVERNAME</a>.</span>"
</Macro>
</IfModule>
# Permanent redirect to https.
<IfModule mod_rewrite.c>
# RewriteEngine On
# RewriteRule ^(.*)$ https://%{HTTP_HOST}$1 [R=301,L]
</IfModule>
# No static service.
# Sequence matters: http://httpd.apache.org/docs/2.4/sections.html#file-and-web
# <Location />
# Require all denied
# # Custom error message.
# <IfModule mod_macro.c>
# Use try_other 403
# Use try_other 404
# </IfModule>
# </Location>
# Let's Encrypt (acme.sh) support.
<Location /.well-known/>
<IfModule mod_proxy.c>
Require all granted
ProxyPreserveHost On
ProxyPass http://$PAR_ACMEHOST:$PAR_ACMEPORT/
ProxyPassReverse http://$PAR_ACMEHOST:$PAR_ACMEPORT/
# Custom error message.
<IfModule mod_macro.c>
Use try_later 500
Use try_later 502
Use try_later 503
Use try_later 504
</IfModule>
</IfModule>
<IfModule !mod_proxy.c>
# Custom error message.
<IfModule mod_macro.c>
Use try_other 403
Use try_other 404
</IfModule>
</IfModule>
</Location>
<Location /$PAR_LOCATION>
<IfModule mod_proxy.c>
Require all granted
ProxyPreserveHost On
ProxyPass http://$PAR_PROXYHOST:$PAR_PROXYPORT/$PAR_LOCATION
ProxyPassReverse http://$PAR_PROXYHOST:$PAR_PROXYPORT/$PAR_LOCATION
# Custom error message.
<IfModule mod_macro.c>
Use try_later 500
Use try_later 502
Use try_later 503
Use try_later 504
</IfModule>
</IfModule>
<IfModule !mod_proxy.c>
# Custom error message.
<IfModule mod_macro.c>
Use try_later 403
Use try_later 404
</IfModule>
</IfModule>
</Location>
# XSS protection
<IfModule mod_headers.c>
Header set X-Frame-Options SAMEORIGIN
Header set X-Content-Type-Options nosniff
</IfModule>
</Virtualhost>

111
.templates/apache2/apache2_ssl.conf Normal file
View File

@ -0,0 +1,111 @@
# Apache2 SSL configuration for a virtualhost proxied to a Docker service.
# Uses https://github.com/acmesh-official/acme.sh to manage SSL certificates.
<IfModule mod_ssl.c>
<VirtualHost *:443 >
ServerAdmin webmaster@$PAR_SERVERNAME
ServerName $PAR_SERVERNAME
# ServerAlias $PAR_SERVERNAMES
DocumentRoot /var/www/html
# Common log settings.
ErrorLog $PAR_SERVICE/logs/web/error.log
CustomLog $PAR_SERVICE/logs/web/access.log combined
# Custom error messages.
<IfModule mod_macro.c>
<Macro try_other $response>
ErrorDocument $response "<span style='font-size: x-large'>Sorry try <a href='http://$PAR_SERVERNAME/$PAR_LOCATION'>http://$PAR_SERVERNAME/$PAR_LOCATION</a> instead.</span>"
</Macro>
<Macro try_later $response>
ErrorDocument $response "<span style='font-size: x-large'>Sorry something went wrong. Try again a bit later.<br>\
You may report this at <a href='mailto:webmaster@$PAR_SERVERNAME'>webmaster@$PAR_SERVERNAME</a>.</span>"
</Macro>
</IfModule>
# No static service.
# Sequence matters: http://httpd.apache.org/docs/2.4/sections.html#file-and-web
# <Location />
# Require all denied
# # Custom error message.
# <IfModule mod_macro.c>
# Use try_other 403
# Use try_other 404
# </IfModule>
# </Location>
# Let's Encrypt (acme.sh) support.
<Location /.well-known/>
<IfModule mod_proxy.c>
Require all granted
ProxyPreserveHost On
ProxyPass http://$PAR_ACMEHOST:$PAR_ACMEPORT/
ProxyPassReverse http://$PAR_ACMEHOST:$PAR_ACMEPORT/
# Custom error message.
<IfModule mod_macro.c>
Use try_later 500
Use try_later 502
Use try_later 503
Use try_later 504
</IfModule>
</IfModule>
<IfModule !mod_proxy.c>
# Custom error message.
<IfModule mod_macro.c>
Use try_other 403
Use try_other 404
</IfModule>
</IfModule>
</Location>
<Location /$PAR_LOCATION>
<IfModule mod_proxy.c>
Require all granted
ProxyPreserveHost On
ProxyPass http://$PAR_PROXYHOST:$PAR_PROXYPORT/$PAR_LOCATION
ProxyPassReverse http://$PAR_PROXYHOST:$PAR_PROXYPORT/$PAR_LOCATION
# Custom error message.
<IfModule mod_macro.c>
Use try_later 500
Use try_later 502
Use try_later 503
Use try_later 504
</IfModule>
</IfModule>
<IfModule !mod_proxy.c>
# Custom error message.
<IfModule mod_macro.c>
Use try_later 403
Use try_later 404
</IfModule>
</IfModule>
</Location>
##################################################################################
# The SSL part
# https://ssl-config.mozilla.org/
SSLEngine on
SSLCertificateFile $PAR_SERVICE/configs/acme/$PAR_SERVERNAME/fullchain.cer
SSLCertificateKeyFile $PAR_SERVICE/configs/acme/$PAR_SERVERNAME/$PAR_SERVERNAME.key
# Settings to achieve 'A' grade on https://www.ssllabs.com/ssltest/
SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1
SSLCipherSuite ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
SSLHonorCipherOrder off
SSLSessionTickets off
# HTTP/2, if available.
<IfModule mod_http2.c>
Protocols h2 http/1.1
</IfModule>
# HTTP Strict Transport Security and XSS protection.
<IfModule mod_headers.c>
Header always set Strict-Transport-Security "max-age=63072000"
Header set X-Frame-Options SAMEORIGIN
Header set X-Content-Type-Options nosniff
</IfModule>
</Virtualhost>
</IfModule>

2
.templates/apache2/sites-available/services.conf Normal file
View File

@ -0,0 +1,2 @@
# Includes the Docker services' configurations.
IncludeOptional $PAR_SERVICEBASE/.apache2/*.conf

176
.templates/bin/.launcher Executable file
View File

@ -0,0 +1,176 @@
#!/bin/bash
#
# Simple launcher script to start a worker script within a service or
# one for every service.
#
# Services are subfolders of the $SERVICE_BASE parameter set below.
# The worker script is an executable file within the service's $TOOLS_BASE
# folder. The launcher supposes that the filename of the worker script
# is the same as the launcher's.
#
# Without command line parameters the launcher enumerates all services
# and launches every service's worker script. Optionally waits between
# launches if is prescribed in $SLEEP_BETWEEN parameter. This operation
# needs to be forced by --doit command line option.
#
# With at least one command line parameter the launcher launches only one
# worker script, which is within the service named in the first parameter.
# If the supposed service or the worker script doesn't exist the script
# silently does nothing.
#
# Normally the launcher starts the worker(s) in background, then exits.
# Using the --wait command line option you may wait for the worker(s)
# and get exit code 1 if any worker has reported some error.
#
# You may call this script from the cron - in this case must be set
# the CRON environment variable in crontab and must be exported to the script.
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# mod: "instance" => "service"
# 2022-08-03 v0.6
# new: it optionally waits for the worker(s) and returns with exit code 1,
# if it has got at least one non-zero exit code from them.
# 2021-02-15 v0.5
# fix: omits the error message when there is no services at all
# 2021-02-05 v0.4
# fix: proper $PATH settings in Debian environment as well
# 2021-02-04 v0.3
# fix: decimal point trouble in non-english environments (eg. Hungarian)
# 2021-01-05 v0.2
# fix: LANG=C and LC_ALL=C initialisations have removed, because these may
# interfere with UTF-8 file name encoding in Java calls:
# https://ogris.de/howtos/java-utf8-filenames.html
# 2020-11-20 v0.1 Initial release
# Accepted environment variables and their defaults.
#
CRON=${CRON-""} # Does it run from cron?
SERVICE_BASE=${SERVICE_BASE-"$HOME/services"} # Services' folder path
SLEEP_BETWEEN=${SLEEP_BETWEEN-"0"} # Secs between forks
# There is nothing to configure below (I hope).
###############################################
# Messages.
#
MSG_BADOPT="Invalid option"
MSG_MISSINGDEP="Fatal: missing dependency"
MSG_USAGE="For one service: $($(which basename) "$0") [--wait] servicename\n"
MSG_USAGE+="For all services: $($(which basename) "$0") [--wait] --doit\n"
MSG_USAGE+="Environment variables:\n"
MSG_USAGE+=" SERVICE_BASE Absolute path to the folder containing services\n"
MSG_USAGE+=" (default: \$HOME/services)\n"
MSG_USAGE+=" SLEEP_BETWEEN Secs (may fraction) to wait between multiple launches\n"
MSG_USAGE+=" (default: 0.0)"
# Getting command line options.
DOIT="" # Enables launch processes for all available services.
WAIT="" # Wait for the result of the actually launched process.
while getopts ":-:" option
do
case ${option} in
"-" )
if [ "$OPTARG" = "doit" ]; then DOIT="yes"
elif [ "$OPTARG" = "wait" ]; then WAIT="yes"
else echo "$MSG_BADOPT --$OPTARG" >&2; exit 1
fi
;;
\? )
echo "$MSG_BADOPT -$OPTARG" >&2; exit 1
;;
esac
done; shift $((OPTIND -1))
# Done with options.
# Basic environment settings.
#
# Corrects the PATH if the operating system didn't loaded yet;
# it is a bug with cron pam_env, I think.
if [ -n "$CRON" ]; then
[[ -r "/etc/profile" ]] && source "/etc/profile"
# Ubuntu gets the initial environment from a separate file.
if [ -r "/etc/environment" ]; then
# Extracts from this file, strips the right part w/o quotes.
includepath=$(cat "/etc/environment" | $(which egrep) '^PATH=')
includepath=${includepath:5}
includepath="${includepath%\"}"; includepath="${includepath#\"}"
[[ -n "$includepath" ]] && PATH="$PATH:$includepath"
unset includepath
fi
# We need the $HOME/bin as well.
PATH="$HOME/bin:$PATH"
fi
# We need also the sbin directories.
if ! [[ "$PATH" =~ '/sbin:' ]]; then PATH="$PATH:/usr/local/sbin:/usr/sbin:/sbin"; fi
# Checks the dependencies.
#
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
#for item in basename df egrep head mail printf sleep
for item in basename printf sleep
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Initializations and sanitizations.
#
if [ -z "$SERVICE_BASE" -o ! -d "$SERVICE_BASE" ]; then exit 1; fi
SLEEP_BETWEEN=$("$PRINTF" '%.2f' "$SLEEP_BETWEEN" 2>/dev/null) # To float
TOOLS_BASE="tools" # Relative path
WORKERFILE="$("$BASENAME" "$0")" # Same filename
# Collects the service(s).
if [ -n "$1" ]; then
# One service.
SERVICES="$1/"; shift
elif [ -n "$DOIT" ]; then
# All services when has forced.
SERVICES="$(cd "$SERVICE_BASE"; ls -d */ 2>/dev/null)"
else
echo -e "$MSG_USAGE" >&2; exit 1
fi
# Prepares to save the result codes.
declare -A JOBS
# Eumerates the service folders.
[[ -z "$SERVICES" ]] && exit
for service in $SERVICES ""
do
# Safety first...
if [ -n "$service" ]; then
# Forks the worker if it does exist and does runnable.
if [ -x "$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" ]; then
# Sets PATH and USER, passes all remaining command line parameters.
[[ -n "$CRON" ]] && export PATH USER
"$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" "$@" &
PID="$!"
# Optionally waits for the worker and saves the result.
if [ -n "$WAIT" ]
then wait $PID; JOBS["$service"]=$?; fi
# Optionally reduces the fork frequency.
"$SLEEP" ${SLEEP_BETWEEN//,/.} # decimal point need
else
# Optionally reports the failure.
if [ -n "$WAIT" ]
then JOBS["$service"]=1; fi
fi
fi
done
# Optionally returns with exit code 1, if it has got at least one
# non-zero exit code from workers.
if [ -n "$WAIT" ]; then
for key in "${!JOBS[@]}"
do [[ "${JOBS[$key]}" -gt 0 ]] && exit 1
done
fi
# That's all, Folks! :)

7952
.templates/bin/acme.sh Executable file
View File

File diff suppressed because it is too large Load Diff

1
.templates/bin/build Symbolic link
View File

@ -0,0 +1 @@
.launcher

93
.templates/bin/copytruncate Executable file
View File

@ -0,0 +1,93 @@
#!/bin/bash
#
# A replacement of the logrotate's copytruncate method to use when:
# * we haven't permission to change ownership, so the built-in copytruncate
# method would fail (this is a bug in logrotate, I think);
# * we haven't permission to reload the service, so the create new log method
# doesn't work - the service would still write to the already rotated file.
#
# This script, when called as a prerotate script from logrotate configuration:
# * copytruncates a file having $LOGEXT (see below) extesion and compresses it
# (and returns with 1 exit code, thus, the logrotate will skip this file).
# * does nothing with files having any other extensions - e.g .1, .2 and so on
# (and returns with 0 exit code, thus, the logrotate can process this file);
#
# In other words, with default settings it simulates the effect of logrotate
# configuration options below:
# <pathname>/*.log {
# dateext
# dateyesterday
# dateformat %Y-%m-%d.
# extension log
# compress
# copytruncate
# [...]
# }
# but doesn't stop if failed to set the permissions during copytruncate.
#
# The script receives the file to process as it's first command line parameter.
# Lack of the parameter it simply does nothing.
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# 2020-11-09 v0.1 Initial release
# Configurable parameters - must be consistent with the logrotate configuration.
#
DATESTRING="$(date "+%Y-%m-%d" -d '1 day ago')" # Yesterday goes to the filename
LOGEXT="log" # Processes only *.$LOGEXT files
# Messages.
MSG_MISSINGDEP="Fatal: missing dependency"
# Checks the dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename cp dirname gzip
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
#
# An additional bugfix (use "$(which gzip)" instead of "$GZIP"):
# https://www.gnu.org/software/gzip/manual/html_node/Environment.html
GZIP=""
# Lack of the parameter (unlikely) it simply does nothing.
# The script returns an OK status, so the logrotate will continue.
[[ -z "$1" ]] && exit 0
# Parses the parameter.
LOGDIR="$("$DIRNAME" $1)"
LOGFILE="$("$BASENAME" $1)"
NEWFILE="${LOGFILE%.*}.$DATESTRING.${LOGFILE##*.}"
# If the parameter doesn't point to a writeable file it simply does nothing.
[[ ! -w "$LOGDIR/$LOGFILE" ]] && exit 0
# If the log's folder isn't writable it simply does nothing.
[[ ! -w "$LOGDIR" ]] && exit 0
# If the extension doesn't match it simply does nothing.
[[ "${LOGFILE##*.}" != "$LOGEXT" ]] && exit 0
# Cases above must handled by the logrotate itself.
# The script returns an OK status, so the logrotate will continue.
# Doesn't copy if it would overwrite something.
# Returns an error status, so the logrotate won't process this file.
[[ -e "$LOGDIR/$NEWFILE" ]] && exit 1
[[ -e "$LOGDIR/$NEWFILE.gz" ]] && exit 1
# Tries to copy the current logfile.
"$CP" -p "$LOGDIR/$LOGFILE" "$LOGDIR/$NEWFILE" # >/dev/null 2>&1
# On error returns an error status so the logrotate won't process this file.
[[ ! -r "$LOGDIR/$NEWFILE" ]] && exit 1
# Done with copy. Tries to empty the current logfile. Doesn't check the result.
: >"$LOGDIR/$LOGFILE"
# Compresses the rotated logfile. Doesn't check the result.
"$(which gzip)" "$LOGDIR/$NEWFILE" #>/dev/null 2>&1
# Returns a fake error status to prevent the logrotate to process this file.
exit 1
# That's all, Folks! :)

3
.templates/bin/mail Normal file
View File

@ -0,0 +1,3 @@
#!/bin/bash
#
# Dummy file to met the dependencies. Does nothing.

148
.templates/bin/maintenance_daily Executable file
View File

@ -0,0 +1,148 @@
#!/bin/bash
#
# Maintenence operations once by day.
# This script called usually by the cron - in this case the CRON variable
# must be set in crontab and exported to the script.
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# mod: "instance" => "service"
# 2023-02-12 v0.9
# new: shows the ZFS disk partitions (if any) as well.
# 2023-01-20 v0.8
# fix: doesn't try to reload non-existing or stopped web servers.
# 2021-10-08 v0.7
# fix: tries to reload the web server(s) to get in effect the renewed SSL
# certificates (if any).
# 2021-06-02 v0.6
# fix: more accurate egrep for df -h output.
# fix: typo => 98-fsck-at-rebooz
# 2021-02-15 v0.5
# fix: omits the error message when there is no services at all
# 2021-02-05 v0.4
# fix: proper $PATH settings in Debian environment as well
# 2021-02-04 v0.3
# fix: decimal point trouble in non-english environments (eg. Hungarian)
# 2021-01-05 v0.2
# fix: LANG=C and LC_ALL=C initialisations have removed, because these may
# interfere with UTF-8 file name encoding in Java calls:
# https://ogris.de/howtos/java-utf8-filenames.html
# fix: Missing message.
# 2020-11-12 v0.1 Initial release
# Accepted environment variables and their defaults.
#
CRON=${CRON-""} # Does it run from cron?
SERVICE_BASE=${SERVICE_BASE-"$HOME/services"} # Services' folder path
SLEEP_BETWEEN=${SLEEP_BETWEEN-"120"} # Secs between forks
# There is nothing to configure below (I hope).
###############################################
# Messages.
#
MSG_MISSINGDEP="Fatal: missing dependency"
# Basic environment settings.
#
# Corrects the PATH if the operating system didn't loaded yet;
# it is a bug with cron pam_env, I think.
if [ -n "$CRON" ]; then
[[ -r "/etc/profile" ]] && source "/etc/profile"
# Ubuntu gets the initial environment from a separate file.
if [ -r "/etc/environment" ]; then
# Extracts from this file, strips the right part w/o quotes.
includepath=$(cat "/etc/environment" | $(which egrep) '^PATH=')
includepath=${includepath:5}
includepath="${includepath%\"}"; includepath="${includepath#\"}"
[[ -n "$includepath" ]] && PATH="$PATH:$includepath"
unset includepath
fi
# We need the $HOME/bin as well.
PATH="$HOME/bin:$PATH"
fi
# We need also the sbin directories.
if ! [[ "$PATH" =~ '/sbin:' ]]; then PATH="$PATH:/usr/local/sbin:/usr/sbin:/sbin"; fi
# Checks the dependencies.
#
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename cut df egrep head mail printf sleep
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Checks and sanitizations.
#
if [ -z "$SERVICE_BASE" -o ! -d "$SERVICE_BASE" ]; then exit 1; fi
SLEEP_BETWEEN=$("$PRINTF" '%.2f' "$SLEEP_BETWEEN" 2>/dev/null) # To float
# Sends a mail report to the Linux user itself.
# Hopes the appropriate message forward rule has been set.
if [ -x "$MAIL" ]; then
subject="[Maintenance] $HOSTNAME daily report"
message="This is a report from $("$BASENAME" "$0") script on $HOSTNAME.\n\n"
[[ -x "/etc/update-motd.d/00-header" ]] \
&& message+="$("/etc/update-motd.d/00-header")\n\n"
message+="$("$DF" -h | "$HEAD" -n1)\n"
message+="$("$DF" -h | "$EGREP" '^/dev/' | "$EGREP" -v 'loop')\n"
if [ -n "$(which zpool)" -a -x "$(which zpool)" ]; then
# Includes ZFS partitions (if any).
for pool in $("$(which zpool)" list -H | "$CUT" -f1) "" #"
do [[ -n "$pool" ]] && message+="$("$DF" -h | "$EGREP" "^$pool/")\n"; done
fi
[[ -x "/etc/update-motd.d/90-updates-available" ]] \
&& message+="$("/etc/update-motd.d/90-updates-available")\n"
[[ -x "/etc/update-motd.d/98-fsck-at-reboot" ]] \
&& message+="$("/etc/update-motd.d/98-fsck-at-reboot")\n"
[[ -x "/etc/update-motd.d/99-reboot-required" ]] \
&& message+="$("/etc/update-motd.d/99-reboot-required")\n"
message+="\nBest regards: the Maintenance Bot"
echo -e "$message" | "$MAIL" -s "$subject" "$USER"
fi
# Done with mail.
# Tries to reload the webserver(s) to get in effect
# the renewed SSL certificates (if any).
if [ -n "$(which sudo)" -a -n "$(which systemctl)" ]; then
for webserver in apache2 nginx
do
if [[ $( systemctl status $webserver >/dev/null 2>&1; echo $? ) -eq 0 ]]; then
sudo -n systemctl reload $webserver >/dev/null 2>&1
fi
done
fi
# Done with the webserver(s).
# Launches the worker scripts with the same name, one for every services.
# Services are subfolders of the $SERVICE_BASE folder set above.
TOOLS_BASE="tools" # Relative path to worker scripts
WORKERFILE="$("$BASENAME" "$0")" # Same filename
# Gets the service folders, give up if none.
SERVICES="$(cd "$SERVICE_BASE"; ls -d */ 2>/dev/null)"
[[ -z "$SERVICES" ]] && exit
# Eumerates the service folders.
for service in $SERVICES ""
do
# Safety first...
if [ -n "$service" ]; then
# Forks the worker if it does exist and does runnable.
# Passes all remaining command line parameters.
if [ -x "$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" ]; then
[[ -n "$CRON" ]] && export PATH USER
"$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" "$@" &
# Optionally reduces the fork frequency.
"$SLEEP" ${SLEEP_BETWEEN//,/.} # decimal point need
fi
fi
done
# Done with workers.
# That's all, Folks! :)

101
.templates/bin/maintenance_midnight Executable file
View File

@ -0,0 +1,101 @@
#!/bin/bash
#
# Maintenence operations at midnight.
# This script called usually by the cron - in this case the CRON variable
# must be set in crontab and exported to the script.
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# mod: "instance" => "service"
# 2021-02-15 v0.5
# fix: omits the error message when there is no services at all
# 2021-02-05 v0.4
# fix: proper $PATH settings in Debian environment as well
# 2021-02-04 v0.3
# fix: decimal point trouble in non-english environments (eg. Hungarian)
# 2021-01-05 v0.2
# fix: LANG=C and LC_ALL=C initialisations have removed, because these may
# interfere with UTF-8 file name encoding in Java calls:
# https://ogris.de/howtos/java-utf8-filenames.html
# fix: Missing message.
# 2020-11-12 v0.1 Initial release
# Accepted environment variables and their defaults.
#
CRON=${CRON-""} # Does it run from cron?
SERVICE_BASE=${SERVICE_BASE-"$HOME/services"} # Services' folder path
SLEEP_BETWEEN=${SLEEP_BETWEEN-"1"} # Secs between forks
# There is nothing to configure below (I hope).
###############################################
# Messages.
#
MSG_MISSINGDEP="Fatal: missing dependency"
# Basic environment settings.
#
# Corrects the PATH if the operating system didn't loaded yet;
# it is a bug with cron pam_env, I think.
if [ -n "$CRON" ]; then
[[ -r "/etc/profile" ]] && source "/etc/profile"
# Ubuntu gets the initial environment from a separate file.
if [ -r "/etc/environment" ]; then
# Extracts from this file, strips the right part w/o quotes.
includepath=$(cat "/etc/environment" | $(which egrep) '^PATH=')
includepath=${includepath:5}
includepath="${includepath%\"}"; includepath="${includepath#\"}"
[[ -n "$includepath" ]] && PATH="$PATH:$includepath"
unset includepath
fi
# We need the $HOME/bin as well.
PATH="$HOME/bin:$PATH"
fi
# We need also the sbin directories.
if ! [[ "$PATH" =~ '/sbin:' ]]; then PATH="$PATH:/usr/local/sbin:/usr/sbin:/sbin"; fi
# Checks the dependencies.
#
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename printf sleep
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Checks and sanitizations.
#
if [ -z "$SERVICE_BASE" -o ! -d "$SERVICE_BASE" ]; then exit 1; fi
SLEEP_BETWEEN=$("$PRINTF" '%.2f' "$SLEEP_BETWEEN" 2>/dev/null) # To float
# Launches the worker scripts with the same name, one for every services.
# Services are subfolders of the $SERVICE_BASE folder set above.
TOOLS_BASE="tools" # Relative path to worker scripts
WORKERFILE="$("$BASENAME" "$0")" # Same filename
# Gets the service folders, give up if none.
SERVICES="$(cd "$SERVICE_BASE"; ls -d */ 2>/dev/null)"
[[ -z "$SERVICES" ]] && exit
# Eumerates the service folders.
for service in $SERVICES ""
do
# Safety first...
if [ -n "$service" ]; then
# Forks the worker if it does exist and does runnable.
# Passes all remaining command line parameters.
if [ -x "$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" ]; then
[[ -n "$CRON" ]] && export PATH USER
"$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" "$@" &
# Optionally reduces the fork frequency.
"$SLEEP" ${SLEEP_BETWEEN//,/.} # decimal point need
fi
fi
done
# Done with workers.
# That's all, Folks! :)

112
.templates/bin/maintenance_reboot Executable file
View File

@ -0,0 +1,112 @@
#!/bin/bash
#
# Maintenence operations at reboot.
# This script called usually by the cron - in this case the CRON variable
# must be set in crontab and exported to the script.
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# mod: "instance" => "service"
# 2021-02-15 v0.5
# fix: omits the error message when there is no services at all
# 2021-02-05 v0.4
# fix: proper $PATH settings in Debian environment as well
# 2021-02-04 v0.3
# fix: decimal point trouble in non-english environments (eg. Hungarian)
# 2021-01-05 v0.2
# fix: LANG=C and LC_ALL=C initialisations have removed, because these may
# interfere with UTF-8 file name encoding in Java calls:
# https://ogris.de/howtos/java-utf8-filenames.html
# fix: Missing message.
# 2020-11-12 v0.1 Initial release
# Accepted environment variables and their defaults.
#
CRON=${CRON-""} # Does it run from cron?
SERVICE_BASE=${SERVICE_BASE-"$HOME/services"} # Services' folder path
SLEEP_BETWEEN=${SLEEP_BETWEEN-"60"} # Secs between forks
# There is nothing to configure below (I hope).
###############################################
# Messages.
#
MSG_MISSINGDEP="Fatal: missing dependency"
# Basic environment settings.
#
# Corrects the PATH if the operating system didn't loaded yet;
# it is a bug with cron pam_env, I think.
if [ -n "$CRON" ]; then
[[ -r "/etc/profile" ]] && source "/etc/profile"
# Ubuntu gets the initial environment from a separate file.
if [ -r "/etc/environment" ]; then
# Extracts from this file, strips the right part w/o quotes.
includepath=$(cat "/etc/environment" | $(which egrep) '^PATH=')
includepath=${includepath:5}
includepath="${includepath%\"}"; includepath="${includepath#\"}"
[[ -n "$includepath" ]] && PATH="$PATH:$includepath"
unset includepath
fi
# We need the $HOME/bin as well.
PATH="$HOME/bin:$PATH"
fi
# We need also the sbin directories.
if ! [[ "$PATH" =~ '/sbin:' ]]; then PATH="$PATH:/usr/local/sbin:/usr/sbin:/sbin"; fi
# Checks the dependencies.
#
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename mail printf sleep
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Checks and sanitizations.
#
if [ -z "$SERVICE_BASE" -o ! -d "$SERVICE_BASE" ]; then exit 1; fi
SLEEP_BETWEEN=$("$PRINTF" '%.2f' "$SLEEP_BETWEEN" 2>/dev/null) # To float
# Sends a mail message to the Linux user itself.
# Hopes the appropriate message forward rule has been set.
if [ -x "$MAIL" ]; then
subject="[Maintenance] $HOSTNAME has been rebooted!"
message="This is a warning from $("$BASENAME" "$0") script on $HOSTNAME.\n"
message+="The machine has been rebooted few minutes ago.\n\n"
message+="Best regards: the Maintenance Bot"
echo -e "$message" | "$MAIL" -s "$subject" "$USER"
fi
# Done with mail.
# Launches the worker scripts with the same name, one for every services.
# Services are subfolders of the $SERVICE_BASE folder set above.
TOOLS_BASE="tools" # Relative path to worker scripts
WORKERFILE="$("$BASENAME" "$0")" # Same filename
# Gets the service folders, give up if none.
SERVICES="$(cd "$SERVICE_BASE"; ls -d */ 2>/dev/null)"
[[ -z "$SERVICES" ]] && exit
# Eumerates the service folders.
for service in $SERVICES ""
do
# Safety first...
if [ -n "$service" ]; then
# Forks the worker if it does exist and does runnable.
# Passes all remaining command line parameters.
if [ -x "$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" ]; then
[[ -n "$CRON" ]] && export PATH USER
"$SERVICE_BASE/$service$TOOLS_BASE/$WORKERFILE" "$@" &
# Optionally reduces the fork frequency.
"$SLEEP" ${SLEEP_BETWEEN//,/.} # decimal point need
fi
fi
done
# Done with workers.
# That's all, Folks! :)

299
.templates/bin/mysql_dumpdb Executable file
View File

@ -0,0 +1,299 @@
#! /bin/bash
#
# Dumps a MySQL database from a native or dockerized MySQL instance running
# on this box. This is a wrapper script to the mysqldump tool.
#
# If the MySQL is dockerized you need call as a Docker manager user
# (member of the docker Linux group).
#
# Accepts few mysql_dump options as well as the optional database password
# and the optional output pathname:
#
# $0 [-u dbuser] [-p dbpass] [-h dbhost] [-P dbport]
# [-C container] [-d database] [-f dumpfile ] [--compress] [--force]
# [database (if not in -d)] [dumpfile (if not in -f)]
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# 2023-02-15 v0.3
# fix: Some updates to MySQL messed with our mysqldump settings.
# https://dev.mysql.com/doc/relnotes/mysql/5.7/en/news-5-7-31.html#mysqld-5-7-31-security
# --no-tablespaces (the recommended option) has been added to fix.
# https://dev.mysql.com/doc/refman/5.7/en/flush.html#flush-tables-with-read-lock
# --single-transaction=false has been added as a temporary workaround.
# 2020-11-12 v0.2
# fix: "$(which gzip)" instad of "$GZIP", see also:
# https://www.gnu.org/software/gzip/manual/html_node/Environment.html
# mod: Accepts a dump folder name as well, instead of a dump file name.
# 2020-09-17 v0.1 Initial release
# Accepted environment variables and their defaults.
#
MYCONTAINER=${MYCONTAINER-""} # Docker container's name
MYDATABASE=${MYDATABASE-""} # Database name to dump
MYDUMP=${MYDUMP-""} # Dump file pathname
MYHOST=${MYHOST:-"localhost"} # Connection parameter
MYOPTIONS=${MYOPTIONS-""} # Options to pass to mysqldump
MYPASSWORD=${MYPASSWORD-""} # Credential for the MySQL user
MYPORT=${MYPORT:-"3306"} # Connection parameter
MYUSER=${MYUSER:-"root"} # MySQL user for this dump
### Temporailly ignored! Need to sanitize.
MYOPTIONS=""
# Other initialisations.
#
MYDUMPFORCED="" # Dumps despite failed checks
# Our default parameters for the mysqldump.
# Content of the MYOPTIONS will also appended during the actual dump.
dumpparameters="--comments --events --routines --triggers "
dumpparameters+="--complete-insert --dump-date --force --no-create-db "
dumpparameters+="--opt --single-transaction "
## https://dev.mysql.com/doc/relnotes/mysql/5.7/en/news-5-7-31.html#mysqld-5-7-31-security
dumpparameters+="--no-tablespaces "
## https://dev.mysql.com/doc/relnotes/mysql/5.7/en/news-5-7-41.html
## a temporary workaround only
dumpparameters+="--single-transaction=false "
# Technical databases which are never dumped.
vetodatabases="information_schema mysql performance_schema sys"
# Messages.
#
MSG_ABORTED="aborted"
MSG_BADCRED="Bad credentials for MySQL"
MSG_BADOPT="Invalid option"
MSG_DOESNOTRUN="Doesn't run the database container"
MSG_DOCKERGRPNEED="You must be a member of the docker group."
MSG_FAILSIZE="Failed to size the database"
MSG_FORCED="but forced to continue"
MSG_MISSINGDEP="Fatal: missing dependency"
MSG_MISSINGDB="Missing database"
MSG_NONWRITE="The target directory isn't writable"
MSG_NOSPACE="Not enough space to dump the database"
MSG_USAGE="Usage: $0 [options] [database [dump_pathname|-]]\n"
MSG_USAGE+="Option:\tENVVAR:\n"
MSG_USAGE+=" -C\tMYCONTAINER\tMySQL Docker container's name\n"
MSG_USAGE+=" -d\tMYDATABASE\tMySQL database to dump\n"
MSG_USAGE+=" -f\tMYDUMP\t\tDumpfile pathname\n"
MSG_USAGE+=" -h\tMYHOST\t\tHostname or IP to connect (localhost)\n"
MSG_USAGE+=" -p\tMYPORT\t\tTCP port to connect (3306)\n"
MSG_USAGE+=" -P\tMYPASSWORD\tMySQL password\n"
MSG_USAGE+=" -u\tMYUSER\t\tMySQL username (root)\n"
MSG_USAGE+="--compress\t\tCompresses with gzip\n"
MSG_USAGE+="--force\t\t\tForces the operation despite the failed checks\n"
# Getting options.
#
while getopts ":-:c:C:d:D:f:F:h:H:p:P:u:U:" option
do
case ${option} in
"-" )
if [ "$OPTARG" = "compress" ]; then compress="yes"
elif [ "$OPTARG" = "force" ]; then MYDUMPFORCED="yes"
elif [ "$OPTARG" = "help" ]; then echo -e "$MSG_USAGE" >&2; exit
else echo "$MSG_BADOPT --$OPTARG" >&2; exit 1
fi
;;
"c" | "C" )
MYCONTAINER="$OPTARG"
;;
"d" | "D" )
MYDATABASE="$OPTARG"
;;
"f" | "F" )
MYDUMP="$OPTARG"
;;
"h" | "H" )
MYHOST="$OPTARG"
;;
"P" )
MYPASSWORD="$OPTARG"
;;
"p" )
MYPORT="$OPTARG"
;;
"u" | "U" )
MYUSER="$OPTARG"
;;
\? )
echo "$MSG_BADOPT -$OPTARG" >&2; exit 1
;;
esac
done; shift $((OPTIND -1))
# All options are processed.
# Checks the dependencies.
#
# Conditional dependencies (according to native or dockerized environment).
[[ -z "$MYCONTAINER" ]] \
&& additem="mysql mysqldump" \
|| additem="docker"
# Common dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in cat cut date df dirname grep gzip hostname id pwd sed tail tee $additem
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
#
# An additional bugfix (use "$(which gzip)" instead of "$GZIP"):
# https://www.gnu.org/software/gzip/manual/html_node/Environment.html
GZIP=""
# Need to be root or a Docker manager user if the DB runs in a container.
#
[[ -n "$MYCONTAINER" ]] && [[ "$USER" != 'root' ]] \
&& [[ -z "$(echo "$("$ID" -Gn "$USER") " | "$GREP" ' docker ')" ]] \
&& echo "$MSG_DOCKERGRPNEED" >&2 && exit 1 #"
# If the MySQL is dockerized the container must be running.
#
[[ -n "$MYCONTAINER" ]] \
&& [[ -z "$("$DOCKER" ps -q -f name=$MYCONTAINER)" ]] \
&& echo "$MSG_DOESNOTRUN $MYCONTAINER" >&2 && exit 1
# Determines the (mandatory) database to dump.
#
# Lack of -d the 1st non-option parameter is the database's name.
if [ -z "$MYDATABASE" -a -n "$1" ]; then MYDATABASE="$1"; shift; fi
if [ -z "$MYDATABASE" ]; then echo -e "$MSG_USAGE" >&2; exit 1; fi
# A humble sanitization.
if [[ ! "$MYDATABASE" =~ ^([[:alnum:]]|[_])*$ ]]; then
echo -e "$MSG_USAGE" >&2; exit 1; fi
# Silently refuses MySQL internal databases.
for veto in $vetodatabases ""
do
[[ "$MYDATABASE" = "$veto" ]] && exit 0
done
# We've a database name to dump.
# Optional backup file pathname, defaults to ./hostname.timestamp.MYDATABASE.sql
#
if [ -z "$MYDUMP" -a -n "$1" ]; then MYDUMP="$1"; shift; fi
if [ -d "$MYDUMP" ]; then
MYDUMP+="/$MYDATABASE.$("$DATE" '+%Y%m%d_%H%M%S').$("$HOSTNAME").sql"
fi
if [ -z "$MYDUMP" ]; then
MYDUMP="$PWD/$MYDATABASE.$("$DATE" '+%Y%m%d_%H%M%S').$("$HOSTNAME").sql"
fi
if [ "$MYDUMP" = "-" ]; then
# If '-' was given as the MYDUMP, we need output to STDOUT.
MYDUMP=""
logfile="/dev/null"
else
# Adds the relevant extension to the MYDUMP and the logfile.
MYDUMP="${MYDUMP%.sql}.sql"
logfile="${MYDUMP%.sql}.log"
fi
# The folder to contain the new files must be writable.
[[ -n "$MYDUMP" ]] && [[ ! -w "$("$DIRNAME" "$MYDUMP")" ]] \
&& echo "$MSG_NONWRITE \"$("$DIRNAME" "$MYDUMP")\"" >&2 && exit 1
# Prepopulates the MySQL commands.
#
my_connect=""
[[ -n "$MYHOST" ]] && my_connect+=" --host=$MYHOST"
[[ -n "$MYPORT" ]] && my_connect+=" --port=$MYPORT"
[[ -n "$MYUSER" ]] && my_connect+=" --user=$MYUSER"
[[ -n "$MYPASSWORD" ]] && my_connect+=" --password=$MYPASSWORD"
# Checks credentials and existence of the database given.
#
[[ -n "$MYCONTAINER" ]] \
&& databases=$("$DOCKER" exec $MYCONTAINER sh -c "mysql -N --batch $my_connect --execute='show databases;'" 2>/dev/null) \
|| databases=$("$MYSQL" -N --batch $my_connect --execute='show databases;' 2>/dev/null )
# Credentials?
[[ -z "$databases" ]] \
&& echo "$MSG_BADCRED ($MYUSER@$([[ -n "$MYCONTAINER" ]] && echo "$MYCONTAINER:")$MYHOST)." >&2 && exit 1
# Existence?
[[ ! "$databases" =~ (^|[[:space:]])"$MYDATABASE"($|[[:space:]]) ]] \
&& echo "$MSG_MISSINGDB \"$MYDATABASE\"." >&2 && exit 1
# We've the database connection and existence checked.
# Do we size the database?
#
dbsize=0
# It isn't relevant when we'll dump to the STDOUT.
if [ -n "$MYDUMP" ]; then
# Calculates the size of the database (KB).
SQLVERB='SELECT table_schema, '
SQLVERB+='ROUND(SUM(data_length + index_length) / 1024, 0) '
SQLVERB+="FROM information_schema.TABLES WHERE table_schema='$MYDATABASE' "
SQLVERB+="GROUP BY table_schema;"
if [ -n "$MYCONTAINER" ]; then
# Dockerized database.
dbsize=$("$DOCKER" exec $MYCONTAINER sh -c "echo \"$SQLVERB\" | mysql -N --batch $my_connect" 2>/dev/null | \
"$CUT" -d$'\t' -f2)
else
# Self-hosted database.
dbsize=$("$MYSQL" -N --batch $my_connect --execute="$SQLVERB" 2>/dev/null | \
"$CUT" -d$'\t' -f2)
fi
# Some sanitization
dbsize="${dbsize//[[:space:]]/}"
[[ -z "$dbsize" ]] && dbsize=0
[[ ! "$dbsize" =~ ^([[:digit:]])*$ ]] && dbsize=0
# On failure aborts here, except if it had forced.
if [ $dbsize -eq 0 ]; then echo -en "$MSG_FAILSIZE" | "$TEE" -a "$logfile"
if [ "$MYDUMPFORCED" ]; then
echo " - $MSG_FORCED" | "$TEE" -a "$logfile"
else
echo " - $MSG_ABORTED" | "$TEE" -a "$logfile"; exit 1
fi
fi
fi
# We've the database size.
# Checks the storage space available.
# Note, that we'll compare the size of the running database, not the dump!
# TODO: find a better estimate.
#
# It isn't relevant when we'll dump to the STDOUT or the database has no size.
if [ -n "$MYDUMP" -a $dbsize -gt 0 ]; then
# KB units
freespace=$("$DF" --output=avail -k "$("$DIRNAME" "$MYDUMP")" | $TAIL -n1) #"
# It is enough?
if [ $freespace -lt $dbsize ]; then echo -en "$MSG_NOSPACE" | "$TEE" -a "$logfile"
# On failure aborts here, except if it had forced.
if [ "$MYDUMPFORCED" ]; then
echo " - $MSG_FORCED" | "$TEE" -a "$logfile"
else
echo " - $MSG_ABORTED" | "$TEE" -a "$logfile"; exit 1
fi
fi
fi
# We've the space checked.
# Some cleanup.
#
[[ -n "$MYDUMP" && -f "$MYDUMP" ]] && rm "$MYDUMP" >/dev/null
[[ -n "$logfile" && -f "$logfile" ]] && rm "$logfile" >/dev/null
#
# Dumping.
#
if [ -n "$MYDUMP" ]; then
# Dumps into a file (then optionally compresses). Writes a separate log too.
# TODO: pipelined compress - doesn't work with Docker yet(?).
[[ -n "$MYCONTAINER" ]] \
&& "$DOCKER" exec $MYCONTAINER sh -c "mysqldump $my_connect $dumpparameters $MYOPTIONS $MYDATABASE" \
>"$MYDUMP" 2>>"$logfile" \
|| "$MYSQLDUMP" $my_connect $dumpparameters $MYOPTIONS $MYDATABASE \
>"$MYDUMP" 2>>"$logfile"
# Optional compression.
[[ -n "$compress" ]] && "$(which gzip)" "$MYDUMP" 2>/dev/null
else
# Dumps to STDOUT without logging.
[[ -n "$MYCONTAINER" ]] \
&& "$DOCKER" exec $MYCONTAINER sh -c "mysqldump $my_connect $dumpparameters $MYOPTIONS $MYDATABASE" \
2>/dev/null \
|| "$MYSQLDUMP" $my_connect $dumpparameters $MYOPTIONS $MYDATABASE \
2>>/dev/null
fi
# That's all, Folks! :)

526
.templates/bin/mysql_restoredb Executable file
View File

@ -0,0 +1,526 @@
#!/bin/bash
#
# Restores a MySQL/MariaDB database from a native or dockerized RDBMS instance
# accessible from this box. Also creates an owner user for this database
# (if it doesn't exist) and grants the appropriate privileges.
#
# Needs MySQL v5.7.6 or MariaDB 10.1.3 (or later).
# To restore a database with the necessary user management and grants,
# needs the superuser privileges on RDBMS.
# * If the RDBMS runs dockerized you need call this script as a Docker manager
# user (member of the docker Linux group).
# * If we're using a native MySQL/MariaDB, you need call this script as a
# Linux user whom the superuser role has been already granted within RDBMS
# (via unix_socket authentication) or you need provide the superuser's
# credentials as well.
# Lack of this the script will skip the user management and grant steps.
#
# Usage:
# $0 [-U dbuser] [-P dbpass] [-h dbhost] [-p dbport]
# [-A dbadminuser] [-a dbadminpass] [-c characterset]
# [-C container] [-d database] [-f dumpfile ]
# [database (if not in -d)] [dumpfile (if not in -f)]
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# 2022-04-07 v0.4
# new: An option and a guess mechanism has added to set the default character set
# of the restored database.
# 2021-08-30 v0.3
# fix: Uses the defaults when MYDBA* variables aren't set.
# 2021-03-22 v0.2
# fix: A duplicated SQLVERB has blocked setting password for a newly created user.
# The unnecessary PASSWORD() call has been removed as well.
# fix: Typos.
# 2021-02-18 v0.1 Initial release
# Accepted environment variables and their defaults.
#
MYCONTAINER=${MYCONTAINER-""} # Docker container's name
MYCHARSET=${MYCHARSET-""} # Default character set for DB
MYDATABASE=${MYDATABASE-""} # Database name to restore
MYDBAUSER=${MYDBAUSER:-""} # Database admin superuser
MYDBAPASSWORD=${MYDBAPASSWORD:-""} # Credential for the DBA user
MYDUMP=${MYDUMP-""} # Dump file pathname
MYHOST=${MYHOST:-"localhost"} # Connection parameter
MYOPTIONS=${MYOPTIONS-""} # Options to pass to pg_dump
MYPASSWORD=${MYPASSWORD-""} # Credential for the DB owner
MYPORT=${MYPORT:-"3306"} # Connection parameter
MYUSER=${MYUSER:-"root"} # Owner of the restored DB
### Temporailly ignored! Need to sanitize.
MYOPTIONS=""
# Basic environment settings.
#
LANG=C
LC_ALL=C
# We need also the sbin directories.
if ! [[ "$PATH" =~ '/sbin:' ]]; then
PATH="$PATH:/usr/local/sbin:/usr/sbin:/sbin"; fi
# Other initialisations.
#
LOGSTAMP="\"\$DATE\" +%Y-%m-%d\ %H:%M:%S" # Timestamp format for logs
MARIADBMIN="010001003" # MariaDB minimum version
MYSQLMIN="005007006" # MySQL minimum version
vetodatabases="information_schema mysql performance_schema sys"
# Messages.
#
MSG_AMBIGOUS="The character set used within the dump is ambigous."
MSG_BADDBTYPE="Unknown database type"
MSG_BADDUMP="Doesn't exist or doesn't a dumpfile:"
MSG_BADOPT="Invalid option"
MSG_BADPARAM="Doubtful parameter:"
MSG_BLOCKING="This is a fatal error - restore has been aborted."
MSG_CONNTERM="DB connection(s) have forced to terminate"
MSG_DOCKERGRPNEED="You must be a member of the docker group."
MSG_DOESNOTRUN="Doesn't run the database container"
MSG_EXISTING="did not create exisiting object"
MSG_FAILCONN="Failed to connect the RDBMS."
MSG_FAILGRANT="Failet to grant privileges to user"
MSG_FAILKILL="Failed to kill active connection"
MSG_FAILPASS="Failed to set password to user"
MSG_FAILTOKILL="Failed to retrieve the active connections."
MSG_FAILVER="Failed to get the RDBMS version."
MSG_FAILUSER="Failed to create RDBMS user"
MSG_MISSINGDEP="Fatal: missing dependency"
MSG_NONBLOCKING="Recoverable error - restore is continuing."
MSG_NONSUPER="DB user hasn't DBA (database superuser) privileges."
MSG_NONZERO="The result code is non zero"
MSG_OLDRDBMS="RDBMS version is too old"
MSG_PERCENT="Hint: you may use percent-encoding (e.g %40 instead of @)"
MSG_SUPERNEED="user must have DBA (database superuser) privileges."
MSG_USAGE="Usage: $0 [options] [database [dump_pathname]]\n"
MSG_USAGE+="Option:\tENVVAR:\n"
MSG_USAGE+=" -A\tMYDBAUSER \tMySQL/MariaDB DB admin superuser\n"
MSG_USAGE+=" -a\tMYDBAPASSWORD \tMySQL/MariaDB DB admin password\n"
MSG_USAGE+=" -c\tMYCHARSET \tMySQL/MariaDB DB character set\n"
MSG_USAGE+=" -C\tMYCONTAINER \tMySQL/MariaDB Docker container's name\n"
MSG_USAGE+=" -d\tMYDATABASE \tMySQL/MariaDB database to restore\n"
MSG_USAGE+=" -f\tMYDUMPFILE \tDumpfile pathname\n"
MSG_USAGE+=" -h\tMYHOST \tHostname or IP to connect (localhost)\n"
MSG_USAGE+=" -p\tMYPORT \tTCP port to connect (3306)\n"
MSG_USAGE+=" -P\tMYPASSWORD \tMySQL/MariaDB password\n"
MSG_USAGE+=" -U\tMYUSER \tMySQL/MariaDB username ($MYUSER)\n"
# Getting options.
#
while getopts ":-:a:A:c:C:d:D:f:h:H:p:P:u:U:" option
do
case ${option} in
"-" )
if [ "$OPTARG" = "help" ]; then echo -e "$MSG_USAGE" >&2; exit
else echo "$MSG_BADOPT --$OPTARG" >&2; exit 1
fi
;;
"A" ) MYDBAUSER="$OPTARG" ;;
"a" ) MYDBAPASSWORD="$OPTARG" ;;
"c" ) MYCHARSET="$OPTARG" ;;
"C" ) MYCONTAINER="$OPTARG" ;;
"d" | "D" ) MYDATABASE="$OPTARG" ;;
"f" ) MYDUMPFILE="$OPTARG" ;;
"h" | "H" ) MYHOST="$OPTARG" ;;
"P" ) MYPASSWORD="$OPTARG" ;;
"p" ) MYPORT="$OPTARG" ;;
"u" | "U" ) MYUSER="$OPTARG" ;;
\? ) echo "$MSG_BADOPT -$OPTARG" >&2; exit 1
;;
esac
done; shift $((OPTIND -1))
# All options have been processed.
# Checks the dependencies.
#
# Conditional dependencies (according to native or dockerized environment).
[[ -z "$MYCONTAINER" ]] \
&& additem="mysql" \
|| additem="docker"
# Common dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename cat cut date dirname file grep gunzip head id locale \
readlink printf sed sort tail tee wc $additem
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Sanitizing the parameters.
# Most of them are only arbitrary restrictions (reliable source: TODO!)
#
[[ -n "$MYDBAUSER" ]] && [[ ! "$MYDBAUSER" =~ ^([[:alnum:]]|[.-_\\+])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYDBAUSER\n$MSG_USAGE" >&2 && exit 1
#
[[ -n "$MYDBAPASSWORD" ]] && [[ ! "$MYDBAPASSWORD" =~ ^([[:alnum:]]|[ !~&#$<>()%+-_.])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYDBAPASSWORD\n$MSG_PERCENT\n$MSG_USAGE" >&2 && exit 1
#
[[ -n "$MYCONTAINER" ]] && [[ ! "$MYCONTAINER" =~ ^([[:alnum:]]|[-_])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYCONTAINER\n$MSG_USAGE" >&2 && exit 1
#
[[ -n "$MYDATABASE" ]] && [[ ! "$MYDATABASE" =~ ^([[:alnum:]]|[_])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYDATABASE\n$MSG_USAGE" >&2 && exit 1
#
[[ -n "$MYDUMPFILE" ]] && [[ ! "$MYDUMPFILE" =~ ^([[:alnum:]]|[ .-_/])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYDUMPFILE\n$MSG_USAGE" >&2 && exit 1
# https://tools.ietf.org/html/rfc1123#page-13 (relaxed)
[[ -z "$MYHOST" ]] && MYHOST="localhost"
[[ -n "$MYHOST" ]] && [[ ! "$MYHOST" =~ ^([[:alnum:]]|[.-])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYHOST\n$MSG_USAGE" >&2 && exit 1
# https://tools.ietf.org/html/rfc6056 (relaxed)
[[ -z "$MYPORT" ]] && MYPORT=3306
[[ -n "$MYPORT" ]] && [[ ! "$MYPORT" =~ ^[1-9]([[:digit:]]){0,4}$ ]] \
&& echo -e "$MSG_BADPARAM $MYPORT\n$MSG_USAGE" >&2 && exit 1
#
[[ -n "$MYPASSWORD" ]] && [[ ! "$MYPASSWORD" =~ ^([[:alnum:]]|[ !~&#$<>()%+-_.])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYPASSWORD\n$MSG_PERCENT\n$MSG_USAGE" >&2 && exit 1
#
[[ -n "$MYUSER" ]] && [[ ! "$MYUSER" =~ ^([[:alnum:]]|[.-_\\+])*$ ]] \
&& echo -e "$MSG_BADPARAM $MYUSER\n$MSG_USAGE" >&2 && exit 1
# We've at least a minimally checked parameters.
# Need to be root or a Docker manager user if the DB runs in a container.
#
[[ -n "$MYCONTAINER" ]] && [[ "$USER" != 'root' ]] \
&& [[ -z "$(echo "$("$ID" -Gn "$USER") " | "$GREP" ' docker ')" ]] \
&& echo "$MSG_DOCKERGRPNEED" >&2 && exit 1 #"
# If the RDBMS is dockerized the container must be running.
#
[[ -n "$MYCONTAINER" ]] \
&& [[ -z "$("$DOCKER" ps -q -f name=$MYCONTAINER)" ]] \
&& echo "$MSG_DOESNOTRUN $MYCONTAINER" >&2 && exit 1
# Determines the database to restore.
#
# Lack of -d the 1st non-option parameter is the database's name.
if [ -z "$MYDATABASE" -a -n "$1" ]; then MYDATABASE="$1"; shift; fi
# The database's name is mandatory.
if [ -z "$MYDATABASE" ]
then echo -e "$MSG_USAGE" >&2; exit 1; fi
# A humble sanitization.
if [[ ! "$MYDATABASE" =~ ^([[:alnum:]]|[_])*$ ]]
then echo -e "$MSG_USAGE" >&2; exit 1; fi
# Silently refuses the MySQL/MariaDB internal databases.
for veto in $vetodatabases ""
do
[[ "$MYDATABASE" = "$veto" ]] && exit 0
done
# We've a database name to restore.
# Determines the dumpfile.
#
# Lack of -f the 2nd non-option parameter is the database's name.
if [ -z "$MYDUMPFILE" -a -n "$1" ]; then MYDUMPFILE="$1"; shift; fi
# The dumpfile is mandatory.
if [ -z "$MYDUMPFILE" ]
then echo -e "$MSG_USAGE" >&2; exit 1; fi
# The MYDUMPFILE must point to a readable file.
# If it is an existing symlink dereferences it to ensure, it points to a file.
if [ -h "$MYDUMPFILE" ]; then
if [[ "$("$READLINK" "$MYDUMPFILE")" != /* ]]
# relative path in symlink
then MYDUMPFILE="$("$DIRNAME" "$MYDUMPFILE")/$("$READLINK" "$MYDUMPFILE")"
# absolute path in symlink
else MYDUMPFILE="$("$READLINK" "$MYDUMPFILE")"; fi
fi
# Let's check it!
if [ ! -r "$MYDUMPFILE" -o ! -f "$MYDUMPFILE" ]
then echo -e "$MSG_BADDUMP $MYDUMPFILE"; exit 1; fi
# We've an existing dumpfile.
# Tries to get the locale settings (actually CHARACTER SET) of this dump.
#
if [ -z "$MYCHARSET" ]; then
# Let's identify the file is gzipped or not.
UNPACKER=$("$FILE" --mime-type "$MYDUMPFILE")
UNPACKER=${UNPACKER##* } # The last word is the MIME-type.
# We'll use gunzip or cat (a dummy unzipper), according to the MIME type.
[[ "$UNPACKER" = 'application/gzip' ]] \
&& UNPACKER="$GUNZIP" \
|| UNPACKER="$CAT"
# Collects all character set adjustments from the dumpfile.
MYCHARSET=$("$CAT" "$MYDUMPFILE" | "$UNPACKER" | "$GREP" -B2 -i 'CREATE TABLE' | \
"$GREP" -i 'character_set_client =' | "$SORT" -u)
# Trims the character set's name itself (the first word after the equal sign).
[[ -n "$MYCHARSET" ]] && MYCHARSET=$(echo -e "$MYCHARSET" | "$SED" 's/^.*= \(.*\) .*$/\1/') #'
fi
# We've a raw guess about the character sets used.
# Finds the LOGFILE to use.
#
# If the folder containing the MYDUMPFILE is writable, we will use a
# logfile with the same name as the dumpfile but with .log extension.
[[ -w "$("$DIRNAME" "$MYDUMPFILE")" ]] \
&& LOGFILE="${MYDUMPFILE%.gz}" \
&& LOGFILE="${LOGFILE%.*}.log" \
|| LOGFILE="/dev/null"
# We've a suitable logfile.
# Opens the log and takes care to close it when finish.
#
echo "$(eval $LOGSTAMP) Starting job #$$ $("$TR" '\0' ' ' < /proc/$$/cmdline)" | \
"$TEE" -a "$LOGFILE"
# Sets a trap to make always a corresponding exit log entry as well.
function close_log() {
echo -e "$(eval $LOGSTAMP) Finished job #$$ $("$TR" '\0' ' ' < /proc/$$/cmdline)\n" | \
"$TEE" -a "$LOGFILE"
}
trap -- 'close_log' EXIT
# We started logging.
# Prepopulates the SQL command skeleton (macro).
#
# This skeleton makes the SQL calls independent to the environment
# (native or dockerized) and credentials. We need only actualize the
# CONNECT, DATABASE and SQLVERB clauses then eval $DO_SQLVERB.
# Warning: the parameters must had been sanitized!
DO_SQLVERB=""
DO_SQLVERB+="export MYSQL_PWD; "
DO_SQLVERB+="\"\$MYSQL\" \$CONNECT -N \$DATABASE "
DO_SQLVERB+="-e \"\$SQLVERB\""
# We've a suitable SQL macro.
# Do we connect the database as a DBA?
#
SQLVERB="SELECT 1;"
# Sets the default DBA username for dockerized and native RDBMS as well.
if [ -z "$MYDBAUSER" ]; then
[[ -n "$MYCONTAINER" ]] \
&& MYDBAUSER="root" \
|| MYDBAUSER="$USER"
fi
#
# We'll try the local connection (Unix-domain socket) first.
CONNECT=""
DATABASE=""
#result=$(eval "$DO_SQLVERB" 2>/dev/null); excode=$?
result="${result//[[:space:]]/}"
if [ "$result" != "1" ]; then
#
# On failure we'll try the TCP connection.
MYSQL_PWD="$MYDBAPASSWORD"
CONNECT="-u $MYDBAUSER -h $MYHOST -P $MYPORT"
result=$(eval "$DO_SQLVERB" 2>/dev/null); excode=$?
result="${result//[[:space:]]/}"
if [ "$result" != "1" ]; then
#
# On failure we'll try the TCP connection with non-DBA credentials.
MYSQL_PWD="$MYPASSWORD"
CONNECT="-u $MYUSER -h $MYHOST -P $MYPORT"
result=$(eval "$DO_SQLVERB" 2>/dev/null); excode=$?
result="${result//[[:space:]]/}"
if [ "$result" != "1" ]; then
#
# On failure we'll give up here.
[[ "$result" != "1" ]] \
&& echo -e "$MSG_FAILCONN" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
fi
fi
fi
# We've valid MYSQL_PWD and CONNECT clauses.
# Checks the superuser privilege.
# Better check: TODO!
ISDBA=false
DATABASE=""
SQLVERB="SHOW GRANTS;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
if [[ $excode -eq 0 && \
"$result" =~ ^GRANTALLPRIVILEGESON\*\.\*.*WITHGRANTOPTION$ ]]; then
ISDBA=true
else
echo -e "$MSG_NONSUPER" | "$TEE" -a "$LOGFILE" >&2
echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
fi
# We know we're a DB superuser or not.
# Following steps need the superuser privileges.
# Lack of this we'll skip them.
if $ISDBA; then
DATABASE="mysql"
# Checks the minimal MySQL/MariaDB version.
#
SQLVERB="SELECT version();"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
[[ -z "$result" ]] \
&& echo -e "$MSG_FAILVER" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# Let's extract a comparable RDBMS version from the result.
dbversion=${result%%-*} # strips anyone after 1st dash (including)
dbversion=(${dbversion//./ }) # converts to an array
dbversion=$("$PRINTF" '%03d%03d%03d' ${dbversion[@]}) # 3 times 3 digits 0-padded
if [ -n "$(echo "$result" | "$GREP" -i "mariadb")" ]; then
# MariaDB version check.
(( dbversion < MARIADBMIN )) \
&& echo -e "$MSG_OLDRDBMS: $result" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
else
# MySQL version check.
(( dbversion < MYSQLMIN )) \
&& echo -e "$MSG_OLDRDBMS: $result" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
fi
# RDBMS version is proper.
# Creates the database user (owner) if it doesn't exist.
#
echo -e "CREATE USER" | "$TEE" -a "$LOGFILE"
SQLVERB=" CREATE USER '$MYUSER'@'$MYHOST'; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
if [[ $excode -ne 0 ]]; then
# Already exists (or something went wrong).
echo -e "$MSG_FAILUSER $MYUSER@$MYHOST" | "$TEE" -a "$LOGFILE" >&2
echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
else
# Sets the password only if the user has just created.
echo -e "SET PASSWORD" | "$TEE" -a "$LOGFILE"
SQLVERB="SET PASSWORD FOR '$MYUSER'@'$MYHOST' = '$MYPASSWORD'; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_FAILPASS $MYUSER@$MYHOST" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
fi
#
# Grants all privileges on the database to the user.
#
echo -e "GRANT" | "$TEE" -a "$LOGFILE"
SQLVERB="GRANT ALL PRIVILEGES ON $MYDATABASE.* TO '$MYUSER'@'$MYHOST'; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_FAILGRANT $MYUSER@$MYHOST" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# We've the database user with the proper password.
# Drops all existing connections to the database being restored.
#
echo -e "KILL CONNECTIONS" | "$TEE" -a "$LOGFILE"
# List of the active connections.
SQLVERB="SELECT id FROM information_schema.processlist "
SQLVERB+="WHERE db = '$MYDATABASE';"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
if [[ $excode -ne 0 ]]; then
echo -e "$MSG_FAILTOKILL" | "$TEE" -a "$LOGFILE" >&2
echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
else
# Enumerates and tries to kill these connections.
for connection in $result ""
do
if [ -n "$connection" ]; then
SQLVERB="KILL $connection;"
eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2); excode=$?
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_FAILKILL $connection" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
fi
done
fi
# Connections have eliminated (we hope).
fi
# Done with the superuser part.
# Drops the database.
#
echo -e "DROP DATABASE" | "$TEE" -a "$LOGFILE"
DATABASE=""
SQLVERB="DROP DATABASE IF EXISTS $MYDATABASE;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# Recreates the database.
#
echo -e "CREATE DATABASE" | "$TEE" -a "$LOGFILE"
DATABASE=""
SQLVERB="CREATE DATABASE $MYDATABASE;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# We've an empty database.
# Sets the default character set.
#
if [ -n "$MYCHARSET" ]; then
echo -e "ALTER CHARACTER SET" | "$TEE" -a "$LOGFILE"
# If it is ambigous, we'll ignore it.
if [ "$(echo -e "$MYCHARSET" | "$WC" -l)" -ne 1 ]; then
echo -e "$MSG_AMBIGOUS" | "$TEE" -a "$LOGFILE" >&2
echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
MYCHARSET=""
else
# Let's set it.
DATABASE="$MYDATABASE"
SQLVERB="ALTER DATABASE $MYDATABASE CHARACTER SET $MYCHARSET;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
fi
fi
# We've the character set adjusted.
# Restores the database from the dump.
#
# This isn't so straightforward as in PostgreSQL.
# We'll use the database user's credentials, not the superuser's
# to mitigate the effect of an unsanitized dump.
echo -e "RESTORE" | "$TEE" -a "$LOGFILE"
# Let's identify the file is gzipped or not.
UNPACKER=$("$FILE" --mime-type "$MYDUMPFILE")
UNPACKER=${UNPACKER##* } # The last word is the MIME-type.
# We'll use gunzip or cat (a dummy unzipper), according to the MIME type.
[[ "$UNPACKER" = 'application/gzip' ]] \
&& UNPACKER="$GUNZIP" \
|| UNPACKER="$CAT"
# This is a sed expression to modify the security definers within the dump.
MOD_DEFINER="s/DEFINER=.*@[^ ]*/DEFINER=\`$MYUSER\`@\`$MYHOST\`/"
# Considers the RDBMS environment.
if [ -n "$MYCONTAINER" ]; then
# Dockerized RDBMS.
echo "MySQL dockerized - TODO!" | "$TEE" -a "$LOGFILE" >&2
else
# Native RDBMS.
# Reads the dump, on the fly unpacks it and modifies the scurity definer,
# then passes the data stream to the MySQL client.
"$CAT" "$MYDUMPFILE" | "$UNPACKER" | "$SED" "$MOD_DEFINER" | \
"$MYSQL" -u "$MYUSER" -p$MYPASSWORD -h "$MYHOST" -P "$MYPORT" \
-f -D "$MYDATABASE" \
>/dev/null 2> >("$TEE" -a "$LOGFILE" >&2); excode=$?
# Unfortunately the result code doesn't differentiate the
# blocking and non-blocking states.
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONZERO: $excode" | "$TEE" -a "$LOGFILE" >&2
fi
# We had a try to restore the database - the result isn't properly defined.
# Closing log entry will be handled via EXIT trap.
#
# That's all, Folks! :)

396
.templates/bin/psql_dumpdb Executable file
View File

@ -0,0 +1,396 @@
#!/bin/bash
#
# Dumps a PostgreSQL database from a native or dockerized PostgreSQL instance
# running on this box. This is a wrapper script to the pg_dump command.
# Uses only the TCP connection, therefore you must enable this in pg_hba file.
#
# If the PostgreSQL is dockerized you need call as a Docker manager user
# (member of the docker Linux group).
#
# Accepts few pg_dump options as well as the optional database password
# and the optional output pathname:
#
# $0 [-U dbuser] [-P dbpass] [-h dbhost] [-p dbport]
# [-C container] [-d database] [-f dumpfile ] [-F dumpformat ]
# [--acl ] [--force]
# [database (if not in -d)] [dumpfile (if not in -f)]
#
# A special -F9 option makes a PostgreSQL 9.x compatible plain text dump.
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# 2022-11-03 v0.4
# mod: May use the peer authentication as a secondary preference.
# fix: pg_restore output overwrited the previous log lines, this has been fixed.
# 2022-01-23 v0.3
# mod: More sophisticated estimate made on storage space needed.
# 2021-03-19 v0.2
# new: Option --acl (include ACLs as well) has been added.
# new: Option -F9 (PSQL 9.x compatible plain dump) has been added (native only).
# mod: Plain text dumps are compressed now.
# mod: Typos and comments.
# fix: The sbin directories has appended to the $PATH (Debian doesn't add them).
# 2020-09-17 v0.1 Initial release
# Accepted environment variables and their defaults.
#
PGCONTAINER=${PGCONTAINER-""} # Docker container's name
PGDATABASE=${PGDATABASE-""} # Database name to dump
PGDUMP=${PGDUMP-""} # Dump file pathname
PGDUMPFORMAT=${PGDUMPFORMAT-"c"} # Dump file format
PGHOST=${PGHOST:-"localhost"} # Connection parameter
PGOPTIONS=${PGOPTIONS-""} # Options to pass to pg_dump
PGPASSWORD=${PGPASSWORD-""} # Credential for the DB user
PGPORT=${PGPORT:-"5432"} # Connection parameter
PGUSER=${PGUSER:-"postgres"} # DB user for this dump
### Temporailly ignored! Need to sanitize.
PGOPTIONS=""
# Other initialisations.
#
PGDUMPACLS="--no-acl --no-owner" # Excludes the ACLs and grants.
PGDUMPFORCED="" # Dumps despite failed checks
vetodatabases="postgres template0 template1" # Technical DBs aren't to dump
# Messages.
#
MSG_ABORTED="aborted"
MSG_BADCRED="Bad credentials for MySQL"
MSG_BADDUMPPATH="Dumpfile's directory isn't writable"
MSG_BADOPT="Invalid option"
MSG_DOESNOTRUN="Doesn't run the database container"
MSG_DOCKERGRPNEED="You must be a member of the docker group."
MSG_FAILBKP="Archiver exited with error code"
MSG_FAILDB="Unable to dump the database"
MSG_FAILCONN="Failed to connect the database"
MSG_FAILSIZE="Failed to size the database"
MSG_FORCED="but forced to continue"
MSG_MISSINGDEP="Fatal: missing dependency"
MSG_NOCOMP="Fatal: missing component"
MSG_NODIRDUMP="Directory format doesn't implemented with a dockerized database."
MSG_NOSPACE="Not enough space to dump the database"
MSG_PEERAUTH="Peer authentication has been used."
MSG_USAGE="Usage: $0 [options] [database [dump_pathname|-]]\n"
MSG_USAGE+="Option:\tENVVAR:\n"
MSG_USAGE+=" -C\tPGCONTAINER\tPostgres Docker container's name\n"
MSG_USAGE+=" -d\tPGDATABASE\tPostgres database to dump ($USER)\n"
MSG_USAGE+=" -f\tPGDUMP\t\tDumpfile pathname\n"
MSG_USAGE+=" -F\tPGDUMPFORMAT\tDumpfile format ($PGDUMPFORMAT)\n"
MSG_USAGE+=" -h\tPGHOST\t\tHostname or IP to connect (localhost)\n"
MSG_USAGE+=" -p\tPGPORT\t\tTCP port to connect (5432)\n"
MSG_USAGE+=" -P\tPGPASSWORD\tPostgres password\n"
MSG_USAGE+=" -U\tPGUSER\t\tPostgres username ($PGUSER)\n"
MSG_USAGE+="--acl\t\t\tIncludes the grants and ACLs as well\n"
MSG_USAGE+="--force\t\t\tForces the operation despite the failed checks\n"
# Basic environment settings.
LANG=C
LC_ALL=C
# We need also the sbin directories.
if ! [[ "$PATH" =~ '/sbin:' ]]; then
PATH="$PATH:/usr/local/sbin:/usr/sbin:/sbin"; fi
# Getting options.
#
while getopts ":-:C:d:D:f:F:h:H:p:P:u:U:" option
do
case ${option} in
"-" )
if [ "$OPTARG" = "acl" ]; then PGDUMPACLS=""
elif [ "$OPTARG" = "force" ]; then PGDUMPFORCED="yes"
elif [ "$OPTARG" = "help" ]; then echo -e "$MSG_USAGE" >&2; exit
else echo "$MSG_BADOPT --$OPTARG" >&2; exit 1
fi
;;
"C" ) PGCONTAINER="$OPTARG" ;;
"d" | "D" ) PGDATABASE="$OPTARG" ;;
"f" ) PGDUMPFILE="$OPTARG" ;;
"F" ) PGDUMPFORMAT="$OPTARG" ;;
"h" | "H" ) PGHOST="$OPTARG" ;;
"P" ) PGPASSWORD="$OPTARG" ;;
"p" ) PGPORT="$OPTARG" ;;
"u" | "U" ) PGUSER="$OPTARG" ;;
\? )
echo "$MSG_BADOPT -$OPTARG" >&2; exit 1
;;
esac
done; shift $((OPTIND -1))
# All options have been processed.
# Checks the dependencies.
#
# Conditional dependencies (according to native or dockerized environment).
[[ -z "$PGCONTAINER" ]] \
&& additem="psql pg_dump" \
|| additem="docker"
# Common dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename df date dirname egrep grep gzip hostname id \
pwd sed tail tee $additem
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
#
# An additional bugfix (use "$(which gzip)" instead of "$GZIP"):
# https://www.gnu.org/software/gzip/manual/html_node/Environment.html
GZIP=""
# Need to be root or a Docker manager user if the DB runs in a container.
#
[[ -n "$PGCONTAINER" ]] && [[ "$USER" != 'root' ]] \
&& [[ -z "$(echo "$("$ID" -Gn "$USER") " | "$GREP" ' docker ')" ]] \
&& echo "$MSG_DOCKERGRPNEED" >&2 && exit 1 #"
# If the PostgreSQL is dockerized the container must be running.
#
[[ -n "$PGCONTAINER" ]] \
&& [[ -z "$("$DOCKER" ps -q -f name=$PGCONTAINER)" ]] \
&& echo "$MSG_DOESNOTRUN $PGCONTAINER" >&2 && exit 1
# Determines the database to dump.
#
# Lack of -d the 1st non-option parameter is the database's name.
if [ -z "$PGDATABASE" -a -n "$1" ]; then PGDATABASE="$1"; shift; fi
# The last resort is the Linux user's name.
if [ -z "$PGDATABASE" ]; then PGDATABASE="$USER"; fi
# A humble sanitization.
if [[ ! "$PGDATABASE" =~ ^([[:alnum:]]|[_])*$ ]]; then
echo -e "$MSG_USAGE" >&2; exit 1; fi
# Silently refuses the PostgreSQL internal databases.
for veto in $vetodatabases ""
do
[[ "$PGDATABASE" = "$veto" ]] && exit 0
done
# We've a database name to dump.
# Determines the output file (or maybe a target directory) and the logfile.
#
# A generated file- or directory name maybe necessary.
DUMPNAME="$PGDATABASE.$("$DATE" '+%Y%m%d_%H%M%S').$("$HOSTNAME")"
# Lack of -f the next non-option parameter is the dumpfile's pathname.
if [ -z "$PGDUMPFILE" -a -n "$1" ]; then PGDUMPFILE="$1"; shift; fi
# The last resort is the generated pathname.
if [ -z "$PGDUMPFILE" ]; then PGDUMPFILE="$("$PWD")/$DUMPNAME"; fi
#
# Let's make some checks.
#
# Dumping to the STDOUT is invalid with the directory format, we'll dump
# into a newly created directory instead.
[[ "$PGDUMPFILE" = "-" ]] && [[ "$PGDUMPFORMAT" = "d" ]] \
&& PGDUMPFILE="$("$PWD")/$DUMPNAME"
# If the given pathname is an existing directory, we need append
# a generated target filename (or directory name for Fd format).
[[ -d "$PGDUMPFILE" ]] && PGDUMPFILE+="/$DUMPNAME"
#
# Here we go with the desired pathname.
#
if [ "$PGDUMPFILE" = "-" ]; then
# If '-' was given as the PGDUMPFILE, we'll write to STDOUT w/o logs.
PGDUMPFILE=""
logfile="/dev/null"
else
# We'll write and log to a directory within the filesystem.
PGDUMPDIR="$("$DIRNAME" "$PGDUMPFILE")"
# This directory must be exist and be writable.
if [ -n "$PGDUMPDIR" ] && [ ! -d "$PGDUMPDIR" -o ! -x "$PGDUMPDIR" ]; then
echo "$MSG_BADDUMPPATH: $PGDUMPDIR" >&2; exit 1
fi
# Extends the output files properly.
if [ "$PGDUMPFORMAT" = "d" ]; then
logfile="$PGDUMPFILE.log"
elif [ "$PGDUMPFORMAT" = "9" -o "$PGDUMPFORMAT" = "p" ]; then
PGDUMPFILE="${PGDUMPFILE%.sql}.sql"
logfile="${PGDUMPFILE%.sql}.log"
elif [ "$PGDUMPFORMAT" = "t" ]; then
PGDUMPFILE="${PGDUMPFILE%.tar}.tar"
logfile="${PGDUMPFILE%.tar}.log"
else
PGDUMPFILE="${PGDUMPFILE%.dmp}.dmp"
logfile="${PGDUMPFILE%.dmp}.log"
fi
fi
# We've a suitable output and log pathname (or we've to use the STDOUT w/o logs).
# Do we connect the database?
#
if [ -n "$PGCONTAINER" ]; then
# Dockerized database.
result=$("$DOCKER" exec $PGCONTAINER \
sh -c "export PGPASSWORD=\"$PGPASSWORD\"; \
psql -U \"$PGUSER\" -w -h \"$PGHOST\" -p \"$PGPORT\" \
-d \"$PGDATABASE\" -t -c \"SELECT 1\"" 2>/dev/null)
result="${result//[[:space:]]/}"
[[ $result -ne 1 ]] && echo -e "$MSG_FAILCONN." >&2 | "$TEE" -a "$logfile" && exit 1
else
# Self-hosted database.
# Preferred method: TCP with username and password.
CONNECT="-U $PGUSER -w -h $PGHOST -p $PGPORT"
export PGPASSWORD
"$PSQL" $CONNECT -d "$PGDATABASE" -c "SELECT 1" >/dev/null 2>&1; result=$?
if [[ $result -ne 0 ]]; then
# On failure we will try the peer authentication.
CONNECT=""
"$PSQL" $CONNECT -d "$PGDATABASE" -c "SELECT 1" >/dev/null 2>&1; result=$?
[[ $result -ne 0 ]] && echo -e "$MSG_FAILCONN." >&2 | "$TEE" -a "$logfile" && exit $result
# Leaves a warning about using the peer authentication.
echo -e "$MSG_PEERAUTH" >>"$logfile"
fi
# We've a valid CONNECT clause.
fi
# We've the database connect checked.
# Do we size the database?
#
dbsize=0
# It isn't relevant when we'll dump to the STDOUT.
if [ -n "$PGDUMPFILE" ]; then
# Calculates the size of the database (MB).
if [ -n "$PGCONTAINER" ]; then
# Dockerized database.
dbsize=$("$DOCKER" exec $PGCONTAINER \
sh -c "export PGPASSWORD=\"$PGPASSWORD\"; \
psql -U \"$PGUSER\" -w -h \"$PGHOST\" -p \"$PGPORT\" -d \"$PGDATABASE\" \
-t -c \"SELECT pg_database_size('$PGDATABASE');\"" 2>/dev/null)
else
# Self-hosted database.
export PGPASSWORD
dbsize=$("$PSQL" $CONNECT -d "$PGDATABASE" \
-t -c "SELECT pg_database_size('$PGDATABASE');" 2>/dev/null)
fi
# Some sanitization
dbsize="${dbsize//[[:space:]]/}"
[[ -z "$dbsize" ]] && dbsize=0
[[ ! "$dbsize" =~ ^([[:digit:]])*$ ]] && dbsize=0
# KB units
dbsize=$(( dbsize /1024 ))
# On failure aborts here, except if it had forced.
if [ $dbsize -eq 0 ]; then echo -en "$MSG_FAILSIZE" | "$TEE" -a "$logfile"
if [ "$PGDUMPFORCED" ]; then
echo " - $MSG_FORCED" | "$TEE" -a "$logfile"
else
echo " - $MSG_ABORTED" | "$TEE" -a "$logfile"; exit 1
fi
fi
fi
# We've the database size.
# Checks the storage space available.
#
# It isn't relevant when we'll dump to the STDOUT or the database has no size.
if [ -n "$PGDUMPFILE" -a "$dbsize" -gt 0 ]; then
# Let's estimate the dump size.
dumpsize=$(( dbsize / 10 * 8 ))
# We'll estimate 1:4 ratio on compression on the fly.
[[ "$PGDUMPFORMAT" = "c" ]] && dumpsize=$(( dumpsize / 4 ))
# We'll estimate 5:4 ratio on native dump followed by the compression.
[[ "$PGDUMPFORMAT" = "9" ]] && dumpsize=$(( dumpsize / 4 * 5 ))
# Let's calculate the available space (KB units).
freespace=$("$DF" --output=avail -k "$("$DIRNAME" "$PGDUMPFILE")" | $TAIL -n1) #"
# Is it enough?
if [ $freespace -lt $dumpsize ]; then
echo -en "$MSG_NOSPACE" | "$TEE" -a "$logfile"
# On failure aborts here, except if it had forced.
if [ "$PGDUMPFORCED" ]; then
echo " - $MSG_FORCED" | "$TEE" -a "$logfile"
else
echo " - $MSG_ABORTED" | "$TEE" -a "$logfile"; exit 1
fi
fi
fi
# We've the space checked.
# Let's dump!
# Writes the database as it was ordered, into the dump file or to the STDOUT.
#
if [ -n "$PGCONTAINER" ]; then
# Dockerized database.
if [ -z "$PGDUMPFILE" ]; then
# STDOUT
"$DOCKER" exec $PGCONTAINER \
sh -c "export PGPASSWORD=\"$PGPASSWORD\"; \
pg_dump -U \"$PGUSER\" -w -h \"$PGHOST\" -p \"$PGPORT\" \
-F$PGDUMPFORMAT $PGDUMPACLS -d \"$PGDATABASE\""
elif [ "$PGDUMPFORMAT" = "d" ]; then
# Directory format doesn't implemented with a dockerized database.
echo "$MSG_NODIRDUMP" | "$TEE" -a "$logfile"; exit 1
else
# File
"$DOCKER" exec $PGCONTAINER \
sh -c "export PGPASSWORD=\"$PGPASSWORD\"; \
pg_dump -U \"$PGUSER\" -w -h \"$PGHOST\" -p \"$PGPORT\" \
-F$PGDUMPFORMAT $PGDUMPACLS -d \"$PGDATABASE\"" \
>"$PGDUMPFILE" 2>>"$logfile"
# If it is a plain dump, compresses it.
if [ "${PGDUMPFILE##*.}" = "sql" ]; then
"$(which gzip)" "$PGDUMPFILE"
fi
fi
# This is a fake result - TODO!
result=0
else
# Self-hosted database.
export PGPASSWORD
if [ "$PGDUMPFORMAT" = "9" ]; then
# Backward-compatible SQL dump.
if [ -z "$PGDUMPFILE" ]; then
# STDOUT
# 1st the schema with some arbitrary conversions.
"$PG_DUMP" $CONNECT \
$PGDUMPACLS --schema-only -d "$PGDATABASE" | \
"$EGREP" -iv '^SET idle_in_transaction_session_timeout =' | \
"$EGREP" -iv '^SET default_table_access_method =' | \
"$SED" 's/FUNCTION =/PROCEDURE = /' | \
"$SED" "s/CURRENT_DATE/\('now'::text\)::date/g"
# 2nd the data as COPY statements.
"$PG_DUMP" $CONNECT \
$PGDUMPACLS --data-only -d "$PGDATABASE"
else
# File
# 1st the schema with some arbitrary conversions.
"$PG_DUMP" $CONNECT \
$PGDUMPACLS --schema-only -d "$PGDATABASE" | \
"$EGREP" -iv '^SET idle_in_transaction_session_timeout =' | \
"$EGREP" -iv '^SET default_table_access_method =' | \
"$SED" 's/FUNCTION =/PROCEDURE = /' | \
"$SED" "s/CURRENT_DATE/\('now'::text\)::date/g" \
>"$PGDUMPFILE" 2>>"$logfile"; result=$?
# 2nd the data as COPY statements.
"$PG_DUMP" $CONNECT \
$PGDUMPACLS --data-only -d "$PGDATABASE" \
>>"$PGDUMPFILE" 2>>"$logfile"; result=$?
# Finally compresses it.
"$(which gzip)" "$PGDUMPFILE"
fi
elif [ -z "$PGDUMPFILE" ]; then
# STDOUT
"$PG_DUMP" $CONNECT \
-F"$PGDUMPFORMAT" $PGDUMPACLS -d "$PGDATABASE" \
2>>"$logfile"; result=$?
elif [ "$PGDUMPFORMAT" = "d" ]; then
# Directory
"$PG_DUMP" $CONNECT \
-F"$PGDUMPFORMAT" $PGDUMPACLS -d "$PGDATABASE" \
-f "$PGDUMPFILE" 2>>"$logfile"; result=$?
else
# File
"$PG_DUMP" $CONNECT \
-F"$PGDUMPFORMAT" $PGDUMPACLS -d "$PGDATABASE" \
>"$PGDUMPFILE" 2>>"$logfile"; result=$?
# If it is a plain dump, compresses it.
if [ "${PGDUMPFILE##*.}" = "sql" ]; then
"$(which gzip)" "$PGDUMPFILE"
fi
fi
fi
exit $result
# That's all, Folks! :)

701
.templates/bin/psql_restoredb Executable file
View File

@ -0,0 +1,701 @@
#!/bin/bash
#
# Restores a PostgreSQL database from a native or dockerized RDBMS instance
# accessible from this box. Also creates an owner user for this database
# (if it doesn't exist) and grants the appropriate privileges.
#
# Mainly, this is a pretty fatty wrapper script to the pg_restore command.
#
# Needs PostgreSQL v9.5 (or later).
# To restore a database with the necessary user management and grants,
# needs the superuser privileges on RDBMS.
# * If the RDBMS runs dockerized you need call this script as a Docker manager
# user (member of the docker Linux group).
# * If we're using a native PostgreSQL, you need call this script as a Linux
# user whom the superuser role has been already granted within PostgreSQL,
# or you need provide the superuser credentials as well. You must enable
# th desired connection in pg_hba file.
#
# Accepts few pg_restore options as well as the optional database password
# and database admin credentials with the optional output pathname:
#
# $0 [-U dbuser] [-P dbpass] [-h dbhost] [-p dbport]
# [-A dbadminuser] [-a dbadminpass] [-r dbreadonlyrole] [-R]
# [-C container] [-d database] [-f dumpfile ] [-F dumpformat]
# [database (if not in -d)] [dumpfile (if not in -f)]
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# 2022-12-01 v0.9
# fix: Coworker user's support.
# The reassign function has been totally rewritten. Now considers only
# that object what has been actually changed. Does nothing when the
# owner is already the designated role, so doesn't disturb the ERP's
# own functionality (e.g doesn't cause deadlocks).
# While now isn't strictly necessary, transferring the event triggers
# to the postgres user has remained, because of better portability.
# 2022-11-11 v0.8
# new: Coworker users' support.
# An event trigger automatism has been set to ensure, the owner of the
# objects to be created by coworkers is the database owner. This avoids
# permission problems when a foreign role (e.g a DBA or a coworker user)
# creates new objects.
# fix: Reassigning the restored objects failed, when an object have to
# owned by a DBA user (e.g an event trigger). Now we transfer
# these objects' ownership (actually the event trigger objects' only)
# to the postgres user before trying the global reassign.
# 2022-11-03 v0.7
# new: Automatic DB username and password generation if them had not
# specified. NOTE: if the user has just been created, the password appears
# in the log as a plain text - even if it was given (so it is known).
# new: Added a new parameter: -R to create a non-login R/O role, named same
# as DB username with a -ro suffix. The R/O privileges will be granted
# automatically to this role. This works same as -r parameter but
# you doesn't need to specify the role name.
# mod: Now prefers the TCP authentication over the peer authentication.
# mod: Improved notification messages.
# 2022-04-21 v0.6
# mod: R/O role doesn't created by default but on demand only.
# mod: Enhanced REASSIGN on the just loaded DB to avoid shared memory
# exhaustion.
# 2022-01-24 v0.5
# new: Creates a R/O role as well if it doesn't exist yet.
# 2021-10-25 v0.4
# fix: A typo has blocked receiving DB admin username from the environment.
# 2021-06-27 v0.3
# fix: Honors properly the given DBA credentials.
# fix: Handles given but empty parameters better.
# mod: Slash is allowed in hostname for Unix sockets.
# 2021-01-11 v0.2
# fix: The --no-acl option has been added to the pg_restore call.
# mod: Honors the DB locale settings (if any) included into dumpfile.
# mod: The minimum PostgreSQL version has been relaxed to 9.5.
# 2020-12-17 v0.1 Initial release
# Accepted environment variables and their defaults.
#
PGCONTAINER=${PGCONTAINER-""} # Docker container's name
PGDATABASE=${PGDATABASE-""} # Database name to restore
PGDBAUSER=${PGDBAUSER:-""} # Database admin superuser
PGDBAPASSWORD=${PGDBAPASSWORD:-""} # Credentials for the DBA user
PGDUMP=${PGDUMP-""} # Dump file pathname
PGDUMPFORMAT=${PGDUMPFORMAT:-"c"} # Dump file format
PGHOST=${PGHOST:-"localhost"} # Connection parameter
PGOPTIONS=${PGOPTIONS-""} # Options to pass to pg_dump
PGPASSWORD=${PGPASSWORD-""} # Credential for the DB owner
PGPORT=${PGPORT:-"5432"} # Connection parameter
PGROROLE=${PGROROLE:-""} # R/O role to the restored DB
PGUSER=${PGUSER:-""} # Owner of the restored DB
### Temporailly ignored! Need to sanitize.
PGOPTIONS=""
# Basic environment settings.
#
LANG=C
LC_ALL=C
# Other initialisations.
#
LOGSTAMP="\"\$DATE\" +%Y-%m-%d\ %H:%M:%S" # Timestamp format for logs
postgres='postgres' # Main DBA user
vetodatabases="postgres template0 template1" # Tech DBs aren't to restore
# Sets the flag: we need a R/O role as well.
[[ -n "$PGROROLE" ]] && PGRONEED='yes' || PGRONEED=''
# Messages.
#
MSG_BADDBTYPE="Unknown database type"
MSG_BADDUMP="Doesn't exist or doesn't a dumpfile:"
MSG_BADOPT="Invalid option"
MSG_BADPARAM="Doubtful parameter:"
MSG_BLOCKING="This is a fatal error - restore has been aborted."
MSG_CONNTERM="DB connection(s) have forced to terminate"
MSG_DOCKERGRPNEED="You must be a member of the docker group."
MSG_DOESNOTRUN="Doesn't run the database container"
MSG_EVTCHGFAIL="Failed to change ownership of the event trigger "
MSG_EXISTING="did not create exisiting object"
MSG_FAILCONN="Failed to connect the RDBMS."
MSG_MISSINGDEP="Fatal: missing dependency"
MSG_NEWPASS="a new password has been set:"
MSG_NEWROLE="a new role has been generated:"
MSG_NONBLOCKING="Recoverable error - restore is continuing."
MSG_OLDRDBMS="RDBMS version is too old"
MSG_PEERAUTH="Peer authentication has been used."
MSG_PERCENT="Hint: you may use percent-encoding (e.g %40 instead of @)"
MSG_SUPERNEED="user must have DBA (database superuser) privileges."
MSG_USAGE="Usage: $0 [options] [database [dump_pathname]]\n"
MSG_USAGE+="Option:\tENVVAR:\n"
MSG_USAGE+=" -A\tPGDBAUSER \tPostgres DB admin superuser\n"
MSG_USAGE+=" -a\tPGDBAPASSWORD \tPostgres DB admin password\n"
MSG_USAGE+=" -C\tPGCONTAINER \tPostgres Docker container's name\n"
MSG_USAGE+=" -d\tPGDATABASE \tPostgres database to restore\n"
MSG_USAGE+=" -f\tPGDUMPFILE \tDumpfile pathname\n"
#MSG_USAGE+=" -F\tPGDUMPFORMAT \tDumpfile format ($PGDUMPFORMAT)\n"
MSG_USAGE+=" -h\tPGHOST \tHostname or IP to connect (localhost)\n"
MSG_USAGE+=" -p\tPGPORT \tTCP port to connect (5432)\n"
MSG_USAGE+=" -P\tPGPASSWORD \tPostgres password\n"
MSG_USAGE+=" -r\tPGROROLE \tPostgres R/O rolename\n"
MSG_USAGE+=" -R\t \tPostgres R/O role (names it)\n"
MSG_USAGE+=" -U\tPGUSER \tPostgres username\n"
# Getting options.
#
while getopts ":-:a:A:C:d:D:f:F:h:H:p:P:r:Ru:U:" option
do
case ${option} in
"-" )
if [ "$OPTARG" = "help" ]; then echo -e "$MSG_USAGE" >&2; exit
else echo "$MSG_BADOPT --$OPTARG" >&2; exit 1
fi
;;
"A" ) PGDBAUSER="$OPTARG" ;;
"a" ) PGDBAPASSWORD="$OPTARG" ;;
"C" ) PGCONTAINER="$OPTARG" ;;
"d" | "D" ) PGDATABASE="$OPTARG" ;;
"f" ) PGDUMPFILE="$OPTARG" ;;
"F" ) PGDUMPFORMAT="$OPTARG" ;;
"h" | "H" ) PGHOST="$OPTARG" ;;
"P" ) PGPASSWORD="$OPTARG" ;;
"p" ) PGPORT="$OPTARG" ;;
"r" ) PGRONEED="yes"; PGROROLE="$OPTARG" ;;
"R" ) PGRONEED="yes" ;;
"u" | "U" ) PGUSER="$OPTARG" ;;
\? ) echo "$MSG_BADOPT -$OPTARG" >&2; exit 1
;;
esac
done; shift $((OPTIND -1))
# All options has been processed.
# Checks the dependencies.
#
# Conditional dependencies (according to native or dockerized environment).
[[ -z "$PGCONTAINER" ]] \
&& additem="psql pg_restore" \
|| additem="docker"
# Common dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename cat cut dd date dirname grep head id locale readlink sed \
tail tee $additem
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Sanitizing the parameters.
#
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html (modded)
[[ -n "$PGDBAUSER" ]] && [[ ! "$PGDBAUSER" =~ ^([[:alnum:]]|[.-_\\+])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGDBAUSER\n$MSG_USAGE" >&2 && exit 1
# This is only an arbitrary restriction, partially enforces percent-encoding.
[[ -n "$PGDBAPASSWORD" ]] && [[ ! "$PGDBAPASSWORD" =~ ^([[:alnum:]]|[ !~&#$<>()%+-_.])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGDBAPASSWORD\n$MSG_PERCENT\n$MSG_USAGE" >&2 && exit 1
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html (modded)
[[ -n "$PGCONTAINER" ]] && [[ ! "$PGCONTAINER" =~ ^([[:alnum:]]|[-_])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGCONTAINER\n$MSG_USAGE" >&2 && exit 1
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html (modded)
[[ -n "$PGDATABASE" ]] && [[ ! "$PGDATABASE" =~ ^([[:alnum:]]|[_])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGDATABASE\n$MSG_USAGE" >&2 && exit 1
# This is only an arbitrary restriction.
[[ -n "$PGDUMPFILE" ]] && [[ ! "$PGDUMPFILE" =~ ^([[:alnum:]]|[ .-_/])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGDUMPFILE\n$MSG_USAGE" >&2 && exit 1
# https://tools.ietf.org/html/rfc1123#page-13 (relaxed)
[[ -n "$PGHOST" ]] && [[ ! "$PGHOST" =~ ^([[:alnum:]]|[.-/])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGHOST\n$MSG_USAGE" >&2 && exit 1
# https://tools.ietf.org/html/rfc6056 (relaxed)
[[ -z "$PGPORT" ]] && PGPORT="5432"
[[ -n "$PGPORT" ]] && [[ ! "$PGPORT" =~ ^[1-9]([[:digit:]]){0,4}$ ]] \
&& echo -e "$MSG_BADPARAM $PGPORT\n$MSG_USAGE" >&2 && exit 1
# This is only an arbitrary restriction, partially enforces percent-encoding.
[[ -n "$PGPASSWORD" ]] && [[ ! "$PGPASSWORD" =~ ^([[:alnum:]]|[ !~&#$<>()%+-_.])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGPASSWORD\n$MSG_PERCENT\n$MSG_USAGE" >&2 && exit 1
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html (modded)
[[ -n "$PGROROLE" ]] && [[ ! "$PGROROLE" =~ ^([[:alnum:]]|[.-_\\+])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGROROLE\n$MSG_USAGE" >&2 && exit 1
# https://www.postgresql.org/docs/current/sql-syntax-lexical.html (modded)
[[ -n "$PGUSER" ]] && [[ ! "$PGUSER" =~ ^([[:alnum:]]|[.-_\\+])*$ ]] \
&& echo -e "$MSG_BADPARAM $PGUSER\n$MSG_USAGE" >&2 && exit 1
# We've at least a minimally checked parameters.
# Need to be root or a Docker manager user if the DB runs in a container.
#
[[ -n "$PGCONTAINER" ]] && [[ "$USER" != 'root' ]] \
&& [[ -z "$(echo "$("$ID" -Gn "$USER") " | "$GREP" ' docker ')" ]] \
&& echo "$MSG_DOCKERGRPNEED" >&2 && exit 1 #"
# If the PostgreSQL is dockerized the container must be running.
#
[[ -n "$PGCONTAINER" ]] \
&& [[ -z "$("$DOCKER" ps -q -f name=$PGCONTAINER)" ]] \
&& echo "$MSG_DOESNOTRUN $PGCONTAINER" >&2 && exit 1
# Determines the database to restore.
#
# Lack of -d the 1st non-option parameter is the database's name.
if [ -z "$PGDATABASE" -a -n "$1" ]; then PGDATABASE="$1"; shift; fi
# The database's name is mandatory.
if [ -z "$PGDATABASE" ]
then echo -e "$MSG_USAGE" >&2; exit 1; fi
# A humble sanitization.
if [[ ! "$PGDATABASE" =~ ^([[:alnum:]]|[_])*$ ]]
then echo -e "$MSG_USAGE" >&2; exit 1; fi
# Silently refuses the PostgreSQL internal databases.
for veto in $vetodatabases ""
do
[[ "$PGDATABASE" = "$veto" ]] && exit 0
done
# We've a database name to restore.
# Determines the database owner's username.
#
# If it isn't given we suppose the name is identical to the database
if [ -z "$PGUSER" ]; then
PGUSER="$PGDATABASE"
# We'll generate a random password for this user. This will be relevant only
# when it doesn't exist and we need to create it.
if [ -z "$PGPASSWORD" ]; then
PGPASSWORD=$("$DD" if=/dev/urandom bs=64 count=1 2>/dev/null | "$TR" -dc 'a-zA-Z0-9')
PGPASSWORD=${PGPASSWORD:0:16}
fi
fi
# We've the database owner's username.
# Determines the R/O database role's username.
#
# If it isn't given we use the owner's name appended with a "_ro" postfix.
[[ -z "$PGROROLE" ]] && PGROROLE="${PGUSER}_ro"
# We've the R/O role's name (but maybe we will not use it).
# Determines the dumpfile.
#
# Lack of -f the 2nd non-option parameter is the dumpfile's pathname.
if [ -z "$PGDUMPFILE" -a -n "$1" ]; then PGDUMPFILE="$1"; shift; fi
# The dumpfile is mandatory.
if [ -z "$PGDUMPFILE" ]
then echo -e "$MSG_USAGE" >&2; exit 1; fi
# The PGDUMPFILE must point to a readable file.
# If it is an existing symlink dereferences it to ensure, it points to a file.
if [ -h "$PGDUMPFILE" ]; then
if [[ "$("$READLINK" "$PGDUMPFILE")" != /* ]]
# relative path in symlink
then PGDUMPFILE="$("$DIRNAME" "$PGDUMPFILE")/$("$READLINK" "$PGDUMPFILE")"
# absolute path in symlink
else PGDUMPFILE="$("$READLINK" "$PGDUMPFILE")"; fi
fi
# Let's check it!
if [ ! -r "$PGDUMPFILE" -o ! -f "$PGDUMPFILE" ]
then echo -e "$MSG_BADDUMP $PGDUMPFILE"; exit 1; fi
# We've an existing dumpfile.
# Tries to get the locale settings of this dump.
#
PGDUMPPROPS=""
LOCALES=$("$LOCALE" -a | "$TR" [:upper:] [:lower:])
[[ -z "$PG_RESTORE" ]] && PG_RESTORE="$(which pg_restore)"
if [ -n "$PG_RESTORE" -a -x "$PG_RESTORE" ]; then
# Gets the CREATE DATABASE line and parses it.
createline=$("$PG_RESTORE" -C -s -f - "$PGDUMPFILE" | \
"$GREP" -i 'create database')
# Encoding (always).
property=$(echo "$createline" | \
"$SED" -n "s/^CREATE DATABASE .* ENCODING = \('*[[:alnum:]]*'*\).*$/\1/ip")
[[ -n "$property" ]] && \
PGDUMPPROPS+=" ENCODING = $property"
# Collation (only if it is available in current locales).
property=$(echo "$createline" | \
"$SED" -n "s/^CREATE DATABASE .* LC_COLLATE = \('*[[:alnum:]._-]*'*\).*$/\1/ip")
# Tricky because of slightly different locale naming in Linux and Postgres.
[[ -n "$property" ]] && \
[[ " $(echo $LOCALES) " =~ " $(echo "$property" | "$SED" 's/utf-8/utf8/i' | \
"$TR" -d \' | "$TR" [:upper:] [:lower:]) " ]] && \
PGDUMPPROPS+=" LC_COLLATE = $property"
# CType (only if it is available in current locales).
property=$(echo "$createline" | \
"$SED" -n "s/^CREATE DATABASE .* LC_CTYPE = \('*[[:alnum:]._-]*'*\).*$/\1/ip")
# Tricky because of slightly different locale naming in Linux and Postgres.
[[ -n "$property" ]] && \
[[ " $(echo $LOCALES) " =~ " $(echo "$property" | "$SED" 's/utf-8/utf8/i' | \
"$TR" -d \' | "$TR" [:upper:] [:lower:]) " ]] && \
PGDUMPPROPS+=" LC_CTYPE = $property"
fi
# Maybe we have a guess about the dump's encoding.
# Finds the LOGFILE to use.
#
# If the folder containing the PGDUMPFILE is writable, we will use a
# logfile with the same name as the dumpfile but with .log extension.
[[ -w "$("$DIRNAME" "$PGDUMPFILE")" ]] \
&& LOGFILE="${PGDUMPFILE%.*}.log" \
|| LOGFILE="/dev/null"
# We've a suitable logfile.
# Opens the log and takes care to close it when finish.
#
echo "$(eval $LOGSTAMP) Starting job #$$ $("$TR" '\0' ' ' < /proc/$$/cmdline)" | \
"$TEE" -a "$LOGFILE"
# Sets a trap to make always a corresponding exit log entry as well.
function close_log() {
echo -e "$(eval $LOGSTAMP) Finished job #$$ $("$TR" '\0' ' ' < /proc/$$/cmdline)\n" | \
"$TEE" -a "$LOGFILE"
}
trap -- 'close_log' EXIT
# We started logging.
# Prepopulates the SQL command skeleton (macro).
#
# This skeleton makes the SQL calls independent to the environment
# (native or dockerized) and credentials. We need only actualize the
# CONNECT, DATABASE and SQLVERB clauses then eval $DO_SQLVERB.
# Warning: the parameters must had been sanitized!
DO_SQLVERB=""
DO_SQLVERB+="export PGPASSWORD=\"\$PGDBAPASSWORD\"; "
DO_SQLVERB+="echo -e \"\$SQLVERB\" | "
DO_SQLVERB+="\"\$PSQL\" \$CONNECT -t -d \$DATABASE "
# We've a suitable SQL macro.
# Do we connect the database as a DBA?
#
DATABASE="postgres"
SQLVERB="SELECT 1;"
# Sets the default DBA username for dockerized and native RDBMS as well.
if [ -z "$PGDBAUSER" ]; then
[[ -n "$PGACONTAINER" ]] \
&& PGDBAUSER="postgres" \
|| PGDBAUSER="$USER"
fi
#
# We'll try the TCP connection first.
CONNECT="-U $PGDBAUSER -w -h $PGHOST -p $PGPORT"
result=$(eval "$DO_SQLVERB" 2>/dev/null)
result="${result//[[:space:]]/}"
if [ "$result" != "1" ]; then
# On failure we'll try the local connection (Unix-domain socket) as well.
CONNECT=""
result=$(eval "$DO_SQLVERB" 2>/dev/null); excode=$?
result="${result//[[:space:]]/}"
[[ "$result" != "1" ]] \
&& echo -e "$MSG_FAILCONN" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# Leaves a warning about using the peer authentication.
echo -e "$MSG_PEERAUTH" | "$TEE" -a "$LOGFILE" >&2
fi
# We've a valid CONNECT clause.
# Checks the superuser privilege.
#
DATABASE="postgres"
SQLVERB="SHOW is_superuser;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
[[ "$result" != "on" ]] \
&& echo -e "$PGDBAUSER $MSG_SUPERNEED" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# We're a DB superuser.
# Checks the PostgreSQL version - 9.5 or later needed.
#
DATABASE="postgres"
SQLVERB="SELECT current_setting('server_version_num')::INTEGER;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
(( result < 90500 )) \
&& echo -e "$MSG_OLDRDBMS: $result" | "$TEE" -a "$LOGFILE" >&2 \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# RDBMS version is proper.
# Creates the database user (owner) if it doesn't exist.
#
echo -e "CREATE ROLE" | "$TEE" -a "$LOGFILE"
DATABASE="postgres"
SQLVERB="
DO LANGUAGE plpgsql \$\$
BEGIN
IF NOT EXISTS (SELECT * FROM pg_user WHERE usename = '$PGUSER') THEN
CREATE ROLE $PGUSER WITH LOGIN ENCRYPTED PASSWORD '$PGPASSWORD';
RAISE NOTICE '$MSG_NEWROLE $PGUSER';
RAISE NOTICE '$MSG_NEWPASS $PGPASSWORD';
ELSE
RAISE NOTICE '$MSG_EXISTING $PGUSER';
END IF;
END;
\$\$;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# We've the database user.
# On demand creates a (non-login) R/O role as well if it doesn't exist.
#
if [ -n "$PGRONEED" ]; then
echo -e "CREATE ROLE (R/O)" | "$TEE" -a "$LOGFILE"
DATABASE="postgres"
SQLVERB="
DO LANGUAGE plpgsql \$\$
BEGIN
IF NOT EXISTS (SELECT * FROM pg_roles WHERE rolname = '$PGROROLE') THEN
CREATE ROLE $PGROROLE
NOSUPERUSER INHERIT NOCREATEDB NOCREATEROLE NOREPLICATION;
RAISE NOTICE '$MSG_NEWROLE $PGROROLE';
ELSE
RAISE NOTICE '$MSG_EXISTING $PGROROLE';
END IF;
END;
\$\$; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
# We don't consider the failure as blocking.
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# We hope that we've the R/O role defined.
fi
# Drops all existing connections to the database being restored,
# then (re)creates the database from scratch using template0.
#
echo -e "DROP DATABASE" | "$TEE" -a "$LOGFILE"
DATABASE="postgres"
SQLVERB="
DO LANGUAGE plpgsql \$\$
DECLARE conn_terminated SMALLINT;
BEGIN
SELECT COUNT(pg_terminate_backend(pid))
FROM pg_stat_activity
INTO conn_terminated
WHERE datname='$PGDATABASE';
IF conn_terminated > 0 THEN
RAISE NOTICE '% $MSG_CONNTERM', conn_terminated ;
END IF;
END
\$\$; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
#
# Drops the database.
#
DATABASE="postgres"
SQLVERB="DROP DATABASE IF EXISTS $PGDATABASE; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
#
# Recreates the database, honors the locale properties (if any).
#
echo -e "CREATE DATABASE" | "$TEE" -a "$LOGFILE"
DATABASE="postgres"
SQLVERB="CREATE DATABASE $PGDATABASE "
SQLVERB+="WITH TEMPLATE = template0 OWNER = $PGUSER $PGDUMPPROPS; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_BLOCKING" | "$TEE" -a "$LOGFILE" >&2 \
&& exit 1
# We've an empty database.
# Grants all privileges on this database, and transfers the public
# schema's ownership to the database user.
#
echo -e "GRANT" | "$TEE" -a "$LOGFILE"
DATABASE="$PGDATABASE"
SQLVERB="GRANT ALL PRIVILEGES ON DATABASE $PGDATABASE TO $PGUSER; "
SQLVERB+="ALTER SCHEMA public OWNER TO $PGUSER; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# Ownership and grants have set on the empty database.
# Restores the database from the dump.
#
echo -e "RESTORE" | "$TEE" -a "$LOGFILE"
if [ -n "$PGCONTAINER" ]; then
echo "PSQL dockerized - TODO!"
else
export PGPASSWORD="$PGDBAPASSWORD"
"$PG_RESTORE" $CONNECT \
--no-owner --no-acl --disable-triggers \
-d $PGDATABASE "$PGDUMPFILE" \
>/dev/null 2> >("$TEE" -a "$LOGFILE" >&2)
fi
# Unfortunately the result code doesn't differentiate the
# blocking and non-blocking states.
[[ $? -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# We had a try to restore the database - the result isn't properly defined.
# Tries to transfer the ownership of the restored objects to the database user,
# but we've some exceptions to deal first.
echo -e "REASSIGN EVENT TRIGGERS" | "$TEE" -a "$LOGFILE"
DATABASE="$PGDATABASE"
# The event triggers (if any) have to owned by a DBA user.
# We try to transfer the ownership of this trigger to the postgres user,
# whom by default we will not use for daily operations.
SQLVERB="
DO LANGUAGE plpgsql \$\$
DECLARE evtrg text;
BEGIN
FOR evtrg IN EXECUTE 'SELECT evtname FROM pg_event_trigger'
LOOP
BEGIN
EXECUTE 'ALTER EVENT TRIGGER ' || evtrg || ' OWNER TO $postgres';
-- EXCEPTION
-- WHEN others THEN RAISE NOTICE '$MSG_EVTCHGFAIL %', evt;
END;
END LOOP;
END;
\$\$;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# Exceptions have dealt, we're moving to the regular objects.
#
# To avoid exhausting the shared memory or running out from
# max_locks_per_transaction first we'll enumerate all the tables and change
# its owner one by one.
echo -e "REASSIGN" | "$TEE" -a "$LOGFILE"
# Creates separate ALTER commands for each table (in all non-technical schemas).
SQLVERB="SELECT 'ALTER TABLE '|| schemaname || '.' || tablename || ' OWNER TO $PGUSER;' "
SQLVERB+="FROM pg_tables WHERE NOT schemaname IN ('pg_catalog', 'information_schema') "
SQLVERB+="ORDER BY schemaname, tablename; "
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
if [ $excode -ne 0 ]; then
# On error simply skips this step.
echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
else
# Runs the generated ALTER commands.
SQLVERB="$result"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
fi
#
# Reassigns all remaining objects at once.
SQLVERB="REASSIGN OWNED BY CURRENT_USER TO $PGUSER;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# Ownership and R/W privileges has been set.
# Sets a function and a trigger to ensure, the objects to be created in the future
# will be owned by the database owner. This avoids permission problems, when
# a foreign role (e.g a DBA) modifies any objects.
# Based on https://stackoverflow.com/questions/64046147
#
echo -e "SET EVENT TRIGGER" | "$TEE" -a "$LOGFILE"
DATABASE="$PGDATABASE"
# The automatic reassign procedure for a future use.
SQLVERB="
CREATE OR REPLACE FUNCTION public.trg_set_owner()
RETURNS event_trigger
LANGUAGE plpgsql
AS \$\$
DECLARE
event_tuple record;
schema varchar;
name varchar;
kind varchar;
owner varchar;
BEGIN
-- Enumerates the queries involved in this event.
FOR event_tuple IN SELECT * FROM pg_event_trigger_ddl_commands()
LOOP
-- Retrieves the relevant properties of the object.
SELECT relnamespace::regnamespace::name,
relname::text,
relkind,
relowner::regrole
INTO schema, name, kind, owner
FROM pg_catalog.pg_class
WHERE oid = event_tuple.objid;
IF NOT owner = '$PGUSER' THEN
-- Reassigns this object.
CASE
WHEN kind = 'r' THEN
EXECUTE 'ALTER TABLE IF EXISTS ' || schema || '.' || name || ' OWNER TO $PGUSER';
WHEN kind = 'v' OR kind = 'm' THEN
EXECUTE 'ALTER VIEW IF EXISTS ' || schema || '.' || name || ' OWNER TO $PGUSER';
END CASE;
END IF;
END LOOP;
END;
\$\$; "
# The event trigger whitch will call the procedure above.
SQLVERB+="
DROP EVENT TRIGGER IF EXISTS trg_set_owner;
CREATE EVENT TRIGGER trg_set_owner
ON ddl_command_end
WHEN tag IN ('CREATE TABLE', 'CREATE TABLE AS', 'CREATE VIEW')
EXECUTE PROCEDURE public.trg_set_owner(); "
# Transfers the ownership of this trigger to the postgres DBA user,
# whom by default we will not use for daily operations.
SQLVERB+="
ALTER EVENT TRIGGER trg_set_owner OWNER TO $postgres; "
# Let's execute (failure doesn't blocks).
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# Done with the reassign automatism.
# Grants SELECT on all tables and sequences in all schemas to the R/O role (if it does exist).
# https://dba.stackexchange.com/questions/95867/grant-usage-on-all-schemas-in-a-database
#
if [ -n "$PGRONEED" ]; then
echo -e "GRANT SELECT (R/O)" | "$TEE" -a "$LOGFILE"
DATABASE="$PGDATABASE"
SQLVERB="GRANT CONNECT ON DATABASE $PGDATABASE TO ${PGROROLE}; "
SQLVERB+="
DO LANGUAGE plpgsql \$do\$
DECLARE
sch text;
BEGIN
FOR sch IN SELECT nspname FROM pg_namespace where nspname != 'pg_toast'
and nspname != 'pg_temp_1' and nspname != 'pg_toast_temp_1'
and nspname != 'pg_statistic' and nspname != 'pg_catalog'
and nspname != 'information_schema'
LOOP
EXECUTE format(\$\$ GRANT USAGE ON SCHEMA %I TO ${PGROROLE} \$\$, sch);
EXECUTE format(\$\$ GRANT SELECT ON ALL TABLES IN SCHEMA %I TO ${PGROROLE} \$\$, sch);
EXECUTE format(\$\$ GRANT SELECT ON ALL SEQUENCES IN SCHEMA %I TO ${PGROROLE} \$\$, sch);
EXECUTE format(\$\$ ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT SELECT ON TABLES TO ${PGROROLE} \$\$, sch);
EXECUTE format(\$\$ ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT SELECT ON SEQUENCES TO ${PGROROLE} \$\$, sch);
END LOOP;
END;
\$do\$;"
result=$(eval "$DO_SQLVERB" 2> >("$TEE" -a "$LOGFILE" >&2)); excode=$?
result="${result//[[:space:]]/}"
[[ $excode -ne 0 ]] \
&& echo -e "$MSG_NONBLOCKING" | "$TEE" -a "$LOGFILE" >&2
# R/O grants has been set.
fi
# Done with grants.
# Done with restore.
# Closing log entry will be handled via EXIT trap.
#
# That's all, Folks! :)

431
.templates/bin/rotate_folder Executable file
View File

@ -0,0 +1,431 @@
#!/bin/bash
#
# Helper script to rotate contents of a folder with a daily-weekly-monthly plan.
# TL;DR: rotate_folder -f path/to/my/folder # Shows what would be done.
# rotate_folder -f path/to/my/folder --doit # Makes the job.
#
# By default the script makes a dry run - doesn't delete anything, only lists
# the operations would be done. You may force the execution by --doit command
# line parameter. Another optional command line parameter is -f followed by
# the pathname of the folder intended to be rotated. Lack of it the script
# assumes the current folder (pwd).
#
# You may configure the script by environment variables and/or by a
# configuration textfile. This file should be placed into the folder intended
# to be rotated. It's name should be a dot followed by the script's name and a
# .conf extension (.rotate_folder.conf by default). The script will create a
# default config file automatically on first (dry) run, if it doesn't exist.
#
# The configurable parameters and their defaults are:
# BACKUP_FOLDER="" # pathname of the folder intended to be rotated
# CLASSES_PATTERN="" # see below
# DOIT="" # if empty the script makes a dry run
# RETAIN_DAYS=7 # retains all files created within that many days
# RETAIN_WEEKS=4 # retains one file per week/month,
# RETAIN_MONTHS=12 # created within that many weeks/months
#
# If you specify a CLASSES_PATTERN the script will classify the files in folder
# and rotate the files class by class independently. A pattern is a regexp:
# * the script considers only the filenames matching the whole regexp;
# * the regexp must contain parts in capturing parentheses (classifiers).
# A class is a set of filenames where the matching part to the all classifiers
# are the same. For example, if CLASSES_PATTERN='^(.*)-[0-9].tgz'
# then "alpha-1.tgz alpha-2.tgz ... alpha-9.tgz" are members of a class;
# "beta-1.tgz beta-2.tgz ... beta-9.tgz" are members of another class.
# "beta-10.tgz gamma-1.log" won't be processed beacuse they don't match
# the pattern at all.
# In this example the "alpha" and "beta" files will be rotated independently.
#
# The rotating rules are:
# * all files created within RETAIN_DAYS will be retained.
# * furthermore from files created within RETAIN_WEEKS, only one file
# (the oldest) will be retained for every 7 days period.
# * furthermore from files created within RETAIN_MONTHS, only one file
# (the oldest) will be retained for every 30 days period.
#
# On dry run the script lists all the files of the class with following
# abbreviations:
# DR filename - would be retained by daily rule
# WR filename - would be retained by weekly rule
# WX filename - would be deleted by weekly rule
# MR filename - would be retained by monthly rule
# MX filename - would be deleted by monthly rule
# AX filename - would be deleted no rule match it, because is too old
#
# Author: Kovács Zoltán <kovacs.zoltan@smartfront.hu>
# Kovács Zoltán <kovacsz@marcusconsulting.hu>
# License: GNU/GPL v3+ (https://www.gnu.org/licenses/gpl-3.0.en.html)
# 2023-06-18 v1.0
# new: forked from the "SMARTERP_skeleton" repository.
# 2021.02.12 v0.3
# add: Multiple classes (mainly rewritten).
# mod: Accepts the first command line parameter as a folder (doesn't
# need the -f option). But the folder doesn't defaults to the $PWD.
# 2020-11-24 v0.2
# fix: Typos.
# mod: Warnings also go to the STDERR.
# 2020-11-02 v0.1 Initial release
# Accepted environment variables and their defaults.
#
BACKUP_FOLDER=${BACKUP_FOLDER-""}
CLASSES_PATTERN=${CLASSES_PATTERN-""}
RETAIN_DAYS=${RETAIN_DAYS-"7"}
RETAIN_WEEKS=${RETAIN_WEEKS-"4"}
RETAIN_MONTHS=${RETAIN_MONTHS-"12"}
# Other initialisations (maybe overridden by configuration).
#
DOIT=""
# Messages (maybe overriden by configuration).
#
MSG_BADFOLDER="Doesn't exist or doesn't writable"
MSG_BADOPT="Invalid option"
MSG_BADPATTERN="The pattern given seems to be illegal"
MSG_CREATED="A new, empty configuration has been created.\n"
MSG_CREATED+="Feel free to fill in and rerun this program!\n"
MSG_CREATED+="You may force the execution unconfigurated with --doit option."
MSG_DELDRY="Dry run - these files would have been deleted:"
MSG_DELREAL="These files have been deleted:"
MSG_FAILCREATE="Failed to create a new, empty configuration file.\n"
MSG_FAILCREATE+="You may force the execution unconfigurated with --doit option."
MSG_MISSINGDEP="Fatal: missing dependency"
MSG_NOCONF="Didn't find the configuration file"
MSG_NOCLASSES="Didn't find suitable classes according to pattern"
MSG_NOFILES="Didn't found files to rotate."
MSG_SCHEDULE="Dry run - this is the schedule:"
MSG_TODOIT="Dry run - you may force the execution with --doit option."
# There is nothing to configure below (I hope).
###############################################
# Getting command line options.
while getopts ":-:f:" option
do
case ${option} in
"-" )
if [ "$OPTARG" = "doit" ]; then DOIT="yes"
else echo "$MSG_BADOPT --$OPTARG" >&2; exit 1
fi
;;
"f" ) BACKUP_FOLDER="$OPTARG" ;;
\? ) echo "$MSG_BADOPT -$OPTARG" >&2; exit 1 ;;
esac
done
# Done with options.
# Checks the dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in basename date dirname egrep sed seq sort stat xargs
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# All dependencies are available via "$THECOMMAND" (upper case) call.
# Checks the backup folder.
# If wasn't defined yet accepts the 1st command line parameter as well.
if [ -z "$BACKUP_FOLDER" ]; then BACKUP_FOLDER="$1"; shift; fi
# Removes the trailing slash (if any).
BACKUP_FOLDER=${BACKUP_FOLDER%/}
# Checks and gives up here if fails.
if [ -z "$BACKUP_FOLDER" -o ! -d "$BACKUP_FOLDER" -o ! -w "$BACKUP_FOLDER" ]
then echo -e "$MSG_BADFOLDER $BACKUP_FOLDER" >&2; exit 1; fi
# Gets the configuration (if any).
BACKUP_CONF="$BACKUP_FOLDER/.$("$BASENAME" "$0").conf"
if [ -r $BACKUP_CONF ]; then . "$BACKUP_CONF"
else
# Warns about failure.
echo -e "$MSG_NOCONF $BACKUP_CONF"
# When on dry run tries to write a new file with some help text and defaults.
if [ -z "$DOIT" -a -z "$CLASSES_PATTERN" ]; then
cat > "$BACKUP_CONF" 2>/dev/null << EOF
# This is a shell script excerpt for configuration purposes only.
# Handle with care! Please don't put code here, only variables.
# The configurable parameters for $("$BASENAME" "$0") script and their defaults are:
# CLASSES_PATTERN="" # see below
# DOIT="" # if empty the script makes a dry run
# RETAIN_DAYS=7 # retains all files created within that many days
# RETAIN_WEEKS=4 # retains one file per week/month,
# RETAIN_MONTHS=12 # created within that many weeks/months
# If you specify a CLASSES_PATTERN the script will classify the files in folder
# and rotates the files class by class independently. A pattern is a regexp:
# * the script considers only the filenames matching the whole regexp;
# * the regexp must contain parts in capturing parentheses (classifiers).
# A class is a set of filenames where the matching part to the all classifiers
# is the same. For example, if CLASSES_PATTERN='^(.*)-[0-9].tgz'
# then "alpha-1.tgz alpha-2.tgz ... alpha-9.tgz" are members of a class;
# "beta-1.tgz beta-2.tgz ... beta-9.tgz" are members of another class.
# "beta-10.tgz gamma-1.log" won't be processed beacuse they don't match
# the pattern at all.
# In this example the "alpha" and "beta" files will be rotated independently.
#
# The rotating rules are:
# * all files have created within RETAIN_DAYS will be retained.
# * furthermore from files created within RETAIN_WEEKS, only one file
# (the oldest) will be retained for every 7 days period.
# * furthermore from files created within RETAIN_MONTHS, only one file
# (the oldest) will be retained for every 30 days period.
#
# On dry run the script lists all the files of the class with following
# abbreviations:
# DR filename - would be retained by daily rule
# WR filename - would be retained by weekly rule
# WX filename - would be deleted by weekly rule
# MR filename - would be retained by monthly rule
# MX filename - would be deleted by monthly rule
# AX filename - would be deleted no rule match it, because is too old
EOF
# Reports the success or failure and stops here.
if [ -r "$BACKUP_CONF" ];
then echo -e "$MSG_CREATED" >&2; exit
else echo -e "$MSG_FAILCREATE" >&2; exit 1; fi
fi
fi
# Configuration file has been handled.
# Initialisations which are protected from configuration.
(( SECS_DAY = 60*60*24 ))
(( SECS_WEEK = 7*SECS_DAY ))
(( SECS_MONTH = 30*SECS_DAY ))
TIMESTAMP=$("$DATE" '+%s')
# This function rotates the files matching to its parameter
# which is a definite regexp (without parenthesised parts).
function rotate_class {
local CLASSES_PATTERN="$1"; shift
local files
# Selection of files to rotate.
#
# We consider only the files matching to the pattern.
# If the pattern is empty, we'll consider all files.
if [ -z "$CLASSES_PATTERN" ]; then
# All non-hidden files but no subfolders, symlinks, etc.
files=$(cd "$BACKUP_FOLDER"; \
ls -1 -t --file-type | "$XARGS" -0 | "$EGREP" -v '[/=>@|$]$' )
else
# Non-hidden files (but no subfolders, symlinks, etc.) matching to the pattern.
files=$(cd "$BACKUP_FOLDER"; \
ls -1 -t --file-type | "$XARGS" -0 | "$EGREP" "$CLASSES_PATTERN" )
fi
# Lack of files gives it up here.
[[ -z "$files" ]] && return
# Converts the list into an array.
local class_files=($files)
# We need to process the files listed within the class_files array.
# The list is ordered by modification time, reverse.
# We'll start with the youngest and step toward the oldest.
# Collectcs the list of files to delete within this class.
#
local delete_files="" # list of filenames to delete
local pointer=0 # class_files index to process
local file_mtime
local file_toretain
local threshold
# Starts with the daily schedule.
# We'll retain all files within this schedule.
[[ -z "$DOIT" ]] && echo -e "$MSG_SCHEDULE"
local last_retained=""
for day in $("$SEQ" 1 "$RETAIN_DAYS")
do
# Finishes if we've no more files.
[[ $pointer -ge ${#class_files[@]} ]] && break
(( threshold = TIMESTAMP - (day * SECS_DAY) ))
file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
# We'll retain all files of this day.
while [[ $file_mtime -ge $threshold ]]
do
[[ -z "$DOIT" ]] && echo "DR ${class_files[$pointer]}"
last_retained="$file_mtime"
# Next file; finishes if we're no more files.
(( pointer++ )); [[ $pointer -ge ${#class_files[@]} ]] && break
file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
done
# This day concluded.
done
# The daily schedule concluded.
# If we didn't save any file within this schedule we'll retain this file.
if [[ -z "$last_retained" && $pointer -lt ${#class_files[@]} ]]; then
last_retained="$file_mtime"
[[ -z "$DOIT" ]] && echo "DR ${class_files[$pointer]}"
(( pointer++ ))
[[ $pointer -lt ${#class_files[@]} ]] \
&& file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
fi
# The weekly schedule.
# We'll retain only the oldest file from a week within this schedule.
last_retained=""
for week in $("$SEQ" 1 "$RETAIN_WEEKS")
do
file_toretain=""
# Finishes if we've no more files.
[[ $pointer -ge ${#class_files[@]} ]] && break
(( threshold = TIMESTAMP - (week * SECS_WEEK) ))
file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
while [[ $file_mtime -ge $threshold ]]
do
if [ -z "$file_toretain" ]; then
# This is the first file from this week.
# marks it to retain temporailly.
file_toretain="${class_files[$pointer]}"
else
# This is an older file from this week than the previous.
# Changes the marker, the previous file should be deleted.
delete_files+="$file_toretain\n"
[[ -z "$DOIT" ]] && echo "WX $file_toretain"
file_toretain="${class_files[$pointer]}"
fi
# Next file; finishes if we're no more files.
(( pointer++ )); [[ $pointer -ge ${#class_files[@]} ]] && break
file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
done
# The marked file from the week passed has been retained.
if [ -n "$file_toretain" ]; then
last_retained=$file_mtime # a cheat but it isn't important here
[[ -z "$DOIT" ]] && echo "WR $file_toretain"
fi
# This week concluded.
done
# The weekly schedule concluded.
# If we didn't save any file within this schedule we'll retain this file.
if [[ -z "$last_retained" && $pointer -lt ${#class_files[@]} ]]; then
last_retained="$file_mtime"
[[ -z "$DOIT" ]] && echo "WR ${class_files[$pointer]}"
(( pointer++ ))
[[ $pointer -lt ${#class_files[@]} ]] \
&& file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
fi
# The monthly schedule.
# We'll retain only the oldest file from a month within this schedule.
last_retained=""
for month in $("$SEQ" 1 "$RETAIN_MONTHS")
do
file_toretain=""
# Finishes if we've no more files.
[[ $pointer -ge ${#class_files[@]} ]] && break
(( threshold = TIMESTAMP - (month * SECS_MONTH) ))
file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
while [[ $file_mtime -ge $threshold ]]
do
if [ -z "$file_toretain" ]; then
# This is the first file from this month.
# marks it to retain temporailly.
file_toretain="${class_files[$pointer]}"
else
# This is an older file from this month than the previous.
# Changes the marker, the previous file should be deleted.
delete_files+="$file_toretain\n"
[[ -z "$DOIT" ]] && echo "MX $file_toretain"
file_toretain="${class_files[$pointer]}"
fi
# Next file; finishes if we're no more files.
(( pointer++ )); [[ $pointer -ge ${#class_files[@]} ]] && break
file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
done
# The marked file from the month passed has been retained.
if [ -n "$file_toretain" ]; then
last_retained=$file_mtime # a cheat but it isn't important here
[[ -z "$DOIT" ]] && echo "MR $file_toretain"
fi
# This month concluded.
done
# The monthly schedule concluded.
# If we didn't save any file within this schedule we'll retain this file.
if [[ -z "$last_retained" && $pointer -lt ${#class_files[@]} ]]; then
last_retained="$file_mtime"
[[ -z "$DOIT" ]] && echo "MR ${class_files[$pointer]}"
(( pointer++ ))
[[ $pointer -lt ${#class_files[@]} ]] \
&& file_mtime=$("$STAT" -c %Y "$BACKUP_FOLDER/${class_files[$pointer]}")
fi
# All the schedules have been processed.
# The remaining files will be deleted all.
while [[ $pointer -lt ${#class_files[@]} ]]
do
delete_files+="${class_files[$pointer]}\n"
[[ -z "$DOIT" ]] && echo "AX ${class_files[$pointer]}"
(( pointer ++ ))
done
# The delete_files contain the list of iles to delete according this class.
if [ -n "$delete_files" ]; then
if [ -z "$DOIT" ]; then
# Simulated deletion.
echo -e "\n$MSG_DELDRY\n$delete_files"
else
# Actual deletion file by file.
for file in $(echo -e "$delete_files")
do [[ -n "$file" ]] && rm "$BACKUP_FOLDER/$file" #2>/dev/null
done
echo -e "\n$MSG_DELREAL\n$delete_files"
fi
else
# Uniform output formatting.
[[ -z "$DOIT" ]] && echo
fi
}
# This function parses the given class pattern, recursively explores
# the classes, subclasses, sub-subclasses and so on, then calls the
# rotator function for each definite class.
function rotate_classes {
local CLASSES_PATTERN="$1"; shift
[[ -z "$CLASSES_PATTERN" ]] && return # unusable
# Tries to validate the pattern.
# Test calls simulate the later use.
if [ -n "$CLASSES_PATTERN" ]; then
echo "test" | "$EGREP" "$CLASSES_PATTERN" >/dev/null 2>&1
[[ $? -gt 1 ]] && return # unusable
fi
# Does contain unexplored classifiers?
echo "test" | "$SED" -E "s/$CLASSES_PATTERN/\1/" >/dev/null 2>&1
if [[ $? -gt 0 ]]; then
# It is a definite classifier, let's call the rotator function.
rotate_class "$CLASSES_PATTERN"
else
# Needs further exploring.
# Non-hidden files (but no subfolders, symlinks, etc.) matching to the pattern.
local files=$(cd "$BACKUP_FOLDER"; \
ls -1 -t --file-type | "$XARGS" -0 | "$EGREP" "$CLASSES_PATTERN" )
# Selects the qualifier substrings which actually have matching files.
local classes=$(echo -e "$files" | "$SED" -E "s/$CLASSES_PATTERN/\1/" | "$SORT" -u)
# Enumerates these qualifiers.
for class in $classes
do
# This is same as the CLASSES_PATTERN but contains the definite qualifier instead of
# the parenthesised expression - e.g one of tgz and log instad of (tgz|log)
local class_pattern=$(echo -e "$CLASSES_PATTERN" | "$SED" -E "s/\([^)]*\)/$class/") #"
# Recurses for further exploring.
rotate_classes "$class_pattern"
done
fi
}
# Rotates the classes, subclasses and so on with a recursive function call.
if [ -z "$CLASSES_PATTERN" ]; then
# All files considered within the same class.
rotate_class
else
# Tries to validate the pattern (loosely).
echo "test" | "$EGREP" "$CLASSES_PATTERN" >/dev/null 2>&1
[[ $? -gt 1 ]] && echo -e "$MSG_BADPATTERN $CLASSES_PATTERN" >&2 && exit 1
# Seems to be valid, go on!
rotate_classes "$CLASSES_PATTERN"
fi
# A final thought about the dry run.
[[ -z "$DOIT" ]] && echo -e "$MSG_TODOIT"
# That's all, Folks :).

1
.templates/bin/shutdown Symbolic link
View File

@ -0,0 +1 @@
.launcher

1
.templates/bin/startup Symbolic link
View File

@ -0,0 +1 @@
.launcher

81
.templates/bin/watchtower_redeploy Executable file
View File

@ -0,0 +1,81 @@
#!/bin/bash
#
# https://containrrr.dev/watchtower/
# https://hub.docker.com/r/containrrr/watchtower
# https://github.com/containrrr/watchtower
#
# Email notification settings below assume the gateway acting
# as a smarthost and Docker version v20.10+ is required.
image="containrrr/watchtower:latest"
instance="watchtower"
networks=""
outfile=""
volume=""
MSG_MISSINGDEP="Fatal: missing dependency"
# Checks the dependencies.
TR=$(which tr 2>/dev/null)
if [ -z "$TR" ]; then echo "$MSG_MISSINGDEP tr."; exit 1 ; fi
for item in cat dirname docker hostname
do
if [ -n "$(which $item)" ]
then export $(echo $item | "$TR" '[:lower:]' '[:upper:]')=$(which $item)
else echo "$MSG_MISSINGDEP $item." >&2; exit 1; fi
done
# Stops and removes the container (if necessary).
if [ -n "$("$DOCKER" ps -q -f name=$instance)" ]
then "$DOCKER" stop "$instance"; fi
if [ -n "$("$DOCKER" ps -a -q -f name=$instance)" ]
then "$DOCKER" rm "$instance"; fi
# Checks for an upgrade.
$DOCKER pull "$image"
# Creates the container.
$DOCKER create \
-e TZ=$("$CAT" "/etc/timezone") \
-e WATCHTOWER_CLEANUP=true \
-e WATCHTOWER_DEBUG=false \
-e WATCHTOWER_INCLUDE_STOPPED=true \
-e WATCHTOWER_LABEL_ENABLE=true \
-e WATCHTOWER_MONITOR_ONLY=true \
-e WATCHTOWER_REVIVE_STOPPED=false \
-e WATCHTOWER_NO_PULL=false \
-e WATCHTOWER_SCHEDULE="0 0 1 * * *" \
-e WATCHTOWER_WARN_ON_HEAD_FAILURE="never" \
-e WATCHTOWER_NOTIFICATIONS=email \
-e WATCHTOWER_NOTIFICATION_EMAIL_FROM="$USER@$(hostname -f)" \
-e WATCHTOWER_NOTIFICATION_EMAIL_TO="$USER@$(hostname)" \
-e WATCHTOWER_NOTIFICATION_EMAIL_SERVER="host.docker.internal" \
-e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_PORT=25 \
-e WATCHTOWER_NOTIFICATION_EMAIL_SERVER_TLS_SKIP_VERIFY=true \
-e WATCHTOWER_NOTIFICATION_EMAIL_DELAY=15 \
-e WATCHTOWER_NOTIFICATION_EMAIL_SUBJECTTAG="[Watchtower $("$HOSTNAME")]" \
-v /var/run/docker.sock:/var/run/docker.sock \
--add-host="host.docker.internal:host-gateway" \
--restart unless-stopped \
--label "com.centurylinklabs.watchtower.enable=true" \
--name $instance $image
# Connects it to the network(s).
if [ -n "$networks" ]; then
for network in $networks
do
# Checks the network, creates it if necessary.
if [ -z "$("$DOCKER" network ls -q -f name=$network)" ]
then "$DOCKER" network create -d bridge "$network"; fi
# Then connects.
$DOCKER network connect $network $instance
done
fi
# Finally launches it.
$DOCKER start $instance
if [ -n "$outfile" -a -w "$("$DIRNAME" "$outfile")" ]; then
# Sets a background process to collect the image's output.
# This process will automatically terminate when the image stops.
"$DOCKER" logs -f $instance >>"$outfile" 2>&1 &
fi

3
.templates/crontab Normal file
View File

@ -0,0 +1,3 @@
@reboot CRON=1 USER=$LOGNAME SLEEP_BETWEEN=60 $HOME/bin/maintenance_reboot
01 00 * * * CRON=1 USER=$LOGNAME SLEEP_BETWEEN=5 $HOME/bin/maintenance_midnight
00 04 * * * CRON=1 USER=$LOGNAME SLEEP_BETWEEN=120 $HOME/bin/maintenance_daily

5
.templates/nginx/.nginx/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
# Ignore everything else in this directory.
*
!.gitignore
!_default.conf
!_general.conf

14
.templates/nginx/.nginx/_default.conf Normal file
View File

@ -0,0 +1,14 @@
# A restrictive default virtualhost configuration.
server {
listen 80 default_server;
server_name _;
root /var/www/html;
index index.html index.htm index.nginx-debian.html;
location / {
default_type text/html;
return 404 "<span style='font-size: large'>$hostname</span>";
}
}

14
.templates/nginx/.nginx/_general.conf Normal file
View File

@ -0,0 +1,14 @@
# NGINX general configuration for Docker services.
# Doesn't explore unnecessary info.
server_tokens off;
# Depends on ngx_http_security_headers_module.so
#hide_server_tokens on;
# https://amalgjose.com/2020/05/15/how-to-set-the-allowed-url-length-for-a-nginx-request-error-code-414-uri-too-large/
large_client_header_buffers 4 32k;
# https://serverfault.com/questions/602201/nginx-possible-bug-with-dashes-in-server-name
# https://trac.nginx.org/nginx/ticket/571
#server_names_hash_bucket_size 64;

2
.templates/nginx/conf.d/services.conf Normal file
View File

@ -0,0 +1,2 @@
# Docker services
include $PAR_SERVICEBASE/.nginx/*.conf;

98
.templates/nginx/nginx.conf Normal file
View File

@ -0,0 +1,98 @@
# NGINX configuration for a virtualhost proxied to a Docker service.
# Uses https://github.com/acmesh-official/acme.sh to manage SSL certificates.
# Flags the non 2xx or non 3xx (probably error) responses.
map $status $errorlog { ~^[23] 0; default 1; }
# Virtualhost's configuration follows.
server {
listen 80;
# listen 443 ssl;
server_name_in_redirect on;
server_name $PAR_SERVERNAME;
set $server_admin webmaster@$server_name;
# access log and error log.
# Any requests getting a non 2xx or non 3xx response will go to the error log as well.
access_log $PAR_SERVICE/logs/web/access.log combined;
access_log $PAR_SERVICE/logs/web/error.log combined if=$errorlog;
# Let's Encrypt (acme.sh) support.
location /.well-known/ {
proxy_pass http://$PAR_ACMEHOST:$PAR_ACMEPORT;
error_page 500 502 503 504 @proxy_error;
}
# Forced redirect to https.
# if ($scheme = http) {
# return 301 https://$host$request_uri;
# }
# Webapp's configuration.
charset utf-8;
location /$PAR_LOCATION {
proxy_pass http://$PAR_PROXYHOST:$PAR_PROXYPORT/$PAR_LOCATION;
error_page 500 502 503 504 @proxy_error;
client_max_body_size 1G;
keepalive_timeout 30;
proxy_read_timeout 300;
proxy_request_buffering on;
proxy_buffers 2048 16k;
proxy_buffer_size 16k;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
#websockets
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
}
location @proxy_error {
default_type text/html;
return 500
"<span style='font-size: x-large'>Sorry something went wrong. Try again a bit later.<br>
You may report this at <a href='mailto:$server_admin'>$server_admin</a>.</span>";
}
# No static service.
# location / {
# default_type text/html;
# return 404 "<span style='font-size: x-large'>Sorry try <a href='$scheme://$server_name/$PAR_LOCATION'>$scheme://$server_name/$PAR_LOCATION</a> instead.</span>";
# }
##################################################################################
# The SSL part
# https://ssl-config.mozilla.org/
# https://community.letsencrypt.org/t/howto-a-with-all-100-s-on-ssl-labs-test-using-nginx-mainline-stable/55033
# ssl_certificate $PAR_SERVICE/configs/acme/$PAR_SERVERNAME/fullchain.cer;
# ssl_certificate_key $PAR_SERVICE/configs/acme/$PAR_SERVERNAME/$PAR_SERVERNAME.key;
# Settings to achieve 'A' grade on https://www.ssllabs.com/ssltest/
ssl_session_timeout 1440m;
ssl_session_tickets off;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_prefer_server_ciphers off;
ssl_ciphers "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-SHA";
ssl_stapling on;
ssl_stapling_verify on;
# Read before activating: https://blog.g3rt.nl/nginx-add_header-pitfall.html
# add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
# add_header X-Frame-Options SAMEORIGIN;
# add_header X-Content-Type-Options nosniff;
}
# That's all.

3
.templates/sudoers.d/user Normal file
View File

@ -0,0 +1,3 @@
# Permits to reload the webserver.
$PAR_USER ALL=(ALL) NOPASSWD: /usr/sbin/apachectl configtest, /usr/bin/systemctl reload apache2
$PAR_USER ALL=(ALL) NOPASSWD: /usr/sbin/nginx -t, /usr/bin/systemctl reload nginx