Carousel-Backup mit rsync

Vor längerem habe ich ein Backup-Script geschrieben mit dem ich Webseiten über rsync sichern und wiederherstellen kann.

Die hier publizierten Scripts sind ohne jegliches Gewähr.

Aufruf des Scripts

# Backup
backup.sh /pfad/zu/credentials.txt

# Restore
restore.sh /pfad/zu/credentials.txt /pfad/zu/backup/daily/2023.11.01

credentials.txt

name=domain.ch
email=vorname@domain.ch
server=domain.ch

ssh_user=username
ssh_privkey=/home/username/.ssh/bak_id_rsa
ssh_port=22

rsync_source=/home/domain/public_html/
basedir=/home/username/backup/domain.ch

mysql_server=localhost
mysql_user=database_username
mysql_pw='database_pw'
mysql_db=database_name

keep_backup_hours=48
keep_backup_days=14
keep_backup_weeks=8
keep_backup_months=12

Parameter

• name – Name der Webseite die gesichert werden soll
• email – E-Mailadresse an welche Fehler gesendet werden sollen
• server – in der Regel die Domain der Webseite oder eine IP des Servers.
• ssh_user – Benutzername mit dem man sich per SSH auf dem Server der Webseite einloggen kann
• ssh_privkey – Pfad zum SSH Private Key
• ssh_port – (optional) – Default: 22 – SSH-Port
• rsync_source – Pfad auf dem Webserver. Wichtig, muss einen Slash am Ende enthalten, damit der direkte Inhalt des angegebenen Ordners synchronisiert wird.
• basedir – Ordner auf dem lokalen Backup-Server in dem das Backup abgelegt wird
• mysql_server – (optional) – Host des MySQL/MariaDB Servers
• mysql_user – (optional) – Datenbank Benutzer
• mysql_pw – (optional) – Passwort des Datenbank Benutzers
• mysql_db – (optional) – Datenbank
• mysql_port – (optional) – Default: 3306 – Port über den die Datenbank angesprochen wird
• mysql_direct – (optional) – Default=false – bei true wird die Datenbank direkt verbunden und nicht über die SSH Verbindung
• keep_backup_hours – (optional) – Default 48 – Anzahl Stunden in denen jedes Backup aufbewart wird
• keep_backup_days – (optional) – Default: 14 – Anzahl Tage in denen das letzte Backup des Tages aufbewahrt wird
• keep_backup_weeks – (optional) – Default: 8 – Anzahl Wochen an denen das sonntägliche Backup aufbewahrt wird – sollte es kein Backup am Sonntag geben, so wird das nächste danach erstellte Backup aufbewahrt
• keep_backup_months – (optional) – Default:12 – Anzahl Monate an denen das Backup am ersten Tag des Monats aufbewahrt wird – sollte es kein Backup am am ersten Tag des Monats geben, so wird das nächste danach erstellte Backup aufbewahrt

Aufbau Datenstruktur

├── domain.ch
│   ├── credentials.txt
│   ├── backup.log
│   ├── current
│   │   ├── backup.log
│   │   ├── database
│   │   │   └── database.sql.gz
│   │   ├── files
│   │   │   ├── Datei 1
│   │   │   ├── Datei 2
│   │   │   └── etc...
│   ├── daily
│   │   ├── 2011.01.01
│   │   │   ├── backup.log
│   │   │   ├── database
│   │   │   │   └── database.sql.gz
│   │   │   ├── files
│   │   │   │   ├── Datei 1
│   │   │   │   ├── Datei 2
│   │   │   │   └── etc...
│   │   ├── 2011.01.02
│   │   │   ├── ....
│   ├── hourly
│   │   ├── 2023.11.01_22.00.20
│   │   │   ├── ...
│   │   ├── 2023.11.01_23.00.10
│   ├── monthly
│   │   ├── 2023.10.01
│   │   │   ├── ...
│   │   ├── 2023.11.01
│   ├── weekly
│   │   ├── 2023.10.29
│   │   │   ├── ...
│   │   ├── 2023.11.05

backup.sh

#!/bin/bash

# ------------------------
# Backup Script 
# Variablen:
#   name
#   email
#   server
#   ssh_user
#   ssh_privkey
#   ssh_port
#   rsync_source
#   rsync_dest
#   mysql_server
#   mysql_user
#   mysql_pw
#   mysql_db
#   mysql_port
#   mysql_direct=false
# ------------------------

# load

name=''
email='info@blattertech.ch'
server=''
ssh_user=''
ssh_privkey=''
ssh_port=''
rsync_source=''
mysql_server=localhost
mysql_user=''
mysql_pw=''
mysql_db=''
mysql_port=''
mysql_direct=false
basedir=''



if [ $# -eq 0 ]; then
    echo "No configuration file defined"
    exit 1
fi

if [ ! -f $1 ]; then
    echo "Configuration file $1 does not exist"
    exit 1
fi



# load configuration
source $1

# check if basedir exist
if [ ! -d ${basedir} ]; then
    echo "Basedir ${basedir} does not exist"
    exit 1
fi

# Check if private ssh key exist
if [ ! -f ${ssh_privkey} ]; then
    echo "SSH private key ${ssh_privkey} does not exist"
    exit 1
fi

lockfile=${basedir}/backup.lock

# Check is Lock File exists, if not create it and set trap on exit
if { set -C; 2>/dev/null >${lockfile}; }; then
	trap "rm -f ${lockfile}" EXIT
else
	echo "Lock file exists… exiting"
	exit
fi


# set directorys
current=$basedir/current
hourly=$basedir/hourly
daily=$basedir/daily
weekly=$basedir/weekly
monthly=$basedir/monthly


# set default values
keep_backup_hours=${keep_backup_hours:-'72'}
keep_backup_days=${keep_backup_days:-'14'}
keep_backup_weeks=${keep_backup_weeks:-'8'}
keep_backup_months=${keep_backup_months:-'15'}

ssh_port=${ssh_port:-'22'}
mysql_port=${mysql_port:-'3306'}
mysql_server=${mysql_server:-$server}

rsync_dest=$current/files
mysql_dest=$current/database
mysql_dest_file=$mysql_dest/database.sql.gz

logfile=$current/backup.log
logfile_basic=$basedir/backup.log


# delete existing logfile
if [ -f "$logfile" ] ; then
    rm "$logfile"
fi

log () {
    if [ -z "$1" ]; then
        cat
    else
        printf '%s\n' "$@" 
    fi | tee -a "$logfile" "$logfile_basic"
}

senderror () {
    tail -1000 "$logfile" | mail -s "Backup $name failed" $email
}


# create folders if not exist
mkdir -p $rsync_dest
mkdir -p $hourly
mkdir -p $daily
mkdir -p $weekly
mkdir -p $monthly 


# sync files

log "-------------------------- start backup $name  --------------------------"
log ' '
log "Date: $(date)"
log ' '
log 'Start rsync'


rsync -e "ssh -p $ssh_port -i $ssh_privkey" --links --recursive --times --out-format="%n" --human-readable --partial --delete --compress --stats $ssh_user@$server:$rsync_source $rsync_dest 2> >(tee -a $logfile $logfile_basic >&2) | log

if [ $? -eq 0 ]; then
	log 'Rsync finished'
else
	log 'An error has occurred during rsync'
        log '================= ERROR ================='
	senderror
	exit 1
fi

log ' '

# backup remote database
if [ $mysql_db != '' ]; then
	log 'Start database backup'

	mkdir -p $mysql_dest
	rm -rv $mysql_dest/* | log
	# Forward Port dynamisch nach Minute und Sekunde definieren, damit parallel laufende Scripts nicht den gleichen Port nutzen -> liesse sich mit Milisekunden allenfalls noch verbessern
	# on Mac and BSD: jot -r 1  2000 65000
	# other way: seq 2000 65000 | sort -R | head -n 1
	# https://stackoverflow.com/questions/2556190/random-number-from-a-range-in-a-bash-script

	if [ "$mysql_direct" = true ]; then
		mysqldump -h $mysql_server -P ${mysql_port} --max_allowed_packet=512M --no-tablespaces --extended-insert --compress --quote-names --single-transaction --routines --triggers --opt -u $mysql_user --password="$mysql_pw" $mysql_db  2> >(tee -a $logfile $logfile_basic >&2) | gzip -c > $mysql_dest_file
 	else
		forward_port=$(shuf -i 2000-65000 -n 1)

		#ssh -p $ssh_port -i $ssh_privkey $ssh_user@$server "mysqldump -h '$mysql_server' --extended-insert --compress --quote-names --single-transaction --routines --triggers --opt -u $mysql_user --password='$mysql_pw' $mysql_db || exit 42;"  | gzip -c > $mysql_dest_file
		ssh -C -f -L${forward_port}:$mysql_server:$mysql_port -p $ssh_port -i $ssh_privkey $ssh_user@$server sleep 5
		# weitere Optionen: --events
		mysqldump -h 127.0.0.1 -P ${forward_port} --max_allowed_packet=512M --no-tablespaces --extended-insert --compress --quote-names --single-transaction --routines --triggers --opt -u $mysql_user --password="$mysql_pw" $mysql_db  2> >(tee -a $logfile $logfile_basic >&2) | gzip -c > $mysql_dest_file
	fi
	mysql_retval=${PIPESTATUS[0]}

	if [ ${mysql_retval} -ne 0 ]; then
		log 'Database backup FAILED'
		log '================= ERROR ================='
		senderror
		exit 1
	else
		#file_size_kb=`du -k "$mysql_dest_file" | cut -f1`
		file_size=$(ls -lh $mysql_dest_file | awk '{print  $5}')
		log "Database backup created successfully with $file_size"
	 fi
	log ' '
fi

log "start carousel"


#calculate backup days
let keep_backup_hours*=60
let keep_backup_weeks*=7
let keep_backup_months*=30

#https://nicaw.wordpress.com/2013/04/18/bash-backup-rotation-script/

# destination file names
date_hourly=`date +"%Y.%m.%d_%H.%M.%S"`
date_daily=`date +"%Y.%m.%d"`
#date_weekly=`date +"%V sav. %m-%Y"`
#date_monthly=`date +"%m-%Y"`

# Get current month and week day number
month_day=`date +"%d"`
week_day=`date +"%u"`

# It is logical to run this script daily. We take files from source folder and move them to
# appropriate destination folder

# On first month day do
if [ "$month_day" -eq 1 ] ; then
	destination=$monthly/$date_daily
	mkdir -p $destination
	cp -al $current/* $destination
	log "monthly backup first day to $destination"

else # if there is no folder in the last 31 days, create a month backup
	hasMonthlyFolder=$(find $monthly -maxdepth 1 -mtime -31 -type d -printf '.' | wc -c)
	if [ $hasMonthlyFolder -le 1 ] ; then
		destination=$monthly/$date_daily
		mkdir -p $destination
		cp -al $current/* $destination
		log "monthly backup other day to $destination"
	fi
fi
# copy weekly on sundays
if [ "$week_day" -eq 7 ] ; then
	destination=$weekly/$date_daily
	mkdir -p $destination
	cp -al $current/* $destination
	log "weekly backup on day 7 to $destination"
else # if there is no folder in the last 7 days, create a week backup
	hasWeeklyFolder=$(find $weekly -maxdepth 1 -mtime -7 -type d -printf '.' | wc -c)
	if [ $hasWeeklyFolder -le 1 ] ; then
		destination=$weekly/$date_daily
		mkdir -p $destination
		cp -al $current/* $destination
		log "weekly backup other day to $destination"
	fi
fi

#cron hourly
destination=$hourly/$date_hourly
mkdir -p $destination
cp -al $current/* $destination
log "hourly backup to $destination"

# copy daily
destination=$daily/$date_daily
# copy files with hardlinks
mkdir -p $destination
cp -al $current/* $destination
log "daily backup to $destination"
log "end carousel"
log ' '
log "Endtime: $(date)"
log ' '
log "------------------------- end backup $name -----------------------------"
log ' '

echo "start cleaning carousel"

# hourly - keep for defined hours
find $hourly/ -maxdepth 1 -cmin +$keep_backup_hours -type d -exec rm -rf {} \;

# daily - keep for defined days
find $daily/ -maxdepth 1 -mtime +$keep_backup_days -type d -exec rm -rf {} \;

# weekly - keep for defined weeks
find $weekly/ -maxdepth 1 -mtime +$keep_backup_weeks -type d -exec rm -rf {} \;

# monthly - keep for defined yearss
find $monthly/ -maxdepth 1 -mtime +$keep_backup_months -type d -exec rm -rf {} \;

echo "end cleaning carousel"

restore.sh

#!/bin/bash

# ------------------------
# Backup Script 
# Variablen:
#   server
#   ssh_user
#   ssh_privkey
#   ssh_port
#   rsync_source
#   rsync_dest
#   mysql_server
#   mysql_user
#   mysql_pw
#   mysql_db
# ------------------------

# load

server=''
ssh_user=''
ssh_privkey=''
rsync_source=''
mysql_server=$server
mysql_user=''
mysql_pw=''
mysql_db=''
mysql_port=''
ssh_port=''
basedir=''

if [ $# -eq 0 ]; then
    echo "No configuration file defined"
    exit 1
fi

# Check if configuration file exist
if [ ! -f $1 ]; then
    echo "Conficuration file '$1' not found!"
    exit 1 
fi

# check if source directory exist
if [[ ! -d "$2" ]]; then
    echo "The source directory '$2' not found!"
    exit 1
fi

# load configuration
source $1
current=$2

rsync_restore_dest=$rsync_source
rsync_restore_source=$current/files/
mysql_source_file=$current/database/database.sql.gz

ssh_port=${ssh_port:-'22'}
mysql_port=${mysql_port:-'3306'}
mysql_server=${mysql_server:-$server}

success=1

logfile_basic=$basedir/restore.log

log () {
    if [ -z "$1" ]; then
        cat
    else
        printf '%s\n' "$@" 
    fi | tee -a "$logfile_basic"
}


# sync files

log '-------------------------- start restore data --------------------------' 
log ' '
log "Date: $(date)"
log ' '
log 'Start rsync'

{
   rsync -e "ssh -p $ssh_port -i $ssh_privkey" --recursive --times --progress --partial --delete --stats $rsync_restore_source $ssh_user@$server:$rsync_restore_dest
} | log

if [ $? = 0 ]; then
  log 'Rsync restore files finished'
else
  log 'An error has occurred during rsync'
  success=0
fi
log ' '

# backup remote database
log 'Start database restore'
{
   gunzip -c $mysql_source_file | ssh -p $ssh_port -i $ssh_privkey $ssh_user@$server "mysql -h $mysql_server -P $mysql_port -u $mysql_user --password='$mysql_pw' $mysql_db" 
} | log
if [ $? = 0 ]; then
  log 'Database restore successfully'
else
  log 'Database restore FAILED'
  success=0
fi

log ' '
log "Endtime: $(date)"
log ' '
log '------------------------- end restore -----------------------------'
log ' '