#!/bin/ksh -u
# Backup a machine using rsync. Keep backups for different days distinct.
#
#
#      /\
#     /  \            (C) Copyright 2006, 2009 Parliament Hill Computers Ltd.
#     \  /            All rights reserved.
#      \/
#       .             Author: Alain Williams, <addw@phcomp.co.uk> April 2006
#       .
#        .            SCCS: @(#)RsyncBackup	1.38 11/10/11 01:13:33
#          .
#


PROGNAME=${0##*/}

Remote=			# Name of the remote machine
RemoteLocalName=	# Used for local purposes - default $Remote
RemoteSrc=backup	# The rsync 'source' on $Remote
RemoteDirs=		# Space separated list of directories to backup
RemoteDirsNonDistinct=	# Space separated list of directories to backup - not date distinct
LocalDir=/arch		# Where to copy to on this machine
SnapshotDir=SNAPSHOT	# Directory where non date distinct snapshots go
BandWidth=		# Rsync bandwidth option

# Defeat SCCS:
yyyymmdd=$( date '+%Y''%m%d' )
hhmm=$( date '+%H''%M' )


CleanDays=14		# Backups older than this (days) will be cleaned (removed)
Leave1st=n		# Leave any backup done on 1st of the month
DoCopyLog=n		# Copy output to stderr as well as $logdir
clean=n			# Don't perform a clean operation
CreateLinks=y		# Create a symlink of LATEST to today's archive
compress=-z		# Compress data transfer
SuccessFile=		# Touch this iff backup successful
OneFileSystem=		# Rsync option
CleanYears=		# Number of years old a 1st of the month backup must be to be cleaned
CleanBeforeYear=	# yyyymmdd for absolute clean out
MailTo=			# Who to email results to
MailFail=n		# Only MailTo on failure
TotalPercent=n		# What percentage for -t. 'n' means option not chosen
OutputToFile=n		# Set y once has been redirected. Old stderr is copied to FD 3
# -A not for all versions of ksh
#typeset -A Succ	# Rsync succeeded. Index by directory name. Values 'y' or 'n'

logRoot=/var/log/backups # Log files under here

# Print the message to what used to be stderr AND current stderr and exit.
# Do not call this unt
function Die {
	[[ $OutputToFile = y ]] && echo "$PROGNAME: $*" >&3
	echo "$PROGNAME: $*" >&2
	exit 2
}

# Generate a note that will be seen by the user -- even if redirected
function Note {
	[[ $OutputToFile = y ]] && echo "$PROGNAME: $*" >&3
	echo "$PROGNAME: $*" >&2
}

function Usage {
	(( $# > 0 )) && echo "$PROGNAME: $*" >&2
	cat <<-!
	Backup machine \$RemoteMachine::$RemoteSrc to local directory $LocalDir/\$RemoteMachine.
	Usage: $PROGNAME [-opts]
	-1	Don't clean a backup made on 1st of the month
	-b kbps	Set average bandwidth to kbsp Kilobytes per second. Default no limit
	-c	Clean out old backups that are more than $CleanDays old
	-C days	Set the number of days older than which backups will be Cleaned to days
	-d dirs	Directory list to backup is 'dirs'. Backups on different days will be stored distinctly
	-D dirs	Directory list to backup is 'dirs'. Backups on different days will be NOT stored distinctly
	-f	mail only on Failure (-M)
	-F	do not cross mount points onto other File systems
	-l dir	root Local directory, default is '$LocalDir'
	-M addr Mail a success/fail message and the log file to here
	-o	Copy output to the current stderr as well as something in $logRoot/\$Remote/
	-r name	Backup the RemoteMachine machine 'name'
	-R name	Name to use for local storage of archive. Default is value of -r
	-s src	rsync source name is 'src' - default: $RemoteSrc
	-S file	Touch file with the Start time of backup iff the backup was successful
	.      If not absolute, relative to: $LocalDir/\$RemoteMachine.
	.      If name starts with '_', relative to: $LocalDir/\$RemoteMachine/LATEST.
	-t n	compare Total number of files and disk used to be within n% of yesterday. 10 is a good n.
	-y yrs	Clean out 1st of the month backups that are more than yrs years old
	-x	eXplain
	-Z	Don't compress rsync data transfer
	--help	Help message
	Directories backed up: $RemoteDirs
	Version: 1.38 11/10/11, latest from: http://www.phcomp.co.uk/Packages/RsyncBackup.html
	!
}

# Parse options, recognise --help
while	[[ $# -ge $OPTIND ]] && eval A1=\$$OPTIND || A1=
	if [[ $A1 = --help ]]
	then	opt=x	# --help is -x
	else	getopts :1b:cC:d:D:fFl:M:or:R:s:S:t:xZy: opt
	fi
do	case "$opt" in
	1)	Leave1st=y ;;
	b)	BandWidth="--bwlimit=$OPTARG" ;;
	c)	clean=y ;;
	C)	CleanDays=$OPTARG ;;
	d)	RemoteDirs="$OPTARG" ;;
	D)	RemoteDirsNonDistinct="$OPTARG" ;;
	f)	MailFail=y ;;
	F)	OneFileSystem=--one-file-system ;;
	l)	LocalDir="$OPTARG" ;;
	M)	MailTo="$OPTARG" ;;
	o)	DoCopyLog=y ;;
	r)	Remote="$OPTARG" ;;
	R)	RemoteLocalName="$OPTARG" ;;
	s)	RemoteSrc="$OPTARG" ;;
	S)	SuccessFile="$OPTARG" ;;
	t)	TotalPercent="$OPTARG" ;;
	x)	Usage
		exit;;
	Z)	compress= ;;
	y)	CleanYears="$OPTARG" ;;
	:)	Usage "Missing argument to option '$OPTARG'" ; exit 2;;
	\?)	Usage "Unknown option '$OPTARG'" ; exit 2;;
	*)	Usage "Internal program error, unrecognised argument '$opt'"
		exit 2;;
	esac
done
shift $((OPTIND - 1))

[[ -z $Remote ]] &&
	echo "Machine to backup not specified (-r)" >&2 &&
	Usage &&
	exit 2

[[ -z $RemoteSrc ]] &&
	echo "Rsync source on machine $Remote not specified (-s)" >&2 &&
	Usage &&
	exit 2

[[ -z $RemoteDirs ]] &&
	echo "Directories to backup not specified (-d)" >&2 &&
	Usage &&
	exit 2

# If -R not given:
[[ -z $RemoteLocalName ]] && RemoteLocalName="$Remote"

LocalPath=$LocalDir/$RemoteLocalName

# Where we (may) log what we do:
logdir=$logRoot/$RemoteLocalName/

# What date do we clean before ?
CleanBefore=$( perl -e '$Now = time - 86400 * '$CleanDays';
		my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime $Now;
		printf "%4.4d%2.2d%2.2d\n", $year + 1900, $mon + 1, $mday' )

# Be conservative, deem a year to be 366 days long:
[[ -n $CleanYears ]] &&
	CleanBeforeYear=$( perl -e '$Now = time - 86400 * 366 * '$CleanYears';
		my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday, $isdst) = localtime $Now;
		printf "%4.4d%2.2d%2.2d\n", $year + 1900, $mon + 1, $mday' )


# Fiddle with where the output of this program goes to.
# The point is that under normal operation we want to capture/redirect
# everything to a log file, when run manually we might want to do that
# but still have everything come to the terminal as well.
[[ -d $logdir ]] || mkdir -p $logdir || Die "Can't create logging directory: $logdir"
exec 3>&2	# So that we can use it later
if [[ $DoCopyLog = y ]]
then	tee /proc/$$/fd/3 > $logdir$yyyymmdd-$hhmm |&
	exec 1>&p 2>&1
	OutputToFile=y
else
	# If being run non interactively, send output to a log file:
	[[ ! -t 2 ]] && exec > $logdir$yyyymmdd-$hhmm 2>&1 && OutputToFile=y
fi

# Go to where we put the backup:
cd $LocalPath || Die "Can't cd to $LocalPath"

# Check that we don't have an archive already today.
# This would screw up the versioning.
if [[ -L LATEST ]]
then	Ldate=$( ls -l LATEST | awk '{print $NF}' )
	[[ $Ldate = $yyyymmdd ]] &&
		Die "$( date ): Archive already done today - aborting"
fi

# Note the start time - in the format that touch likes -- also something for us humans:
eval $( date '+StartTime=%Y''%m%d%H''%M.%S StartTimeHuman="%Y/%m/%d %H:%M:%S"' )

echo "Backup starting at $( date ) ($PROGNAME version 1.38)"
echo "Backup for $Remote::$RemoteSrc to local directory $(uname -n)::$LocalPath"


Success=y	# Let's be optimistic

# Count directories - traverse $RemoteDirs $RemoteDirsNonDistinct - in that order!
# Some versions of ksh cannot use a string as an array index.
DirIndex=0

for dir in $RemoteDirs
do	echo ""
	echo "****"
	echo "Backup starting $dir at $( date )"
	mkdir -p "$yyyymmdd/$dir" || Die "Can't make $yyyymmdd/$dir"

	# Can we link from y/day's backup, break links where files differ:
	[[ -d LATEST/$dir ]] && link="--link-dest=$LocalPath/LATEST/$dir" || link=-H

	Succ[$DirIndex]=y	# Be optimistic

	rsync -aH --numeric-ids --stats $BandWidth $OneFileSystem $compress "$link" "$Remote::$RemoteSrc/$dir/" "$yyyymmdd/$dir/" || {
		echo "Rsync failed, exit code: $?"
		Success=n
		Succ[$DirIndex]=n
	}

	((DirIndex++))

	echo "Completed backup of $dir at $( date )"
done

echo ""

# These just go into one directory, a snapshot with the very latests - no history:
for dir in $RemoteDirsNonDistinct
do	echo ""
	echo "****"
	echo "Snapshot backup starting $dir at $( date )"
	mkdir -p "$SnapshotDir/$dir" || Die "Can't make $SnapshotDir/$dir"

	Succ[$DirIndex]=y	# Be optimistic

	rsync -aH --delete --numeric-ids --stats $BandWidth $OneFileSystem $compress "$Remote::$RemoteSrc/$dir/" "$SnapshotDir/$dir/" || {
		echo "Rsync failed, exit code: $?"
		echo "This does not count as a failure from the point of view of LATEST link creation"
		Succ[$DirIndex]=n
	}

	((DirIndex++))

	echo "Completed backup of $dir at $( date )"
done

echo ""


# Create/maintain the symlink LATEST to the latest *complete* backup.
# If the above backup failed, don't reassign to today's since the next one that works
# will not find the files that it needs to link to - so we will get them recreated
# which takes more execution time and loads of disk to hold the files that we already
# have an unchanged copy of.
if [[ $CreateLinks = y && $Success = y ]]
then	echo "Creating links; LATEST -> $yyyymmdd"
	rm -f $LocalPath/LATEST
	ln -s $yyyymmdd $LocalPath/LATEST
else	Note "Not creating links due to rsync error"
fi

# If there is a latest, get what it points at:
[[ -L LATEST ]] && lastestdir=$(find LATEST -printf %l) || lastestdir=

# Clean out old backups ?
if [[ $clean = y ]]
then	echo ""
	echo "Cleaning old backups - Clean before: $CleanBefore"
	[[ -n $lastestdir ]] && echo "Protect LATEST=$lastestdir"
	[[ $Leave1st = y ]]  && echo "Protect backups done on 1st of the month"
	echo ""
	for dir in $( ls | grep '^2' )	# Y3K bug here
	do	# Leave if too young:
		[[ $dir -gt $CleanBefore ]] && continue

		# Leave if first of the month.
		# If we clean 1st of the month that are too old, check the date:
		[[ $Leave1st = y && $dir = *01 ]] &&
			[[ -z $CleanBeforeYear || $dir -gt $CleanBeforeYear ]] && continue

		# Leave if it is pointed to be LATEST symlink, if backups have been
		# failing for some time this could be quite old:
		[[ $lastestdir = $dir ]] && continue

		echo "Cleaning old backup: $dir"
		rm -rf "$dir"
	done

	echo "Completed cleanup of old backups at $( date )"
fi

# Set the modification time on some file ?
if [[ -n $SuccessFile && $Success = y ]]
then	# If the name starts '_' put in the directory that we just wrote into
	[[ $SuccessFile = _* ]] &&
		SuccessFile="LATEST/${SuccessFile#?}"

	touch -t $StartTime "$SuccessFile"
	echo ""
	echo "Date file $SuccessFile modified time set to backup start time: $StartTimeHuman"
fi

# Record disk usage. Twice - once for machine analysis, once for humans:
echo ""
# Output: device mount-point dev-size blocks-used blocks-free blocks-%age-used inode-total inode-used inode-free inode-%age-used
# Blocks in 1KiByte block units.
echo "Usage $( (df -P .;df -Pi .) | sed -e 1d -e 3d -e 's/ \+/ /g' -e '2{s/^\([^ ]\+\) \(.*\) \([^ ]\+\)$/\1 \3 \2/; h; d}' -e '4{s/^[^ ]* \(.*\) [^ ]*$/\1/; H; g; s/\n/ /}' )"
echo ""
echo "Disk usage:"
df -h .
df -hi .
echo ""

# Are we to check the # files & disk used with last successful bacup ?
# This is a good check for something having gone wrong.
# Totals are one file per directory backed up in a subdirectory: Totals
if [[ $TotalPercent != n ]]
then	DirIndex=-1	# Reset for retraverse. NB do increment first thing.

	[[ -d Totals ]] || mkdir Totals || Die "Can't make Totals directory"
	for dir in $RemoteDirs $RemoteDirsNonDistinct
	do	((DirIndex++))

		# Don't if the backup of that directory failed
		[[ ${Succ[$DirIndex]} = n ]] &&
			echo "NOT comparing last successful backup size with today's for $dir as the copy failed" &&
			continue

		echo "Comparing last successful backup size with today's for $dir"

		# Summary files in 'Totals' dir have / replaced by -
		summary=$( echo "$dir" | sed -e 's:^/::' -e 's:/:-:g' )

		# The directory that we are totaling is either under "$yyyymmdd or $SnapshotDir
		realDir=
		[[ -d "$yyyymmdd/$dir" ]] && realDir="$yyyymmdd/$dir"
		[[ -d "$SnapshotDir/$dir" ]] && realDir="$SnapshotDir/$dir"
		[[ -z $realDir ]] && Die "Can't find real directory of $dir"

		nfiles=$( find "$realDir" | grep -c ^ )
		d_used=$( du -ks "$realDir" | sed -e 's/^\([0-9]\+\).*/\1/' )

		# Compare with y/day:
		if [[ ! -s Totals/$summary ]]
		then	if [[ ! -f Totals/$summary ]]
			then	Note "No previous totals for $dir - creating afresh, files $nfiles, Kb $d_used used"
			else	Note "Empty previous totals file for $dir - creating afresh, files $nfiles, Kb $d_used used"
			fi
		else	Msg=$( perl -wln -e "\$nf=$nfiles; \$du=$d_used; \$per=$TotalPercent; \$dir='$dir';" -e '
				next if /^#/;
				$date = $1 if /^Date (.+)/;
				($y_nf, $y_du) = ($1, $2) if /^(\d+) +(\d+)/;
				END {
					unless(defined($y_nf) and defined($y_du)) {
						print qq[Format error in Totals file for /$dir\n];
						exit;
					}
					$date = q[yesterday] unless defined $date;
					print qq[/$dir: File count differs greatly: $date: $y_nf, today: $nf]     if(abs($y_nf - $nf) > $nf / 100 * $per);
					print qq[/$dir: Disc used (Kb) differs greatly: $date: $y_du, today: $du] if(abs($y_du - $du) > $du / 100 * $per);
				}' Totals/$summary )
			[[ $Msg != '' ]] && Note "$Msg"
		fi

		# Create today's summary:
		cat <<-! > Totals/$summary
			# Auto Generated by $PROGNAME -- do not hand edit
			# Date generated: $( date )
			Date $( date '+%d %b' )
			# Number-of-files disk-used
			$nfiles $d_used
		!
	done

	echo "Completed comparison of size with last successful backups at $( date )"
fi

echo "Completed backup at $( date )"

# If something went wrong and output is NOT going to stderr (ie on a terminal) - copy it there.
# This is prob being run from cron which will mail the output somewhere suitable.
if [[ $Success = n && $OutputToFile = y ]]
then	( echo "**** Error in backup ****"
	  echo ""
	  cat $logdir$yyyymmdd-$hhmm 
	) >&3
fi

# If we have an email address to mail to use it.
# Don't mail if this worked unless we are to always email.
# We will prob run under cron, but maybe not - it is important that failure messages get noticed.
if [[ -n $MailTo && ( $Success = n || $MailFail = n ) ]]
then	( [[ $Success = y ]] && Worked=Success || Worked=Failure
	  echo "**** $Worked in backup ****"
	  echo ""
	  cat $logdir$yyyymmdd-$hhmm 
	) | mail -s "Backup $Worked" "$MailTo"
fi

# end
