#!/bin/bash

# VCS Backup

# Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program. If not, see <http://www.gnu.org/licenses/>.

# VCS Backup is a configuration for setting up a version control system mirrors.

# Currently Mercurial (Hg) and Git are supported.

# Features:

#  - mirrors remote repositories

#  - creates Btrfs subvolume for each repository

#  - does periodic pull to keep mirrors up to date

#  - does periodic Btrfs snapshot to keep history (git push --force done on the remote repository will lead to modifications or deletions in our current mirror, but previous versions will be kept in the snapshots)

#  - provides web interface for remote clonning of our mirrors (see systemd and etc folders)

#  - can be controlled over SSH by a sane person / owner of the system

#  - provides reports in the recfile format (to be processed using GNU Recutils or Relational pipes):

#     - list of repositories/mirrors

#     - results of pull operations

# This is an asynchronous message-driven shell script that runs distributed across two machines and four user accounts. You have been warned :-)

# Server-side configuration: / loadServerConfigFile()

VCS_BACKUP_DATA_DIR="/mnt/data";

VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";

VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";

VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";

VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";

VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";

VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}

VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";

VCS_BACKUP_USER="vcs-backup";

VCS_BACKUP_MANAGER="vcs-backup-manager";

# Client-side configuration: / see loadClientConfigFile()

VCS_BACKUP_SERVER="$VCS_BACKUP_MANAGER@localhost";

VCS_BACKUP_SSH_COMMAND=(ssh "$VCS_BACKUP_SERVER");

# Installation – check and do it by hand:

# There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR

installInstructions() {

cp vcs-backup.sh /usr/local/bin/

adduser --disabled-password "$VCS_BACKUP"

adduser --disabled-password "$VCS_BACKUP_MANAGER"

mkdir "$VCS_BACKUP_CURRENT_DIR";

mkdir "$VCS_BACKUP_CONFIG_DIR";

mkdir "$VCS_BACKUP_SNAPSHOT_DIR";

mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"

chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"

chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"

}

# --- Private functions: ---------------------------------------------------------------------------

# Environment: all

# $1 = VCS type: hg, git

# $2 = URL

isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }

# Environment: all

# $1 = path to the config file

loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }

loadClientConfigFile() { loadConfigFile ~/.config/vcs-backup/client.cfg; }

loadServerConfigFile() { loadConfigFile "/etc/vcs-backup/server.cfg"; }

# Environment: server

# $1 = URL

urlToRelativeDirectoryPath() {

	echo "$1" | sed -E 's@^[^:]+://@@g';

}

# Environment: server

# Lists relative paths (starting wiht hg or git) of all configured repositories

allRepositories() { find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs --no-run-if-empty dirname; }

# Environment: all

# $1 = optional value (if missing, reads STDIN)

escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }

# Environment: all

# $1 = key

# $2 = value

printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }

# --- Public interface functions: ------------------------------------------------------------------

# Environment: client

# $1 = VCS type: hg, git

# $2 = URL

# $3 = "public" or "private" (default), whether the repository should be available through the public web interface

# $4 = "clone" (optional), if present, will also clone the backup locally

vcs_backup_public_clientSubmitBackupRequest() {

	if isValidTypeAndURL "$1" "$2"; then

		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"

		if [[ "$4" == "clone" ]]; then

			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";

			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";

			fi

		fi

	else

		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

	fi

}

# Environment: server

# User: $VCS_BACKUP_MANAGER

# has same parameters as clientSubmitBackupRequest (see above)

vcs_backup_public_serverSubmitBackupRequest() {

	if isValidTypeAndURL "$1" "$2"; then

		relativePath=$1/$(urlToRelativeDirectoryPath "$2");

		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";

		# TODO: stop if directory already exists / only add public link?

		mkdir -p "$absolutePath";

		echo "$2" > "$absolutePath/url.txt"

		echo "submited" > "$absolutePath/state.txt"

		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"

		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"

		if [[ "$3" == "public" ]]; then

			cd "$VCS_BACKUP_PUBLIC_DIR";

			mkdir -p "$(dirname $relativePath)";

			ln -rs "../current/$relativePath" "$(dirname $relativePath)";

		fi

		if [[ "$4" == "clone" ]]; then

			callBackSocket=$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET}

			socat -u "unix-recvfrom:$callBackSocket,mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777 ?

				echo "Message from the service: $m";

			done &

			callBackPID=$!;

		fi

		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};

		if [[ "$4" == "clone" ]]; then

			echo "Waiting for a message from the service on $callBackSocket (PID $callBackPID)";

			wait -n $callBackPID;

		fi

	else

		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

	fi

}

# Environment: server

# User: root

# Should be started as a systemd/init service.

# - reads messages from from the subvolume socket – message contains the relative directory path

# - creates a subvolume for given repository + necesary parent directories

# - sends a message to the clone service → start cloning into the created subvolume

vcs_backup_public_serverStartSubvolumeService() {

	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do

		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");

		if [[ -e "$VCS_BACKUP_CURRENT_DIR/$d" ]]; then

			callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";

			if [[ -e "$callBackSocket" ]]; then

				echo "alreadyDone" | socat -u - unix-send:"$callBackSocket";

			fi

		else

			btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \

			echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \

			chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \

			echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};

		fi

	done

}

# Environment: server

# User: $VCS_BACKUP_USER

# should be started as a systemd/init service

vcs_backup_public_serverStartCloneService() {

	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do

		vcsType=$(echo "$d" | sed 's@/.*@@g');

		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

		if isValidTypeAndURL "$vcsType" "$url"; then

			if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

			elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

			fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";

		else

			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;

		fi

		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";

		if [[ -e "$callBackSocket" ]]; then

			echo "done" | socat -u - unix-send:"$callBackSocket";

		fi

	done

}

# Environment: client

# prints list of repositories in recfile format

# usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular

vcs_backup_public_clientListRepositories() {

	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;

}

# Environment: server

# User: $VCS_BACKUP_MANAGER

vcs_backup_public_serverListRepositories() {

	printRecfileKeyValue "%rec"  "repository";

	printRecfileKeyValue "%type" "bytes int";

	printRecfileKeyValue "%type" "public bool";

	echo;

	allRepositories | while read d; do

		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

		vcsType=$(echo "$d" | sed 's@/.*@@g');

		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);

		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";

		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);

		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai"); 

		else lastCommit=""; fi

		printRecfileKeyValue "type"            "$vcsType";

		printRecfileKeyValue "url"             "$url";

		printRecfileKeyValue "state"           "$state";

		printRecfileKeyValue "public"          "$public";

		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";

		printRecfileKeyValue "size"            "$sizeBytes";

		printRecfileKeyValue "lastCommit"      "$lastCommit";

		echo;

	done

}

# Environment: server

# User: $VCS_BACKUP_USER

# should be called from cron (usually every day)

vcs_backup_public_serverPullCronTask() {

	printRecfileKeyValue "%rec"  "pull";

	printRecfileKeyValue "%type" "started date";

	printRecfileKeyValue "%type" "finished date";

	printRecfileKeyValue "%type" "duration int";

	printRecfileKeyValue "%type" "resultCode int";

	allRepositories | while read d; do

		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

		vcsType=$(echo "$d" | sed 's@/.*@@g');

		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";

		pullStarted=$(date --iso-8601=s);

		pullStartedMiliseconds=$(($(date +%s%N)/1000000));

		pullFinished="";

		pullDuration="";

		pullResult="";

		pullResultCode="";

		if [[ "$state" == "cloned" ]]; then

			if   [[ "$vcsType" == "hg" ]];  then pullResult=$(hg pull --force --repository "$absolutePath" 2>&1); pullResultCode=$?;

			elif [[ "$vcsType" == "git" ]]; then pullResult=$(git -C "$absolutePath" fetch 2>&1); pullResultCode=$?;

			fi

			pullFinished=$(date --iso-8601=s);

			pullFinishedMiliseconds=$(($(date +%s%N)/1000000));

			pullDuration=$(( $pullFinishedMiliseconds - $pullStartedMiliseconds ));

		fi

		printRecfileKeyValue "serverPath"      "$absolutePath";

		printRecfileKeyValue "type"            "$vcsType";

		printRecfileKeyValue "state"           "$state";

		printRecfileKeyValue "started"         "$pullStarted";

		printRecfileKeyValue "finished"        "$pullFinished";

		printRecfileKeyValue "duration"        "$pullDuration";

		printRecfileKeyValue "resultCode"      "$pullResultCode";

		printRecfileKeyValue "message"         "$pullResult";

		echo;

	done

}

# Environment: server

# User: root

# should be called from cron (usually every day) after Pull (see above)

vcs_backup_public_serverSnapshotCronTask() {

	printRecfileKeyValue "%rec"  "snapshot";

	printRecfileKeyValue "%type" "started date";

	printRecfileKeyValue "%type" "finished date";

	printRecfileKeyValue "%type" "duration int";

	printRecfileKeyValue "%type" "resultCode int";

	allRepositories | while read d; do

		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

		vcsType=$(echo "$d" | sed 's@/.*@@g');

		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";

		started=$(date --iso-8601=s);

		startedMiliseconds=$(($(date +%s%N)/1000000));

		finished="";

		duration="";

		result="";

		resultCode="";

		snapshotPath="";

		if [[ "$state" == "cloned" ]]; then

			snapshotPath="$VCS_BACKUP_SNAPSHOT_DIR/$d/$(date --iso-8601=date)";

			mkdir -p $(dirname "$snapshotPath");

			result=$(btrfs subvolume snapshot -r "$absolutePath" "$snapshotPath" 2>&1);

			resultCode=$?;

			finished=$(date --iso-8601=s);

			finishedMiliseconds=$(($(date +%s%N)/1000000));

			duration=$(( $finishedMiliseconds - $startedMiliseconds ));

		fi

		printRecfileKeyValue "currentPath"     "$absolutePath";

		printRecfileKeyValue "snapshotPath"    "$snapshotPath";

		printRecfileKeyValue "type"            "$vcsType";

		printRecfileKeyValue "state"           "$state";

		printRecfileKeyValue "started"         "$started";

		printRecfileKeyValue "finished"        "$finished";

		printRecfileKeyValue "duration"        "$duration";

		printRecfileKeyValue "resultCode"      "$resultCode";

		printRecfileKeyValue "message"         "$result";

		echo;

	done

}

# --- Single entry-point: --------------------------------------------------------------------------

loadClientConfigFile;

loadServerConfigFile;

PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";

PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";

if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then

	"$PUBLIC_FUNCTION_PREFIX${@:1}";

elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;

elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;

elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;

elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;

else

	echo "Unsupported sub-command: $1" >&2

	echo "Available sub-commands:" >&2

	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2

	exit 1;

fi