franta-hg@0: #!/bin/bash
franta-hg@0: 
franta-hg@0: # VCS Backup
franta-hg@0: # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
franta-hg@0: #
franta-hg@0: # This program is free software: you can redistribute it and/or modify
franta-hg@0: # it under the terms of the GNU General Public License as published by
franta-hg@0: # the Free Software Foundation, either version 3 of the License, or
franta-hg@0: # (at your option) any later version.
franta-hg@0: #
franta-hg@0: # This program is distributed in the hope that it will be useful,
franta-hg@0: # but WITHOUT ANY WARRANTY; without even the implied warranty of
franta-hg@0: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
franta-hg@0: # GNU General Public License for more details.
franta-hg@0: #
franta-hg@0: # You should have received a copy of the GNU General Public License
franta-hg@0: # along with this program. If not, see <http://www.gnu.org/licenses/>.
franta-hg@0: 
franta-hg@0: 
franta-hg@13: # VCS Backup is a configuration for setting up a version control system mirrors.
franta-hg@13: # Currently Mercurial (Hg) and Git are supported.
franta-hg@13: # Features:
franta-hg@13: #  - mirrors remote repositories
franta-hg@13: #  - creates Btrfs subvolume for each repository
franta-hg@13: #  - does periodic pull to keep mirrors up to date
franta-hg@13: #  - does periodic Btrfs snapshot to keep history (git push --force done on the remote repository will lead to modifications or deletions in our current mirror, but previous versions will be kept in the snapshots)
franta-hg@13: #  - provides web interface for remote clonning of our mirrors (see systemd and etc folders)
franta-hg@13: #  - can be controlled over SSH by a sane person / owner of the system
franta-hg@13: #  - provides reports in the recfile format (to be processed using GNU Recutils or Relational pipes):
franta-hg@13: #     - list of repositories/mirrors
franta-hg@13: #     - results of pull operations
franta-hg@13: 
franta-hg@13: 
franta-hg@13: # This is an asynchronous message-driven shell script that runs distributed across two machines and four user accounts. You have been warned :-)
franta-hg@0: # Server-side configuration:
franta-hg@0: VCS_BACKUP_DATA_DIR="/mnt/data";
franta-hg@0: VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";
franta-hg@2: VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";
franta-hg@0: VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";
franta-hg@0: VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";
franta-hg@0: VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";
franta-hg@7: VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}
franta-hg@3: VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";
franta-hg@0: VCS_BACKUP_USER="vcs-backup";
franta-hg@0: VCS_BACKUP_MANAGER="vcs-backup-manager";
franta-hg@0: 
franta-hg@0: # Installation – check and do it by hand:
franta-hg@0: # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR
franta-hg@0: installInstructions() {
franta-hg@0: cp vcs-backup.sh /usr/local/bin/
franta-hg@0: adduser --disabled-password "$VCS_BACKUP"
franta-hg@0: adduser --disabled-password "$VCS_BACKUP_MANAGER"
franta-hg@0: 
franta-hg@0: mkdir "$VCS_BACKUP_CURRENT_DIR";
franta-hg@0: mkdir "$VCS_BACKUP_CONFIG_DIR";
franta-hg@0: mkdir "$VCS_BACKUP_SNAPSHOT_DIR";
franta-hg@5: mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"
franta-hg@0: 
franta-hg@5: chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"
franta-hg@0: chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"
franta-hg@0: }
franta-hg@0: 
franta-hg@0: 
franta-hg@0: # --- Private functions: ---------------------------------------------------------------------------
franta-hg@0: 
franta-hg@0: # Environment: all
franta-hg@0: # $1 = VCS type: hg, git
franta-hg@0: # $2 = URL
franta-hg@0: isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }
franta-hg@0: 
franta-hg@0: # Environment: all
franta-hg@0: # $1 = path to the config file
franta-hg@5: loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }
franta-hg@0: 
franta-hg@0: # Environment: server
franta-hg@0: # $1 = URL
franta-hg@0: urlToRelativeDirectoryPath() {
franta-hg@0: 	echo "$1" | sed -E 's@^[^:]+://@@g';
franta-hg@0: }
franta-hg@0: 
franta-hg@9: # Environment: all
franta-hg@9: # $1 = optional value (if missing, reads STDIN)
franta-hg@9: escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }
franta-hg@9: 
franta-hg@9: # Environment: all
franta-hg@9: # $1 = key
franta-hg@9: # $2 = value
franta-hg@9: printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }
franta-hg@9: 
franta-hg@0: # --- Public interface functions: ------------------------------------------------------------------
franta-hg@0: 
franta-hg@0: # Environment: client
franta-hg@0: # $1 = VCS type: hg, git
franta-hg@0: # $2 = URL
franta-hg@2: # $3 = "public" or "private" (default), whether the repository should be available through the public web interface
franta-hg@10: # $4 = "clone" (optional), if present, will also clone the backup locally
franta-hg@0: vcs_backup_public_clientSubmitBackupRequest() {
franta-hg@0: 	if isValidTypeAndURL "$1" "$2"; then
franta-hg@0: 		loadConfigFile ~/.config/vcs-backup/client.cfg
franta-hg@3: 		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"
franta-hg@3: 		if [[ "$4" == "clone" ]]; then
franta-hg@10: 			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";
franta-hg@10: 			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";
franta-hg@3: 			fi
franta-hg@3: 		fi
franta-hg@0: 	else
franta-hg@0: 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
franta-hg@0: 	fi
franta-hg@0: }
franta-hg@0: 
franta-hg@0: # Environment: server
franta-hg@5: # User: $VCS_BACKUP_MANAGER
franta-hg@2: # has same parameters as clientSubmitBackupRequest (see above)
franta-hg@0: vcs_backup_public_serverSubmitBackupRequest() {
franta-hg@0: 	if isValidTypeAndURL "$1" "$2"; then
franta-hg@0: 		loadConfigFile "/etc/vcs-backup/server.cfg";
franta-hg@0: 		relativePath=$1/$(urlToRelativeDirectoryPath "$2");
franta-hg@0: 		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";
franta-hg@0: 		mkdir -p "$absolutePath";
franta-hg@0: 		echo "$2" > "$absolutePath/url.txt"
franta-hg@0: 		echo "submited" > "$absolutePath/state.txt"
franta-hg@0: 		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"
franta-hg@0: 		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"
franta-hg@3: 
franta-hg@2: 		if [[ "$3" == "public" ]]; then
franta-hg@2: 			cd "$VCS_BACKUP_PUBLIC_DIR";
franta-hg@2: 			mkdir -p "$(dirname $relativePath)";
franta-hg@2: 			ln -rs "../current/$relativePath" "$(dirname $relativePath)";
franta-hg@2: 		fi
franta-hg@3: 
franta-hg@3: 		if [[ "$4" == "clone" ]]; then
franta-hg@12: 			callBackSocket=$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET}
franta-hg@12: 			socat -u "unix-recvfrom:$callBackSocket,mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777 ?
franta-hg@12: 				echo "Message from the service: $m";
franta-hg@12: 			done &
franta-hg@12: 			callBackPID=$!;
franta-hg@12: 		fi
franta-hg@12: 
franta-hg@12: 		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};
franta-hg@12: 
franta-hg@12: 		if [[ "$4" == "clone" ]]; then
franta-hg@12: 			echo "Waiting for a message from the service on $callBackSocket (PID $callBackPID)";
franta-hg@12: 			wait -n $callBackPID;
franta-hg@3: 		fi
franta-hg@0: 	else
franta-hg@0: 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
franta-hg@0: 	fi
franta-hg@0: }
franta-hg@0: 
franta-hg@0: # Environment: server
franta-hg@5: # User: root
franta-hg@0: # Should be started as a systemd/init service.
franta-hg@0: # - reads messages from from the subvolume socket – message contains the relative directory path
franta-hg@0: # - creates a subvolume for given repository + necesary parent directories
franta-hg@0: # - sends a message to the clone service → start cloning into the created subvolume
franta-hg@0: vcs_backup_public_serverStartSubvolumeService() {
franta-hg@0: 	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do
franta-hg@0: 		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");
franta-hg@12: 		if [[ -e "$VCS_BACKUP_CURRENT_DIR/$d" ]]; then
franta-hg@12: 			callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";
franta-hg@12: 			if [[ -e "$callBackSocket" ]]; then
franta-hg@12: 				echo "alreadyDone" | socat -u - unix-send:"$callBackSocket";
franta-hg@12: 			fi
franta-hg@12: 		else
franta-hg@12: 			btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \
franta-hg@12: 			echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \
franta-hg@12: 			chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \
franta-hg@12: 			echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};
franta-hg@12: 		fi
franta-hg@0: 	done
franta-hg@0: }
franta-hg@0: 
franta-hg@0: # Environment: server
franta-hg@5: # User: $VCS_BACKUP_USER
franta-hg@0: # should be started as a systemd/init service
franta-hg@0: vcs_backup_public_serverStartCloneService() {
franta-hg@0: 	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do
franta-hg@2: 		vcsType=$(echo "$d" | sed 's@/.*@@g');
franta-hg@0: 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
franta-hg@3: 
franta-hg@0: 		if isValidTypeAndURL "$vcsType" "$url"; then
franta-hg@8: 			if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@8: 			elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@0: 			fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";
franta-hg@0: 		else
franta-hg@0: 			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;
franta-hg@0: 		fi
franta-hg@3: 
franta-hg@3: 		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";
franta-hg@3: 		if [[ -e "$callBackSocket" ]]; then
franta-hg@3: 			echo "done" | socat -u - unix-send:"$callBackSocket";
franta-hg@3: 		fi
franta-hg@0: 	done
franta-hg@0: }
franta-hg@0: 
franta-hg@7: # Environment: client
franta-hg@7: # prints list of repositories in recfile format
franta-hg@7: # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular
franta-hg@7: vcs_backup_public_clientListRepositories() {
franta-hg@7: 	loadConfigFile ~/.config/vcs-backup/client.cfg;
franta-hg@7: 	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;
franta-hg@7: }
franta-hg@7: 
franta-hg@7: # Environment: server
franta-hg@7: # User: $VCS_BACKUP_MANAGER
franta-hg@7: vcs_backup_public_serverListRepositories() {
franta-hg@7: 	echo "%rec: repositories";
franta-hg@7: 	echo "%type: bytes int";
franta-hg@7: 	echo "%type: public bool";
franta-hg@7: 	echo;
franta-hg@7: 
franta-hg@7: 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do
franta-hg@7: 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
franta-hg@7: 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
franta-hg@7: 		vcsType=$(echo "$d" | sed 's@/.*@@g');
franta-hg@7: 		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);
franta-hg@7: 		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";
franta-hg@7: 		
franta-hg@7: 		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);
franta-hg@7: 		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai"); 
franta-hg@7: 		else lastCommit=""; fi
franta-hg@7: 		
franta-hg@9: 		printRecfileKeyValue "type"            "$vcsType";
franta-hg@9: 		printRecfileKeyValue "url"             "$url";
franta-hg@9: 		printRecfileKeyValue "state"           "$state";
franta-hg@9: 		printRecfileKeyValue "public"          "$public";
franta-hg@9: 		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@9: 		printRecfileKeyValue "size"            "$sizeBytes";
franta-hg@9: 		printRecfileKeyValue "lastCommit"      "$lastCommit";
franta-hg@7: 		echo;
franta-hg@7: 	done
franta-hg@7: }
franta-hg@7: 
franta-hg@6: # Environment: server
franta-hg@6: # User: $VCS_BACKUP_USER
franta-hg@6: # should be called from cron (usually every day)
franta-hg@6: vcs_backup_public_serverPullCronTask() {
franta-hg@11: 	echo "%rec: pull";
franta-hg@11: 	echo "%type: started date";
franta-hg@11: 	echo "%type: finished date";
franta-hg@11: 	echo "%type: duration int";
franta-hg@11: 	echo "%type: resultCode int";
franta-hg@11: 
franta-hg@11: 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do
franta-hg@11: 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
franta-hg@11: 		vcsType=$(echo "$d" | sed 's@/.*@@g');
franta-hg@11: 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@11: 
franta-hg@11: 
franta-hg@11: 		pullStarted=$(date --iso-8601=s);
franta-hg@11: 		pullStartedMiliseconds=$(($(date +%s%N)/1000000));
franta-hg@11: 		pullFinished="";
franta-hg@11: 		pullDuration="";
franta-hg@11: 		pullResult="";
franta-hg@11: 		pullResultCode="";
franta-hg@11: 		if [[ "$state" == "cloned" ]]; then
franta-hg@11: 			if   [[ "$vcsType" == "hg" ]];  then pullResult=$(hg pull --force --repository "$absolutePath" 2>&1); pullResultCode=$?;
franta-hg@11: 			elif [[ "$vcsType" == "git" ]]; then pullResult=$(git -C "$absolutePath" fetch 2>&1); pullResultCode=$?;
franta-hg@11: 			fi
franta-hg@11: 			pullFinished=$(date --iso-8601=s);
franta-hg@11: 			pullFinishedMiliseconds=$(($(date +%s%N)/1000000));
franta-hg@11: 			pullDuration=$(( $pullFinishedMiliseconds - $pullStartedMiliseconds ));
franta-hg@11: 		fi
franta-hg@11: 
franta-hg@11: 		printRecfileKeyValue "serverPath"      "$absolutePath";
franta-hg@11: 		printRecfileKeyValue "type"            "$vcsType";
franta-hg@11: 		printRecfileKeyValue "state"           "$state";
franta-hg@11: 		printRecfileKeyValue "started"         "$pullStarted";
franta-hg@11: 		printRecfileKeyValue "finished"        "$pullFinished";
franta-hg@11: 		printRecfileKeyValue "duration"        "$pullDuration";
franta-hg@11: 		printRecfileKeyValue "resultCode"      "$pullResultCode";
franta-hg@11: 		printRecfileKeyValue "message"         "$pullResult";
franta-hg@11: 		echo;
franta-hg@11: 	done
franta-hg@6: }
franta-hg@6: 
franta-hg@6: # Environment: server
franta-hg@11: # User: root
franta-hg@6: # should be called from cron (usually every day) after Pull (see above)
franta-hg@6: vcs_backup_public_serverSnapshotCronTask() {
franta-hg@6: 	return;
franta-hg@6: }
franta-hg@6: 
franta-hg@0: # --- Single entry-point: --------------------------------------------------------------------------
franta-hg@0: 
franta-hg@0: PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
franta-hg@0: PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";
franta-hg@0: if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then
franta-hg@0: 	"$PUBLIC_FUNCTION_PREFIX${@:1}";
franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;
franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;
franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;
franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;
franta-hg@0: else
franta-hg@0: 	echo "Unsupported sub-command: $1" >&2
franta-hg@0: 	echo "Available sub-commands:" >&2
franta-hg@0: 	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2
franta-hg@0: 	exit 1;
franta-hg@0: fi