vcs-backup: vcs-backup.sh@e7279b13a071

     1 #!/bin/bash

     3 # VCS Backup

     4 # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)

5 #

     6 # This program is free software: you can redistribute it and/or modify

     7 # it under the terms of the GNU General Public License as published by

     8 # the Free Software Foundation, either version 3 of the License, or

     9 # (at your option) any later version.

    10 #

    11 # This program is distributed in the hope that it will be useful,

    12 # but WITHOUT ANY WARRANTY; without even the implied warranty of

    13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

    14 # GNU General Public License for more details.

    15 #

    16 # You should have received a copy of the GNU General Public License

    17 # along with this program. If not, see <http://www.gnu.org/licenses/>.

    20 # VCS Backup is a configuration for setting up a version control system mirrors.

    21 # Currently Mercurial (Hg) and Git are supported.

    22 # Features:

    23 #  - mirrors remote repositories

    24 #  - creates Btrfs subvolume for each repository

    25 #  - does periodic pull to keep mirrors up to date

    26 #  - does periodic Btrfs snapshot to keep history (git push --force done on the remote repository will lead to modifications or deletions in our current mirror, but previous versions will be kept in the snapshots)

    27 #  - provides web interface for remote clonning of our mirrors (see systemd and etc folders)

    28 #  - can be controlled over SSH by a sane person / owner of the system

    29 #  - provides reports in the recfile format (to be processed using GNU Recutils or Relational pipes):

    30 #     - list of repositories/mirrors

    31 #     - results of pull operations

    34 # This is an asynchronous message-driven shell script that runs distributed across two machines and four user accounts. You have been warned :-)

    37 # Server-side configuration:

    38 VCS_BACKUP_DATA_DIR="/mnt/data";

    39 VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";

    40 VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";

    41 VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";

    42 VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";

    43 VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";

    44 VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}

    45 VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";

    46 VCS_BACKUP_USER="vcs-backup";

    47 VCS_BACKUP_MANAGER="vcs-backup-manager";

    49 # Installation – check and do it by hand:

    50 # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR

    51 installInstructions() {

    52 cp vcs-backup.sh /usr/local/bin/

    53 adduser --disabled-password "$VCS_BACKUP"

    54 adduser --disabled-password "$VCS_BACKUP_MANAGER"

    56 mkdir "$VCS_BACKUP_CURRENT_DIR";

    57 mkdir "$VCS_BACKUP_CONFIG_DIR";

    58 mkdir "$VCS_BACKUP_SNAPSHOT_DIR";

    59 mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"

    61 chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"

    62 chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"

    63 }

    66 # --- Private functions: ---------------------------------------------------------------------------

    68 # Environment: all

    69 # $1 = VCS type: hg, git

    70 # $2 = URL

    71 isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }

    73 # Environment: all

    74 # $1 = path to the config file

    75 loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }

    77 # Environment: server

    78 # $1 = URL

    79 urlToRelativeDirectoryPath() {

    80 	echo "$1" | sed -E 's@^[^:]+://@@g';

    81 }

    83 # Environment: all

    84 # $1 = optional value (if missing, reads STDIN)

    85 escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }

    87 # Environment: all

    88 # $1 = key

    89 # $2 = value

    90 printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }

    92 # --- Public interface functions: ------------------------------------------------------------------

    94 # Environment: client

    95 # $1 = VCS type: hg, git

    96 # $2 = URL

    97 # $3 = "public" or "private" (default), whether the repository should be available through the public web interface

    98 # $4 = "clone" (optional), if present, will also clone the backup locally

    99 vcs_backup_public_clientSubmitBackupRequest() {

   100 	if isValidTypeAndURL "$1" "$2"; then

   101 		loadConfigFile ~/.config/vcs-backup/client.cfg

   102 		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"

   103 		if [[ "$4" == "clone" ]]; then

   104 			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";

   105 			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";

   106 			fi

   107 		fi

   108 	else

   109 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

   110 	fi

   111 }

   113 # Environment: server

   114 # User: $VCS_BACKUP_MANAGER

   115 # has same parameters as clientSubmitBackupRequest (see above)

   116 vcs_backup_public_serverSubmitBackupRequest() {

   117 	if isValidTypeAndURL "$1" "$2"; then

   118 		loadConfigFile "/etc/vcs-backup/server.cfg";

   119 		relativePath=$1/$(urlToRelativeDirectoryPath "$2");

   120 		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";

   121 		# TODO: stop if directory already exists / only add public link?

   122 		mkdir -p "$absolutePath";

   123 		echo "$2" > "$absolutePath/url.txt"

   124 		echo "submited" > "$absolutePath/state.txt"

   125 		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"

   126 		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"

   128 		if [[ "$3" == "public" ]]; then

   129 			cd "$VCS_BACKUP_PUBLIC_DIR";

   130 			mkdir -p "$(dirname $relativePath)";

   131 			ln -rs "../current/$relativePath" "$(dirname $relativePath)";

   132 		fi

   134 		if [[ "$4" == "clone" ]]; then

   135 			callBackSocket=$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET}

   136 			socat -u "unix-recvfrom:$callBackSocket,mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777 ?

   137 				echo "Message from the service: $m";

   138 			done &

   139 			callBackPID=$!;

   140 		fi

   142 		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};

   144 		if [[ "$4" == "clone" ]]; then

   145 			echo "Waiting for a message from the service on $callBackSocket (PID $callBackPID)";

   146 			wait -n $callBackPID;

   147 		fi

   148 	else

   149 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

   150 	fi

   151 }

   153 # Environment: server

   154 # User: root

   155 # Should be started as a systemd/init service.

   156 # - reads messages from from the subvolume socket – message contains the relative directory path

   157 # - creates a subvolume for given repository + necesary parent directories

   158 # - sends a message to the clone service → start cloning into the created subvolume

   159 vcs_backup_public_serverStartSubvolumeService() {

   160 	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do

   161 		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");

   162 		if [[ -e "$VCS_BACKUP_CURRENT_DIR/$d" ]]; then

   163 			callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";

   164 			if [[ -e "$callBackSocket" ]]; then

   165 				echo "alreadyDone" | socat -u - unix-send:"$callBackSocket";

   166 			fi

   167 		else

   168 			btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \

   169 			echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \

   170 			chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \

   171 			echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};

   172 		fi

   173 	done

   174 }

   176 # Environment: server

   177 # User: $VCS_BACKUP_USER

   178 # should be started as a systemd/init service

   179 vcs_backup_public_serverStartCloneService() {

   180 	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do

   181 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   182 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

   184 		if isValidTypeAndURL "$vcsType" "$url"; then

   185 			if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

   186 			elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

   187 			fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";

   188 		else

   189 			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;

   190 		fi

   192 		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";

   193 		if [[ -e "$callBackSocket" ]]; then

   194 			echo "done" | socat -u - unix-send:"$callBackSocket";

   195 		fi

   196 	done

   197 }

   199 # Environment: client

   200 # prints list of repositories in recfile format

   201 # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular

   202 vcs_backup_public_clientListRepositories() {

   203 	loadConfigFile ~/.config/vcs-backup/client.cfg;

   204 	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;

   205 }

   207 # Environment: server

   208 # User: $VCS_BACKUP_MANAGER

   209 vcs_backup_public_serverListRepositories() {

   210 	printRecfileKeyValue "%rec"  "repository";

   211 	printRecfileKeyValue "%type" "bytes int";

   212 	printRecfileKeyValue "%type" "public bool";

   213 	echo;

   215 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do

   216 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

   217 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   218 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   219 		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);

   220 		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";

   222 		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);

   223 		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai");

   224 		else lastCommit=""; fi

   226 		printRecfileKeyValue "type"            "$vcsType";

   227 		printRecfileKeyValue "url"             "$url";

   228 		printRecfileKeyValue "state"           "$state";

   229 		printRecfileKeyValue "public"          "$public";

   230 		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";

   231 		printRecfileKeyValue "size"            "$sizeBytes";

   232 		printRecfileKeyValue "lastCommit"      "$lastCommit";

   233 		echo;

   234 	done

   235 }

   237 # Environment: server

   238 # User: $VCS_BACKUP_USER

   239 # should be called from cron (usually every day)

   240 vcs_backup_public_serverPullCronTask() {

   241 	printRecfileKeyValue "%rec"  "pull";

   242 	printRecfileKeyValue "%type" "started date";

   243 	printRecfileKeyValue "%type" "finished date";

   244 	printRecfileKeyValue "%type" "duration int";

   245 	printRecfileKeyValue "%type" "resultCode int";

   247 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do

   248 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   249 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   250 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";

   252 		pullStarted=$(date --iso-8601=s);

   253 		pullStartedMiliseconds=$(($(date +%s%N)/1000000));

   254 		pullFinished="";

   255 		pullDuration="";

   256 		pullResult="";

   257 		pullResultCode="";

   258 		if [[ "$state" == "cloned" ]]; then

   259 			if   [[ "$vcsType" == "hg" ]];  then pullResult=$(hg pull --force --repository "$absolutePath" 2>&1); pullResultCode=$?;

   260 			elif [[ "$vcsType" == "git" ]]; then pullResult=$(git -C "$absolutePath" fetch 2>&1); pullResultCode=$?;

   261 			fi

   262 			pullFinished=$(date --iso-8601=s);

   263 			pullFinishedMiliseconds=$(($(date +%s%N)/1000000));

   264 			pullDuration=$(( $pullFinishedMiliseconds - $pullStartedMiliseconds ));

   265 		fi

   267 		printRecfileKeyValue "serverPath"      "$absolutePath";

   268 		printRecfileKeyValue "type"            "$vcsType";

   269 		printRecfileKeyValue "state"           "$state";

   270 		printRecfileKeyValue "started"         "$pullStarted";

   271 		printRecfileKeyValue "finished"        "$pullFinished";

   272 		printRecfileKeyValue "duration"        "$pullDuration";

   273 		printRecfileKeyValue "resultCode"      "$pullResultCode";

   274 		printRecfileKeyValue "message"         "$pullResult";

   275 		echo;

   276 	done

   277 }

   279 # Environment: server

   280 # User: root

   281 # should be called from cron (usually every day) after Pull (see above)

   282 vcs_backup_public_serverSnapshotCronTask() {

   283 	printRecfileKeyValue "%rec"  "snapshot";

   284 	printRecfileKeyValue "%type" "started date";

   285 	printRecfileKeyValue "%type" "finished date";

   286 	printRecfileKeyValue "%type" "duration int";

   287 	printRecfileKeyValue "%type" "resultCode int";

   289 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do

   290 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   291 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   292 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";

   294 		started=$(date --iso-8601=s);

   295 		startedMiliseconds=$(($(date +%s%N)/1000000));

   296 		finished="";

   297 		duration="";

   298 		result="";

   299 		resultCode="";

   300 		snapshotPath="";

   301 		if [[ "$state" == "cloned" ]]; then

   302 			snapshotPath="$VCS_BACKUP_SNAPSHOT_DIR/$d/$(date --iso-8601=date)";

   303 			mkdir -p $(dirname "$snapshotPath");

   304 			result=$(btrfs subvolume snapshot -r "$absolutePath" "$snapshotPath" 2>&1);

   305 			resultCode=$?;

   306 			finished=$(date --iso-8601=s);

   307 			finishedMiliseconds=$(($(date +%s%N)/1000000));

   308 			duration=$(( $finishedMiliseconds - $startedMiliseconds ));

   309 		fi

   311 		printRecfileKeyValue "currentPath"     "$absolutePath";

   312 		printRecfileKeyValue "snapshotPath"    "$snapshotPath";

   313 		printRecfileKeyValue "type"            "$vcsType";

   314 		printRecfileKeyValue "state"           "$state";

   315 		printRecfileKeyValue "started"         "$started";

   316 		printRecfileKeyValue "finished"        "$finished";

   317 		printRecfileKeyValue "duration"        "$duration";

   318 		printRecfileKeyValue "resultCode"      "$resultCode";

   319 		printRecfileKeyValue "message"         "$result";

   320 		echo;

   321 	done

   322 }

   324 # --- Single entry-point: --------------------------------------------------------------------------

   326 PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";

   327 PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";

   328 if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then

   329 	"$PUBLIC_FUNCTION_PREFIX${@:1}";

   330 elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;

   331 elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;

   332 elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;

   333 elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;

   334 else

   335 	echo "Unsupported sub-command: $1" >&2

   336 	echo "Available sub-commands:" >&2

   337 	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2

   338 	exit 1;

   339 fi

author	František Kučera <franta-hg@frantovo.cz>
	Mon, 22 Apr 2019 11:24:43 +0200
branch	v_0
changeset 15	e7279b13a071
parent 14	1e1ba6753d92
child 16	44a8a36ca380
permissions	-rwxr-xr-x