vcs-backup: vcs-backup.sh@b2cb46ae50f7

     1 #!/bin/bash

     3 # VCS Backup

     4 # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)

5 #

     6 # This program is free software: you can redistribute it and/or modify

     7 # it under the terms of the GNU General Public License as published by

     8 # the Free Software Foundation, either version 3 of the License, or

     9 # (at your option) any later version.

    10 #

    11 # This program is distributed in the hope that it will be useful,

    12 # but WITHOUT ANY WARRANTY; without even the implied warranty of

    13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

    14 # GNU General Public License for more details.

    15 #

    16 # You should have received a copy of the GNU General Public License

    17 # along with this program. If not, see <http://www.gnu.org/licenses/>.

    20 # VCS Backup is a configuration for setting up VCS (version control system) mirrors.

    21 # Currently Mercurial (Hg) and Git are supported.

    22 # Features:

    23 #  - mirrors remote repositories

    24 #  - creates Btrfs subvolume for each repository

    25 #  - does periodic pull to keep mirrors up to date

    26 #  - does periodic Btrfs snapshot to keep history (git push --force done on the remote repository will lead to modifications or deletions in our current mirror, but previous versions will be kept in the snapshots)

    27 #  - provides web interface for remote clonning of our mirrors (see systemd and etc folders)

    28 #  - can be controlled over SSH by a sane person / owner of the system

    29 #  - provides reports in the recfile format (to be processed using GNU Recutils or Relational pipes):

    30 #     - list of repositories/mirrors

    31 #     - results of pull operations

    34 # This is an asynchronous message-driven shell script that runs distributed across two machines and four user accounts. You have been warned :-)

    37 # Server-side configuration: / loadServerConfigFile()

    38 VCS_BACKUP_DATA_DIR="/mnt/data";

    39 VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";

    40 VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";

    41 VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";

    42 VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";

    43 VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";

    44 VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}

    45 VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";

    46 VCS_BACKUP_USER="vcs-backup";

    47 VCS_BACKUP_MANAGER="vcs-backup-manager";

    49 # Client-side configuration: / see loadClientConfigFile()

    50 VCS_BACKUP_SERVER="$VCS_BACKUP_MANAGER@localhost";

    51 VCS_BACKUP_SSH_COMMAND=(ssh "$VCS_BACKUP_SERVER");

    53 # Installation – check and do it by hand:

    54 # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR

    55 installInstructions() {

    56 cp vcs-backup.sh /usr/local/bin/

    57 adduser --disabled-password "$VCS_BACKUP"

    58 adduser --disabled-password "$VCS_BACKUP_MANAGER"

    60 mkdir "$VCS_BACKUP_CURRENT_DIR";

    61 mkdir "$VCS_BACKUP_CONFIG_DIR";

    62 mkdir "$VCS_BACKUP_SNAPSHOT_DIR";

    63 mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"

    65 chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"

    66 chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"

    67 }

    70 # --- Private functions: ---------------------------------------------------------------------------

    72 # Environment: all

    73 # $1 = VCS type: hg, git

    74 # $2 = URL

    75 isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }

    77 # Environment: all

    78 # $1 = path to the config file

    79 loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }

    80 loadClientConfigFile() { loadConfigFile ~/.config/vcs-backup/client.cfg; }

    81 loadServerConfigFile() { loadConfigFile "/etc/vcs-backup/server.cfg"; }

    83 # Environment: server

    84 # $1 = URL

    85 urlToRelativeDirectoryPath() {

    86 	echo "$1" | sed -E 's@^[^:]+://@@g';

    87 }

    89 # Environment: server

    90 # Lists relative paths (starting wiht hg or git) of all configured repositories

    91 allRepositories() { find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs --no-run-if-empty dirname; }

    93 # Environment: all

    94 # $1 = optional value (if missing, reads STDIN)

    95 escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }

    97 # Environment: all

    98 # $1 = key

    99 # $2 = value

   100 printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }

   102 # --- Public interface functions: ------------------------------------------------------------------

   104 # Environment: client

   105 # $1 = VCS type: hg, git

   106 # $2 = URL

   107 # $3 = "public" or "private" (default), whether the repository should be available through the public web interface

   108 # $4 = "clone" (optional), if present, will also clone the backup locally

   109 vcs_backup_public_clientSubmitBackupRequest() {

   110 	if isValidTypeAndURL "$1" "$2"; then

   111 		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"

   112 		if [[ "$4" == "clone" ]]; then

   113 			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";

   114 			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";

   115 			fi

   116 		fi

   117 	else

   118 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

   119 	fi

   120 }

   122 # Environment: server

   123 # User: $VCS_BACKUP_MANAGER

   124 # has same parameters as clientSubmitBackupRequest (see above)

   125 vcs_backup_public_serverSubmitBackupRequest() {

   126 	if isValidTypeAndURL "$1" "$2"; then

   127 		relativePath=$1/$(urlToRelativeDirectoryPath "$2");

   128 		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";

   129 		# TODO: stop if directory already exists / only add public link?

   130 		mkdir -p "$absolutePath";

   131 		echo "$2" > "$absolutePath/url.txt"

   132 		echo "submited" > "$absolutePath/state.txt"

   133 		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"

   134 		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"

   136 		if [[ "$3" == "public" ]]; then

   137 			cd "$VCS_BACKUP_PUBLIC_DIR";

   138 			mkdir -p "$(dirname $relativePath)";

   139 			ln -rsf "../current/$relativePath" "$(dirname $relativePath)";

   140 		elif [[ "$3" == "private" && -e "$VCS_BACKUP_PUBLIC_DIR/$relativePath" ]]; then

   141 			rm "$VCS_BACKUP_PUBLIC_DIR/$relativePath";

   142 		fi

   144 		if [[ "$4" == "clone" ]]; then

   145 			callBackSocket=$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET}

   146 			socat -u "unix-recvfrom:$callBackSocket,mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777 ?

   147 				echo "Message from the service: $m";

   148 			done &

   149 			callBackPID=$!;

   150 		fi

   152 		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};

   154 		if [[ "$4" == "clone" ]]; then

   155 			echo "Waiting for a message from the service on $callBackSocket (PID $callBackPID)";

   156 			wait -n $callBackPID;

   157 		fi

   158 	else

   159 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

   160 	fi

   161 }

   163 # Environment: server

   164 # User: root

   165 # Should be started as a systemd/init service.

   166 # - reads messages from from the subvolume socket – message contains the relative directory path

   167 # - creates a subvolume for given repository + necesary parent directories

   168 # - sends a message to the clone service → start cloning into the created subvolume

   169 vcs_backup_public_serverStartSubvolumeService() {

   170 	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do

   171 		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");

   172 		if [[ -e "$VCS_BACKUP_CURRENT_DIR/$d" ]]; then

   173 			echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";

   174 		else

   175 			btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \

   176 			echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \

   177 			chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d"

   178 		fi

   179 		echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};

   180 	done

   181 }

   183 # Environment: server

   184 # User: $VCS_BACKUP_USER

   185 # should be started as a systemd/init service

   186 vcs_backup_public_serverStartCloneService() {

   187 	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do

   188 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   189 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

   190 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   192 		if isValidTypeAndURL "$vcsType" "$url"; then

   193 			if [[ "$state" == "cloned"  ]]; then

   194 				# Already cloned repository → just pull

   195 				if   [[ "$vcsType" == "hg"  ]]; then hg -R "$VCS_BACKUP_CURRENT_DIR/$d" pull;

   196 				elif [[ "$vcsType" == "git" ]]; then git -C "$VCS_BACKUP_CURRENT_DIR/$d" fetch;

   197 				fi

   198 			else

   199 				# New repository → clone

   200 				if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

   201 				elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

   202 				fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";

   203 			fi

   204 		else

   205 			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;

   206 		fi

   208 		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";

   209 		if [[ -e "$callBackSocket" ]]; then

   210 			echo "done" | socat -u - unix-send:"$callBackSocket";

   211 		fi

   212 	done

   213 }

   215 # Environment: client

   216 # prints list of repositories in recfile format

   217 # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular

   218 vcs_backup_public_clientListRepositories() {

   219 	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;

   220 }

   222 # Environment: server

   223 # User: $VCS_BACKUP_MANAGER

   224 vcs_backup_public_serverListRepositories() {

   225 	printRecfileKeyValue "%rec"  "repository";

   226 	printRecfileKeyValue "%type" "bytes int";

   227 	printRecfileKeyValue "%type" "public bool";

   228 	echo;

   230 	allRepositories | while read d; do

   231 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

   232 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   233 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   234 		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);

   235 		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";

   237 		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);

   238 		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai");

   239 		else lastCommit=""; fi

   241 		printRecfileKeyValue "type"            "$vcsType";

   242 		printRecfileKeyValue "url"             "$url";

   243 		printRecfileKeyValue "state"           "$state";

   244 		printRecfileKeyValue "public"          "$public";

   245 		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";

   246 		printRecfileKeyValue "size"            "$sizeBytes";

   247 		printRecfileKeyValue "lastCommit"      "$lastCommit";

   248 		echo;

   249 	done

   250 }

   252 # Environment: server

   253 # User: $VCS_BACKUP_USER

   254 # should be called from cron (usually every day)

   255 vcs_backup_public_serverPullCronTask() {

   256 	printRecfileKeyValue "%rec"  "pull";

   257 	printRecfileKeyValue "%type" "started date";

   258 	printRecfileKeyValue "%type" "finished date";

   259 	printRecfileKeyValue "%type" "duration int";

   260 	printRecfileKeyValue "%type" "resultCode int";

   262 	allRepositories | while read d; do

   263 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   264 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   265 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";

   267 		pullStarted=$(date --iso-8601=s);

   268 		pullStartedMiliseconds=$(($(date +%s%N)/1000000));

   269 		pullFinished="";

   270 		pullDuration="";

   271 		pullResult="";

   272 		pullResultCode="";

   273 		if [[ "$state" == "cloned" ]]; then

   274 			if   [[ "$vcsType" == "hg" ]];  then pullResult=$(hg pull --force --repository "$absolutePath" 2>&1); pullResultCode=$?;

   275 			elif [[ "$vcsType" == "git" ]]; then pullResult=$(git -C "$absolutePath" fetch 2>&1); pullResultCode=$?;

   276 			fi

   277 			pullFinished=$(date --iso-8601=s);

   278 			pullFinishedMiliseconds=$(($(date +%s%N)/1000000));

   279 			pullDuration=$(( $pullFinishedMiliseconds - $pullStartedMiliseconds ));

   280 		fi

   282 		printRecfileKeyValue "type"            "$vcsType";

   283 		printRecfileKeyValue "serverPath"      "$absolutePath";

   284 		printRecfileKeyValue "state"           "$state";

   285 		printRecfileKeyValue "started"         "$pullStarted";

   286 		printRecfileKeyValue "finished"        "$pullFinished";

   287 		printRecfileKeyValue "duration"        "$pullDuration";

   288 		printRecfileKeyValue "resultCode"      "$pullResultCode";

   289 		printRecfileKeyValue "message"         "$pullResult";

   290 		echo;

   291 	done

   292 }

   294 # Environment: server

   295 # User: root

   296 # should be called from cron (usually every day) after Pull (see above)

   297 vcs_backup_public_serverSnapshotCronTask() {

   298 	printRecfileKeyValue "%rec"  "snapshot";

   299 	printRecfileKeyValue "%type" "started date";

   300 	printRecfileKeyValue "%type" "finished date";

   301 	printRecfileKeyValue "%type" "duration int";

   302 	printRecfileKeyValue "%type" "resultCode int";

   304 	allRepositories | while read d; do

   305 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   306 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   307 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";

   309 		started=$(date --iso-8601=s);

   310 		startedMiliseconds=$(($(date +%s%N)/1000000));

   311 		finished="";

   312 		duration="";

   313 		result="";

   314 		resultCode="";

   315 		snapshotPath="";

   316 		if [[ "$state" == "cloned" ]]; then

   317 			snapshotPath="$VCS_BACKUP_SNAPSHOT_DIR/$d/$(date --iso-8601=date)";

   318 			mkdir -p $(dirname "$snapshotPath");

   319 			result=$(btrfs subvolume snapshot -r "$absolutePath" "$snapshotPath" 2>&1);

   320 			resultCode=$?;

   321 			finished=$(date --iso-8601=s);

   322 			finishedMiliseconds=$(($(date +%s%N)/1000000));

   323 			duration=$(( $finishedMiliseconds - $startedMiliseconds ));

   324 		fi

   326 		printRecfileKeyValue "currentPath"     "$absolutePath";

   327 		printRecfileKeyValue "snapshotPath"    "$snapshotPath";

   328 		printRecfileKeyValue "type"            "$vcsType";

   329 		printRecfileKeyValue "state"           "$state";

   330 		printRecfileKeyValue "started"         "$started";

   331 		printRecfileKeyValue "finished"        "$finished";

   332 		printRecfileKeyValue "duration"        "$duration";

   333 		printRecfileKeyValue "resultCode"      "$resultCode";

   334 		printRecfileKeyValue "message"         "$result";

   335 		echo;

   336 	done

   337 }

   339 # --- Single entry-point: --------------------------------------------------------------------------

   341 loadClientConfigFile;

   342 loadServerConfigFile;

   343 PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";

   344 PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";

   345 if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then

   346 	"$PUBLIC_FUNCTION_PREFIX${@:1}";

   347 elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;

   348 elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;

   349 elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;

   350 elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;

   351 else

   352 	echo "Unsupported sub-command: $1" >&2

   353 	echo "Available sub-commands:" >&2

   354 	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2

   355 	exit 1;

   356 fi

author	František Kučera <franta-hg@frantovo.cz>
	Sun, 28 Apr 2019 16:03:30 +0200
branch	v_0
changeset 22	b2cb46ae50f7
parent 21	735961d9596e
child 23	1222714ae08f
permissions	-rwxr-xr-x