vcs-backup.sh
author František Kučera <franta-hg@frantovo.cz>
Sun, 28 Apr 2019 16:03:30 +0200
branchv_0
changeset 22 b2cb46ae50f7
parent 21 735961d9596e
child 23 1222714ae08f
permissions -rwxr-xr-x
apache2-httpd: alias for robots.txt
     1 #!/bin/bash
     2 
     3 # VCS Backup
     4 # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
     5 #
     6 # This program is free software: you can redistribute it and/or modify
     7 # it under the terms of the GNU General Public License as published by
     8 # the Free Software Foundation, either version 3 of the License, or
     9 # (at your option) any later version.
    10 #
    11 # This program is distributed in the hope that it will be useful,
    12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
    13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    14 # GNU General Public License for more details.
    15 #
    16 # You should have received a copy of the GNU General Public License
    17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
    18 
    19 
    20 # VCS Backup is a configuration for setting up VCS (version control system) mirrors.
    21 # Currently Mercurial (Hg) and Git are supported.
    22 # Features:
    23 #  - mirrors remote repositories
    24 #  - creates Btrfs subvolume for each repository
    25 #  - does periodic pull to keep mirrors up to date
    26 #  - does periodic Btrfs snapshot to keep history (git push --force done on the remote repository will lead to modifications or deletions in our current mirror, but previous versions will be kept in the snapshots)
    27 #  - provides web interface for remote clonning of our mirrors (see systemd and etc folders)
    28 #  - can be controlled over SSH by a sane person / owner of the system
    29 #  - provides reports in the recfile format (to be processed using GNU Recutils or Relational pipes):
    30 #     - list of repositories/mirrors
    31 #     - results of pull operations
    32 
    33 
    34 # This is an asynchronous message-driven shell script that runs distributed across two machines and four user accounts. You have been warned :-)
    35 
    36 
    37 # Server-side configuration: / loadServerConfigFile()
    38 VCS_BACKUP_DATA_DIR="/mnt/data";
    39 VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";
    40 VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";
    41 VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";
    42 VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";
    43 VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";
    44 VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}
    45 VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";
    46 VCS_BACKUP_USER="vcs-backup";
    47 VCS_BACKUP_MANAGER="vcs-backup-manager";
    48 
    49 # Client-side configuration: / see loadClientConfigFile()
    50 VCS_BACKUP_SERVER="$VCS_BACKUP_MANAGER@localhost";
    51 VCS_BACKUP_SSH_COMMAND=(ssh "$VCS_BACKUP_SERVER");
    52 
    53 # Installation – check and do it by hand:
    54 # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR
    55 installInstructions() {
    56 cp vcs-backup.sh /usr/local/bin/
    57 adduser --disabled-password "$VCS_BACKUP"
    58 adduser --disabled-password "$VCS_BACKUP_MANAGER"
    59 
    60 mkdir "$VCS_BACKUP_CURRENT_DIR";
    61 mkdir "$VCS_BACKUP_CONFIG_DIR";
    62 mkdir "$VCS_BACKUP_SNAPSHOT_DIR";
    63 mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"
    64 
    65 chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"
    66 chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"
    67 }
    68 
    69 
    70 # --- Private functions: ---------------------------------------------------------------------------
    71 
    72 # Environment: all
    73 # $1 = VCS type: hg, git
    74 # $2 = URL
    75 isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }
    76 
    77 # Environment: all
    78 # $1 = path to the config file
    79 loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }
    80 loadClientConfigFile() { loadConfigFile ~/.config/vcs-backup/client.cfg; }
    81 loadServerConfigFile() { loadConfigFile "/etc/vcs-backup/server.cfg"; }
    82 
    83 # Environment: server
    84 # $1 = URL
    85 urlToRelativeDirectoryPath() {
    86 	echo "$1" | sed -E 's@^[^:]+://@@g';
    87 }
    88 
    89 # Environment: server
    90 # Lists relative paths (starting wiht hg or git) of all configured repositories
    91 allRepositories() { find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs --no-run-if-empty dirname; }
    92 
    93 # Environment: all
    94 # $1 = optional value (if missing, reads STDIN)
    95 escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }
    96 
    97 # Environment: all
    98 # $1 = key
    99 # $2 = value
   100 printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }
   101 
   102 # --- Public interface functions: ------------------------------------------------------------------
   103 
   104 # Environment: client
   105 # $1 = VCS type: hg, git
   106 # $2 = URL
   107 # $3 = "public" or "private" (default), whether the repository should be available through the public web interface
   108 # $4 = "clone" (optional), if present, will also clone the backup locally
   109 vcs_backup_public_clientSubmitBackupRequest() {
   110 	if isValidTypeAndURL "$1" "$2"; then
   111 		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"
   112 		if [[ "$4" == "clone" ]]; then
   113 			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";
   114 			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)";
   115 			fi
   116 		fi
   117 	else
   118 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
   119 	fi
   120 }
   121 
   122 # Environment: server
   123 # User: $VCS_BACKUP_MANAGER
   124 # has same parameters as clientSubmitBackupRequest (see above)
   125 vcs_backup_public_serverSubmitBackupRequest() {
   126 	if isValidTypeAndURL "$1" "$2"; then
   127 		relativePath=$1/$(urlToRelativeDirectoryPath "$2");
   128 		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";
   129 		# TODO: stop if directory already exists / only add public link?
   130 		mkdir -p "$absolutePath";
   131 		echo "$2" > "$absolutePath/url.txt"
   132 		echo "submited" > "$absolutePath/state.txt"
   133 		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"
   134 		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"
   135 
   136 		if [[ "$3" == "public" ]]; then
   137 			cd "$VCS_BACKUP_PUBLIC_DIR";
   138 			mkdir -p "$(dirname $relativePath)";
   139 			ln -rsf "../current/$relativePath" "$(dirname $relativePath)";
   140 		elif [[ "$3" == "private" && -e "$VCS_BACKUP_PUBLIC_DIR/$relativePath" ]]; then
   141 			rm "$VCS_BACKUP_PUBLIC_DIR/$relativePath";
   142 		fi
   143 
   144 		if [[ "$4" == "clone" ]]; then
   145 			callBackSocket=$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET}
   146 			socat -u "unix-recvfrom:$callBackSocket,mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777 ?
   147 				echo "Message from the service: $m";
   148 			done &
   149 			callBackPID=$!;
   150 		fi
   151 
   152 		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};
   153 
   154 		if [[ "$4" == "clone" ]]; then
   155 			echo "Waiting for a message from the service on $callBackSocket (PID $callBackPID)";
   156 			wait -n $callBackPID;
   157 		fi
   158 	else
   159 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
   160 	fi
   161 }
   162 
   163 # Environment: server
   164 # User: root
   165 # Should be started as a systemd/init service.
   166 # - reads messages from from the subvolume socket – message contains the relative directory path
   167 # - creates a subvolume for given repository + necesary parent directories
   168 # - sends a message to the clone service → start cloning into the created subvolume
   169 vcs_backup_public_serverStartSubvolumeService() {
   170 	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do
   171 		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");
   172 		if [[ -e "$VCS_BACKUP_CURRENT_DIR/$d" ]]; then
   173 			echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";
   174 		else
   175 			btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \
   176 			echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \
   177 			chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d"
   178 		fi
   179 		echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};
   180 	done
   181 }
   182 
   183 # Environment: server
   184 # User: $VCS_BACKUP_USER
   185 # should be started as a systemd/init service
   186 vcs_backup_public_serverStartCloneService() {
   187 	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do
   188 		vcsType=$(echo "$d" | sed 's@/.*@@g');
   189 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
   190 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
   191 
   192 		if isValidTypeAndURL "$vcsType" "$url"; then
   193 			if [[ "$state" == "cloned"  ]]; then
   194 				# Already cloned repository → just pull
   195 				if   [[ "$vcsType" == "hg"  ]]; then hg -R "$VCS_BACKUP_CURRENT_DIR/$d" pull;
   196 				elif [[ "$vcsType" == "git" ]]; then git -C "$VCS_BACKUP_CURRENT_DIR/$d" fetch;
   197 				fi
   198 			else
   199 				# New repository → clone
   200 				if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
   201 				elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
   202 				fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";
   203 			fi
   204 		else
   205 			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;
   206 		fi
   207 
   208 		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";
   209 		if [[ -e "$callBackSocket" ]]; then
   210 			echo "done" | socat -u - unix-send:"$callBackSocket";
   211 		fi
   212 	done
   213 }
   214 
   215 # Environment: client
   216 # prints list of repositories in recfile format
   217 # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular
   218 vcs_backup_public_clientListRepositories() {
   219 	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;
   220 }
   221 
   222 # Environment: server
   223 # User: $VCS_BACKUP_MANAGER
   224 vcs_backup_public_serverListRepositories() {
   225 	printRecfileKeyValue "%rec"  "repository";
   226 	printRecfileKeyValue "%type" "bytes int";
   227 	printRecfileKeyValue "%type" "public bool";
   228 	echo;
   229 
   230 	allRepositories | while read d; do
   231 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
   232 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
   233 		vcsType=$(echo "$d" | sed 's@/.*@@g');
   234 		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);
   235 		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";
   236 		
   237 		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);
   238 		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai"); 
   239 		else lastCommit=""; fi
   240 		
   241 		printRecfileKeyValue "type"            "$vcsType";
   242 		printRecfileKeyValue "url"             "$url";
   243 		printRecfileKeyValue "state"           "$state";
   244 		printRecfileKeyValue "public"          "$public";
   245 		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";
   246 		printRecfileKeyValue "size"            "$sizeBytes";
   247 		printRecfileKeyValue "lastCommit"      "$lastCommit";
   248 		echo;
   249 	done
   250 }
   251 
   252 # Environment: server
   253 # User: $VCS_BACKUP_USER
   254 # should be called from cron (usually every day)
   255 vcs_backup_public_serverPullCronTask() {
   256 	printRecfileKeyValue "%rec"  "pull";
   257 	printRecfileKeyValue "%type" "started date";
   258 	printRecfileKeyValue "%type" "finished date";
   259 	printRecfileKeyValue "%type" "duration int";
   260 	printRecfileKeyValue "%type" "resultCode int";
   261 
   262 	allRepositories | while read d; do
   263 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
   264 		vcsType=$(echo "$d" | sed 's@/.*@@g');
   265 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";
   266 
   267 		pullStarted=$(date --iso-8601=s);
   268 		pullStartedMiliseconds=$(($(date +%s%N)/1000000));
   269 		pullFinished="";
   270 		pullDuration="";
   271 		pullResult="";
   272 		pullResultCode="";
   273 		if [[ "$state" == "cloned" ]]; then
   274 			if   [[ "$vcsType" == "hg" ]];  then pullResult=$(hg pull --force --repository "$absolutePath" 2>&1); pullResultCode=$?;
   275 			elif [[ "$vcsType" == "git" ]]; then pullResult=$(git -C "$absolutePath" fetch 2>&1); pullResultCode=$?;
   276 			fi
   277 			pullFinished=$(date --iso-8601=s);
   278 			pullFinishedMiliseconds=$(($(date +%s%N)/1000000));
   279 			pullDuration=$(( $pullFinishedMiliseconds - $pullStartedMiliseconds ));
   280 		fi
   281 
   282 		printRecfileKeyValue "type"            "$vcsType";
   283 		printRecfileKeyValue "serverPath"      "$absolutePath";
   284 		printRecfileKeyValue "state"           "$state";
   285 		printRecfileKeyValue "started"         "$pullStarted";
   286 		printRecfileKeyValue "finished"        "$pullFinished";
   287 		printRecfileKeyValue "duration"        "$pullDuration";
   288 		printRecfileKeyValue "resultCode"      "$pullResultCode";
   289 		printRecfileKeyValue "message"         "$pullResult";
   290 		echo;
   291 	done
   292 }
   293 
   294 # Environment: server
   295 # User: root
   296 # should be called from cron (usually every day) after Pull (see above)
   297 vcs_backup_public_serverSnapshotCronTask() {
   298 	printRecfileKeyValue "%rec"  "snapshot";
   299 	printRecfileKeyValue "%type" "started date";
   300 	printRecfileKeyValue "%type" "finished date";
   301 	printRecfileKeyValue "%type" "duration int";
   302 	printRecfileKeyValue "%type" "resultCode int";
   303 	
   304 	allRepositories | while read d; do
   305 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
   306 		vcsType=$(echo "$d" | sed 's@/.*@@g');
   307 		absolutePath="$VCS_BACKUP_CURRENT_DIR/$d";
   308 
   309 		started=$(date --iso-8601=s);
   310 		startedMiliseconds=$(($(date +%s%N)/1000000));
   311 		finished="";
   312 		duration="";
   313 		result="";
   314 		resultCode="";
   315 		snapshotPath="";
   316 		if [[ "$state" == "cloned" ]]; then
   317 			snapshotPath="$VCS_BACKUP_SNAPSHOT_DIR/$d/$(date --iso-8601=date)";
   318 			mkdir -p $(dirname "$snapshotPath");
   319 			result=$(btrfs subvolume snapshot -r "$absolutePath" "$snapshotPath" 2>&1);
   320 			resultCode=$?;
   321 			finished=$(date --iso-8601=s);
   322 			finishedMiliseconds=$(($(date +%s%N)/1000000));
   323 			duration=$(( $finishedMiliseconds - $startedMiliseconds ));
   324 		fi
   325 
   326 		printRecfileKeyValue "currentPath"     "$absolutePath";
   327 		printRecfileKeyValue "snapshotPath"    "$snapshotPath";
   328 		printRecfileKeyValue "type"            "$vcsType";
   329 		printRecfileKeyValue "state"           "$state";
   330 		printRecfileKeyValue "started"         "$started";
   331 		printRecfileKeyValue "finished"        "$finished";
   332 		printRecfileKeyValue "duration"        "$duration";
   333 		printRecfileKeyValue "resultCode"      "$resultCode";
   334 		printRecfileKeyValue "message"         "$result";
   335 		echo;
   336 	done
   337 }
   338 
   339 # --- Single entry-point: --------------------------------------------------------------------------
   340 
   341 loadClientConfigFile;
   342 loadServerConfigFile;
   343 PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
   344 PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";
   345 if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then
   346 	"$PUBLIC_FUNCTION_PREFIX${@:1}";
   347 elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;
   348 elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;
   349 elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;
   350 elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;
   351 else
   352 	echo "Unsupported sub-command: $1" >&2
   353 	echo "Available sub-commands:" >&2
   354 	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2
   355 	exit 1;
   356 fi