vcs-backup: vcs-backup.sh@66728b983b41

     1 #!/bin/bash

     3 # VCS Backup

     4 # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)

5 #

     6 # This program is free software: you can redistribute it and/or modify

     7 # it under the terms of the GNU General Public License as published by

     8 # the Free Software Foundation, either version 3 of the License, or

     9 # (at your option) any later version.

    10 #

    11 # This program is distributed in the hope that it will be useful,

    12 # but WITHOUT ANY WARRANTY; without even the implied warranty of

    13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

    14 # GNU General Public License for more details.

    15 #

    16 # You should have received a copy of the GNU General Public License

    17 # along with this program. If not, see <http://www.gnu.org/licenses/>.

    20 # Server-side configuration:

    21 VCS_BACKUP_DATA_DIR="/mnt/data";

    22 VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";

    23 VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";

    24 VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";

    25 VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";

    26 VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";

    27 VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}

    28 VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";

    29 VCS_BACKUP_USER="vcs-backup";

    30 VCS_BACKUP_MANAGER="vcs-backup-manager";

    32 # Installation – check and do it by hand:

    33 # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR

    34 installInstructions() {

    35 cp vcs-backup.sh /usr/local/bin/

    36 adduser --disabled-password "$VCS_BACKUP"

    37 adduser --disabled-password "$VCS_BACKUP_MANAGER"

    39 mkdir "$VCS_BACKUP_CURRENT_DIR";

    40 mkdir "$VCS_BACKUP_CONFIG_DIR";

    41 mkdir "$VCS_BACKUP_SNAPSHOT_DIR";

    42 mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"

    44 chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"

    45 chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"

    46 }

    49 # --- Private functions: ---------------------------------------------------------------------------

    51 # Environment: all

    52 # $1 = VCS type: hg, git

    53 # $2 = URL

    54 isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }

    56 # Environment: all

    57 # $1 = path to the config file

    58 loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }

    60 # Environment: server

    61 # $1 = URL

    62 urlToRelativeDirectoryPath() {

    63 	echo "$1" | sed -E 's@^[^:]+://@@g';

    64 }

    66 # Environment: all

    67 # $1 = optional value (if missing, reads STDIN)

    68 escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }

    70 # Environment: all

    71 # $1 = key

    72 # $2 = value

    73 printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }

    75 # --- Public interface functions: ------------------------------------------------------------------

    77 # Environment: client

    78 # $1 = VCS type: hg, git

    79 # $2 = URL

    80 # $3 = "public" or "private" (default), whether the repository should be available through the public web interface

    81 vcs_backup_public_clientSubmitBackupRequest() {

    82 	if isValidTypeAndURL "$1" "$2"; then

    83 		loadConfigFile ~/.config/vcs-backup/client.cfg

    84 		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"

    85 		if [[ "$4" == "clone" ]]; then

    86 			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";

    87 			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";

    88 			fi

    89 		fi

    90 	else

    91 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

    92 	fi

    93 }

    95 # Environment: server

    96 # User: $VCS_BACKUP_MANAGER

    97 # has same parameters as clientSubmitBackupRequest (see above)

    98 vcs_backup_public_serverSubmitBackupRequest() {

    99 	if isValidTypeAndURL "$1" "$2"; then

   100 		loadConfigFile "/etc/vcs-backup/server.cfg";

   101 		relativePath=$1/$(urlToRelativeDirectoryPath "$2");

   102 		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";

   103 		mkdir -p "$absolutePath";

   104 		echo "$2" > "$absolutePath/url.txt"

   105 		echo "submited" > "$absolutePath/state.txt"

   106 		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"

   107 		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"

   108 		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};

   110 		if [[ "$3" == "public" ]]; then

   111 			cd "$VCS_BACKUP_PUBLIC_DIR";

   112 			mkdir -p "$(dirname $relativePath)";

   113 			ln -rs "../current/$relativePath" "$(dirname $relativePath)";

   114 		fi

   116 		if [[ "$4" == "clone" ]]; then

   117 			socat -u "unix-recvfrom:$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET},mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777

   118 				echo "Message from the clone service: $m";

   119 			done;

   120 		fi

   121 	else

   122 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;

   123 	fi

   124 }

   126 # Environment: server

   127 # User: root

   128 # Should be started as a systemd/init service.

   129 # - reads messages from from the subvolume socket – message contains the relative directory path

   130 # - creates a subvolume for given repository + necesary parent directories

   131 # - sends a message to the clone service → start cloning into the created subvolume

   132 vcs_backup_public_serverStartSubvolumeService() {

   133 	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do

   134 		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");

   135 		btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \

   136 		echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \

   137 		chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \

   138 		echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};

   139 	done

   140 }

   142 # Environment: server

   143 # User: $VCS_BACKUP_USER

   144 # should be started as a systemd/init service

   145 vcs_backup_public_serverStartCloneService() {

   146 	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do

   147 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   148 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

   150 		if isValidTypeAndURL "$vcsType" "$url"; then

   151 			if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

   152 			elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";

   153 			fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";

   154 		else

   155 			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;

   156 		fi

   158 		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";

   159 		if [[ -e "$callBackSocket" ]]; then

   160 			echo "done" | socat -u - unix-send:"$callBackSocket";

   161 		fi

   162 	done

   163 }

   165 # Environment: client

   166 # prints list of repositories in recfile format

   167 # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular

   168 vcs_backup_public_clientListRepositories() {

   169 	loadConfigFile ~/.config/vcs-backup/client.cfg;

   170 	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;

   171 }

   173 # Environment: server

   174 # User: $VCS_BACKUP_MANAGER

   175 vcs_backup_public_serverListRepositories() {

   176 	echo "%rec: repositories";

   177 	echo "%type: bytes int";

   178 	echo "%type: public bool";

   179 	echo;

   181 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do

   182 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");

   183 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");

   184 		vcsType=$(echo "$d" | sed 's@/.*@@g');

   185 		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);

   186 		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";

   188 		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);

   189 		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai");

   190 		else lastCommit=""; fi

   192 		printRecfileKeyValue "type"            "$vcsType";

   193 		printRecfileKeyValue "url"             "$url";

   194 		printRecfileKeyValue "state"           "$state";

   195 		printRecfileKeyValue "public"          "$public";

   196 		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";

   197 		printRecfileKeyValue "size"            "$sizeBytes";

   198 		printRecfileKeyValue "lastCommit"      "$lastCommit";

   199 		echo;

   200 	done

   201 }

   203 # Environment: server

   204 # User: $VCS_BACKUP_USER

   205 # should be called from cron (usually every day)

   206 vcs_backup_public_serverPullCronTask() {

   207 	return;

   208 }

   210 # Environment: server

   211 # User: $VCS_BACKUP_USER

   212 # should be called from cron (usually every day) after Pull (see above)

   213 vcs_backup_public_serverSnapshotCronTask() {

   214 	return;

   215 }

   217 # --- Single entry-point: --------------------------------------------------------------------------

   219 PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";

   220 PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";

   221 if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then

   222 	"$PUBLIC_FUNCTION_PREFIX${@:1}";

   223 elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;

   224 elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;

   225 elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;

   226 elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;

   227 else

   228 	echo "Unsupported sub-command: $1" >&2

   229 	echo "Available sub-commands:" >&2

   230 	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2

   231 	exit 1;

   232 fi

author	František Kučera <franta-hg@frantovo.cz>
	Sat, 20 Apr 2019 22:17:38 +0200
branch	v_0
changeset 9	66728b983b41
parent 8	2c10a7d30ffd
child 10	5454e8a0c53e
permissions	-rwxr-xr-x