vcs-backup.sh
author František Kučera <franta-hg@frantovo.cz>
Sat, 20 Apr 2019 18:51:34 +0200
branchv_0
changeset 7 ca71e981bdb4
parent 6 308978165cc3
child 8 2c10a7d30ffd
permissions -rwxr-xr-x
list repositories command
     1 #!/bin/bash
     2 
     3 # VCS Backup
     4 # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
     5 #
     6 # This program is free software: you can redistribute it and/or modify
     7 # it under the terms of the GNU General Public License as published by
     8 # the Free Software Foundation, either version 3 of the License, or
     9 # (at your option) any later version.
    10 #
    11 # This program is distributed in the hope that it will be useful,
    12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
    13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
    14 # GNU General Public License for more details.
    15 #
    16 # You should have received a copy of the GNU General Public License
    17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
    18 
    19 
    20 # Server-side configuration:
    21 VCS_BACKUP_DATA_DIR="/mnt/data";
    22 VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";
    23 VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";
    24 VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";
    25 VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";
    26 VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";
    27 VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}
    28 VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";
    29 VCS_BACKUP_USER="vcs-backup";
    30 VCS_BACKUP_MANAGER="vcs-backup-manager";
    31 
    32 # Installation – check and do it by hand:
    33 # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR
    34 installInstructions() {
    35 cp vcs-backup.sh /usr/local/bin/
    36 adduser --disabled-password "$VCS_BACKUP"
    37 adduser --disabled-password "$VCS_BACKUP_MANAGER"
    38 
    39 mkdir "$VCS_BACKUP_CURRENT_DIR";
    40 mkdir "$VCS_BACKUP_CONFIG_DIR";
    41 mkdir "$VCS_BACKUP_SNAPSHOT_DIR";
    42 mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"
    43 
    44 chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"
    45 chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"
    46 }
    47 
    48 
    49 # --- Private functions: ---------------------------------------------------------------------------
    50 
    51 # Environment: all
    52 # $1 = VCS type: hg, git
    53 # $2 = URL
    54 isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }
    55 
    56 # Environment: all
    57 # $1 = path to the config file
    58 loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }
    59 
    60 # Environment: server
    61 # $1 = URL
    62 urlToRelativeDirectoryPath() {
    63 	echo "$1" | sed -E 's@^[^:]+://@@g';
    64 }
    65 
    66 # --- Public interface functions: ------------------------------------------------------------------
    67 
    68 # Environment: client
    69 # $1 = VCS type: hg, git
    70 # $2 = URL
    71 # $3 = "public" or "private" (default), whether the repository should be available through the public web interface
    72 vcs_backup_public_clientSubmitBackupRequest() {
    73 	if isValidTypeAndURL "$1" "$2"; then
    74 		loadConfigFile ~/.config/vcs-backup/client.cfg
    75 		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"
    76 		if [[ "$4" == "clone" ]]; then
    77 			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";
    78 			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";
    79 			fi
    80 		fi
    81 	else
    82 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
    83 	fi
    84 }
    85 
    86 # Environment: server
    87 # User: $VCS_BACKUP_MANAGER
    88 # has same parameters as clientSubmitBackupRequest (see above)
    89 vcs_backup_public_serverSubmitBackupRequest() {
    90 	if isValidTypeAndURL "$1" "$2"; then
    91 		loadConfigFile "/etc/vcs-backup/server.cfg";
    92 		relativePath=$1/$(urlToRelativeDirectoryPath "$2");
    93 		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";
    94 		mkdir -p "$absolutePath";
    95 		echo "$2" > "$absolutePath/url.txt"
    96 		echo "submited" > "$absolutePath/state.txt"
    97 		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"
    98 		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"
    99 		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};
   100 
   101 		if [[ "$3" == "public" ]]; then
   102 			cd "$VCS_BACKUP_PUBLIC_DIR";
   103 			mkdir -p "$(dirname $relativePath)";
   104 			ln -rs "../current/$relativePath" "$(dirname $relativePath)";
   105 		fi
   106 
   107 		if [[ "$4" == "clone" ]]; then
   108 			socat -u "unix-recvfrom:$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET},mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777
   109 				echo "Message from the clone service: $m";
   110 			done;
   111 		fi
   112 	else
   113 		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
   114 	fi
   115 }
   116 
   117 # Environment: server
   118 # User: root
   119 # Should be started as a systemd/init service.
   120 # - reads messages from from the subvolume socket – message contains the relative directory path
   121 # - creates a subvolume for given repository + necesary parent directories
   122 # - sends a message to the clone service → start cloning into the created subvolume
   123 vcs_backup_public_serverStartSubvolumeService() {
   124 	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do
   125 		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");
   126 		btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \
   127 		echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \
   128 		chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \
   129 		echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};
   130 	done
   131 }
   132 
   133 # Environment: server
   134 # User: $VCS_BACKUP_USER
   135 # should be started as a systemd/init service
   136 vcs_backup_public_serverStartCloneService() {
   137 	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do
   138 		vcsType=$(echo "$d" | sed 's@/.*@@g');
   139 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
   140 
   141 		if isValidTypeAndURL "$vcsType" "$url"; then
   142 			if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U     "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
   143 			elif [[ "$vcsType" == "git" ]]; then git clone --bare "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
   144 			fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";
   145 		else
   146 			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;
   147 		fi
   148 
   149 		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";
   150 		if [[ -e "$callBackSocket" ]]; then
   151 			echo "done" | socat -u - unix-send:"$callBackSocket";
   152 		fi
   153 	done
   154 }
   155 
   156 # Environment: client
   157 # prints list of repositories in recfile format
   158 # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular
   159 vcs_backup_public_clientListRepositories() {
   160 	loadConfigFile ~/.config/vcs-backup/client.cfg;
   161 	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;
   162 }
   163 
   164 # Environment: server
   165 # User: $VCS_BACKUP_MANAGER
   166 vcs_backup_public_serverListRepositories() {
   167 	echo "%rec: repositories";
   168 	echo "%type: bytes int";
   169 	echo "%type: public bool";
   170 	echo;
   171 
   172 	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do
   173 		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
   174 		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
   175 		vcsType=$(echo "$d" | sed 's@/.*@@g');
   176 		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);
   177 		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";
   178 		
   179 		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);
   180 		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai"); 
   181 		else lastCommit=""; fi
   182 		
   183 		echo "type: $vcsType";
   184 		echo "url: $url";
   185 		echo "state: $state";
   186 		echo "public: $public";
   187 		echo "serverPath: $VCS_BACKUP_CURRENT_DIR/$d";
   188 		echo "size: $sizeBytes";
   189 		echo "lastCommit: $lastCommit";
   190 		echo;
   191 	done
   192 }
   193 
   194 # Environment: server
   195 # User: $VCS_BACKUP_USER
   196 # should be called from cron (usually every day)
   197 vcs_backup_public_serverPullCronTask() {
   198 	return;
   199 }
   200 
   201 # Environment: server
   202 # User: $VCS_BACKUP_USER
   203 # should be called from cron (usually every day) after Pull (see above)
   204 vcs_backup_public_serverSnapshotCronTask() {
   205 	return;
   206 }
   207 
   208 # --- Single entry-point: --------------------------------------------------------------------------
   209 
   210 PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
   211 PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";
   212 if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then
   213 	"$PUBLIC_FUNCTION_PREFIX${@:1}";
   214 elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;
   215 elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;
   216 elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;
   217 elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;
   218 else
   219 	echo "Unsupported sub-command: $1" >&2
   220 	echo "Available sub-commands:" >&2
   221 	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2
   222 	exit 1;
   223 fi