vcs-backup.sh
author František Kučera <franta-hg@frantovo.cz>
Sat, 20 Apr 2019 22:17:38 +0200
branchv_0
changeset 9 66728b983b41
parent 8 2c10a7d30ffd
child 10 5454e8a0c53e
permissions -rwxr-xr-x
recfile multi-line value escaping
franta-hg@0
     1
#!/bin/bash
franta-hg@0
     2
franta-hg@0
     3
# VCS Backup
franta-hg@0
     4
# Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
franta-hg@0
     5
#
franta-hg@0
     6
# This program is free software: you can redistribute it and/or modify
franta-hg@0
     7
# it under the terms of the GNU General Public License as published by
franta-hg@0
     8
# the Free Software Foundation, either version 3 of the License, or
franta-hg@0
     9
# (at your option) any later version.
franta-hg@0
    10
#
franta-hg@0
    11
# This program is distributed in the hope that it will be useful,
franta-hg@0
    12
# but WITHOUT ANY WARRANTY; without even the implied warranty of
franta-hg@0
    13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
franta-hg@0
    14
# GNU General Public License for more details.
franta-hg@0
    15
#
franta-hg@0
    16
# You should have received a copy of the GNU General Public License
franta-hg@0
    17
# along with this program. If not, see <http://www.gnu.org/licenses/>.
franta-hg@0
    18
franta-hg@0
    19
franta-hg@0
    20
# Server-side configuration:
franta-hg@0
    21
VCS_BACKUP_DATA_DIR="/mnt/data";
franta-hg@0
    22
VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";
franta-hg@2
    23
VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";
franta-hg@0
    24
VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";
franta-hg@0
    25
VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";
franta-hg@0
    26
VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";
franta-hg@7
    27
VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}
franta-hg@3
    28
VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";
franta-hg@0
    29
VCS_BACKUP_USER="vcs-backup";
franta-hg@0
    30
VCS_BACKUP_MANAGER="vcs-backup-manager";
franta-hg@0
    31
franta-hg@0
    32
# Installation – check and do it by hand:
franta-hg@0
    33
# There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR
franta-hg@0
    34
installInstructions() {
franta-hg@0
    35
cp vcs-backup.sh /usr/local/bin/
franta-hg@0
    36
adduser --disabled-password "$VCS_BACKUP"
franta-hg@0
    37
adduser --disabled-password "$VCS_BACKUP_MANAGER"
franta-hg@0
    38
franta-hg@0
    39
mkdir "$VCS_BACKUP_CURRENT_DIR";
franta-hg@0
    40
mkdir "$VCS_BACKUP_CONFIG_DIR";
franta-hg@0
    41
mkdir "$VCS_BACKUP_SNAPSHOT_DIR";
franta-hg@5
    42
mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"
franta-hg@0
    43
franta-hg@5
    44
chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"
franta-hg@0
    45
chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"
franta-hg@0
    46
}
franta-hg@0
    47
franta-hg@0
    48
franta-hg@0
    49
# --- Private functions: ---------------------------------------------------------------------------
franta-hg@0
    50
franta-hg@0
    51
# Environment: all
franta-hg@0
    52
# $1 = VCS type: hg, git
franta-hg@0
    53
# $2 = URL
franta-hg@0
    54
isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }
franta-hg@0
    55
franta-hg@0
    56
# Environment: all
franta-hg@0
    57
# $1 = path to the config file
franta-hg@5
    58
loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }
franta-hg@0
    59
franta-hg@0
    60
# Environment: server
franta-hg@0
    61
# $1 = URL
franta-hg@0
    62
urlToRelativeDirectoryPath() {
franta-hg@0
    63
	echo "$1" | sed -E 's@^[^:]+://@@g';
franta-hg@0
    64
}
franta-hg@0
    65
franta-hg@9
    66
# Environment: all
franta-hg@9
    67
# $1 = optional value (if missing, reads STDIN)
franta-hg@9
    68
escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi }
franta-hg@9
    69
franta-hg@9
    70
# Environment: all
franta-hg@9
    71
# $1 = key
franta-hg@9
    72
# $2 = value
franta-hg@9
    73
printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; }
franta-hg@9
    74
franta-hg@0
    75
# --- Public interface functions: ------------------------------------------------------------------
franta-hg@0
    76
franta-hg@0
    77
# Environment: client
franta-hg@0
    78
# $1 = VCS type: hg, git
franta-hg@0
    79
# $2 = URL
franta-hg@2
    80
# $3 = "public" or "private" (default), whether the repository should be available through the public web interface
franta-hg@0
    81
vcs_backup_public_clientSubmitBackupRequest() {
franta-hg@0
    82
	if isValidTypeAndURL "$1" "$2"; then
franta-hg@0
    83
		loadConfigFile ~/.config/vcs-backup/client.cfg
franta-hg@3
    84
		${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"
franta-hg@3
    85
		if [[ "$4" == "clone" ]]; then
franta-hg@3
    86
			if   [[ "$1" == "hg"  ]]; then  hg clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";
franta-hg@3
    87
			elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";
franta-hg@3
    88
			fi
franta-hg@3
    89
		fi
franta-hg@0
    90
	else
franta-hg@0
    91
		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
franta-hg@0
    92
	fi
franta-hg@0
    93
}
franta-hg@0
    94
franta-hg@0
    95
# Environment: server
franta-hg@5
    96
# User: $VCS_BACKUP_MANAGER
franta-hg@2
    97
# has same parameters as clientSubmitBackupRequest (see above)
franta-hg@0
    98
vcs_backup_public_serverSubmitBackupRequest() {
franta-hg@0
    99
	if isValidTypeAndURL "$1" "$2"; then
franta-hg@0
   100
		loadConfigFile "/etc/vcs-backup/server.cfg";
franta-hg@0
   101
		relativePath=$1/$(urlToRelativeDirectoryPath "$2");
franta-hg@0
   102
		absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";
franta-hg@0
   103
		mkdir -p "$absolutePath";
franta-hg@0
   104
		echo "$2" > "$absolutePath/url.txt"
franta-hg@0
   105
		echo "submited" > "$absolutePath/state.txt"
franta-hg@0
   106
		setfacl -m u:${VCS_BACKUP_USER}:r  "$absolutePath/url.txt"
franta-hg@0
   107
		setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"
franta-hg@0
   108
		echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};
franta-hg@3
   109
franta-hg@2
   110
		if [[ "$3" == "public" ]]; then
franta-hg@2
   111
			cd "$VCS_BACKUP_PUBLIC_DIR";
franta-hg@2
   112
			mkdir -p "$(dirname $relativePath)";
franta-hg@2
   113
			ln -rs "../current/$relativePath" "$(dirname $relativePath)";
franta-hg@2
   114
		fi
franta-hg@3
   115
franta-hg@3
   116
		if [[ "$4" == "clone" ]]; then
franta-hg@3
   117
			socat -u "unix-recvfrom:$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET},mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777
franta-hg@3
   118
				echo "Message from the clone service: $m";
franta-hg@3
   119
			done;
franta-hg@3
   120
		fi
franta-hg@0
   121
	else
franta-hg@0
   122
		echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
franta-hg@0
   123
	fi
franta-hg@0
   124
}
franta-hg@0
   125
franta-hg@0
   126
# Environment: server
franta-hg@5
   127
# User: root
franta-hg@0
   128
# Should be started as a systemd/init service.
franta-hg@0
   129
# - reads messages from from the subvolume socket – message contains the relative directory path
franta-hg@0
   130
# - creates a subvolume for given repository + necesary parent directories
franta-hg@0
   131
# - sends a message to the clone service → start cloning into the created subvolume
franta-hg@0
   132
vcs_backup_public_serverStartSubvolumeService() {
franta-hg@0
   133
	socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do
franta-hg@0
   134
		mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");
franta-hg@0
   135
		btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \
franta-hg@0
   136
		echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \
franta-hg@0
   137
		chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \
franta-hg@0
   138
		echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};
franta-hg@0
   139
	done
franta-hg@0
   140
}
franta-hg@0
   141
franta-hg@0
   142
# Environment: server
franta-hg@5
   143
# User: $VCS_BACKUP_USER
franta-hg@0
   144
# should be started as a systemd/init service
franta-hg@0
   145
vcs_backup_public_serverStartCloneService() {
franta-hg@0
   146
	socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do
franta-hg@2
   147
		vcsType=$(echo "$d" | sed 's@/.*@@g');
franta-hg@0
   148
		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
franta-hg@3
   149
franta-hg@0
   150
		if isValidTypeAndURL "$vcsType" "$url"; then
franta-hg@8
   151
			if   [[ "$vcsType" == "hg"  ]]; then  hg clone -U       "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@8
   152
			elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@0
   153
			fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";
franta-hg@0
   154
		else
franta-hg@0
   155
			echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;
franta-hg@0
   156
		fi
franta-hg@3
   157
franta-hg@3
   158
		callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";
franta-hg@3
   159
		if [[ -e "$callBackSocket" ]]; then
franta-hg@3
   160
			echo "done" | socat -u - unix-send:"$callBackSocket";
franta-hg@3
   161
		fi
franta-hg@0
   162
	done
franta-hg@0
   163
}
franta-hg@0
   164
franta-hg@7
   165
# Environment: client
franta-hg@7
   166
# prints list of repositories in recfile format
franta-hg@7
   167
# usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular
franta-hg@7
   168
vcs_backup_public_clientListRepositories() {
franta-hg@7
   169
	loadConfigFile ~/.config/vcs-backup/client.cfg;
franta-hg@7
   170
	${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;
franta-hg@7
   171
}
franta-hg@7
   172
franta-hg@7
   173
# Environment: server
franta-hg@7
   174
# User: $VCS_BACKUP_MANAGER
franta-hg@7
   175
vcs_backup_public_serverListRepositories() {
franta-hg@7
   176
	echo "%rec: repositories";
franta-hg@7
   177
	echo "%type: bytes int";
franta-hg@7
   178
	echo "%type: public bool";
franta-hg@7
   179
	echo;
franta-hg@7
   180
franta-hg@7
   181
	find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do
franta-hg@7
   182
		url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
franta-hg@7
   183
		state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
franta-hg@7
   184
		vcsType=$(echo "$d" | sed 's@/.*@@g');
franta-hg@7
   185
		sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);
franta-hg@7
   186
		[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";
franta-hg@7
   187
		
franta-hg@7
   188
		if [[ "$vcsType" == "hg"  ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);
franta-hg@7
   189
		elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai"); 
franta-hg@7
   190
		else lastCommit=""; fi
franta-hg@7
   191
		
franta-hg@9
   192
		printRecfileKeyValue "type"            "$vcsType";
franta-hg@9
   193
		printRecfileKeyValue "url"             "$url";
franta-hg@9
   194
		printRecfileKeyValue "state"           "$state";
franta-hg@9
   195
		printRecfileKeyValue "public"          "$public";
franta-hg@9
   196
		printRecfileKeyValue "serverPath"      "$VCS_BACKUP_CURRENT_DIR/$d";
franta-hg@9
   197
		printRecfileKeyValue "size"            "$sizeBytes";
franta-hg@9
   198
		printRecfileKeyValue "lastCommit"      "$lastCommit";
franta-hg@7
   199
		echo;
franta-hg@7
   200
	done
franta-hg@7
   201
}
franta-hg@7
   202
franta-hg@6
   203
# Environment: server
franta-hg@6
   204
# User: $VCS_BACKUP_USER
franta-hg@6
   205
# should be called from cron (usually every day)
franta-hg@6
   206
vcs_backup_public_serverPullCronTask() {
franta-hg@6
   207
	return;
franta-hg@6
   208
}
franta-hg@6
   209
franta-hg@6
   210
# Environment: server
franta-hg@6
   211
# User: $VCS_BACKUP_USER
franta-hg@6
   212
# should be called from cron (usually every day) after Pull (see above)
franta-hg@6
   213
vcs_backup_public_serverSnapshotCronTask() {
franta-hg@6
   214
	return;
franta-hg@6
   215
}
franta-hg@6
   216
franta-hg@0
   217
# --- Single entry-point: --------------------------------------------------------------------------
franta-hg@0
   218
franta-hg@0
   219
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
franta-hg@0
   220
PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";
franta-hg@0
   221
if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then
franta-hg@0
   222
	"$PUBLIC_FUNCTION_PREFIX${@:1}";
franta-hg@4
   223
elif [[ $(basename $0) == "vcs-backup-clone-private-hg"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" private clone;
franta-hg@4
   224
elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;
franta-hg@4
   225
elif [[ $(basename $0) == "vcs-backup-clone-public-hg"   ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg  "$1" public  clone;
franta-hg@4
   226
elif [[ $(basename $0) == "vcs-backup-clone-public-git"  ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public  clone;
franta-hg@0
   227
else
franta-hg@0
   228
	echo "Unsupported sub-command: $1" >&2
franta-hg@0
   229
	echo "Available sub-commands:" >&2
franta-hg@0
   230
	declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/  /g" >&2
franta-hg@0
   231
	exit 1;
franta-hg@0
   232
fi