franta-hg@0
|
1 |
#!/bin/bash
|
franta-hg@0
|
2 |
|
franta-hg@0
|
3 |
# VCS Backup
|
franta-hg@0
|
4 |
# Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info)
|
franta-hg@0
|
5 |
#
|
franta-hg@0
|
6 |
# This program is free software: you can redistribute it and/or modify
|
franta-hg@0
|
7 |
# it under the terms of the GNU General Public License as published by
|
franta-hg@0
|
8 |
# the Free Software Foundation, either version 3 of the License, or
|
franta-hg@0
|
9 |
# (at your option) any later version.
|
franta-hg@0
|
10 |
#
|
franta-hg@0
|
11 |
# This program is distributed in the hope that it will be useful,
|
franta-hg@0
|
12 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
franta-hg@0
|
13 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
franta-hg@0
|
14 |
# GNU General Public License for more details.
|
franta-hg@0
|
15 |
#
|
franta-hg@0
|
16 |
# You should have received a copy of the GNU General Public License
|
franta-hg@0
|
17 |
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
franta-hg@0
|
18 |
|
franta-hg@0
|
19 |
|
franta-hg@0
|
20 |
# Server-side configuration:
|
franta-hg@0
|
21 |
VCS_BACKUP_DATA_DIR="/mnt/data";
|
franta-hg@0
|
22 |
VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current";
|
franta-hg@2
|
23 |
VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public";
|
franta-hg@0
|
24 |
VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config";
|
franta-hg@0
|
25 |
VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot";
|
franta-hg@0
|
26 |
VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume";
|
franta-hg@7
|
27 |
VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER}
|
franta-hg@3
|
28 |
VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback";
|
franta-hg@0
|
29 |
VCS_BACKUP_USER="vcs-backup";
|
franta-hg@0
|
30 |
VCS_BACKUP_MANAGER="vcs-backup-manager";
|
franta-hg@0
|
31 |
|
franta-hg@0
|
32 |
# Installation – check and do it by hand:
|
franta-hg@0
|
33 |
# There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR
|
franta-hg@0
|
34 |
installInstructions() {
|
franta-hg@0
|
35 |
cp vcs-backup.sh /usr/local/bin/
|
franta-hg@0
|
36 |
adduser --disabled-password "$VCS_BACKUP"
|
franta-hg@0
|
37 |
adduser --disabled-password "$VCS_BACKUP_MANAGER"
|
franta-hg@0
|
38 |
|
franta-hg@0
|
39 |
mkdir "$VCS_BACKUP_CURRENT_DIR";
|
franta-hg@0
|
40 |
mkdir "$VCS_BACKUP_CONFIG_DIR";
|
franta-hg@0
|
41 |
mkdir "$VCS_BACKUP_SNAPSHOT_DIR";
|
franta-hg@5
|
42 |
mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)"
|
franta-hg@0
|
43 |
|
franta-hg@5
|
44 |
chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)"
|
franta-hg@0
|
45 |
chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR"
|
franta-hg@0
|
46 |
}
|
franta-hg@0
|
47 |
|
franta-hg@0
|
48 |
|
franta-hg@0
|
49 |
# --- Private functions: ---------------------------------------------------------------------------
|
franta-hg@0
|
50 |
|
franta-hg@0
|
51 |
# Environment: all
|
franta-hg@0
|
52 |
# $1 = VCS type: hg, git
|
franta-hg@0
|
53 |
# $2 = URL
|
franta-hg@0
|
54 |
isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; }
|
franta-hg@0
|
55 |
|
franta-hg@0
|
56 |
# Environment: all
|
franta-hg@0
|
57 |
# $1 = path to the config file
|
franta-hg@5
|
58 |
loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi }
|
franta-hg@0
|
59 |
|
franta-hg@0
|
60 |
# Environment: server
|
franta-hg@0
|
61 |
# $1 = URL
|
franta-hg@0
|
62 |
urlToRelativeDirectoryPath() {
|
franta-hg@0
|
63 |
echo "$1" | sed -E 's@^[^:]+://@@g';
|
franta-hg@0
|
64 |
}
|
franta-hg@0
|
65 |
|
franta-hg@0
|
66 |
# --- Public interface functions: ------------------------------------------------------------------
|
franta-hg@0
|
67 |
|
franta-hg@0
|
68 |
# Environment: client
|
franta-hg@0
|
69 |
# $1 = VCS type: hg, git
|
franta-hg@0
|
70 |
# $2 = URL
|
franta-hg@2
|
71 |
# $3 = "public" or "private" (default), whether the repository should be available through the public web interface
|
franta-hg@0
|
72 |
vcs_backup_public_clientSubmitBackupRequest() {
|
franta-hg@0
|
73 |
if isValidTypeAndURL "$1" "$2"; then
|
franta-hg@0
|
74 |
loadConfigFile ~/.config/vcs-backup/client.cfg
|
franta-hg@3
|
75 |
${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4"
|
franta-hg@3
|
76 |
if [[ "$4" == "clone" ]]; then
|
franta-hg@3
|
77 |
if [[ "$1" == "hg" ]]; then hg clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";
|
franta-hg@3
|
78 |
elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}//mnt/data/current/$1/$(urlToRelativeDirectoryPath $2)";
|
franta-hg@3
|
79 |
fi
|
franta-hg@3
|
80 |
fi
|
franta-hg@0
|
81 |
else
|
franta-hg@0
|
82 |
echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
|
franta-hg@0
|
83 |
fi
|
franta-hg@0
|
84 |
}
|
franta-hg@0
|
85 |
|
franta-hg@0
|
86 |
# Environment: server
|
franta-hg@5
|
87 |
# User: $VCS_BACKUP_MANAGER
|
franta-hg@2
|
88 |
# has same parameters as clientSubmitBackupRequest (see above)
|
franta-hg@0
|
89 |
vcs_backup_public_serverSubmitBackupRequest() {
|
franta-hg@0
|
90 |
if isValidTypeAndURL "$1" "$2"; then
|
franta-hg@0
|
91 |
loadConfigFile "/etc/vcs-backup/server.cfg";
|
franta-hg@0
|
92 |
relativePath=$1/$(urlToRelativeDirectoryPath "$2");
|
franta-hg@0
|
93 |
absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath";
|
franta-hg@0
|
94 |
mkdir -p "$absolutePath";
|
franta-hg@0
|
95 |
echo "$2" > "$absolutePath/url.txt"
|
franta-hg@0
|
96 |
echo "submited" > "$absolutePath/state.txt"
|
franta-hg@0
|
97 |
setfacl -m u:${VCS_BACKUP_USER}:r "$absolutePath/url.txt"
|
franta-hg@0
|
98 |
setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt"
|
franta-hg@0
|
99 |
echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET};
|
franta-hg@3
|
100 |
|
franta-hg@2
|
101 |
if [[ "$3" == "public" ]]; then
|
franta-hg@2
|
102 |
cd "$VCS_BACKUP_PUBLIC_DIR";
|
franta-hg@2
|
103 |
mkdir -p "$(dirname $relativePath)";
|
franta-hg@2
|
104 |
ln -rs "../current/$relativePath" "$(dirname $relativePath)";
|
franta-hg@2
|
105 |
fi
|
franta-hg@3
|
106 |
|
franta-hg@3
|
107 |
if [[ "$4" == "clone" ]]; then
|
franta-hg@3
|
108 |
socat -u "unix-recvfrom:$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET},mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777
|
franta-hg@3
|
109 |
echo "Message from the clone service: $m";
|
franta-hg@3
|
110 |
done;
|
franta-hg@3
|
111 |
fi
|
franta-hg@0
|
112 |
else
|
franta-hg@0
|
113 |
echo "Unsupported VCS type: '$1' or URL: '$2'" >&2;
|
franta-hg@0
|
114 |
fi
|
franta-hg@0
|
115 |
}
|
franta-hg@0
|
116 |
|
franta-hg@0
|
117 |
# Environment: server
|
franta-hg@5
|
118 |
# User: root
|
franta-hg@0
|
119 |
# Should be started as a systemd/init service.
|
franta-hg@0
|
120 |
# - reads messages from from the subvolume socket – message contains the relative directory path
|
franta-hg@0
|
121 |
# - creates a subvolume for given repository + necesary parent directories
|
franta-hg@0
|
122 |
# - sends a message to the clone service → start cloning into the created subvolume
|
franta-hg@0
|
123 |
vcs_backup_public_serverStartSubvolumeService() {
|
franta-hg@0
|
124 |
socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do
|
franta-hg@0
|
125 |
mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d");
|
franta-hg@0
|
126 |
btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \
|
franta-hg@0
|
127 |
echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \
|
franta-hg@0
|
128 |
chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \
|
franta-hg@0
|
129 |
echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET};
|
franta-hg@0
|
130 |
done
|
franta-hg@0
|
131 |
}
|
franta-hg@0
|
132 |
|
franta-hg@0
|
133 |
# Environment: server
|
franta-hg@5
|
134 |
# User: $VCS_BACKUP_USER
|
franta-hg@0
|
135 |
# should be started as a systemd/init service
|
franta-hg@0
|
136 |
vcs_backup_public_serverStartCloneService() {
|
franta-hg@0
|
137 |
socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do
|
franta-hg@2
|
138 |
vcsType=$(echo "$d" | sed 's@/.*@@g');
|
franta-hg@0
|
139 |
url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
|
franta-hg@3
|
140 |
|
franta-hg@0
|
141 |
if isValidTypeAndURL "$vcsType" "$url"; then
|
franta-hg@8
|
142 |
if [[ "$vcsType" == "hg" ]]; then hg clone -U "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
|
franta-hg@8
|
143 |
elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d";
|
franta-hg@0
|
144 |
fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt";
|
franta-hg@0
|
145 |
else
|
franta-hg@0
|
146 |
echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2;
|
franta-hg@0
|
147 |
fi
|
franta-hg@3
|
148 |
|
franta-hg@3
|
149 |
callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET";
|
franta-hg@3
|
150 |
if [[ -e "$callBackSocket" ]]; then
|
franta-hg@3
|
151 |
echo "done" | socat -u - unix-send:"$callBackSocket";
|
franta-hg@3
|
152 |
fi
|
franta-hg@0
|
153 |
done
|
franta-hg@0
|
154 |
}
|
franta-hg@0
|
155 |
|
franta-hg@7
|
156 |
# Environment: client
|
franta-hg@7
|
157 |
# prints list of repositories in recfile format
|
franta-hg@7
|
158 |
# usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular
|
franta-hg@7
|
159 |
vcs_backup_public_clientListRepositories() {
|
franta-hg@7
|
160 |
loadConfigFile ~/.config/vcs-backup/client.cfg;
|
franta-hg@7
|
161 |
${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories;
|
franta-hg@7
|
162 |
}
|
franta-hg@7
|
163 |
|
franta-hg@7
|
164 |
# Environment: server
|
franta-hg@7
|
165 |
# User: $VCS_BACKUP_MANAGER
|
franta-hg@7
|
166 |
vcs_backup_public_serverListRepositories() {
|
franta-hg@7
|
167 |
echo "%rec: repositories";
|
franta-hg@7
|
168 |
echo "%type: bytes int";
|
franta-hg@7
|
169 |
echo "%type: public bool";
|
franta-hg@7
|
170 |
echo;
|
franta-hg@7
|
171 |
|
franta-hg@7
|
172 |
find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs dirname | while read d; do
|
franta-hg@7
|
173 |
url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt");
|
franta-hg@7
|
174 |
state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt");
|
franta-hg@7
|
175 |
vcsType=$(echo "$d" | sed 's@/.*@@g');
|
franta-hg@7
|
176 |
sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1);
|
franta-hg@7
|
177 |
[[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false";
|
franta-hg@7
|
178 |
|
franta-hg@7
|
179 |
if [[ "$vcsType" == "hg" ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null);
|
franta-hg@7
|
180 |
elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai");
|
franta-hg@7
|
181 |
else lastCommit=""; fi
|
franta-hg@7
|
182 |
|
franta-hg@7
|
183 |
echo "type: $vcsType";
|
franta-hg@7
|
184 |
echo "url: $url";
|
franta-hg@7
|
185 |
echo "state: $state";
|
franta-hg@7
|
186 |
echo "public: $public";
|
franta-hg@7
|
187 |
echo "serverPath: $VCS_BACKUP_CURRENT_DIR/$d";
|
franta-hg@7
|
188 |
echo "size: $sizeBytes";
|
franta-hg@7
|
189 |
echo "lastCommit: $lastCommit";
|
franta-hg@7
|
190 |
echo;
|
franta-hg@7
|
191 |
done
|
franta-hg@7
|
192 |
}
|
franta-hg@7
|
193 |
|
franta-hg@6
|
194 |
# Environment: server
|
franta-hg@6
|
195 |
# User: $VCS_BACKUP_USER
|
franta-hg@6
|
196 |
# should be called from cron (usually every day)
|
franta-hg@6
|
197 |
vcs_backup_public_serverPullCronTask() {
|
franta-hg@6
|
198 |
return;
|
franta-hg@6
|
199 |
}
|
franta-hg@6
|
200 |
|
franta-hg@6
|
201 |
# Environment: server
|
franta-hg@6
|
202 |
# User: $VCS_BACKUP_USER
|
franta-hg@6
|
203 |
# should be called from cron (usually every day) after Pull (see above)
|
franta-hg@6
|
204 |
vcs_backup_public_serverSnapshotCronTask() {
|
franta-hg@6
|
205 |
return;
|
franta-hg@6
|
206 |
}
|
franta-hg@6
|
207 |
|
franta-hg@0
|
208 |
# --- Single entry-point: --------------------------------------------------------------------------
|
franta-hg@0
|
209 |
|
franta-hg@0
|
210 |
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
franta-hg@0
|
211 |
PUBLIC_FUNCTION_PREFIX="vcs_backup_public_";
|
franta-hg@0
|
212 |
if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then
|
franta-hg@0
|
213 |
"$PUBLIC_FUNCTION_PREFIX${@:1}";
|
franta-hg@4
|
214 |
elif [[ $(basename $0) == "vcs-backup-clone-private-hg" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg "$1" private clone;
|
franta-hg@4
|
215 |
elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone;
|
franta-hg@4
|
216 |
elif [[ $(basename $0) == "vcs-backup-clone-public-hg" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg "$1" public clone;
|
franta-hg@4
|
217 |
elif [[ $(basename $0) == "vcs-backup-clone-public-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public clone;
|
franta-hg@0
|
218 |
else
|
franta-hg@0
|
219 |
echo "Unsupported sub-command: $1" >&2
|
franta-hg@0
|
220 |
echo "Available sub-commands:" >&2
|
franta-hg@0
|
221 |
declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/ /g" >&2
|
franta-hg@0
|
222 |
exit 1;
|
franta-hg@0
|
223 |
fi
|