franta-hg@0: #!/bin/bash franta-hg@0: franta-hg@0: # VCS Backup franta-hg@0: # Copyright © 2019 František Kučera (Frantovo.cz, GlobalCode.info) franta-hg@0: # franta-hg@0: # This program is free software: you can redistribute it and/or modify franta-hg@0: # it under the terms of the GNU General Public License as published by franta-hg@0: # the Free Software Foundation, either version 3 of the License, or franta-hg@0: # (at your option) any later version. franta-hg@0: # franta-hg@0: # This program is distributed in the hope that it will be useful, franta-hg@0: # but WITHOUT ANY WARRANTY; without even the implied warranty of franta-hg@0: # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the franta-hg@0: # GNU General Public License for more details. franta-hg@0: # franta-hg@0: # You should have received a copy of the GNU General Public License franta-hg@0: # along with this program. If not, see . franta-hg@0: franta-hg@0: franta-hg@13: # VCS Backup is a configuration for setting up a version control system mirrors. franta-hg@13: # Currently Mercurial (Hg) and Git are supported. franta-hg@13: # Features: franta-hg@13: # - mirrors remote repositories franta-hg@13: # - creates Btrfs subvolume for each repository franta-hg@13: # - does periodic pull to keep mirrors up to date franta-hg@13: # - does periodic Btrfs snapshot to keep history (git push --force done on the remote repository will lead to modifications or deletions in our current mirror, but previous versions will be kept in the snapshots) franta-hg@13: # - provides web interface for remote clonning of our mirrors (see systemd and etc folders) franta-hg@13: # - can be controlled over SSH by a sane person / owner of the system franta-hg@13: # - provides reports in the recfile format (to be processed using GNU Recutils or Relational pipes): franta-hg@13: # - list of repositories/mirrors franta-hg@13: # - results of pull operations franta-hg@13: franta-hg@13: franta-hg@13: # This is an asynchronous message-driven shell script that runs distributed across two machines and four user accounts. You have been warned :-) franta-hg@14: franta-hg@14: franta-hg@16: # Server-side configuration: / loadServerConfigFile() franta-hg@0: VCS_BACKUP_DATA_DIR="/mnt/data"; franta-hg@0: VCS_BACKUP_CURRENT_DIR="$VCS_BACKUP_DATA_DIR/current"; franta-hg@2: VCS_BACKUP_PUBLIC_DIR="$VCS_BACKUP_DATA_DIR/public"; franta-hg@0: VCS_BACKUP_CONFIG_DIR="$VCS_BACKUP_DATA_DIR/config"; franta-hg@0: VCS_BACKUP_SNAPSHOT_DIR="$VCS_BACKUP_DATA_DIR/snapshot"; franta-hg@0: VCS_BACKUP_SUBVOLUME_SOCKET="/run/vcs-backup-subvolume"; franta-hg@7: VCS_BACKUP_CLONE_SOCKET="/run/vcs-backup-clone/socket"; # the directory will be writable by ${VCS_BACKUP_USER} franta-hg@3: VCS_BACKUP_CLONE_CALLBACK_SOCKET="clone-callback"; franta-hg@0: VCS_BACKUP_USER="vcs-backup"; franta-hg@0: VCS_BACKUP_MANAGER="vcs-backup-manager"; franta-hg@0: franta-hg@16: # Client-side configuration: / see loadClientConfigFile() franta-hg@16: VCS_BACKUP_SERVER="$VCS_BACKUP_MANAGER@localhost"; franta-hg@16: VCS_BACKUP_SSH_COMMAND=(ssh "$VCS_BACKUP_SERVER"); franta-hg@16: franta-hg@0: # Installation – check and do it by hand: franta-hg@0: # There should be already mounted Btrfs at $VCS_BACKUP_DATA_DIR franta-hg@0: installInstructions() { franta-hg@0: cp vcs-backup.sh /usr/local/bin/ franta-hg@0: adduser --disabled-password "$VCS_BACKUP" franta-hg@0: adduser --disabled-password "$VCS_BACKUP_MANAGER" franta-hg@0: franta-hg@0: mkdir "$VCS_BACKUP_CURRENT_DIR"; franta-hg@0: mkdir "$VCS_BACKUP_CONFIG_DIR"; franta-hg@0: mkdir "$VCS_BACKUP_SNAPSHOT_DIR"; franta-hg@5: mkdir "$(dirname VCS_BACKUP_CLONE_SOCKET)" franta-hg@0: franta-hg@5: chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$(dirname VCS_BACKUP_CLONE_SOCKET)" franta-hg@0: chown "${VCS_BACKUP_MANAGER}:${VCS_BACKUP_MANAGER}" "$VCS_BACKUP_CONFIG_DIR" franta-hg@0: } franta-hg@0: franta-hg@0: franta-hg@0: # --- Private functions: --------------------------------------------------------------------------- franta-hg@0: franta-hg@0: # Environment: all franta-hg@0: # $1 = VCS type: hg, git franta-hg@0: # $2 = URL franta-hg@0: isValidTypeAndURL() { ([[ "$1" == "hg" || "$1" == "git" ]]) && [[ $(echo "$2" | wc -l) == 1 ]] && [[ $(echo "$2" | grep -E '^(http|https|ssh)://([a-zA-Z0-9_-][a-zA-Z0-9_-.]*/?)+$' | wc -l) == 1 ]]; } franta-hg@0: franta-hg@0: # Environment: all franta-hg@0: # $1 = path to the config file franta-hg@5: loadConfigFile() { if [ -f "$1" ]; then . "$1"; fi } franta-hg@16: loadClientConfigFile() { loadConfigFile ~/.config/vcs-backup/client.cfg; } franta-hg@16: loadServerConfigFile() { loadConfigFile "/etc/vcs-backup/server.cfg"; } franta-hg@0: franta-hg@0: # Environment: server franta-hg@0: # $1 = URL franta-hg@0: urlToRelativeDirectoryPath() { franta-hg@0: echo "$1" | sed -E 's@^[^:]+://@@g'; franta-hg@0: } franta-hg@0: franta-hg@17: # Environment: server franta-hg@17: # Lists relative paths (starting wiht hg or git) of all configured repositories franta-hg@17: allRepositories() { find "$VCS_BACKUP_CONFIG_DIR" -name url.txt -printf '%P\n' | sort | xargs --no-run-if-empty dirname; } franta-hg@17: franta-hg@9: # Environment: all franta-hg@9: # $1 = optional value (if missing, reads STDIN) franta-hg@9: escapeRecfileValue() { if [[ $# = 0 ]]; then awk '{ if (NR > 1) { printf "+ " } print $_ }'; else echo "${@}" | ${FUNCNAME[0]}; fi } franta-hg@9: franta-hg@9: # Environment: all franta-hg@9: # $1 = key franta-hg@9: # $2 = value franta-hg@9: printRecfileKeyValue() { echo -n "$1: "; escapeRecfileValue "$2"; } franta-hg@9: franta-hg@0: # --- Public interface functions: ------------------------------------------------------------------ franta-hg@0: franta-hg@0: # Environment: client franta-hg@0: # $1 = VCS type: hg, git franta-hg@0: # $2 = URL franta-hg@2: # $3 = "public" or "private" (default), whether the repository should be available through the public web interface franta-hg@10: # $4 = "clone" (optional), if present, will also clone the backup locally franta-hg@0: vcs_backup_public_clientSubmitBackupRequest() { franta-hg@0: if isValidTypeAndURL "$1" "$2"; then franta-hg@3: ${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverSubmitBackupRequest "$1" "$2" "$3" "$4" franta-hg@3: if [[ "$4" == "clone" ]]; then franta-hg@10: if [[ "$1" == "hg" ]]; then hg clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)"; franta-hg@10: elif [[ "$1" == "git" ]]; then git clone "ssh://${VCS_BACKUP_SERVER}/$VCS_BACKUP_CURRENT_DIR/$1/$(urlToRelativeDirectoryPath $2)"; franta-hg@3: fi franta-hg@3: fi franta-hg@0: else franta-hg@0: echo "Unsupported VCS type: '$1' or URL: '$2'" >&2; franta-hg@0: fi franta-hg@0: } franta-hg@0: franta-hg@0: # Environment: server franta-hg@5: # User: $VCS_BACKUP_MANAGER franta-hg@2: # has same parameters as clientSubmitBackupRequest (see above) franta-hg@0: vcs_backup_public_serverSubmitBackupRequest() { franta-hg@0: if isValidTypeAndURL "$1" "$2"; then franta-hg@0: relativePath=$1/$(urlToRelativeDirectoryPath "$2"); franta-hg@0: absolutePath="$VCS_BACKUP_CONFIG_DIR/$relativePath"; franta-hg@15: # TODO: stop if directory already exists / only add public link? franta-hg@0: mkdir -p "$absolutePath"; franta-hg@0: echo "$2" > "$absolutePath/url.txt" franta-hg@0: echo "submited" > "$absolutePath/state.txt" franta-hg@0: setfacl -m u:${VCS_BACKUP_USER}:r "$absolutePath/url.txt" franta-hg@0: setfacl -m u:${VCS_BACKUP_USER}:rw "$absolutePath/state.txt" franta-hg@3: franta-hg@2: if [[ "$3" == "public" ]]; then franta-hg@2: cd "$VCS_BACKUP_PUBLIC_DIR"; franta-hg@2: mkdir -p "$(dirname $relativePath)"; franta-hg@18: ln -rsf "../current/$relativePath" "$(dirname $relativePath)"; franta-hg@18: elif [[ "$3" == "private" && -e "$VCS_BACKUP_PUBLIC_DIR/$relativePath" ]]; then franta-hg@18: rm "$VCS_BACKUP_PUBLIC_DIR/$relativePath"; franta-hg@2: fi franta-hg@3: franta-hg@3: if [[ "$4" == "clone" ]]; then franta-hg@12: callBackSocket=$absolutePath/${VCS_BACKUP_CLONE_CALLBACK_SOCKET} franta-hg@12: socat -u "unix-recvfrom:$callBackSocket,mode=777" - | while read m; do # TODO: ,group=${VCS_BACKUP_USER} and no 777 ? franta-hg@12: echo "Message from the service: $m"; franta-hg@12: done & franta-hg@12: callBackPID=$!; franta-hg@12: fi franta-hg@12: franta-hg@12: echo "$relativePath" | socat -u - unix-send:${VCS_BACKUP_SUBVOLUME_SOCKET}; franta-hg@12: franta-hg@12: if [[ "$4" == "clone" ]]; then franta-hg@12: echo "Waiting for a message from the service on $callBackSocket (PID $callBackPID)"; franta-hg@12: wait -n $callBackPID; franta-hg@3: fi franta-hg@0: else franta-hg@0: echo "Unsupported VCS type: '$1' or URL: '$2'" >&2; franta-hg@0: fi franta-hg@0: } franta-hg@0: franta-hg@0: # Environment: server franta-hg@5: # User: root franta-hg@0: # Should be started as a systemd/init service. franta-hg@0: # - reads messages from from the subvolume socket – message contains the relative directory path franta-hg@0: # - creates a subvolume for given repository + necesary parent directories franta-hg@0: # - sends a message to the clone service → start cloning into the created subvolume franta-hg@0: vcs_backup_public_serverStartSubvolumeService() { franta-hg@0: socat -u "unix-recv:${VCS_BACKUP_SUBVOLUME_SOCKET},group=${VCS_BACKUP_MANAGER},mode=770" - | while read d; do franta-hg@0: mkdir -p $(dirname "$VCS_BACKUP_CURRENT_DIR/$d"); franta-hg@12: if [[ -e "$VCS_BACKUP_CURRENT_DIR/$d" ]]; then franta-hg@12: callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET"; franta-hg@12: if [[ -e "$callBackSocket" ]]; then franta-hg@12: echo "alreadyDone" | socat -u - unix-send:"$callBackSocket"; franta-hg@12: fi franta-hg@12: else franta-hg@12: btrfs subvolume create "$VCS_BACKUP_CURRENT_DIR/$d" && \ franta-hg@12: echo "subvolumeCreated" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt" && \ franta-hg@12: chown "${VCS_BACKUP_USER}:${VCS_BACKUP_USER}" "$VCS_BACKUP_CURRENT_DIR/$d" && \ franta-hg@12: echo "$d" | socat -u - unix-send:${VCS_BACKUP_CLONE_SOCKET}; franta-hg@12: fi franta-hg@0: done franta-hg@0: } franta-hg@0: franta-hg@0: # Environment: server franta-hg@5: # User: $VCS_BACKUP_USER franta-hg@0: # should be started as a systemd/init service franta-hg@0: vcs_backup_public_serverStartCloneService() { franta-hg@0: socat -u "unix-recv:${VCS_BACKUP_CLONE_SOCKET},mode=700" - | while read d; do franta-hg@2: vcsType=$(echo "$d" | sed 's@/.*@@g'); franta-hg@0: url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt"); franta-hg@3: franta-hg@0: if isValidTypeAndURL "$vcsType" "$url"; then franta-hg@8: if [[ "$vcsType" == "hg" ]]; then hg clone -U "$url" "$VCS_BACKUP_CURRENT_DIR/$d"; franta-hg@8: elif [[ "$vcsType" == "git" ]]; then git clone --mirror "$url" "$VCS_BACKUP_CURRENT_DIR/$d"; franta-hg@0: fi && echo "cloned" > "$VCS_BACKUP_CONFIG_DIR/$d/state.txt"; franta-hg@0: else franta-hg@0: echo "Unsupported VCS type: '$vcsType' or URL: '$url'" >&2; franta-hg@0: fi franta-hg@3: franta-hg@3: callBackSocket="$VCS_BACKUP_CONFIG_DIR/$d/$VCS_BACKUP_CLONE_CALLBACK_SOCKET"; franta-hg@3: if [[ -e "$callBackSocket" ]]; then franta-hg@3: echo "done" | socat -u - unix-send:"$callBackSocket"; franta-hg@3: fi franta-hg@0: done franta-hg@0: } franta-hg@0: franta-hg@7: # Environment: client franta-hg@7: # prints list of repositories in recfile format franta-hg@7: # usage example: vcs-backup.sh clientListRepositories | relpipe-in-recfile | relpipe-out-tabular franta-hg@7: vcs_backup_public_clientListRepositories() { franta-hg@7: ${VCS_BACKUP_SSH_COMMAND[@]} vcs-backup.sh serverListRepositories; franta-hg@7: } franta-hg@7: franta-hg@7: # Environment: server franta-hg@7: # User: $VCS_BACKUP_MANAGER franta-hg@7: vcs_backup_public_serverListRepositories() { franta-hg@15: printRecfileKeyValue "%rec" "repository"; franta-hg@14: printRecfileKeyValue "%type" "bytes int"; franta-hg@14: printRecfileKeyValue "%type" "public bool"; franta-hg@7: echo; franta-hg@7: franta-hg@17: allRepositories | while read d; do franta-hg@7: url=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/url.txt"); franta-hg@7: state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt"); franta-hg@7: vcsType=$(echo "$d" | sed 's@/.*@@g'); franta-hg@7: sizeBytes=$(du -sb "$VCS_BACKUP_CURRENT_DIR/$d" | cut -f1); franta-hg@7: [[ -e "$VCS_BACKUP_PUBLIC_DIR/$d" ]] && public="true" || public="false"; franta-hg@7: franta-hg@7: if [[ "$vcsType" == "hg" ]]; then lastCommit=$(hg log --limit 1 --template '{date|isodatesec}' -R "$VCS_BACKUP_CURRENT_DIR/$d" 2>/dev/null); franta-hg@7: elif [[ "$vcsType" == "git" ]]; then lastCommit=$(git -C "$VCS_BACKUP_CURRENT_DIR/$d" log --max-count=1 --pretty="%ai"); franta-hg@7: else lastCommit=""; fi franta-hg@7: franta-hg@9: printRecfileKeyValue "type" "$vcsType"; franta-hg@9: printRecfileKeyValue "url" "$url"; franta-hg@9: printRecfileKeyValue "state" "$state"; franta-hg@9: printRecfileKeyValue "public" "$public"; franta-hg@9: printRecfileKeyValue "serverPath" "$VCS_BACKUP_CURRENT_DIR/$d"; franta-hg@9: printRecfileKeyValue "size" "$sizeBytes"; franta-hg@9: printRecfileKeyValue "lastCommit" "$lastCommit"; franta-hg@7: echo; franta-hg@7: done franta-hg@7: } franta-hg@7: franta-hg@6: # Environment: server franta-hg@6: # User: $VCS_BACKUP_USER franta-hg@6: # should be called from cron (usually every day) franta-hg@6: vcs_backup_public_serverPullCronTask() { franta-hg@14: printRecfileKeyValue "%rec" "pull"; franta-hg@14: printRecfileKeyValue "%type" "started date"; franta-hg@14: printRecfileKeyValue "%type" "finished date"; franta-hg@14: printRecfileKeyValue "%type" "duration int"; franta-hg@14: printRecfileKeyValue "%type" "resultCode int"; franta-hg@11: franta-hg@17: allRepositories | while read d; do franta-hg@11: state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt"); franta-hg@11: vcsType=$(echo "$d" | sed 's@/.*@@g'); franta-hg@11: absolutePath="$VCS_BACKUP_CURRENT_DIR/$d"; franta-hg@11: franta-hg@11: pullStarted=$(date --iso-8601=s); franta-hg@11: pullStartedMiliseconds=$(($(date +%s%N)/1000000)); franta-hg@11: pullFinished=""; franta-hg@11: pullDuration=""; franta-hg@11: pullResult=""; franta-hg@11: pullResultCode=""; franta-hg@11: if [[ "$state" == "cloned" ]]; then franta-hg@11: if [[ "$vcsType" == "hg" ]]; then pullResult=$(hg pull --force --repository "$absolutePath" 2>&1); pullResultCode=$?; franta-hg@11: elif [[ "$vcsType" == "git" ]]; then pullResult=$(git -C "$absolutePath" fetch 2>&1); pullResultCode=$?; franta-hg@11: fi franta-hg@11: pullFinished=$(date --iso-8601=s); franta-hg@11: pullFinishedMiliseconds=$(($(date +%s%N)/1000000)); franta-hg@11: pullDuration=$(( $pullFinishedMiliseconds - $pullStartedMiliseconds )); franta-hg@11: fi franta-hg@11: franta-hg@11: printRecfileKeyValue "serverPath" "$absolutePath"; franta-hg@11: printRecfileKeyValue "type" "$vcsType"; franta-hg@11: printRecfileKeyValue "state" "$state"; franta-hg@11: printRecfileKeyValue "started" "$pullStarted"; franta-hg@11: printRecfileKeyValue "finished" "$pullFinished"; franta-hg@11: printRecfileKeyValue "duration" "$pullDuration"; franta-hg@11: printRecfileKeyValue "resultCode" "$pullResultCode"; franta-hg@11: printRecfileKeyValue "message" "$pullResult"; franta-hg@11: echo; franta-hg@11: done franta-hg@6: } franta-hg@6: franta-hg@6: # Environment: server franta-hg@11: # User: root franta-hg@6: # should be called from cron (usually every day) after Pull (see above) franta-hg@6: vcs_backup_public_serverSnapshotCronTask() { franta-hg@15: printRecfileKeyValue "%rec" "snapshot"; franta-hg@15: printRecfileKeyValue "%type" "started date"; franta-hg@15: printRecfileKeyValue "%type" "finished date"; franta-hg@15: printRecfileKeyValue "%type" "duration int"; franta-hg@15: printRecfileKeyValue "%type" "resultCode int"; franta-hg@15: franta-hg@17: allRepositories | while read d; do franta-hg@15: state=$(cat "$VCS_BACKUP_CONFIG_DIR/$d/state.txt"); franta-hg@15: vcsType=$(echo "$d" | sed 's@/.*@@g'); franta-hg@15: absolutePath="$VCS_BACKUP_CURRENT_DIR/$d"; franta-hg@15: franta-hg@15: started=$(date --iso-8601=s); franta-hg@15: startedMiliseconds=$(($(date +%s%N)/1000000)); franta-hg@15: finished=""; franta-hg@15: duration=""; franta-hg@15: result=""; franta-hg@15: resultCode=""; franta-hg@15: snapshotPath=""; franta-hg@15: if [[ "$state" == "cloned" ]]; then franta-hg@15: snapshotPath="$VCS_BACKUP_SNAPSHOT_DIR/$d/$(date --iso-8601=date)"; franta-hg@15: mkdir -p $(dirname "$snapshotPath"); franta-hg@15: result=$(btrfs subvolume snapshot -r "$absolutePath" "$snapshotPath" 2>&1); franta-hg@15: resultCode=$?; franta-hg@15: finished=$(date --iso-8601=s); franta-hg@15: finishedMiliseconds=$(($(date +%s%N)/1000000)); franta-hg@15: duration=$(( $finishedMiliseconds - $startedMiliseconds )); franta-hg@15: fi franta-hg@15: franta-hg@15: printRecfileKeyValue "currentPath" "$absolutePath"; franta-hg@15: printRecfileKeyValue "snapshotPath" "$snapshotPath"; franta-hg@15: printRecfileKeyValue "type" "$vcsType"; franta-hg@15: printRecfileKeyValue "state" "$state"; franta-hg@15: printRecfileKeyValue "started" "$started"; franta-hg@15: printRecfileKeyValue "finished" "$finished"; franta-hg@15: printRecfileKeyValue "duration" "$duration"; franta-hg@15: printRecfileKeyValue "resultCode" "$resultCode"; franta-hg@15: printRecfileKeyValue "message" "$result"; franta-hg@15: echo; franta-hg@15: done franta-hg@6: } franta-hg@6: franta-hg@0: # --- Single entry-point: -------------------------------------------------------------------------- franta-hg@0: franta-hg@16: loadClientConfigFile; franta-hg@16: loadServerConfigFile; franta-hg@0: PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; franta-hg@0: PUBLIC_FUNCTION_PREFIX="vcs_backup_public_"; franta-hg@0: if type -t "$PUBLIC_FUNCTION_PREFIX$1" > /dev/null; then franta-hg@0: "$PUBLIC_FUNCTION_PREFIX${@:1}"; franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-private-hg" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg "$1" private clone; franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-private-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" private clone; franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-public-hg" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" hg "$1" public clone; franta-hg@4: elif [[ $(basename $0) == "vcs-backup-clone-public-git" ]]; then "${PUBLIC_FUNCTION_PREFIX}clientSubmitBackupRequest" git "$1" public clone; franta-hg@0: else franta-hg@0: echo "Unsupported sub-command: $1" >&2 franta-hg@0: echo "Available sub-commands:" >&2 franta-hg@0: declare -F | grep "$PUBLIC_FUNCTION_PREFIX" | sed "s/.*$PUBLIC_FUNCTION_PREFIX/ /g" >&2 franta-hg@0: exit 1; franta-hg@0: fi