-
-
Save huyz/a3711154a5ac476f06240e13a148fcf6 to your computer and use it in GitHub Desktop.
Revisions
-
huyz revised this gist
Sep 22, 2020 . 1 changed file with 4 additions and 4 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -23,20 +23,20 @@ fi printf "# PathsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print PathsExcluded" \ | sed -En 's,^[[:space:]]+,,p' printf "\n# ContentsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print ContentsExcluded" \ | sed -En 's,^[[:space:]]+(.*),\1/\*,p' printf "\n# FileContentsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print FileContentsExcluded" \ | sed -En 's,^[[:space:]]+,,p' \ | xargs -I _ find _ \! -type d printf "\n# UserPathsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print UserPathsExcluded" \ | sed -En 's,^[[:space:]]+,/Users/*/,p' printf "\n# Applications files excluded\n" mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'" -
Sep 22, 2020 . 1 changed file with 31 additions and 31 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -2,41 +2,41 @@ # This script intend to mimic TimeMachine exclude list. # As the exclude list can evolve between backups it has to be rebuilt before every backup # Apple uses 5 types of excludes: # - 4 `/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist` # - and files from applications where metadata says to not backup, these can be found usinr # `sudo mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'"` PLISTBUDDY=/usr/libexec/PlistBuddy RESTIC=/usr/local/bin/restic STDEXCLUSIONS=/System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist if [[ ! -x $PLISTBUDDY ]]; then syslog -s -k Facility com.apple.console Level Error Sender ResticBackupUserScript Message \ "Can't find $PLISTBUDDY" exit 1 fi if [[ $# -eq 1 ]]; then exec > "$1" fi printf "# PathsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print PathsExcluded" \ | sed -En 's,^[[:space:]]+,,p' printf "\n# ContentsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print ContentsExcluded" \ | sed -En 's,^[[:space:]]+(.*),\1/\*,p' printf "\n# FileContentsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print FileContentsExcluded" \ | sed -En 's,^[[:space:]]+,,p' \ | xargs -I _ find _ \! -type d printf "\n# UserPathsExcluded\n" $PLISTBUDDY $STDEXCLUSIONS -c "Print UserPathsExcluded" \ | sed -En 's,^[[:space:]]+,/Users/*/,p' printf "\n# Applications files excluded\n" mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'" -
Vartkat revised this gist
Aug 9, 2018 . 1 changed file with 2 additions and 2 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -49,7 +49,7 @@ function my_trap_handler() MYSELF="$0" # equals to my script name LASTLINE="$1" # argument 1: last line of error occurence LASTERR="$2" # argument 2: error code of last command $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Error in line ${LASTLINE}: exit status of last command: ${LASTERR}\n" $PRINTF "ResticUser encountered an error at `$DATE` in ${MYSELF}: line ${LASTLINE}: exit status of last command: ${LASTERR}\n" >> $LOGFILE } @@ -58,7 +58,7 @@ function my_exit_handler() { $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "End of script. Exiting" $PRINTF "Restic user script on $THISHOST ends at `$DATE`\n" >> $LOGFILE $CAT $LOGFILE | $MAIL -s "Restic User script for $THISHOST" $ADMINMAIL #if lockfile is mine remove temp directory LOCKID=$($CAT $LOCKFILE) if [ $LOCKID -eq $$ ]; then -
Aug 9, 2018 . 1 changed file with 1 addition and 1 deletion.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -33,7 +33,7 @@ EXCLUDEFILE=$TEMPDIR/tmpexcl.txt; LOGFILE=$TEMPDIR/restic.log; THISHOST=$($HOSTNAME -s); ADMINMAIL=YourEmail; REPO=YourRepoRef (i.e. 3:https://us-east-1@s3.wasabisys.com/repo-name); #Restic repo on Wasabi export RESTIC_PASSWORD="YourResticRepoPassword" -
Aug 9, 2018 . No changes.There are no files selected for viewing
-
Aug 9, 2018 . 1 changed file with 27 additions and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,27 @@ <?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>Label</key> <string>org.resticuser.daemon</string> <key>UserName</key> <string>root</string> <key>GroupName</key> <string>admin</string> <key>InitGroups</key> <true/> <key>Program</key> <string>/usr/local/sbin/resticuser.sh</string> <key>RunAtLoad</key> <true/> <key>StandardErrorPath</key> <string>/tmp/org.resticuser.daemon.stderr</string> <key>StandardOutPath</key> <string>/tmp/org.resticuser.daemon.stdout</string> <key>StartInterval</key> <integer>3600</integer> <key>WorkingDirectory</key> <string>/Users/YOURUSERNAME</string> <key>AbandonProcessGroup</key> <true/> </dict> -
Aug 9, 2018 . 1 changed file with 151 additions and 0 deletions.There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,151 @@ #!/bin/bash # This script intend to optimize user dir backup by restic. # It will run hourly (using launchd). Another daily script will do a complete backup. # This script avoids running if another instance is already doing the job # (as sometime a backup can last more than one hour, or the daily one can last). # It builds a special exclude list, inspired by Apple TimeMachine exclude list before running restic. # Edit lines 35 to 42 WHICH=/usr/bin/which; BASENAME=$($WHICH basename); CAT=$($WHICH cat); DATE=$($WHICH date); DSCL=$($WHICH dscl); ECHO=$($WHICH echo); GREP=$($WHICH grep); HOSTNAME=$($WHICH hostname); LOGGER=$($WHICH logger); MAIL=$($WHICH mail); MKDIR=$($WHICH mkdir); PRINTF=$($WHICH printf); RESTIC=$($WHICH restic); RM=$($WHICH rm); SLEEP=$($WHICH sleep); TOUCH=$($WHICH touch); WC=$($WHICH wc); TEMPDIR=/tmp/restictemp; LOCKFILE=$TEMPDIR/restic.pid; EXCLUDEFILE=$TEMPDIR/tmpexcl.txt; LOGFILE=$TEMPDIR/restic.log; THISHOST=$($HOSTNAME -s); ADMINMAIL=YourEmail; REPO=YourRepoRed; #Restic repo on Wasabi export RESTIC_PASSWORD="YourResticRepoPassword" export RESTIC_REPOSITORY="YourResticRepository" export AWS_ACCESS_KEY_ID="YourWasabiAccessKey" export AWS_SECRET_ACCESS_KEY="YourWasabiSecretAccessKey" #set -x # trap handler: print location of last error and process it further # function my_trap_handler() { MYSELF="$0" # equals to my script name LASTLINE="$1" # argument 1: last line of error occurence LASTERR="$2" # argument 2: error code of last command $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Error in line ${LASTLINE}: exit status of last command: ${LASTERR}\n" $PRINTF "ResticUser encountered an error at `$DATE` in ${MYSELF}: line ${LASTLINE}: exit status of last command: ${LASTERR}\n" >> $LOGFILE } function my_exit_handler() { $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "End of script. Exiting" $PRINTF "Restic user script on $THISHOST ends at `$DATE`\n" >> $LOGFILE $CAT $LOGFILE | $MAIL -s "Restic User script for $THISHOST" $ADMINMAIL #if lockfile is mine remove temp directory LOCKID=$($CAT $LOCKFILE) if [ $LOCKID -eq $$ ]; then $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Removing temp directory." [ -d ${TEMPDIR} ] && $RM -rf ${TEMPDIR} fi } function adduserexcludes() { USERDIR=$1; DESTFILE=$2; #Add some user specific excludes $FIND /Users/$USERDIR/.vagrant.d/ -iname '*.vmdk' 2>/dev/null>> $DESTFILE $ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.ova" >> $DESTFILE $ECHO "/Users/$USERDIR/Documents/Compta_D_et_T/Backup/*.tgz" >> $DESTFILE $ECHO "/Users/$USERDIR/Music/iTunes/iTunes Media/Mobile Applications/" >> $DESTFILE $ECHO "/Users/$USERDIR/.Trash" >> $DESTFILE $ECHO "/Users/$USERDIR/Library/Application Support/MobileSync/Backup/*" >> $DESTFILE #Add some forgotten mail V3 excludes copied from V2 $ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index" >> $DESTFILE $ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-journal" >> $DESTFILE $ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/AvailableFeeds" >> $DESTFILE $ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/BackingStoreUpdateJournal" >> $DESTFILE $ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-shm" >> $DESTFILE $ECHO "/Users/$USERDIR/Library/Mail/V3/MailData/Envelope Index-wal" >> $DESTFILE } # trap commands with non-zero exit code # trap script EXIT # trap 'my_trap_handler ${LINENO} $?' ERR trap 'my_exit_handler' EXIT #Verify if lock exists if [ -f $LOCKFILE ]; then $PRINTF "Restic : Other instance running on $THISHOST at `$DATE`, Restic script exits\n" >> $LOGFILE $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "pid="$$ "Other instance is running, exiting." exit 0 fi #Create a tempdir if [ ! -d $TEMPDIR ]; then $MKDIR $TEMPDIR fi #Create pidfile $ECHO $$ > $LOCKFILE #Create system log file if [ ! -f /var/log/restic.log ]; then $TOUCH /var/log/restic.log fi #Build Exclude list $PRINTF "Restic user script on $THISHOST builds exclude list at `$DATE`\n\n" >> $LOGFILE source /usr/local/sbin/buildresticexcludes.sh $EXCLUDEFILE #Add restic temp dir to exclude list $ECHO $TEMPDIR >> $EXCLUDEFILE # As we're only dealing with user directory let's optimize our exclude list mv $EXCLUDEFILE $TEMPDIR/temptemp.txt $CAT $TEMPDIR/temptemp.txt | $GREP '/Users/*' >> $EXCLUDEFILE # Find list of users users=$($DSCL . list /Users | $GREP -v '^_') for i in $users; do # check if some file has been changed during last hour and forget about 'no such dir' errors changed=$($FIND /Users/$i ! -name '.DS_Store' -type f -mmin -60 -print -quit 2>/dev/null | $WC -l) if [ $changed -ne 0 ]; then #As it seems better to not use wildcards in excludes we build the user specific excludes adduserexcludes $i $EXCLUDEFILE # do restic backup for this user $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup begins" $PRINTF "Backup for $THISHOST begins at `$DATE`\n" >> $LOGFILE $RESTIC -r $REPO backup /Users/$i --exclude-file=$EXCLUDEFILE --tag UserDir --cleanup-cache 2>&1| tee -a $LOGFILE $LOGGER -i -t '#ResticUser' -f /var/log/restic.log "Restic backup ends" $PRINTF "Backup for $THISHOST ends at `$DATE`\n" >> $LOGFILE fi done exit 0 -
Aug 9, 2018 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,42 @@ #!/bin/bash # This script intend to mimic TimeMachine exclude list. # As the exclude list can evolve between backups it has to be rebuilt before every backup # Apple uses 5 types of excludes, four from the /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist file # and files from applications where metadata says to not backup, these can be found usinf # sudo mdfind "com_apple_backup_excludeItem = 'com.apple.backupd'" SYSLOG=/usr/bin/syslog; TEMPFILE=$1; PLISTBUDDY=/usr/libexec/PlistBuddy; MDFIND=/usr/bin/mdfind; FIND=/usr/bin/find; RESTIC=/usr/local/bin/restic; ECHO=/bin/echo; # Verify if PlistBuddy is installed if not exit if [ ! -f $PLISTBUDDY ]; then $SYSLOG -s -k Facility com.apple.console Level Error Sender ResticBackupUserScript Message "Can't find PlistBuddy" exit 1 fi # Excludes to keep directory structure without files $FIND /private/var/log -type f > $TEMPFILE $FIND /private/var/spool/cups -type f >> $TEMPFILE $FIND /private/var/spool/fax -type f >> $TEMPFILE $FIND /private/var/spool/uucp -type f >> $TEMPFILE $ECHO -e "# PathsExcluded\n#" >> $TEMPFILE $PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print PathsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*//' >> $TEMPFILE $ECHO -e "#\n# ContentsExcluded\n#" >> $TEMPFILE $PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print ContentsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*//' | sed -e 's/$/\/\*/' >> $TEMPFILE $ECHO -e "#\n# UserPathsExcluded\n#" >> $TEMPFILE $PLISTBUDDY /System/Library/CoreServices/backupd.bundle/Contents/Resources/StdExclusions.plist -c "Print UserPathsExcluded" | grep -v '{' |grep -v '}'| sed -e 's/^[[:space:]]*/\/Users\/\*\//' >> $TEMPFILE $ECHO -e "#\n# Applications files excluded\n#" >> $TEMPFILE $MDFIND "com_apple_backup_excludeItem = 'com.apple.backupd'" >> $TEMPFILE # Exclude DocumentRevision files $ECHO "/.DocumentRevisions-V100/" >> $TEMPFILE