Re: [PATCH] v3 Add archive support to hot-backup

Thanks Chris! I've committed your patch, with some very minor tweaks,
in r18874.

- Dan

On Sat, 04 Mar 2006, Chris Foote wrote:

> Hi Dan,
>
> Here's the latest update of the patch and log.
>
> Regards
> Chris
>
> [[[Log]]]
>
> Add option to archive the backup using gz, bz2 or zip.
>
> * tools/backup/hot-backup.py.in
> Update copyright year.
> (archive_map): New. Supported archive extentions.
> (usage): New. Print out a usage message.
> Redo command line processing using getopt and add option to specify the
> archive type.
> (comparator): If archiving, add the archive extenstion to the regexp to
> find archives as well.
> Step 2, also find archives.
> Step 3, print the error message to stderr.
> Step 4, archive backup.
> Step 5, look for archives to remove as well.
>
>
>
> ----- Original Message -----
> From: "Daniel Rall" <dlr@collab.net>
> To: "Chris Foote" <cfoote@v21.me.uk>
> Cc: <dev@subversion.tigris.org>
> Sent: Tuesday, February 28, 2006 8:19 AM
> Subject: Re: [PATCH] v2 Add archive support to hot-backup
>
> On Tue, 21 Feb 2006, Chris Foote wrote:
>
> > Thanks Dan for the review.
> >
> > Here's an updated version of the patch and log.
>
> Chris, again this looks good. If you're up for it, I'd like a last
> few changes made before committing (related to UI and error handling).
> Comments inline below...
>
> [snip]

> Index: tools/backup/hot-backup.py.in
> ===================================================================
> --- tools/backup/hot-backup.py.in (revision 18714)
> +++ tools/backup/hot-backup.py.in (working copy)
> @@ -7,7 +7,7 @@
> # See http://subversion.tigris.org for more information.
> #
> # ====================================================================
> -# Copyright (c) 2000-2004 CollabNet. All rights reserved.
> +# Copyright (c) 2000-2006 CollabNet. All rights reserved.
> #
> # This software is licensed as described in the file COPYING, which
> # you should have received as part of this distribution. The terms
> @@ -22,7 +22,7 @@
>
> ######################################################################
>
> -import sys, os, shutil, string, re
> +import sys, os, getopt, shutil, string, re
>
> ######################################################################
> # Global Settings
> @@ -36,29 +36,85 @@
> # Number of backups to keep around (0 for "keep them all")
> num_backups = 64
>
> +# Archive types/extensions
> +archive_map = {
> + 'gz' : ".tar.gz",
> + 'bz2' : ".tar.bz2",
> + 'zip' : ".zip"
> + }
> +
> ######################################################################
> # Command line arguments
>
> +def usage(out = sys.stdout):
> + scriptname = os.path.basename(sys.argv[0])
> + out.write(
> +"""USAGE: %s [OPTIONS] REPOS_PATH BACKUP_PATH
>
> -if len(sys.argv) != 3:
> - print "Usage: ", os.path.basename(sys.argv[0]), " <repos_path> <backup_path>"
> - sys.exit(1)
> +Create a backup of the repository at REPOS_PATH in a subdirectory of
> +the BACKUP_PATH location, named after the youngest revision.
>
> +Options:
> + --archive-type=FMT Create an archive of the backup. FMT can be one of:
> + bz2 : Creates a bzip2 compressed tar file.
> + gz : Creates a gzip compressed tar file.
> + zip : Creates a compressed zip file.
> + --help -h Print this help message and exit.
> +
> +""" % (scriptname,))
> +
> +
> +try:
> + opts, args = getopt.gnu_getopt(sys.argv[1:], "h?", ["archive-type=", "help"])
> +except getopt.GetoptError, e:
> + sys.stderr.write("ERROR: " + str(e) + "\n\n")
> + usage(sys.stderr)
> + sys.exit(2)
> +
> +archive_type = None
> +
> +for o, a in opts:
> + if o == "--archive-type":
> + archive_type = a
> + elif o in ("-h", "--help", "-?"):
> + usage()
> + sys.exit()
> +
> +if len(args) != 2:
> + sys.stderr.write("ERROR: only two arguments allowed.\n\n")
> + usage(sys.stderr)
> + sys.exit(2)
> +
> # Path to repository
> -repo_dir = sys.argv[1]
> +repo_dir = args[0]
> repo = os.path.basename(os.path.abspath(repo_dir))
>
> # Where to store the repository backup. The backup will be placed in
> # a *subdirectory* of this location, named after the youngest
> # revision.
> -backup_dir = sys.argv[2]
> +backup_dir = args[1]
>
> +# Added to the filename regexp, set when using --archive-type.
> +ext_re = ""
> +
> +# Do we want to create an archive of the backup
> +if archive_type:
> + if archive_map.has_key(archive_type):
> + # Additionally find files with the archive extension.
> + ext_re = "(" + re.escape(archive_map[archive_type]) + ")?"
> + else:
> + sys.stderr.write("Unknown archive type '%s'.\n\n" % (archive_type,))
> + usage(sys.stderr)
> + sys.exit(2)
> +
> +
> ######################################################################
> # Helper functions
>
> def comparator(a, b):
> # We pass in filenames so there is never a case where they are equal.
> - regexp = re.compile("-(?P<revision>[0-9]+)(-(?P<increment>[0-9]+))?$")
> + regexp = re.compile("-(?P<revision>[0-9]+)(-(?P<increment>[0-9]+))?" +
> + ext_re + "$")
> matcha = regexp.search(a)
> matchb = regexp.search(b)
> reva = int(matcha.groupdict()['revision'])
> @@ -109,7 +165,8 @@
> # rather than start from 1 and increment because the starting
> # increments may have already been removed due to num_backups.
>
> -regexp = re.compile("^" + repo + "-" + youngest + "(-(?P<increment>[0-9]+))?$")
> +regexp = re.compile("^" + repo + "-" + youngest +
> + "(-(?P<increment>[0-9]+))?" + ext_re + "$")
> directory_list = os.listdir(backup_dir)
> young_list = filter(lambda x: regexp.search(x), directory_list)
> if young_list:
> @@ -127,23 +184,78 @@
> print "Backing up repository to '" + backup_subdir + "'..."
> err_code = os.spawnl(os.P_WAIT, svnadmin, "svnadmin", "hotcopy", repo_dir,
> backup_subdir, "--clean-logs")
> -if(err_code != 0):
> - print "Unable to backup the repository."
> +if err_code != 0:
> + sys.stderr.write("Unable to backup the repository.\n")
> sys.exit(err_code)
> else:
> print "Done."
>
>
> -### Step 4: finally, remove all repository backups other than the last
> +### Step 4: Make an archive of the backup if required.
> +if archive_type:
> + archive_path = backup_subdir + archive_map[archive_type]
> + err_msg = ""
> +
> + print "Archiving backup to '" + archive_path + "'..."
> + if archive_type == 'gz' or archive_type == 'bz2':
> + try:
> + import tarfile
> + tar = tarfile.open(archive_path, 'w:' + archive_type)
> + tar.add(backup_subdir, os.path.basename(backup_subdir))
> + tar.close()
> + except ImportError, e:
> + err_msg = "Import failed: " + str(e)
> + err_code = -2
> + except tarfile.TarError, e:
> + err_msg = "Tar failed: " + str(e)
> + err_code = -3
> +
> + elif archive_type == 'zip':
> + try:
> + import zipfile
> +
> + def add_to_zip(baton, dirname, names):
> + zp = baton[0]
> + root = os.path.join(baton[1], '')
> +
> + for file in names:
> + path = os.path.join(dirname, file)
> + if os.path.isfile(path):
> + zp.write(path, path[len(root):])
> + elif os.path.isdir(path) and os.path.islink(path):
> + os.path.walk(path, add_to_zip, (zp, path))
> +
> + zp = zipfile.ZipFile(archive_path, 'w', zipfile.ZIP_DEFLATED)
> + os.path.walk(backup_subdir, add_to_zip, (zp, backup_dir))
> + zp.close()
> + except ImportError, e:
> + err_msg = "Import failed: " + str(e)
> + err_code = -4
> + except zipfile.error, e:
> + err_msg = "Zip failed: " + str(e)
> + err_code = -5
> +
> +
> + if err_code != 0:
> + sys.stderr.write("Unable to create an archive for the backup.\n" + err_msg + "\n")
> + sys.exit(err_code)
> + else:
> + print "Archive created, removing backup '" + backup_subdir + "'..."
> + shutil.rmtree(backup_subdir)
> +
> +### Step 5: finally, remove all repository backups other than the last
> ### NUM_BACKUPS.
>
> if num_backups > 0:
> - regexp = re.compile("^" + repo + "-[0-9]+(-[0-9]+)?$")
> + regexp = re.compile("^" + repo + "-[0-9]+(-[0-9]+)?" + ext_re + "$")
> directory_list = os.listdir(backup_dir)
> old_list = filter(lambda x: regexp.search(x), directory_list)
> old_list.sort(comparator)
> del old_list[max(0,len(old_list)-num_backups):]
> for item in old_list:
> - old_backup_subdir = os.path.join(backup_dir, item)
> - print "Removing old backup: " + old_backup_subdir
> - shutil.rmtree(old_backup_subdir)
> + old_backup_item = os.path.join(backup_dir, item)
> + print "Removing old backup: " + old_backup_item
> + if os.path.isdir(old_backup_item):
> + shutil.rmtree(old_backup_item)
> + else:
> + os.remove(old_backup_item)