[Zodb-checkins] SVN: ZODB/trunk/src/ Merged tseaver-lp143158-feature, which adds the new feature:
Jim Fulton
jim at zope.com
Wed Apr 14 16:12:35 EDT 2010
Log message for revision 110913:
Merged tseaver-lp143158-feature, which adds the new feature:
- Added a '--kill-old-on-full' argument to the backup options: if passed,
remove any older full or incremental backup files from the repository after
doing a full backup. (https://bugs.launchpad.net/zope2/+bug/143158)
Changed:
U ZODB/trunk/src/CHANGES.txt
U ZODB/trunk/src/ZODB/scripts/repozo.py
U ZODB/trunk/src/ZODB/scripts/tests/test_repozo.py
-=-
Modified: ZODB/trunk/src/CHANGES.txt
===================================================================
--- ZODB/trunk/src/CHANGES.txt 2010-04-14 20:12:29 UTC (rev 110912)
+++ ZODB/trunk/src/CHANGES.txt 2010-04-14 20:12:35 UTC (rev 110913)
@@ -5,6 +5,18 @@
3.10.0a2 (2010-??-??)
=====================
+New Features
+------------
+
+- Added a '--kill-old-on-full' argument to the backup options: if passed,
+ remove any older full or incremental backup files from the repository after
+ doing a full backup. (https://bugs.launchpad.net/zope2/+bug/143158)
+
+- When transactions are aborted, new object ids allocated during the
+ transaction are saved and used in subsequent transactions. This can
+ help in situations where object ids are used as BTree keys and the
+ sequential allocation of object ids leads to conflict errors.
+
Bugs Fixed
----------
@@ -14,14 +26,6 @@
- When using using a ClientStorage in a Storage server, there was a
threading bug that caused clients to get disconnected.
-New Features
-------------
-
-- When transactions are aborted, new object ids allocated during the
- transaction are saved and used in subsequent transactions. This can
- help in situations where object ids are used as BTree keys and the
- sequential allocation of object ids leads to conflict errors.
-
3.10.0a1 (2010-02-08)
=====================
Modified: ZODB/trunk/src/ZODB/scripts/repozo.py
===================================================================
--- ZODB/trunk/src/ZODB/scripts/repozo.py 2010-04-14 20:12:29 UTC (rev 110912)
+++ ZODB/trunk/src/ZODB/scripts/repozo.py 2010-04-14 20:12:35 UTC (rev 110913)
@@ -50,6 +50,11 @@
Compress with gzip the backup files. Uses the default zlib
compression level. By default, gzip compression is not used.
+ -k / --kill-old-on-full
+ If a full backup is created, remove any prior full or incremental
+ backup files (and associated metadata files) from the repository
+ directory.
+
Options for -R/--recover:
-D str
--date=str
@@ -109,10 +114,20 @@
def parseargs(argv):
global VERBOSE
try:
- opts, args = getopt.getopt(argv, 'BRvhf:r:FD:o:Qz',
- ['backup', 'recover', 'verbose', 'help',
- 'file=', 'repository=', 'full', 'date=',
- 'output=', 'quick', 'gzip'])
+ opts, args = getopt.getopt(argv, 'BRvhr:f:FQzkD:o:',
+ ['backup',
+ 'recover',
+ 'verbose',
+ 'help',
+ 'repository=',
+ 'file=',
+ 'full',
+ 'quick',
+ 'gzip',
+ 'kill-old-on-full',
+ 'date=',
+ 'output=',
+ ])
except getopt.error, msg:
usage(1, msg)
@@ -125,6 +140,7 @@
output = None # where to write recovered data; None = stdout
quick = False # -Q flag state
gzip = False # -z flag state
+ killold = False # -k flag state
options = Options()
@@ -155,6 +171,8 @@
options.output = arg
elif opt in ('-z', '--gzip'):
options.gzip = True
+ elif opt in ('-k', '--kill-old-on-full'):
+ options.killold = True
else:
assert False, (opt, arg)
@@ -179,6 +197,9 @@
if options.file is not None:
log('--file option is ignored in recover mode')
options.file = None
+ if options.killold is not None:
+ log('--kill-old-on-full option is ignored in recover mode')
+ options.killold = None
return options
@@ -351,7 +372,40 @@
return fn, startpos, endpos, sum
+def delete_old_backups(options):
+ # Delete all full backup files except for the most recent full backup file
+ all = filter(is_data_file, os.listdir(options.repository))
+ all.sort()
+ deletable = []
+ full = []
+ for fname in all:
+ root, ext = os.path.splitext(fname)
+ if ext in ('.fs', '.fsz'):
+ full.append(fname)
+ if ext in ('.fs', '.fsz', '.deltafs', '.deltafsz'):
+ deletable.append(fname)
+
+ # keep most recent full
+ if not full:
+ return
+
+ recentfull = full.pop(-1)
+ deletable.remove(recentfull)
+ root, ext = os.path.splitext(recentfull)
+ dat = root + '.dat'
+ if dat in deletable:
+ deletable.remove(dat)
+
+ for fname in deletable:
+ log('removing old backup file %s (and .dat)', fname)
+ root, ext = os.path.splitext(fname)
+ try:
+ os.unlink(os.path.join(options.repository, root + '.dat'))
+ except OSError:
+ pass
+ os.unlink(os.path.join(options.repository, fname))
+
def do_full_backup(options):
# Find the file position of the last completed transaction.
fs = FileStorage(options.file, read_only=True)
@@ -376,6 +430,8 @@
fp.flush()
os.fsync(fp.fileno())
fp.close()
+ if options.killold:
+ delete_old_backups(options)
def do_incremental_backup(options, reposz, repofiles):
Modified: ZODB/trunk/src/ZODB/scripts/tests/test_repozo.py
===================================================================
--- ZODB/trunk/src/ZODB/scripts/tests/test_repozo.py 2010-04-14 20:12:29 UTC (rev 110912)
+++ ZODB/trunk/src/ZODB/scripts/tests/test_repozo.py 2010-04-14 20:12:35 UTC (rev 110913)
@@ -168,8 +168,94 @@
(correctpath, when, ' '.join(argv)))
self.assertEquals(fguts, gguts, msg)
+class Test_delete_old_backups(unittest.TestCase):
+ _repository_directory = None
+
+ def tearDown(self):
+ if self._repository_directory is not None:
+ from shutil import rmtree
+ rmtree(self._repository_directory)
+
+ def _callFUT(self, options=None, filenames=()):
+ from ZODB.scripts.repozo import delete_old_backups
+ if options is None:
+ options = self._makeOptions(filenames)
+ delete_old_backups(options)
+
+ def _makeOptions(self, filenames=()):
+ import tempfile
+ dir = self._repository_directory = tempfile.mkdtemp()
+ for filename in filenames:
+ fqn = os.path.join(dir, filename)
+ f = open(fqn, 'wb')
+ f.write('testing delete_old_backups')
+ f.close()
+ class Options(object):
+ repository = dir
+ return Options()
+
+ def test_empty_dir_doesnt_raise(self):
+ self._callFUT()
+ self.assertEqual(len(os.listdir(self._repository_directory)), 0)
+
+ def test_no_repozo_files_doesnt_raise(self):
+ FILENAMES = ['bogus.txt', 'not_a_repozo_file']
+ self._callFUT(filenames=FILENAMES)
+ remaining = os.listdir(self._repository_directory)
+ self.assertEqual(len(remaining), len(FILENAMES))
+ for name in FILENAMES:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failUnless(os.path.isfile(fqn))
+
+ def test_doesnt_remove_current_repozo_files(self):
+ FILENAMES = ['2009-12-20-10-08-03.fs', '2009-12-20-10-08-03.dat']
+ self._callFUT(filenames=FILENAMES)
+ remaining = os.listdir(self._repository_directory)
+ self.assertEqual(len(remaining), len(FILENAMES))
+ for name in FILENAMES:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failUnless(os.path.isfile(fqn))
+
+ def test_removes_older_repozo_files(self):
+ OLDER_FULL = ['2009-12-20-00-01-03.fs', '2009-12-20-00-01-03.dat']
+ DELTAS = ['2009-12-21-00-00-01.deltafs', '2009-12-22-00-00-01.deltafs']
+ CURRENT_FULL = ['2009-12-23-00-00-01.fs', '2009-12-23-00-00-01.dat']
+ FILENAMES = OLDER_FULL + DELTAS + CURRENT_FULL
+ self._callFUT(filenames=FILENAMES)
+ remaining = os.listdir(self._repository_directory)
+ self.assertEqual(len(remaining), len(CURRENT_FULL))
+ for name in OLDER_FULL:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failIf(os.path.isfile(fqn))
+ for name in DELTAS:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failIf(os.path.isfile(fqn))
+ for name in CURRENT_FULL:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failUnless(os.path.isfile(fqn))
+
+ def test_removes_older_repozo_files_zipped(self):
+ OLDER_FULL = ['2009-12-20-00-01-03.fsz', '2009-12-20-00-01-03.dat']
+ DELTAS = ['2009-12-21-00-00-01.deltafsz',
+ '2009-12-22-00-00-01.deltafsz']
+ CURRENT_FULL = ['2009-12-23-00-00-01.fsz', '2009-12-23-00-00-01.dat']
+ FILENAMES = OLDER_FULL + DELTAS + CURRENT_FULL
+ self._callFUT(filenames=FILENAMES)
+ remaining = os.listdir(self._repository_directory)
+ self.assertEqual(len(remaining), len(CURRENT_FULL))
+ for name in OLDER_FULL:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failIf(os.path.isfile(fqn))
+ for name in DELTAS:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failIf(os.path.isfile(fqn))
+ for name in CURRENT_FULL:
+ fqn = os.path.join(self._repository_directory, name)
+ self.failUnless(os.path.isfile(fqn))
+
def test_suite():
return unittest.TestSuite([
unittest.makeSuite(RepozoTests),
+ unittest.makeSuite(Test_delete_old_backups),
])
More information about the Zodb-checkins
mailing list