[Zope-CVS] CVS: Packages/PartitionedFileStorage - PartitionedFileStorage_2_6.patch:1.1 PartitionedFileStorage_HEAD.patch:1.1 FileStorage_py_1_129.patch:NONE FileStorage_py_1_98_2_7.patch:NONE
Shane Hathaway
shane@zope.com
Thu, 15 May 2003 15:45:05 -0400
Update of /cvs-repository/Packages/PartitionedFileStorage
In directory cvs.zope.org:/tmp/cvs-serv23920
Added Files:
PartitionedFileStorage_2_6.patch
PartitionedFileStorage_HEAD.patch
Removed Files:
FileStorage_py_1_129.patch FileStorage_py_1_98_2_7.patch
Log Message:
Updated to the latest HEAD and 2_6 branch. PFS now works with the new pack
implementation.
=== Added File Packages/PartitionedFileStorage/PartitionedFileStorage_2_6.patch ===
Index: FileStorage.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/FileStorage.py,v
retrieving revision 1.98.2.9
diff -u -u -r1.98.2.9 FileStorage.py
--- FileStorage.py 13 May 2003 18:04:21 -0000 1.98.2.9
+++ FileStorage.py 15 May 2003 19:40:39 -0000
@@ -196,6 +196,43 @@
packed_version='FS21'
+
+class FileOperations:
+ '''This class implements some basic operations on files.
+ PartitionedFileOperations provides an alternate implementation.
+ '''
+ def open(self, name, mode='r', bufsize=0):
+ global open
+ return open(name, mode, bufsize)
+
+ def exists(self, name):
+ return os.path.exists(name)
+
+ def rename(self, oldname, newname):
+ os.rename(oldname, newname)
+
+ def remove(self, name):
+ os.remove(name)
+
+ def fsync(self, file):
+ global fsync
+ if fsync is not None:
+ fsync(file.fileno())
+
+defaultFops = FileOperations()
+
+
+# Set the PARTITIONED_FILE_SIZE environment variable to use
+# PartitionedFile support. Note: To convert a partitioned file back
+# to a normal file, just 'cat' (Unix) or 'copy' (DOS/Windows) the
+# partitions together in order.
+PARTITIONED_FILE_SIZE = long(os.environ.get('PARTITIONED_FILE_SIZE') or 0)
+if PARTITIONED_FILE_SIZE > 0:
+ from PartitionedFile import PartitionedFileOperations
+ defaultFops = PartitionedFileOperations(partlen=PARTITIONED_FILE_SIZE)
+
+
+
class FileStorage(BaseStorage.BaseStorage,
ConflictResolution.ConflictResolvingStorage):
# default pack time is 0
@@ -204,7 +241,10 @@
_records_before_save = 10000
def __init__(self, file_name, create=0, read_only=0, stop=None,
- quota=None):
+ quota=None, fops=None):
+ if fops is None:
+ fops = defaultFops
+ self.fops = fops
if read_only:
self._is_read_only = 1
@@ -246,7 +286,8 @@
self._file = None
if not create:
try:
- self._file = open(file_name, read_only and 'rb' or 'r+b')
+ self._file = self.fops.open(
+ file_name, read_only and 'rb' or 'r+b')
except IOError, exc:
if exc.errno == errno.EFBIG:
# The file is too big to open. Fail visibly.
@@ -257,15 +298,15 @@
# If something else went wrong, it's hard to guess
# what the problem was. If the file does not exist,
# create it. Otherwise, fail.
- if os.path.exists(file_name):
+ if self.fops.exists(file_name):
raise
else:
create = 1
if self._file is None and create:
- if os.path.exists(file_name):
- os.remove(file_name)
- self._file = open(file_name, 'w+b')
+ if self.fops.exists(file_name):
+ self.fops.remove(file_name)
+ self._file = self.fops.open(file_name, 'w+b')
self._file.write(packed_version)
r = self._restore_index()
@@ -1003,7 +1044,7 @@
file.write(self._tstatus)
file.flush()
- if fsync is not None: fsync(file.fileno())
+ self.fops.fsync(file)
self._pos=nextpos
@@ -1498,7 +1539,8 @@
p = FileStoragePacker(self._file_name, stop,
self._lock_acquire, self._lock_release,
self._commit_lock_acquire,
- self._commit_lock_release)
+ self._commit_lock_release,
+ self.fops)
try:
opos = p.pack()
if opos is None:
@@ -1506,16 +1548,16 @@
oldpath = self._file_name + ".old"
self._file.close()
try:
- if os.path.exists(oldpath):
- os.remove(oldpath)
- os.rename(self._file_name, oldpath)
+ if self.fops.exists(oldpath):
+ self.fops.remove(oldpath)
+ self.fops.rename(self._file_name, oldpath)
except Exception, msg:
- self._file = open(self._file_name, 'r+b')
+ self._file = self.fops.open(self._file_name, 'r+b')
raise
# OK, we're beyond the point of no return
- os.rename(self._file_name + '.pack', self._file_name)
- self._file = open(self._file_name, 'r+b')
+ self.fops.rename(self._file_name + '.pack', self._file_name)
+ self._file = self.fops.open(self._file_name, 'r+b')
self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
self._pos = opos
self._save_index()
@@ -1688,8 +1730,10 @@
return p, s
-def recover(file_name):
- file=open(file_name, 'r+b')
+def recover(file_name, fops=None):
+ if fops is None:
+ fops = defaultFops
+ file=fops.open(file_name, 'r+b')
index={}
vindex={}
tindex={}
@@ -1939,7 +1983,9 @@
# seek to transaction header, where tid is first 8 bytes
return file.read(8)
-def _truncate(file, name, pos):
+def _truncate(file, name, pos, fops=None):
+ if fops is None:
+ fops = defaultFops
seek=file.seek
seek(0,2)
file_size=file.tell()
@@ -1947,11 +1993,11 @@
i=0
while 1:
oname='%s.tr%s' % (name, i)
- if os.path.exists(oname):
+ if fops.exists(oname):
i=i+1
else:
warn("Writing truncated data from %s to %s", name, oname)
- o=open(oname,'wb')
+ o = fops.open(oname, 'wb')
seek(pos)
cp(file, o, file_size-pos)
o.close()
@@ -1986,9 +2032,11 @@
_ltid = z64
_file = None
- def __init__(self, file, start=None, stop=None):
+ def __init__(self, file, start=None, stop=None, fops=None):
+ if fops is None:
+ fops = defaultFops
if isinstance(file, StringType):
- file = open(file, 'rb')
+ file = fops.open(file, 'rb')
self._file = file
if file.read(4) != packed_version:
raise FileStorageFormatError, file.name
Index: fspack.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/fspack.py,v
retrieving revision 1.5.4.1
diff -u -u -r1.5.4.1 fspack.py
--- fspack.py 13 May 2003 18:04:21 -0000 1.5.4.1
+++ fspack.py 15 May 2003 19:40:40 -0000
@@ -606,9 +606,10 @@
class FileStoragePacker(FileStorageFormatter):
- def __init__(self, path, stop, la, lr, cla, clr):
+ def __init__(self, path, stop, la, lr, cla, clr, fops):
+ self.fops = fops
self._name = path
- self._file = open(path, "rb")
+ self._file = fops.open(path, "rb")
self._stop = stop
self._packt = None
self.locked = 0
@@ -659,7 +660,7 @@
# Setup the destination file and copy the metadata.
# XXX rename from _tfile to something clearer
- self._tfile = open(self._name + ".pack", "w+b")
+ self._tfile = self.fops.open(self._name + ".pack", "w+b")
self._file.seek(0)
self._tfile.write(self._file.read(self._metadata_size))
@@ -671,7 +672,7 @@
if ipos == opos:
# pack didn't free any data. there's no point in continuing.
self._tfile.close()
- os.remove(self._name + ".pack")
+ self.fops.remove(self._name + ".pack")
return None
if ipos < self.file_end:
self.copyRest(ipos)
=== Added File Packages/PartitionedFileStorage/PartitionedFileStorage_HEAD.patch ===
Index: FileStorage.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/FileStorage.py,v
retrieving revision 1.131
diff -u -u -r1.131 FileStorage.py
--- FileStorage.py 1 May 2003 17:10:19 -0000 1.131
+++ FileStorage.py 15 May 2003 19:36:48 -0000
@@ -202,6 +202,43 @@
packed_version='FS21'
+
+class FileOperations:
+ '''This class implements some basic operations on files.
+ PartitionedFileOperations provides an alternate implementation.
+ '''
+ def open(self, name, mode='r', bufsize=0):
+ global open
+ return open(name, mode, bufsize)
+
+ def exists(self, name):
+ return os.path.exists(name)
+
+ def rename(self, oldname, newname):
+ os.rename(oldname, newname)
+
+ def remove(self, name):
+ os.remove(name)
+
+ def fsync(self, file):
+ global fsync
+ if fsync is not None:
+ fsync(file.fileno())
+
+defaultFops = FileOperations()
+
+
+# Set the PARTITIONED_FILE_SIZE environment variable to use
+# PartitionedFile support. Note: To convert a partitioned file back
+# to a normal file, just 'cat' (Unix) or 'copy' (DOS/Windows) the
+# partitions together in order.
+PARTITIONED_FILE_SIZE = long(os.environ.get('PARTITIONED_FILE_SIZE') or 0)
+if PARTITIONED_FILE_SIZE > 0:
+ from PartitionedFile import PartitionedFileOperations
+ defaultFops = PartitionedFileOperations(partlen=PARTITIONED_FILE_SIZE)
+
+
+
class FileStorage(BaseStorage.BaseStorage,
ConflictResolution.ConflictResolvingStorage):
# default pack time is 0
@@ -210,7 +247,10 @@
_records_before_save = 10000
def __init__(self, file_name, create=0, read_only=0, stop=None,
- quota=None):
+ quota=None, fops=None):
+ if fops is None:
+ fops = defaultFops
+ self.fops = fops
if read_only:
self._is_read_only = 1
@@ -242,7 +282,8 @@
self._file = None
if not create:
try:
- self._file = open(file_name, read_only and 'rb' or 'r+b')
+ self._file = self.fops.open(
+ file_name, read_only and 'rb' or 'r+b')
except IOError, exc:
if exc.errno == errno.EFBIG:
# The file is too big to open. Fail visibly.
@@ -253,15 +294,15 @@
# If something else went wrong, it's hard to guess
# what the problem was. If the file does not exist,
# create it. Otherwise, fail.
- if os.path.exists(file_name):
+ if self.fops.exists(file_name):
raise
else:
create = 1
if self._file is None and create:
- if os.path.exists(file_name):
- os.remove(file_name)
- self._file = open(file_name, 'w+b')
+ if self.fops.exists(file_name):
+ self.fops.remove(file_name)
+ self._file = self.fops.open(file_name, 'w+b')
self._file.write(packed_version)
r = self._restore_index()
@@ -1009,7 +1050,7 @@
file.write(self._tstatus)
file.flush()
- if fsync is not None: fsync(file.fileno())
+ self.fops.fsync(file)
self._pos = nextpos
@@ -1454,7 +1495,8 @@
p = FileStoragePacker(self._file_name, stop,
self._lock_acquire, self._lock_release,
self._commit_lock_acquire,
- self._commit_lock_release)
+ self._commit_lock_release,
+ self.fops)
try:
opos = p.pack()
if opos is None:
@@ -1462,16 +1504,16 @@
oldpath = self._file_name + ".old"
self._file.close()
try:
- if os.path.exists(oldpath):
- os.remove(oldpath)
- os.rename(self._file_name, oldpath)
+ if self.fops.exists(oldpath):
+ self.fops.remove(oldpath)
+ self.fops.rename(self._file_name, oldpath)
except Exception, msg:
- self._file = open(self._file_name, 'r+b')
+ self._file = self.fops.open(self._file_name, 'r+b')
raise
# OK, we're beyond the point of no return
- os.rename(self._file_name + '.pack', self._file_name)
- self._file = open(self._file_name, 'r+b')
+ self.fops.rename(self._file_name + '.pack', self._file_name)
+ self._file = self.fops.open(self._file_name, 'r+b')
self._initIndex(p.index, p.vindex, p.tindex, p.tvindex)
self._pos = opos
self._save_index()
@@ -1649,8 +1691,10 @@
return p, s
-def recover(file_name):
- file=open(file_name, 'r+b')
+def recover(file_name, fops=None):
+ if fops is None:
+ fops = defaultFops
+ file=fops.open(file_name, 'r+b')
index={}
vindex={}
tindex={}
@@ -1898,7 +1942,9 @@
# seek to transaction header, where tid is first 8 bytes
return file.read(8)
-def _truncate(file, name, pos):
+def _truncate(file, name, pos, fops=None):
+ if fops is None:
+ fops = defaultFops
seek=file.seek
seek(0,2)
file_size=file.tell()
@@ -1906,11 +1952,11 @@
i=0
while 1:
oname='%s.tr%s' % (name, i)
- if os.path.exists(oname):
+ if fops.exists(oname):
i=i+1
else:
warn("Writing truncated data from %s to %s", name, oname)
- o=open(oname,'wb')
+ o = fops.open(oname, 'wb')
seek(pos)
cp(file, o, file_size-pos)
o.close()
@@ -1945,9 +1991,11 @@
_ltid = z64
_file = None
- def __init__(self, file, start=None, stop=None):
+ def __init__(self, file, start=None, stop=None, fops=None):
+ if fops is None:
+ fops = defaultFops
if isinstance(file, StringType):
- file = open(file, 'rb')
+ file = fops.open(file, 'rb')
self._file = file
if file.read(4) != packed_version:
raise FileStorageFormatError, file.name
Index: fspack.py
===================================================================
RCS file: /cvs-repository/Zope/lib/python/ZODB/fspack.py,v
retrieving revision 1.5
diff -u -u -r1.5 fspack.py
--- fspack.py 13 May 2003 16:29:21 -0000 1.5
+++ fspack.py 15 May 2003 19:36:49 -0000
@@ -605,9 +605,10 @@
class FileStoragePacker(FileStorageFormatter):
- def __init__(self, path, stop, la, lr, cla, clr):
+ def __init__(self, path, stop, la, lr, cla, clr, fops):
+ self.fops = fops
self._name = path
- self._file = open(path, "rb")
+ self._file = fops.open(path, "rb")
self._stop = stop
self._packt = None
self.locked = 0
@@ -658,7 +659,7 @@
# Setup the destination file and copy the metadata.
# XXX rename from _tfile to something clearer
- self._tfile = open(self._name + ".pack", "w+b")
+ self._tfile = self.fops.open(self._name + ".pack", "w+b")
self._file.seek(0)
self._tfile.write(self._file.read(self._metadata_size))
@@ -670,7 +671,7 @@
if ipos == opos:
# pack didn't free any data. there's no point in continuing.
self._tfile.close()
- os.remove(self._name + ".pack")
+ self.fops.remove(self._name + ".pack")
return None
if ipos < self.file_end:
self.copyRest(ipos)
=== Removed File Packages/PartitionedFileStorage/FileStorage_py_1_129.patch ===
=== Removed File Packages/PartitionedFileStorage/FileStorage_py_1_98_2_7.patch ===