[Zope-CVS] CVS: Products/Ape/lib/apelib/fs - oidtable.py:1.2
__init__.py:1.2 annotated.py:1.5 base.py:1.8
classification.py:1.5 connection.py:1.8 fileops.py:1.2
interfaces.py:1.5 params.py:1.2 properties.py:1.5
security.py:1.4 structure.py:1.8
Shane Hathaway
shane at zope.com
Sat Feb 28 15:06:58 EST 2004
Update of /cvs-repository/Products/Ape/lib/apelib/fs
In directory cvs.zope.org:/tmp/cvs-serv30293/lib/apelib/fs
Modified Files:
__init__.py annotated.py base.py classification.py
connection.py fileops.py interfaces.py params.py properties.py
security.py structure.py
Added Files:
oidtable.py
Log Message:
Merged ape-fs-oid-branch.
Ape now uses arbitrary OIDs on the filesystem, rather than using paths
as OIDs. This solves problems with moving and replacing objects and
further unifies SQL and filesystem databases.
=== Products/Ape/lib/apelib/fs/oidtable.py 1.1 => 1.2 ===
--- /dev/null Sat Feb 28 15:06:58 2004
+++ Products/Ape/lib/apelib/fs/oidtable.py Sat Feb 28 15:06:27 2004
@@ -0,0 +1,133 @@
+##############################################################################
+#
+# Copyright (c) 2003 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+##############################################################################
+"""OIDTable class.
+
+$Id$
+"""
+
+class OIDTable:
+ """Table of (parent_oid, filename, child_oid).
+
+ The oid and filename columns form the primary key. Maintains an index
+ on the child_oid column to allow fast reverse lookups.
+ """
+
+ def __init__(self):
+ self.fwd = {} # { parent_oid : {filename : child_oid} }
+ self.back = {} # { child_oid : [(parent_oid, filename)] }
+
+ def add(self, parent_oid, filename, child_oid):
+ """Adds an association from a parent and filename to a child.
+ """
+ d = self.fwd.get(parent_oid)
+ if d is None:
+ d = {}
+ self.fwd[parent_oid] = d
+ if d.has_key(filename):
+ if d[filename] != child_oid:
+ raise KeyError(
+ "'%s' already has a child named '%s', with OID '%s'"
+ % (parent_oid, filename, d[filename]))
+ else:
+ d[filename] = child_oid
+ p = self.back.get(child_oid)
+ key = (parent_oid, filename)
+ if p is None:
+ p = [key]
+ self.back[child_oid] = p
+ elif key not in p:
+ p.append(key)
+
+ def remove(self, parent_oid, filename):
+ """Removes an association between a parent and a child.
+ """
+ d = self.fwd.get(parent_oid)
+ if not d:
+ return
+ child_oid = d.get(filename)
+ if not child_oid:
+ return
+ del d[filename]
+ if not d:
+ del self.fwd[parent_oid]
+ p = self.back.get(child_oid)
+ key = (parent_oid, filename)
+ if key in p:
+ p.remove(key)
+ if not p:
+ del self.back[child_oid]
+
+ def setChildren(self, parent_oid, new_children):
+ """Updates all children for a parent.
+
+ new_children is {filename: child_oid}. Calls self.add() and
+ self.remove() to make all changes.
+ """
+ old_children = self.fwd.get(parent_oid)
+ if old_children is not None:
+ # The dictionary in the table will change as children are
+ # added/removed, so make a copy.
+ old_children = old_children.copy()
+ else:
+ old_children = {}
+ for filename, child_oid in new_children.items():
+ if old_children.has_key(filename):
+ if old_children[filename] != child_oid:
+ # Change this child to a new OID.
+ self.remove(parent_oid, filename)
+ self.add(parent_oid, filename, child_oid)
+ del old_children[filename]
+ else:
+ # Add a new child.
+ self.add(parent_oid, filename, child_oid)
+ # Remove the filenames left over in old_children.
+ for filename, child_oid in old_children.items():
+ self.remove(parent_oid, filename)
+
+ def getPath(self, ancestor_oid, descendant_oid):
+ """Returns the primary path that connects two OIDs.
+
+ The primary path follows the first parent of each OID.
+ """
+ parts = []
+ back_get = self.back.get
+ parts_append = parts.append
+ oid = descendant_oid
+ while oid != ancestor_oid:
+ p = back_get(oid)
+ if not p:
+ # The primary lineage doesn't reach the ancestor.
+ return None
+ # Follow only the first parent.
+ oid, filename = p[0]
+ if oid == descendant_oid:
+ # Circular OID chain.
+ return None
+ parts_append(filename)
+ parts.reverse()
+ return parts
+
+ def getChildren(self, parent_oid):
+ """Returns the children of an OID as a mapping of {filename: oid}.
+
+ Do not modify the return value.
+ """
+ return self.fwd.get(parent_oid)
+
+ def getParents(self, child_oid):
+ """Returns the parents of an OID as a list of (oid, filename).
+
+ Do not modify the return value.
+ """
+ return self.back.get(child_oid)
=== Products/Ape/lib/apelib/fs/__init__.py 1.1 => 1.2 ===
=== Products/Ape/lib/apelib/fs/annotated.py 1.4 => 1.5 ===
--- Products/Ape/lib/apelib/fs/annotated.py:1.4 Fri Feb 20 22:30:39 2004
+++ Products/Ape/lib/apelib/fs/annotated.py Sat Feb 28 15:06:27 2004
@@ -170,7 +170,7 @@
return 0
return 1
- def computeDirectoryContents(self, path, allow_missing=0):
+ def computeContents(self, path, allow_missing=0):
"""Returns the name translations for a directory. Caches the results.
Returns ({filename: name}, {name: filename}).
=== Products/Ape/lib/apelib/fs/base.py 1.7 => 1.8 ===
=== Products/Ape/lib/apelib/fs/classification.py 1.4 => 1.5 ===
--- Products/Ape/lib/apelib/fs/classification.py:1.4 Mon Feb 2 10:07:20 2004
+++ Products/Ape/lib/apelib/fs/classification.py Sat Feb 28 15:06:27 2004
@@ -31,40 +31,41 @@
def load(self, event):
fs_conn = self.getConnection(event)
- p = event.oid
- classification = {'node_type': fs_conn.readNodeType(p)}
- text = fs_conn.readAnnotation(p, 'classification', '')
+ oid = event.oid
+ classification = {'node_type': fs_conn.readNodeType(oid)}
+ text = fs_conn.readAnnotation(oid, 'classification', '')
if text:
lines = text.split('\n')
for line in lines:
if '=' in line:
k, v = line.split('=', 1)
classification[k.strip()] = v.strip()
- classification['extension'] = fs_conn.getExtension(p)
+ classification['extension'] = fs_conn.getExtension(oid)
+ classification['subpath'] = fs_conn.getSubpath(oid)
return classification, text.strip()
def store(self, event, state):
# state is a classification
fs_conn = self.getConnection(event)
- p = event.oid
+ oid = event.oid
if event.is_new:
# Don't overwrite existing data
try:
- fs_conn.readNodeType(p)
+ fs_conn.readNodeType(oid)
except LoadError:
# Nothing exists yet.
pass
else:
# Something exists. Don't overwrite it.
- raise OIDConflictError(p)
+ raise OIDConflictError(oid)
items = state.items()
items.sort()
text = []
for k, v in items:
if k == 'extension':
- fs_conn.suggestExtension(p, v)
+ fs_conn.suggestExtension(oid, v)
else:
text.append('%s=%s' % (k, v))
text = '\n'.join(text)
- fs_conn.writeAnnotation(p, 'classification', text)
+ fs_conn.writeAnnotation(oid, 'classification', text)
return text.strip()
=== Products/Ape/lib/apelib/fs/connection.py 1.7 => 1.8 ===
--- Products/Ape/lib/apelib/fs/connection.py:1.7 Fri Feb 20 22:30:39 2004
+++ Products/Ape/lib/apelib/fs/connection.py Sat Feb 28 15:06:27 2004
@@ -11,15 +11,20 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
-"""Full-featured Filesystem connection class.
+"""Filesystem connection class.
$Id$
"""
from apelib.core.interfaces import ITPCConnection, ISourceRepository, LoadError
-from interfaces import IFSReader, IFSWriter, FSWriteError
+
+from interfaces import IFSReader, IFSWriter, FSReadError, FSWriteError
from fileops import StandardFileOperations
from annotated import AnnotatedFilesystem, object_names_ann
+from oidtable import OIDTable
+
+DEBUG = 0
+
# For a node_type_ann, the value is 'f' (file) or 'd' (directory)
node_type_ann = '@node_type'
@@ -40,86 +45,137 @@
__implements__ = IFSReader, IFSWriter, ITPCConnection, ISourceRepository
basepath = ''
+ root_oid = '0'
+
+ # When app_filename is set, FSConnection translates paths, placing
+ # the application object at basepath and the root at
+ # (basepath)/_root.
+ app_filename = 'Application'
def __init__(self, basepath, annotation_prefix='.', hidden_filenames='_',
ops=None):
+ # These attributes are used for both reading and writing.
self.basepath = basepath
if ops is None:
ops = StandardFileOperations()
self.ops = ops
self.afs = AnnotatedFilesystem(
ops, annotation_prefix, hidden_filenames)
- self._final = 0
+ self.table = OIDTable()
+
+ # These attributes are used only for writing.
+ self._final = 0 # True if second phase of commit.
# _pending holds the data to be written.
- # _pending: { subpath string -> { annotation_name -> data } }
+ # _pending: { oid -> { annotation_name -> data } }
self._pending = {}
+ self._script = None # [(instruction, *args)]
+ self._tmp_subpaths = {} # { oid: subpath }
+
+ def reset(self):
+ self._final = 0
+ self._pending.clear()
+ self.afs.clearCache()
+ self._script = None
+ self._tmp_subpaths.clear()
#
# IFSReader implementation.
#
- def getPath(self, subpath):
- if self.basepath:
- while subpath.startswith('/') or subpath.startswith('\\'):
- subpath = subpath[1:]
- path = self.ops.join(self.basepath, subpath)
+ def getSubpath(self, oid):
+ p = self.table.getPath(self.root_oid, oid)
+ if p is None:
+ return self._tmp_subpaths.get(oid)
+ if self.app_filename:
+ # Translate paths.
+ if p and p[0] == self.app_filename:
+ # Place the application object at basepath.
+ return p[1:]
+ else:
+ # Everything else goes in "_root".
+ return ['_root'] + p
else:
- # unchanged.
- path = subpath
- if not self.ops.exists(path):
- dir_path, obj_name = self.ops.split(path)
- if '.' not in obj_name:
- # This object might have an automatic filename extension.
- contents = self.afs.computeDirectoryContents(dir_path, 1)
- fn_to_name, name_to_fn = contents
- fn = name_to_fn.get(obj_name)
- if fn:
- # Use the filename with an extension.
- path = self.ops.join(dir_path, fn)
- return path
+ return p
- def readNodeType(self, subpath):
- path = self.getPath(subpath)
+ def getPath(self, oid):
+ p = self.getSubpath(oid)
+ if p is None:
+ raise LoadError(oid)
+ return self.ops.join(self.basepath, *p)
+
+ def readNodeType(self, oid):
+ path = self.getPath(oid)
if not self.ops.exists(path):
raise LoadError("%s does not exist" % path)
return self.ops.isdir(path) and 'd' or 'f'
- def readData(self, subpath, allow_missing=0, as_text=0):
- path = self.getPath(subpath)
+ def readData(self, oid, allow_missing=0, as_text=0):
# Return a string.
try:
+ path = self.getPath(oid)
return self.ops.readfile(path, as_text)
- except IOError:
+ except (LoadError, IOError):
if allow_missing:
return None
raise
- def readDirectory(self, subpath, allow_missing=0):
- path = self.getPath(subpath)
- # Return a sequence of object names.
- contents = self.afs.computeDirectoryContents(path, allow_missing)
+ def readDirectory(self, oid, allow_missing=0):
+ # Return a sequence of (object_name, child_oid).
+ path = self.getPath(oid)
+ contents = self.afs.computeContents(path, allow_missing)
fn_to_name, name_to_fn = contents
- return name_to_fn.keys()
+ children = self.table.getChildren(oid)
+ if children is None:
+ children = {}
+ # Remove vanished children from the OID table.
+ for filename, child_oid in children.items():
+ if not fn_to_name.has_key(filename):
+ self.table.remove(oid, filename)
+ # XXX Need to garbage collect descendants.
+ # Return the current children.
+ return [(objname, children.get(filename))
+ for filename, objname in fn_to_name.items()]
- def readAnnotation(self, subpath, name, default=None):
- self.afs.checkAnnotationName(name)
- path = self.getPath(subpath)
+ def readAnnotation(self, oid, name, default=None):
+ path = self.getPath(oid)
annotations = self.afs.getAnnotations(path)
return annotations.get(name, default)
- def getExtension(self, subpath):
- path = self.getPath(subpath)
+ def readObjectName(self, oid):
+ parents = self.table.getParents(oid)
+ parent_oid, filename = parents[0]
+ parent_path = self.getPath(parent_oid)
+ contents = self.afs.computeContents(parent_path)
+ fn_to_name, name_to_fn = contents
+ return fn_to_name[filename]
+
+ def getExtension(self, oid):
+ path = self.getPath(oid)
stuff, ext = self.ops.splitext(path)
return ext
- def getModTime(self, subpath, default=0):
+ def assignExisting(self, oid, children):
+ """See IFSReader.
+ """
+ dir_path = self.getPath(oid)
+ contents = self.afs.computeContents(dir_path)
+ fn_to_name, name_to_fn = contents
+ existing = self.table.getChildren(oid) or {}
+ for name, child_oid in children:
+ assert child_oid
+ if existing.has_key(name) and existing[name] != child_oid:
+ raise FSReadError("assignExisting() doesn't override")
+ filename = name_to_fn[name]
+ self.table.add(oid, filename, child_oid)
+
+ def getModTime(self, oid, default=0):
"""Returns the time an object was last modified.
Since objects are split into up to three files, this
implementation returns the modification time of the most
recently modified of the three.
"""
- path = self.getPath(subpath)
+ path = self.getPath(oid)
extra = self.afs.getAnnotationPaths(path)
maxtime = -1
for p in (path,) + tuple(extra):
@@ -134,10 +190,6 @@
maxtime = default
return maxtime
- #
- # ISourceRepository implementation.
- #
-
def _get_paths_mtime(self, paths):
t = []
for path in paths:
@@ -147,13 +199,17 @@
t.append(None)
return t
- def getPollSources(self, subpath):
- path = self.getPath(subpath)
+ def getPollSources(self, oid):
+ path = self.getPath(oid)
extra = self.afs.getAnnotationPaths(path)
paths = (path,) + tuple(extra)
t = self._get_paths_mtime(paths)
return {(self, paths): t}
+ #
+ # ISourceRepository implementation.
+ #
+
def poll(self, sources):
"""ISourceRepository implementation.
@@ -171,187 +227,429 @@
#
# IFSWriter implementation.
#
+ def _queue(self, oid, name, data):
+ """Queues data to be written at commit time.
- def writeNodeType(self, subpath, data):
- self._queue(subpath, node_type_ann, data)
-
- def writeData(self, subpath, data, as_text=0):
- self._queue(subpath, data_ann, (data, as_text))
+ 'name' is the name of the annotation.
+ """
+ m = self._pending
+ anns = m.get(oid)
+ if anns is None:
+ anns = {}
+ m[oid] = anns
+ if anns.has_key(name):
+ if anns[name] != data:
+ raise FSWriteError(
+ 'Conflicting data storage at %s (%s)' %
+ (oid, name))
+ else:
+ anns[name] = data
- def writeDirectory(self, subpath, names):
- self._queue(subpath, file_list_ann, names)
+ def writeNodeType(self, oid, data):
+ if data not in ('d', 'f'):
+ raise FSWriteError(
+ 'Node type must be "d" or "f" at %s' % oid)
+ self._queue(oid, node_type_ann, data)
+
+ def writeData(self, oid, data, as_text=0):
+ if not isinstance(data, type('')):
+ raise FSWriteError(
+ 'Data for a file must be a string at %s' % oid)
+ self._queue(oid, data_ann, (data, as_text))
+
+ def writeDirectory(self, oid, data):
+ if isinstance(data, type('')): # XXX Need a better check
+ raise FSWriteError(
+ 'Data for a directory must be a list or tuple at %s' % oid)
+ isLegalFilename = self.afs.isLegalFilename
+ for objname, child_oid in data:
+ assert child_oid, "%s lacks a child_oid" % repr(objname)
+ if not isLegalFilename(objname):
+ raise FSWriteError(
+ 'Not a legal object name: %s' % repr(objname))
+ self._queue(oid, file_list_ann, data)
- def writeAnnotation(self, subpath, name, data):
+ def writeAnnotation(self, oid, name, data):
self.afs.checkAnnotationName(name)
- self._queue(subpath, name, data)
+ self._queue(oid, name, data)
- def suggestExtension(self, subpath, ext):
- self._queue(subpath, suggested_extension_ann, ext)
+ def suggestExtension(self, oid, ext):
+ self._queue(oid, suggested_extension_ann, ext)
- def _writeFinal(self, subpath, anns):
- """Performs an actual write of a file or directory to disk.
- """
- # anns is a mapping.
- path = self.getPath(subpath)
- t = anns[node_type_ann]
- if not self.ops.exists(path):
- if t == 'd':
- self.ops.mkdir(path)
- else:
- fn = self.ops.split(path)[1]
- if '.' not in fn:
- # This object has no extension and doesn't yet exist.
- ext = anns.get(suggested_extension_ann)
- if ext:
- # Try to use the suggested extension.
- if not ext.startswith('.'):
- ext = '.' + ext
- p = path + ext
- if not self.ops.exists(p):
- # No file is in the way.
- # Use the suggested extension.
- path = p
- to_write = {}
- for name, value in anns.items():
- if (name == node_type_ann
- or name == suggested_extension_ann):
- # Doesn't need to be written.
- continue
- elif name == data_ann:
- data, as_text = value
- self.ops.writefile(path, as_text, data)
- elif name == file_list_ann:
- # Change the list of subobjects.
- self._removeUnlinkedItems(path, value)
- to_write[object_names_ann] = '\n'.join(value)
- self._disableConflictingExtensions(subpath, value)
- self.afs.invalidate(path)
- else:
- to_write[name] = value
- self.afs.writeAnnotations(path, to_write)
+ def _prepareContainerChanges(self, path, data):
+ """Prepares the new dictionary of children for a directory.
+ Chooses filenames for all of the directory's children.
+ Prevents filename collisions involving extensions by enforcing
+ the rule that if there is some object named 'foo.*', an object
+ named 'foo' may not have an automatic extension.
- def _removeUnlinkedItems(self, path, names):
- """Removes unused files/subtrees from a directory."""
- linked = {}
- for name in names:
- linked[name] = 1
- fn_to_name, name_to_fn = self.afs.computeDirectoryContents(path)
- for fn, obj_name in fn_to_name.items():
- if not linked.get(obj_name):
- item_fn = self.ops.join(path, fn)
- if self.ops.isdir(item_fn):
- self.ops.rmtree(item_fn)
- else:
- self.ops.remove(item_fn)
- extra_paths = self.afs.getAnnotationPaths(item_fn)
- for p in extra_paths:
- if self.ops.exists(p):
- self.ops.remove(p)
-
-
- def _disableConflictingExtensions(self, subpath, obj_names):
- """Fixes collisions before writing files in a directory.
-
- Enforces the rule: if 'foo.*' is in the database, 'foo' may
- not have an automatic extension. Enforces by un-queuing
- suggested extensions.
- """
- reserved = {} # { object name without extension -> 1 }
- for obj_name in obj_names:
- if '.' in obj_name:
- base, ext = obj_name.split('.', 1)
+ 'path' is a filesystem path or None. 'data' is a list of
+ (objname, child_oid). Returns {filename: child_oid}.
+ """
+ if path:
+ existing = self.afs.computeContents(path)[1]
+ # existing contains {objname: filename}
+ else:
+ existing = {}
+
+ reserved = {} # { object name stripped of extension: 1 }
+ for objname, child_oid in data:
+ if '.' in objname:
+ base, ext = objname.split('.', 1)
reserved[base] = 1
- if not reserved:
- # No objects have extensions.
- return
- while subpath.endswith('/'):
- subpath = subpath[:-1]
- for obj_name in obj_names:
- if reserved.has_key(obj_name):
- # Prevent obj_name from using an automatic extension.
- child_subpath = '%s/%s' % (subpath, obj_name)
- self._queue(child_subpath, suggested_extension_ann,
- '', force=1)
-
-
- def _beforeWrite(self, items):
- """Does some early checking while it's easy to bail out.
-
- This helps avoid exceptions during the second phase of
- transaction commit.
- """
- non_containers = {}
- for subpath, anns in items:
- path = self.getPath(subpath)
- exists = self.ops.exists(path)
- if exists and not self.ops.canwrite(path):
+ new_filenames = {}
+ for objname, child_oid in data:
+ filename = objname
+ if '.' not in filename and not reserved.has_key(objname):
+ # This object is eligible for an automatic extension.
+ fn = existing.get(objname)
+ if fn:
+ # Use the existing filename.
+ filename = fn
+ else:
+ anns = self._pending.get(child_oid)
+ if anns:
+ extension = anns.get(suggested_extension_ann)
+ if extension:
+ if not extension.startswith('.'):
+ extension = '.' + extension
+ filename = objname + extension
+ new_filenames[objname] = filename
+
+ fn_oid = {}
+ for objname, child_oid in data:
+ fn_oid[new_filenames[objname]] = child_oid
+ return fn_oid
+
+
+ def _prepare(self):
+ """Prepares for transaction commit.
+
+ Does some early checking while it's easy to bail out. This
+ helps avoid exceptions during the second phase of transaction
+ commit.
+ """
+ container_changes = {} # {oid: {filename: child_oid}}
+ for oid, anns in self._pending.items():
+ if self.table.getParents(oid) or oid == self.root_oid:
+ # This is an existing object. It has a path.
+ p = self.getSubpath(oid)
+ if p is None:
+ raise FSWriteError(
+ "No path known for OID %s" % repr(oid))
+ if p:
+ info = self.ops.join(*p)
+ path = self.ops.join(self.basepath, info)
+ else:
+ info = '/'
+ path = self.basepath
+ if not self.ops.exists(path):
+ path = None
+ else:
+ # This is a new object. It does not have a path yet.
+ path = None
+ info = 'new object: %s' % repr(oid)
+ if path and not self.ops.canwrite(path):
raise FSWriteError(
- "Can't get write access to %s" % subpath)
+ "Can't get write access. %s" % info)
+
# type must be provided and must always be either 'd' or 'f'.
if not anns.has_key(node_type_ann):
raise FSWriteError(
- 'Node type not specified for %s' % subpath)
+ 'Node type not specified for %s' % info)
t = anns[node_type_ann]
- dir = self.ops.dirname(subpath)
- if non_containers.get(dir):
- raise FSWriteError(
- "Not a directory: %s" % dir)
+ dir = self.ops.dirname(oid)
+
if t == 'f':
+ # Writing a file
data, as_text = anns[data_ann]
if anns.has_key(file_list_ann):
raise FSWriteError(
- "Files can't have directory contents. %s"
- % subpath)
- if exists and self.ops.isdir(path):
+ "Files can't have directory contents. %s" % info)
+ if path and self.ops.isdir(path):
raise FSWriteError(
- "Can't write file data to directory at %s"
- % subpath)
- non_containers[subpath] = 1
- if not isinstance(data, type('')):
- raise FSWriteError(
- 'Data for a file must be a string at %s'
- % subpath)
+ "A directory exists where a file is to be written. %s"
+ % info)
+
elif t == 'd':
+ # Writing a directory
data = anns[file_list_ann]
if anns.has_key(data_ann):
raise FSWriteError(
- "Directories can't have file data. %s"
- % subpath)
- if exists and not self.ops.isdir(path):
+ "Directories can't have file data. %s" % info)
+ if path and not self.ops.isdir(path):
raise FSWriteError(
- "Can't write directory contents to file at %s"
- % subpath)
- if isinstance(data, type('')):
+ "A file exists where a directory is to be written. %s"
+ % info)
+ fn_oid = self._prepareContainerChanges(path, data)
+ container_changes[oid] = fn_oid
+
+ else:
+ raise FSWriteError('Node type must be "d" or "f". %s' % info)
+ self._script = self._generateScript(container_changes)
+
+
+ def _generateScript(self, container_changes):
+ """Generates the script for committing the transaction.
+
+ Returns [(instruction, *args)].
+ """
+ # container_changes is {oid: {filename: child_oid}}
+ # script is [(instruction, *args)]
+ script = []
+ script.append(("clearTemp",))
+
+ # Compute the number of times each relevant child_oid is to
+ # be linked or unlinked.
+ # counts is {child_oid: [link_count, unlink_count]}
+ counts = {}
+ def increment(child_oid, index, counts=counts):
+ c = counts.get(child_oid)
+ if c is None:
+ counts[child_oid] = c = [0, 0]
+ c[index] += 1
+
+ for oid, new_children in container_changes.items():
+ old_children = self.table.getChildren(oid)
+ if old_children is None:
+ old_children = {}
+ for filename, child_oid in new_children.items():
+ if old_children.get(filename) == child_oid:
+ continue # No change.
+ # Adding a link
+ increment(child_oid, 0)
+ if DEBUG:
+ print 'fs: add link %s/%s -> %s' % (
+ oid, filename, child_oid)
+ for filename, child_oid in old_children.items():
+ if new_children.get(filename) == child_oid:
+ continue # No change.
+ # Removing a link
+ increment(child_oid, 1)
+ if DEBUG:
+ print 'fs: del link %s/%s -> %s' % (
+ oid, filename, child_oid)
+
+ # Add steps to the script to move objects to a temporary directory,
+ # then delete objects.
+ to_delete = [] # [oid]
+ for child_oid, (links, unlinks) in counts.items():
+ if not self.table.getParents(child_oid):
+ # A new object should be added once or not at all.
+ if links > 1:
raise FSWriteError(
- 'Data for a directory must be a list or tuple at %s'
- % subpath)
- for item in data:
- if not self.afs.isLegalFilename(item):
- raise FSWriteError(
- 'Not a legal object name: %s' % repr(item))
+ "Multiple links to %s" % repr(child_oid))
else:
- raise FSWriteError(
- 'Node type must be "d" or "f" at %s' % subpath)
+ # An existing object should be moved, removed, or left alone.
+ if links > 1 or (links > 0 and unlinks < 1):
+ raise FSWriteError(
+ "Multiple links to %s" % repr(child_oid))
+ if links > 0:
+ # Moving.
+ script.append(("moveToTemp", child_oid))
+ elif unlinks > 0:
+ # Deleting.
+ to_delete.append(child_oid)
+
+ for child_oid in to_delete:
+ script.append(("delete", child_oid))
+ script.append(("writeAll", container_changes))
+ if self.app_filename and container_changes.has_key(self.root_oid):
+ # Link or unlink the application object.
+ root_changes = container_changes[self.root_oid]
+ script.append(("linkApp", root_changes.has_key(self.app_filename)))
+ script.append(("clearTemp",))
+ return script
+ def _rmrf(self, path):
+ """Delete ala 'rm -rf'.
- def _queue(self, subpath, name, data, force=0):
- """Queues data to be written at commit time"""
- m = self._pending
- anns = m.get(subpath)
- if anns is None:
- anns = {}
- m[subpath] = anns
- if anns.has_key(name) and not force:
- if anns[name] != data:
- raise FSWriteError(
- 'Conflicting data storage at %s (%s)' %
- (subpath, name))
+ If it's a file, remove it. If it's a directory, remove all of it.
+ If it doesn't exist, quietly ignore it.
+ """
+ ops = self.ops
+ if ops.exists(path):
+ if ops.isdir(path):
+ ops.rmtree(path)
+ else:
+ ops.remove(path)
+
+ def _do_clearTemp(self):
+ """Script command: zap the temporary directory.
+ """
+ ops = self.ops
+ path = ops.join(self.basepath, '_tmp')
+ self._rmrf(path)
+ self._tmp_subpaths.clear()
+
+ def _moveBaseContents(self, src, dest):
+ """Move the base directory's contents, but not the directory.
+
+ Also leaves behind the _root and _tmp subdirectories.
+ """
+ ops = self.ops
+ if not ops.exists(dest):
+ ops.makedirs(dest)
+ for fn in ops.listdir(src):
+ if fn not in ('_root', '_tmp'):
+ ops.rename(ops.join(src, fn), ops.join(dest, fn))
+
+ def _moveItem(self, src, dest):
+ """Moves a file or directory.
+
+ For files, also moves annotations next to the file.
+ """
+ ops = self.ops
+ parent = ops.dirname(dest)
+ if not ops.exists(parent):
+ ops.makedirs(parent)
+ if not ops.isdir(src):
+ # Move the annotation files also.
+ extra_src = self.afs.getAnnotationPaths(src)
+ extra_dest = self.afs.getAnnotationPaths(dest)
+ for s, d in zip(extra_src, extra_dest):
+ if ops.exists(s):
+ ops.rename(s, d)
+ ops.rename(src, dest)
+
+ def _do_moveToTemp(self, oid):
+ """Script command: move an object to the temporary directory.
+ """
+ ops = self.ops
+ src = self.getPath(oid)
+ if src == self.basepath:
+ # Move the base by moving most of the contents
+ # instead of the actual directory.
+ dest_sub = ('_tmp', 'base', 'data')
+ dest = ops.join(self.basepath, *dest_sub)
+ self._moveBaseContents(src, dest)
else:
- anns[name] = data
+ # Move an object.
+ dest_sub = ('_tmp', 'oid.%s' % oid, 'data')
+ dest = ops.join(self.basepath, *dest_sub)
+ self._moveItem(src, dest)
+ self._tmp_subpaths[oid] = dest_sub
+ parents = self.table.getParents(oid)
+ for parent_oid, filename in parents:
+ self.table.remove(parent_oid, filename)
+
+ def _restore(self, oid):
+ """Moves an object in the temp directory into the object system.
+ """
+ ops = self.ops
+ dest = self.getPath(oid)
+ src_sub = self._tmp_subpaths[oid]
+ src = ops.join(self.basepath, *src_sub)
+ if dest == self.basepath:
+ self._moveBaseContents(src, dest)
+ else:
+ self._moveItem(src, dest)
+ del self._tmp_subpaths[oid]
+
+ def _do_delete(self, oid):
+ """Script command: delete an object.
+ """
+ ops = self.ops
+ path = self.getPath(oid)
+ if path == self.basepath:
+ # Delete the contents of the base directory.
+ for fn in ops.listdir(path):
+ if not fn in ('_root', '_tmp'):
+ self._rmrf(ops.join(self.basepath, fn))
+ else:
+ # Delete an object.
+ if not ops.isdir(path):
+ # Delete the annotation files also.
+ extra = self.afs.getAnnotationPaths(path)
+ for s in extra:
+ if ops.exists(s):
+ ops.remove(s)
+ self._rmrf(path)
+ if self._tmp_subpaths.has_key(oid):
+ del self._tmp_subpaths[oid]
+ parents = self.table.getParents(oid)
+ for parent_oid, filename in parents:
+ self.table.remove(parent_oid, filename)
+ # XXX Need to garbage collect descendants in the OID table.
+ def _do_writeAll(self, container_changes):
+ """Script command: write all objects.
+
+ Uses multiple passes.
+
+ container_changes: {oid: {filename: child_oid}}
+ """
+ ops = self.ops
+ while self._pending:
+ written = 0
+ for oid, anns in self._pending.items():
+ p = self.getSubpath(oid)
+ if p is None:
+ # Not linked into the object system yet.
+ # Try again on the next pass.
+ continue
+ path = ops.join(self.basepath, *p)
+ t = anns[node_type_ann]
+ if not ops.exists(path):
+ if t == 'd':
+ ops.mkdir(path)
+ to_write = {}
+ for name, value in anns.items():
+ if (name == node_type_ann
+ or name == suggested_extension_ann):
+ # Doesn't need to be written.
+ continue
+ elif name == data_ann:
+ data, as_text = value
+ ops.writefile(path, as_text, data)
+ elif name == file_list_ann:
+ # Prepare the object_names annotation.
+ object_names = []
+ for objname, child_oid in value:
+ object_names.append(objname)
+ to_write[object_names_ann] = '\n'.join(object_names)
+ # Move objects from the temporary directory.
+ fn_oid = container_changes.get(oid)
+ if fn_oid:
+ for filename, child_oid in fn_oid.items():
+ self.table.add(oid, filename, child_oid)
+ if self._tmp_subpaths.has_key(child_oid):
+ self._restore(child_oid)
+ self.afs.invalidate(path)
+ else:
+ to_write[name] = value
+ self.afs.writeAnnotations(path, to_write)
+ self.afs.invalidate(self.ops.dirname(path))
+ # This object has been written.
+ written += 1
+ del self._pending[oid]
+
+ if DEBUG and not written:
+ # Nothing was written in this pass. This means that
+ # the rest of the queued objects are not actually
+ # linked into the object system. Toss them.
+ tossing = self._pending.keys()
+ tossing.sort()
+ print "tossing: %s" % ', '.join(tossing)
+ break
+
+
+ def _do_linkApp(self, app_exists):
+ """Script command: link or unlink the application object at the root.
+ """
+ path = self.ops.join(self.basepath, '_root', self.app_filename)
+ if app_exists:
+ # The root has an application. Represent it with a directory.
+ if not self.ops.exists(path):
+ self.ops.makedirs(path)
+ else:
+ # The root does not have an application. Remove it.
+ if self.ops.exists(path):
+ self.ops.rmtree(path)
+
#
# ITPCConnection implementation
#
@@ -363,8 +661,14 @@
return self.basepath
def connect(self):
- if not self.ops.exists(self.basepath):
- self.ops.makedirs(self.basepath)
+ ops = self.ops
+ if not ops.exists(self.basepath):
+ ops.makedirs(self.basepath)
+ if self.app_filename:
+ # If there are objects at basepath, create a _root
+ # containing an application also.
+ if self.afs.computeContents(self.basepath):
+ self._do_linkApp(1)
def begin(self):
self.afs.clearCache()
@@ -374,26 +678,18 @@
This is done while the transaction can still be vetoed safely.
"""
- items = self._pending.items()
- items.sort() # Ensure that base directories come first.
- self._beforeWrite(items)
+ self._prepare()
self._final = 1
- def reset(self):
- self._final = 0
- self._pending.clear()
- self.afs.clearCache()
-
def abort(self):
self.reset()
def finish(self):
if self._final:
try:
- items = self._pending.items()
- items.sort() # Ensure that base directories come first.
- for subpath, anns in items:
- self._writeFinal(subpath, anns)
+ for code in self._script:
+ m = getattr(self, '_do_%s' % code[0])
+ m(*code[1:])
finally:
self.reset()
=== Products/Ape/lib/apelib/fs/fileops.py 1.1 => 1.2 ===
=== Products/Ape/lib/apelib/fs/interfaces.py 1.4 => 1.5 ===
--- Products/Ape/lib/apelib/fs/interfaces.py:1.4 Fri Feb 20 22:30:39 2004
+++ Products/Ape/lib/apelib/fs/interfaces.py Sat Feb 28 15:06:27 2004
@@ -19,6 +19,9 @@
from Interface import Interface
+class FSReadError (Exception):
+ """Unable to read data"""
+
class FSWriteError (Exception):
"""Unable to write data"""
@@ -27,18 +30,19 @@
"""Filesystem reader that supports annotations.
"""
- def getPath(subpath):
- """Returns the filesystem path for a subpath.
+ def getSubpath(oid):
+ """Returns the tuple path for an oid, relative to the base.
+ """
- May automatically append an extension if the file already
- exists.
+ def getPath(oid):
+ """Returns the filesystem path for an oid.
"""
- def readNodeType(subpath):
+ def readNodeType(oid):
"""Reads the node type of a filesystem node.
"""
- def readData(subpath, allow_missing=0, as_text=0):
+ def readData(oid, allow_missing=0, as_text=0):
"""Reads the main data stream from a file.
If the allow_missing flag is specified, this method returns
@@ -46,28 +50,45 @@
is read in text mode.
"""
- def readDirectory(subpath, allow_missing=0):
+ def readDirectory(oid, allow_missing=0):
"""Reads the contents of a directory.
- Returns a list of object names. If the allow_missing flag is
- specified, this method returns None if no such directory is
- found.
+ Returns a list of (object_name, child_oid). The child_oid is
+ None for objects not seen before. The application should
+ assign unique OIDs to the newly found children, then tell this
+ object about the assignments through the assignNew() method.
+
+ If the allow_missing flag is specified, this method returns
+ None if no such directory is found.
"""
- def readAnnotation(subpath, name, default=None):
+ def readAnnotation(oid, name, default=None):
"""Reads a text-based annotation for a file.
"""
- def getExtension(subpath):
- """Returns the filename extension for a subpath.
+ def readObjectName(oid):
+ """Gets the canonical name for an object.
+
+ Note that this only makes sense when objects can have only one
+ parent.
+ """
+
+ def assignExisting(oid, children):
+ """Assigns OIDs to previously existing objects on the filesystem.
+
+ See readDirectory(). children is a list of (object_name, child_oid).
+ """
+
+ def getExtension(oid):
+ """Returns the filename extension for a file.
"""
- def getModTime(subpath, default=0):
+ def getModTime(oid, default=0):
"""Returns the last-modified time of a file.
"""
- def getPollSources(subpath):
- """Returns source information for a subpath.
+ def getPollSources(oid):
+ """Returns source information for an oid.
The source information is a mapping that maps
(source_repository, path) to a state object. The contents of
@@ -81,29 +102,29 @@
"""Filesystem writer that supports annotations.
"""
- def writeNodeType(subpath, data):
+ def writeNodeType(oid, data):
"""Writes the node type for a filesystem node.
'd' (directory) and 'f' (file) are supported.
"""
- def writeData(subpath, data, as_text=0):
+ def writeData(oid, data, as_text=0):
"""Writes string data to a filesystem node.
If 'as_text' is true, the file is written in text mode.
"""
- def writeDirectory(subpath, names):
+ def writeDirectory(oid, data):
"""Writes data to a directory.
- 'names' is a sequence of object names used for determining filenames..
+ 'data' is a sequence of (object_name, child_oid).
"""
- def writeAnnotation(subpath, name, data):
+ def writeAnnotation(oid, name, data):
"""Writes a text-based annotation for a filesystem node.
"""
- def suggestExtension(subpath, ext):
+ def suggestExtension(oid, ext):
"""Suggests a filename extension for a filesystem node.
The IFSConnection may use this information to store the file
=== Products/Ape/lib/apelib/fs/params.py 1.1 => 1.2 ===
=== Products/Ape/lib/apelib/fs/properties.py 1.4 => 1.5 ===
--- Products/Ape/lib/apelib/fs/properties.py:1.4 Mon Feb 2 10:07:20 2004
+++ Products/Ape/lib/apelib/fs/properties.py Sat Feb 28 15:06:27 2004
@@ -73,9 +73,8 @@
FSGatewayBase.__init__(self, conn_name)
def load(self, event):
- p = event.oid
fs_conn = self.getConnection(event)
- text = fs_conn.readAnnotation(p, self.annotation, '')
+ text = fs_conn.readAnnotation(event.oid, self.annotation, '')
res = []
if text:
lines = text.split('\n')
@@ -96,9 +95,8 @@
lines.append('%s:%s=%s' % (k, t, escape_string(v)))
lines.sort()
text = '\n'.join(lines)
- p = event.oid
fs_conn = self.getConnection(event)
- fs_conn.writeAnnotation(p, self.annotation, text)
+ fs_conn.writeAnnotation(event.oid, self.annotation, text)
state = list(state)
state.sort()
return tuple(state)
@@ -117,8 +115,7 @@
def load(self, event):
fs_conn = self.getConnection(event)
- p = event.oid
- state = fs_conn.readAnnotation(p, self.annotation, '').strip()
+ state = fs_conn.readAnnotation(event.oid, self.annotation, '').strip()
return state, state
def store(self, event, state):
@@ -126,8 +123,6 @@
raise ValueError('Not a string: %s' % repr(state))
state = state.strip()
if state:
- p = event.oid
fs_conn = self.getConnection(event)
- fs_conn.writeAnnotation(p, self.annotation, state)
+ fs_conn.writeAnnotation(event.oid, self.annotation, state)
return state
-
=== Products/Ape/lib/apelib/fs/security.py 1.3 => 1.4 ===
--- Products/Ape/lib/apelib/fs/security.py:1.3 Mon Feb 2 10:07:20 2004
+++ Products/Ape/lib/apelib/fs/security.py Sat Feb 28 15:06:27 2004
@@ -108,9 +108,8 @@
def load(self, event):
c = self.getConnection(event)
- p = event.oid
- assert c.readNodeType(p) == 'f'
- text = c.readData(p)
+ assert c.readNodeType(event.oid) == 'f'
+ text = c.readData(event.oid)
res = []
for line in text.split('\n'):
L = line.strip()
@@ -156,10 +155,11 @@
domainlist = self._joinList(domains)
to_write = '%s:%s:%s:%s' % (id, password, rolelist, domainlist)
replace_lines[id] = to_write
- p = event.oid
+ oid = event.oid
fs_conn = self.getConnection(event)
- fs_conn.writeNodeType(p, 'f')
- text = fs_conn.readData(p, allow_missing=1)
+ fs_conn.writeNodeType(oid, 'f')
+ # Read the existing text only to maintain the current order.
+ text = fs_conn.readData(oid, allow_missing=1)
if text is None:
text = ''
new_lines = []
@@ -181,8 +181,7 @@
new_lines.append(line)
# Write it
text = '\n'.join(new_lines)
- fs_conn.writeData(p, text)
+ fs_conn.writeData(oid, text)
serial = list(state)
serial.sort()
return text
-
=== Products/Ape/lib/apelib/fs/structure.py 1.7 => 1.8 ===
--- Products/Ape/lib/apelib/fs/structure.py:1.7 Thu Feb 19 01:44:03 2004
+++ Products/Ape/lib/apelib/fs/structure.py Sat Feb 28 15:06:27 2004
@@ -42,46 +42,35 @@
def load(self, event):
c = self.getConnection(event)
- p = event.oid
- assert c.readNodeType(p) == 'f'
- state = c.readData(p, as_text=self.text)
+ assert c.readNodeType(event.oid) == 'f'
+ state = c.readData(event.oid, as_text=self.text)
return state, state
def store(self, event, state):
if not isinstance(state, StringType):
raise ValueError('Not a string: %s' % repr(state))
c = self.getConnection(event)
- p = event.oid
- c.writeNodeType(p, 'f')
- c.writeData(p, state, as_text=self.text)
+ c.writeNodeType(event.oid, 'f')
+ c.writeData(event.oid, state, as_text=self.text)
return state
class FSAutoId (FSGatewayBase):
- """Automatic ID gateway based on the key of the item."""
+ """Automatic ID gateway based on the object name in the primary parent.
+ """
__implements__ = IGateway
schema = FieldSchema('id', 'string')
- def getIdFrom(self, event):
- p = event.oid
- pos = p.rfind('/')
- if pos >= 0:
- return p[pos + 1:]
- else:
- return p
-
def load(self, event):
- id = self.getIdFrom(event)
- return id, id
+ id = self.getConnection(event).readObjectName(event.oid)
+ # Disable conflict checking by returning None as the hash value.
+ return id, None
def store(self, event, state):
- id = self.getIdFrom(event)
- if state != id:
- raise ValueError('Mismatched object name: %s != %s' %
- (state, id))
- return id
+ # Ignore.
+ return None
def getPollSources(self, event):
fs_conn = self.getConnection(event)
@@ -99,34 +88,39 @@
schema.addField('classification', 'classification')
def load(self, event):
- p = event.oid
c = self.getConnection(event)
- assert c.readNodeType(p) == 'd'
- names = c.readDirectory(p)
- names.sort()
+ if c.readNodeType(event.oid) != 'd':
+ raise LoadError("Not a directory")
+ data = list(c.readDirectory(event.oid))
+ data.sort()
+ # Assign OIDs to previously existing subobjects.
+ assigned = {}
+ for objname, child_oid in data:
+ if child_oid is None:
+ child_oid = event.conf.oid_gen.new_oid(event, objname, True)
+ assigned[objname] = child_oid
+ if assigned:
+ # Saw existing objects. Tell the connection what their OIDs are.
+ c.assignExisting(event.oid, assigned.items())
+ # Return the results.
res = []
- for name in names:
- oid = event.conf.oid_gen.new_oid(event, name, False)
- classification = event.classify(oid)
+ for objname, child_oid in data:
+ if child_oid is None:
+ child_oid = assigned[objname]
+ classification = event.classify(child_oid)
# Return info about each subobject.
- res.append((name, oid, classification))
- return res, tuple(names)
+ res.append((objname, child_oid, classification))
+ return res, tuple(data)
def store(self, event, state):
- p = event.oid
c = self.getConnection(event)
- c.writeNodeType(p, 'd')
- state = list(state)
- state.sort()
- if __debug__:
- for name, oid, classification in state:
- expect = event.conf.oid_gen.new_oid(event, name, False)
- assert expect == oid, (
- "Child of %s named %s must use OID %s, but used %s" %
- (event.oid, name, expect, oid))
- names = [row[0] for row in state]
- c.writeDirectory(p, names)
- return tuple(names)
+ c.writeNodeType(event.oid, 'd')
+ data = []
+ for objname, child_oid, classification in state:
+ data.append((objname, child_oid))
+ data.sort()
+ c.writeDirectory(event.oid, data)
+ return tuple(data)
class FSModTime (FSGatewayBase):
@@ -137,9 +131,8 @@
schema = FieldSchema('mtime', 'int')
def load(self, event):
- p = event.oid
fs_conn = self.getConnection(event)
- state = long(fs_conn.getModTime(p))
+ state = long(fs_conn.getModTime(event.oid))
return state, None # Use None as the hash (see store())
def store(self, event, state):
@@ -148,66 +141,12 @@
return None
-class RootDirectoryItems (FSGatewayBase):
- """Read/write the root object.
-
- The root object is stored as a normal directory with one special feature:
- the name 'Application' is always present and points to the OID '/'. This
- allows the root object to be stored inside the application object.
- """
-
- __implements__ = IGateway
-
- schema = FSDirectoryItems.schema
-
- def load(self, event):
- p = event.oid
- c = self.getConnection(event)
- try:
- t = c.readNodeType(p)
- except LoadError:
- # The root object doesn't exist, but it's reasonable
- # to infer a state anyway.
- names = []
- else:
- assert t == 'd', 'The root object must be a directory'
- names = c.readDirectory(p)
- names.sort()
- res = [('Application', '/', None)]
- for name in names:
- if name != 'Application':
- oid = event.conf.oid_gen.new_oid(event, name, False)
- classification = event.classify(oid)
- res.append((name, oid, classification))
- return res, tuple(names)
-
- def store(self, event, state):
- p = event.oid
- c = self.getConnection(event)
- c.writeNodeType(p, 'd')
- state = list(state)
- state.sort()
- names = []
- for name, oid, classification in state:
- if name == 'Application':
- expect = '/'
- else:
- expect = event.conf.oid_gen.new_oid(event, name, False)
- names.append(name)
- assert expect == oid, (
- "Child of %s named %s must use OID %s, but used %s" %
- (event.oid, name, expect, oid))
- c.writeDirectory(p, names)
- names.sort()
- return tuple(names)
-
-
def root_mapping():
"""Returns a gateway suitable for storing the root persistent mapping.
"""
from apelib.core.gateways import CompositeGateway
from properties import FSAnnotationData
g = CompositeGateway()
- g.add('references', RootDirectoryItems())
+ g.add('references', FSDirectoryItems())
g.add('others', FSAnnotationData('others'))
return g
More information about the Zope-CVS
mailing list