[Zope-CVS] CVS: Products/AdaptableStorage/gateway_fs - ShortLivedCache.py:1.1.2.1 FSAutoId.py:1.4.2.5 FSClassificationSection.py:1.7.2.5 FSConnection.py:1.6.2.6 FSDirectoryItems.py:1.8.2.5 public.py:1.3.2.5
Christian Zagrodnick
cz@gocept.com
Thu, 6 Feb 2003 08:01:38 -0500
Update of /cvs-repository/Products/AdaptableStorage/gateway_fs
In directory cvs.zope.org:/tmp/cvs-serv14413/gateway_fs
Modified Files:
Tag: zagy-patches
FSAutoId.py FSClassificationSection.py FSConnection.py
FSDirectoryItems.py public.py
Added Files:
Tag: zagy-patches
ShortLivedCache.py
Log Message:
merging HEAD into zagy-patches branch
=== Added File Products/AdaptableStorage/gateway_fs/ShortLivedCache.py ===
##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Simple short-lived object cache.
$Id: ShortLivedCache.py,v 1.1.2.1 2003/02/06 13:01:05 zagy Exp $
"""
from time import time
class ShortLivedCache:
def __init__(self, lifetime=1):
# The default lifetime is 1 second.
self.lifetime = lifetime
self.data = {}
self.expiration = time() + lifetime
def get(self, key, default=None):
now = time()
if now >= self.expiration:
self.data.clear()
return default
res = self.data.get(key, default)
return res
def set(self, key, value):
now = time()
if now >= self.expiration:
self.data.clear()
self.expiration = now + self.lifetime
self.data[key] = value
def invalidate(self, key):
try:
del self.data[key]
except KeyError:
pass
def clear(self):
self.data.clear()
=== Products/AdaptableStorage/gateway_fs/FSAutoId.py 1.4.2.4 => 1.4.2.5 ===
=== Products/AdaptableStorage/gateway_fs/FSClassificationSection.py 1.7.2.4 => 1.7.2.5 ===
--- Products/AdaptableStorage/gateway_fs/FSClassificationSection.py:1.7.2.4 Tue Feb 4 12:29:36 2003
+++ Products/AdaptableStorage/gateway_fs/FSClassificationSection.py Thu Feb 6 08:01:05 2003
@@ -16,6 +16,8 @@
$Id$
"""
+import os
+
from mapper_public import IGateway, FieldSchema
@@ -50,17 +52,21 @@
if '=' in line:
k, v = line.split('=', 1)
classification[k.strip()] = v.strip()
- classification['filename'] = self.getIdFrom(event)
+ filename = self.getIdFrom(event)
+ classification['extension'] = os.path.splitext(filename)[1]
return classification, text.strip()
def store(self, event, state):
# state is a classification
+ p = event.getKeychain()[-1]
items = state.items()
items.sort()
text = []
for k, v in items:
- text.append('%s=%s' % (k, v))
+ if k == 'extension':
+ self.fs_conn.suggestExtension(p, v)
+ else:
+ text.append('%s=%s' % (k, v))
text = '\n'.join(text)
- p = event.getKeychain()[-1]
self.fs_conn.writeSection(p, 'classification', text)
return text.strip()
=== Products/AdaptableStorage/gateway_fs/FSConnection.py 1.6.2.5 => 1.6.2.6 ===
--- Products/AdaptableStorage/gateway_fs/FSConnection.py:1.6.2.5 Tue Feb 4 12:48:34 2003
+++ Products/AdaptableStorage/gateway_fs/FSConnection.py Thu Feb 6 08:01:05 2003
@@ -24,6 +24,7 @@
from interfaces.public import IFSConnection
from exceptions import FSWriteError
from mapper_public import ITPCConnection, NoStateFoundError
+from ShortLivedCache import ShortLivedCache
# Try to decipher this regular expression ;-)
@@ -34,6 +35,8 @@
NODE_TYPE_SECTION = '@node_type' # Data is 'f' (file) or 'd' (directory)
DATA_SECTION = '@data' # Data is a string (file) or list of names (directory)
+SUGGESTED_EXTENSION_SECTION = '@s_ext' # The suggested filename extension
+OBJECT_NAMES_SECTION = 'object_names' # For directories
class FSConnection:
@@ -50,16 +53,88 @@
basepath = ''
- def __init__(self, basepath, hidden_filename_prefix='.',
- hidden_filenames='$f^'):
- # $f^ never matches anything
+ def __init__(self, basepath, metadata_prefix='.', hidden_filenames='_'):
self.basepath = basepath
- self.hidden_filename_prefix = hidden_filename_prefix
- self.hidden_filenames = re.compile(hidden_filenames)
+ self.metadata_prefix = metadata_prefix
+ self.hidden_re = re.compile(hidden_filenames)
self._final = 0
# _pending holds the data to be written.
# _pending: { subpath string -> { section_name -> data } }
self._pending = {}
+ self._props_cache = ShortLivedCache()
+ self._dir_cache = ShortLivedCache()
+
+
+ def computeDirectoryContents(self, path, ignore_error=0):
+ """Computes and returns intermediate directory contents info.
+
+ Returns (filenames, object_names, translations).
+ """
+ res = self._dir_cache.get(path)
+ if res is not None:
+ return res
+
+ filenames = []
+ obj_names = []
+ trans = {} # { base name -> filename with extension or None }
+ try:
+ fns = os.listdir(path)
+ except OSError:
+ if ignore_error:
+ return (filenames, obj_names, trans)
+ metadata_prefix = self.metadata_prefix
+ for fn in fns:
+ if (not fn
+ or fn.startswith(metadata_prefix)
+ or self.hidden_re.match(fn) is not None):
+ continue
+ filenames.append(fn)
+
+ props = self.getPropertiesFromFile(path)
+ text = props.get(OBJECT_NAMES_SECTION)
+ if text:
+ # Prepare a dictionary of translations.
+ for fn in filenames:
+ if '.' in fn:
+ base, ext = fn.split('.', 1)
+ if trans.has_key(base):
+ # Name collision: two or more files have the same base
+ # name. Don't use an extension for this name.
+ trans[base] = None
+ else:
+ trans[base] = fn
+ else:
+ trans[fn] = None
+ obj_names = [line.strip() for line in text.split('\n')]
+ for obj_name in obj_names:
+ if '.' in obj_name:
+ base, ext = obj_name.split('.', 1)
+ trans[base] = None
+
+ res = (filenames, obj_names, trans)
+ self._dir_cache.set(path, res)
+ return res
+
+
+ def listDirectoryAsMapping(self, path, ignore_error=0):
+ """Returns the translated filenames at path.
+
+ The ignore_error flag makes this method return an empty
+ dictionary if the directory is not found.
+
+ Returns {filename -> obj_name}.
+ """
+ filenames, obj_names, trans = self.computeDirectoryContents(
+ path, ignore_error)
+ res = {}
+ for fn in filenames:
+ res[fn] = fn
+ # Translate names.
+ for obj_name in obj_names:
+ fn = trans.get(obj_name)
+ if fn:
+ res[fn] = obj_name
+ return res
def expandPath(self, subpath):
@@ -70,48 +145,45 @@
else:
# unchanged.
path = subpath
+ if not os.path.exists(path):
+ dir_path, obj_name = os.path.split(path)
+ if '.' not in obj_name:
+ # This object might have an automatic filename extension.
+ filenames, obj_names, trans = self.computeDirectoryContents(
+ dir_path, 1)
+ fn = trans.get(obj_name)
+ if fn is not None:
+ # Use the filename with an extension.
+ path = os.path.join(dir_path, fn)
return path
def checkSectionName(self, section_name):
- assert isinstance(section_name, StringType)
- assert '[' not in section_name
- assert ']' not in section_name
- assert '\n' not in section_name
- assert section_name != NODE_TYPE_SECTION
- assert section_name != DATA_SECTION
-
-
- def _write(self, subpath, section_name, data):
- # XXX We should be checking for '..'
- path = self.expandPath(subpath)
- # Do some early checking.
- if os.path.exists(path):
- v = os.access(path, os.W_OK)
- if not v:
- raise FSWriteError(
- "Can't get write access to %s" % subpath)
- self.queue(subpath, section_name, data)
+ if (not isinstance(section_name, StringType)
+ or not section_name
+ or '[' in section_name
+ or ']' in section_name
+ or '\n' in section_name
+ or section_name.startswith('@')
+ or section_name == OBJECT_NAMES_SECTION):
+ raise ValueError, section_name
def writeSection(self, subpath, section_name, data):
self.checkSectionName(section_name)
- self._write(subpath, section_name, data)
+ self.queue(subpath, section_name, data)
def writeNodeType(self, subpath, data):
- path = self.expandPath(subpath)
- # Do some early checking.
- if os.path.exists(path):
- want_dir = (data == 'd')
- if (want_dir != (not not os.path.isdir(path))):
- raise FSWriteError(
- "Can't mix file and directory at %s" % subpath)
self.queue(subpath, NODE_TYPE_SECTION, data)
def writeData(self, subpath, data):
- self._write(subpath, DATA_SECTION, data)
+ self.queue(subpath, DATA_SECTION, data)
+
+
+ def suggestExtension(self, subpath, ext):
+ self.queue(subpath, SUGGESTED_EXTENSION_SECTION, ext)
def readSection(self, subpath, section_name, default=None):
@@ -133,17 +205,10 @@
isdir = os.path.isdir(path)
# Read either the directory listing or the file contents.
if isdir:
- names = []
- prefix = self.hidden_filename_prefix
- for name in os.listdir(path):
- if name.startswith(self.hidden_filename_prefix):
- continue
- if self.hidden_filenames.search(name) is not None:
- continue
- names.append(name)
- # Return a sequence instead of a string.
- return names
+ # Return a sequence of object names.
+ return self.listDirectoryAsMapping(path).values()
else:
+ # Return a string.
try:
f = open(path, 'rb')
except IOError:
@@ -158,24 +223,30 @@
def getPropertiesPath(self, path):
if os.path.isdir(path):
- props_fn = os.path.join(path, self.hidden_filename_prefix +
+ props_fn = os.path.join(path, self.metadata_prefix +
'properties')
else:
dirname, filename = os.path.split(path)
- props_fn = os.path.join(dirname, self.hidden_filename_prefix +
+ props_fn = os.path.join(dirname, self.metadata_prefix +
('%s.properties' % filename))
return props_fn
def getPropertiesFromFile(self, path):
- """Read a properties file next to path."""
+ """Reads a properties file next to path."""
+ res = self._props_cache.get(path)
+ if res is not None:
+ return res
+
props_fn = self.getPropertiesPath(path)
try:
f = open(props_fn, 'rb')
except IOError:
# The file is presumably nonexistent
- return {}
+ res = {}
+ self._props_cache.set(path, res)
+ return res
try:
data = f.read()
finally:
@@ -199,6 +270,7 @@
prev_section_name = match.group(1)
pos = match.end()
+ self._props_cache.set(path, res)
return res
@@ -206,8 +278,23 @@
# sections is a mapping.
path = self.expandPath(subpath)
t = sections[NODE_TYPE_SECTION]
- if t == 'd' and not os.path.exists(path):
- os.mkdir(path)
+ if not os.path.exists(path):
+ if t == 'd':
+ os.mkdir(path)
+ else:
+ fn = os.path.split(path)[1]
+ if '.' not in fn:
+ # This object has no extension and doesn't yet exist.
+ ext = sections.get(SUGGESTED_EXTENSION_SECTION)
+ if ext:
+ # Try to use the suggested extension.
+ if not ext.startswith('.'):
+ ext = '.' + ext
+ p = path + ext
+ if not os.path.exists(p):
+ # No file is in the way.
+ # Use the suggested extension.
+ path = p
props_fn = self.getPropertiesPath(path)
items = sections.items()
items.sort()
@@ -219,38 +306,37 @@
elif name == DATA_SECTION:
if t == 'd':
# Change the list of subobjects.
- # Here we only have to delete.
- # Subobjects will be created later.
- # XXX we might check for dotted names here.
self.removeUnlinkedItems(path, data)
+ writeSection(props_f, OBJECT_NAMES_SECTION,
+ '\n'.join(data))
+ self.disableConflictingExtensions(subpath, data)
+ self._dir_cache.invalidate(path)
else:
- # Change file contents.
+ # Change the file contents.
f = open(path, 'wb')
try:
f.write(data)
finally:
f.close()
+ elif name == SUGGESTED_EXTENSION_SECTION:
+ # This doesn't need to be written.
+ pass
else:
- props_f.write('[%s]\n' % name)
- props_f.write(data.replace('[', '[['))
- if data.endswith('\n'):
- props_f.write('\n')
- else:
- props_f.write('\n\n')
+ writeSection(props_f, name, data)
finally:
props_f.close()
+ self._props_cache.invalidate(path)
+ # The file might be new, so invalidate the directory.
+ self._dir_cache.invalidate(os.path.dirname(path))
def removeUnlinkedItems(self, path, names):
+ """Removes unused files/subtrees from a directory."""
linked = {}
for name in names:
linked[name] = 1
- existing = os.listdir(path)
- prefix = self.hidden_filename_prefix
- for fn in existing:
- if (not fn.startswith(prefix)
- and self.hidden_filenames.search(fn) is None
- and not linked.get(fn)):
+ for fn, obj_name in self.listDirectoryAsMapping(path).items():
+ if not linked.get(obj_name):
item_fn = os.path.join(path, fn)
if os.path.isdir(item_fn):
rmtree(item_fn)
@@ -261,18 +347,50 @@
os.remove(item_pfn)
+ def disableConflictingExtensions(self, subpath, obj_names):
+ """Fixes collisions before writing files in a directory.
+
+ Enforces the rule: if 'foo.*' is in the
+ database, 'foo' may not have an automatic extension.
+ Enforces by removing suggested extensions.
+ """
+ reserved = {} # { object name without extension -> 1 }
+ for obj_name in obj_names:
+ if '.' in obj_name:
+ base, ext = obj_name.split('.', 1)
+ reserved[base] = 1
+ if not reserved:
+ # No objects have extensions.
+ return
+
+ while subpath.endswith('/'):
+ subpath = subpath[:-1]
+ for obj_name in obj_names:
+ if reserved.has_key(obj_name):
+ # Prevent obj_name from using an automatic extension.
+ child_subpath = '%s/%s' % (subpath, obj_name)
+ self.queue(child_subpath, SUGGESTED_EXTENSION_SECTION,
+ '', force=1)
+
+
def beforeWrite(self, items):
+ """Does some early checking while it's easy to bail out.
+
+ This avoids exceptions during the second phase of transaction commit.
+ """
non_containers = {}
for subpath, sections in items:
+ path = self.expandPath(subpath)
+ exists = os.path.exists(path)
+ if exists and not os.access(path, os.W_OK):
+ raise FSWriteError(
+ "Can't get write access to %s" % subpath)
# type must be provided and must always be either 'd' or 'f'.
if (not sections.has_key(NODE_TYPE_SECTION)
or not sections.has_key(DATA_SECTION)):
raise FSWriteError(
'Data or node type not specified for %s' % subpath)
t = sections[NODE_TYPE_SECTION]
- if t not in ('d', 'f'):
- raise FSWriteError(
- 'node type must be "d" or "f" at %s' % subpath)
dir = os.path.dirname(subpath)
if non_containers.get(dir):
raise FSWriteError(
@@ -283,26 +401,44 @@
'The id %s is not allowed.' % (os.path.split(subpath)[1], ))
if t == 'f':
+ if exists and os.path.isdir(path):
+ raise FSWriteError(
+ "Can't write file data to directory at %s"
+ % subpath)
non_containers[subpath] = 1
if not isinstance(sections[DATA_SECTION], StringType):
raise FSWriteError(
'Data for a file must be a string at %s'
% subpath)
- else:
- if isinstance(sections[DATA_SECTION], StringType):
+ elif t == 'd':
+ if exists and not os.path.isdir(path):
+ raise FSWriteError(
+ "Can't write directory contents to file at %s"
+ % subpath)
+ items = sections[DATA_SECTION]
+ if isinstance(items, StringType):
raise FSWriteError(
'Data for a directory must be a list or tuple at %s'
% subpath)
+ for item in items:
+ if (not item
+ or item.startswith(self.metadata_prefix)
+ or self.hidden_re.match(item) is not None):
+ raise FSWriteError(
+ 'Not a legal object name: %s' % repr(item))
+ else:
+ raise FSWriteError(
+ 'Node type must be "d" or "f" at %s' % subpath)
- def queue(self, subpath, section_name, data):
+ def queue(self, subpath, section_name, data, force=0):
"""Queues data to be written at commit time"""
m = self._pending
sections = m.get(subpath)
if sections is None:
sections = {}
m[subpath] = sections
- if sections.has_key(section_name):
+ if sections.has_key(section_name) and not force:
if sections[section_name] != data:
raise FSWriteError(
'Conflicting data storage at %s (%s)' %
@@ -326,7 +462,8 @@
os.makedirs(self.basepath)
def begin(self):
- pass
+ self._props_cache.clear()
+ self._dir_cache.clear()
def vote(self):
"""Do some early verification
@@ -341,6 +478,8 @@
def reset(self):
self._final = 0
self._pending.clear()
+ self._props_cache.clear()
+ self._dir_cache.clear()
def abort(self):
self.reset()
@@ -358,3 +497,12 @@
def close(self):
pass
+
+
+def writeSection(props_f, name, data):
+ props_f.write('[%s]\n' % name)
+ props_f.write(data.replace('[', '[['))
+ if data.endswith('\n'):
+ props_f.write('\n')
+ else:
+ props_f.write('\n\n')
=== Products/AdaptableStorage/gateway_fs/FSDirectoryItems.py 1.8.2.4 => 1.8.2.5 ===
=== Products/AdaptableStorage/gateway_fs/public.py 1.3.2.4 => 1.3.2.5 ===