[Checkins] SVN: zc.catalog/trunk/src/zc/catalog/ Separated a simple
extent from a filter extent.
Albertas Agejevas
alga at pov.lt
Mon Feb 5 14:21:48 EST 2007
Log message for revision 72378:
Separated a simple extent from a filter extent.
Changed:
U zc.catalog/trunk/src/zc/catalog/extentcatalog.py
U zc.catalog/trunk/src/zc/catalog/extentcatalog.txt
U zc.catalog/trunk/src/zc/catalog/interfaces.py
-=-
Modified: zc.catalog/trunk/src/zc/catalog/extentcatalog.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/extentcatalog.py 2007-02-05 17:44:10 UTC (rev 72377)
+++ zc.catalog/trunk/src/zc/catalog/extentcatalog.py 2007-02-05 19:21:48 UTC (rev 72378)
@@ -30,21 +30,20 @@
from zc.catalog import interfaces
-class FilterExtent(persistent.Persistent):
- interface.implements(interfaces.IFilterExtent)
+class Extent(persistent.Persistent):
+ interface.implements(interfaces.IExtent)
__parent__ = None
BTreeAPI = zc.catalog.BTreeAPI32
- def __init__(self, filter):
+ def __init__(self):
self.BTreeAPI = zope.component.queryUtility(
interfaces.IBTreeAPI,
default=zc.catalog.BTreeAPI32)
self.set = self.BTreeAPI.TreeSet()
- self.filter = filter
- def addable(self, uid, obj):
- return self.filter(self, uid, obj)
+ def add(self, uid, obj):
+ self.set.insert(uid)
def clear(self):
self.set.clear()
@@ -92,12 +91,6 @@
def __contains__(self, uid):
return self.set.has_key(uid)
- def add(self, uid, obj):
- if not self.addable(uid, obj):
- raise ValueError
- else:
- self.set.insert(uid)
-
def remove(self, uid):
self.set.remove(uid)
@@ -107,6 +100,24 @@
except KeyError:
pass
+
+class FilterExtent(Extent):
+ interface.implements(interfaces.IFilterExtent)
+
+ def __init__(self, filter):
+ super(FilterExtent, self).__init__()
+ self.filter = filter
+
+ def add(self, uid, obj):
+ if not self.addable(uid, obj):
+ raise ValueError
+ else:
+ self.set.insert(uid)
+
+ def addable(self, uid, obj):
+ return self.filter(self, uid, obj)
+
+
class Catalog(catalog.Catalog):
interface.implements(interfaces.IExtentCatalog)
Modified: zc.catalog/trunk/src/zc/catalog/extentcatalog.txt
===================================================================
--- zc.catalog/trunk/src/zc/catalog/extentcatalog.txt 2007-02-05 17:44:10 UTC (rev 72377)
+++ zc.catalog/trunk/src/zc/catalog/extentcatalog.txt 2007-02-05 19:21:48 UTC (rev 72378)
@@ -1,10 +1,12 @@
-An extent catalog is very similar to a normal catalog except that it only
-indexes items addable to its extent. The extent is both a filter and a
-set that may be merged with other result sets.
+An extent catalog is very similar to a normal catalog except that it
+only indexes items addable to its extent. The extent is both a filter
+and a set that may be merged with other result sets. The filtering is
+an additional feature we will discuss below; we'll begin with a simple
+"do nothing" extent that only supports the second use case.
-To show the extent catalog at work, we need an intid utility, an index,
-some items to index, and a filter that determines what the extent accepts.
-We'll do this within a real ZODB and a real intid utility [#setup]_.
+To show the extent catalog at work, we need an intid utility, an
+index, some items to index. We'll do this within a real ZODB and a
+real intid utility [#setup]_.
>>> import zc.catalog
>>> import zc.catalog.interfaces
@@ -39,8 +41,139 @@
... self.id = name
... self.__parent__ = parent
...
+
+ >>> extent = extentcatalog.Extent()
+ >>> verify.verifyObject(interfaces.IExtent, extent)
+ True
+ >>> root['catalog'] = catalog = extentcatalog.Catalog(extent)
+ >>> verify.verifyObject(interfaces.IExtentCatalog, catalog)
+ True
+ >>> index = DummyIndex()
+ >>> catalog['index'] = index
+ >>> transaction.commit()
+
+Now we have a catalog set up with an index and an extent. We can add
+some data to the extent:
+
+ >>> matches = []
+ >>> for i in range(100):
+ ... c = DummyContent(i, root)
+ ... root[i] = c
+ ... doc_id = intid.register(c)
+ ... catalog.index_doc(doc_id, c)
+ ... matches.append(doc_id)
+ >>> matches.sort()
+ >>> sorted(extent) == sorted(index.uids) == matches
+ True
+
+Unindexing an object that is in the catalog should simply remove it from the
+catalog and index as usual.
+
+ >>> matches[0] in catalog.extent
+ True
+ >>> matches[0] in catalog['index'].uids
+ True
+ >>> catalog.unindex_doc(matches[0])
+ >>> matches[0] in catalog.extent
+ False
+ >>> matches[0] in catalog['index'].uids
+ False
+ >>> doc_id = matches.pop(0)
+ >>> sorted(extent) == sorted(index.uids) == matches
+ True
+
+Clearing the catalog clears both the extent and the contained indexes.
+
+ >>> catalog.clear()
+ >>> list(catalog.extent) == list(catalog['index'].uids) == []
+ True
+
+Updating all indexes and an individual index both also update the extent.
+
+ >>> catalog.updateIndexes()
+ >>> matches.insert(0, doc_id)
+ >>> sorted(extent) == sorted(index.uids) == matches
+ True
+
+ >>> index2 = DummyIndex()
+ >>> catalog['index2'] = index2
+ >>> index2.__parent__ == catalog
+ True
+ >>> index.uids.remove(matches[0]) # to confirm that only index 2 is touched
+ >>> catalog.updateIndex(index2)
+ >>> sorted(extent) == sorted(index2.uids) == matches
+ True
+ >>> matches[0] in index.uids
+ False
+ >>> matches[0] in index2.uids
+ True
+ >>> res = index.uids.insert(matches[0])
+
+But so why have an extent in the first place? It allows indices to
+operate against a reliable collection of the full indexed data;
+therefore, it allows the indices in zc.catalog to perform NOT
+operations.
+
+The extent itself provides a number of merging features to allow its
+values to be merged with other BTrees.IFBTree data structures. These
+include intersection, union, difference, and reverse difference.
+Given an extent named 'extent' and another IFBTree data structure
+named 'data', intersections can be spelled "extent & data" or "data &
+extent"; unions can be spelled "extent | data" or "data | extent";
+differences can be spelled "extent - data"; and reverse differences
+can be spelled "data - extent". Unions and intersections are
+weighted.
+
+ >>> extent = extentcatalog.Extent()
+ >>> for i in range(1, 100, 2):
+ ... extent.add(i, None)
+ ...
+ >>> alt_set = BTreeAPI.TreeSet()
+ >>> alt_set.update(range(0, 166, 33)) # return value is unimportant here
+ 6
+ >>> sorted(alt_set)
+ [0, 33, 66, 99, 132, 165]
+ >>> sorted(extent & alt_set)
+ [33, 99]
+ >>> sorted(alt_set & extent)
+ [33, 99]
+ >>> sorted(extent.intersection(alt_set))
+ [33, 99]
+ >>> original = set(extent)
+ >>> union_matches = original.copy()
+ >>> union_matches.update(alt_set)
+ >>> union_matches = sorted(union_matches)
+ >>> sorted(alt_set | extent) == union_matches
+ True
+ >>> sorted(extent | alt_set) == union_matches
+ True
+ >>> sorted(extent.union(alt_set)) == union_matches
+ True
+ >>> sorted(alt_set - extent)
+ [0, 66, 132, 165]
+ >>> sorted(extent.rdifference(alt_set))
+ [0, 66, 132, 165]
+ >>> original.remove(33)
+ >>> original.remove(99)
+ >>> set(extent - alt_set) == original
+ True
+ >>> set(extent.difference(alt_set)) == original
+ True
+
+[#cleanup]_
+
+
+Catalog with a filter extent
+----------------------------
+
+As discussed at the beginning of this document, extents can not only help
+with index operations, but also act as a filter, so that a given catalog
+can answer questions about a subset of the objects contained in the intids.
+
+The filter extent only stores objects that match a given filter.
+
>>> def filter(extent, uid, ob):
- ... assert interfaces.IExtent.providedBy(extent)
+ ... assert interfaces.IFilterExtent.providedBy(extent)
... # This is an extent of objects with odd-numbered uids without a
... # True ignore attribute
... return uid % 2 and not getattr(ob, 'ignore', False)
@@ -48,7 +181,7 @@
>>> extent = extentcatalog.FilterExtent(filter)
>>> verify.verifyObject(interfaces.IFilterExtent, extent)
True
- >>> root['catalog'] = catalog = extentcatalog.Catalog(extent)
+ >>> root['catalog1'] = catalog = extentcatalog.Catalog(extent)
>>> verify.verifyObject(interfaces.IExtentCatalog, catalog)
True
>>> index = DummyIndex()
@@ -95,25 +228,8 @@
>>> sorted(extent) == sorted(index.uids) == matches
True
-Unindexing an object that is in the catalog should simply remove it from the
-catalog and index as usual.
+Unindexing an object that is not in the catalog should be a no-op.
- >>> matches[0] in catalog.extent
- True
- >>> matches[0] in catalog['index'].uids
- True
- >>> catalog.unindex_doc(matches[0])
- >>> matches[0] in catalog.extent
- False
- >>> matches[0] in catalog['index'].uids
- False
- >>> doc_id = matches.pop(0)
- >>> sorted(extent) == sorted(index.uids) == matches
- True
-
-And similarly, unindexing an object that is not in the catalog should be a
-no-op.
-
>>> fails[0] in catalog.extent
False
>>> catalog.unindex_doc(fails[0])
@@ -122,26 +238,15 @@
>>> sorted(extent) == sorted(index.uids) == matches
True
-Clearing the catalog clears both the extent and the contained indexes.
-
- >>> catalog.clear()
- >>> list(catalog.extent) == list(catalog['index'].uids) == []
- True
-
Updating all indexes and an individual index both also update the extent.
- >>> catalog.updateIndexes()
- >>> matches.insert(0, doc_id)
- >>> sorted(extent) == sorted(index.uids) == matches
- True
-
>>> index2 = DummyIndex()
>>> catalog['index2'] = index2
>>> index2.__parent__ == catalog
True
>>> index.uids.remove(matches[0]) # to confirm that only index 2 is touched
>>> catalog.updateIndex(index2)
- >>> sorted(extent) == sorted(index2.uids) == matches
+ >>> sorted(extent) == sorted(index2.uids)
True
>>> matches[0] in index.uids
False
@@ -172,52 +277,7 @@
... == sorted(index2.uids))
True
-The extent itself provides a number of merging features to allow its values to
-be merged with other BTrees.IFBTree data structures. These include
-intersection, union, difference, and reverse difference. Given an extent
-named 'extent' and another IFBTree data structure named 'data', intersections
-can be spelled "extent & data" or "data & extent"; unions can be spelled
-"extent | data" or "data | extent"; differences can be spelled "extent - data";
-and reverse differences can be spelled "data - extent". Unions and
-intersections are weighted.
- >>> extent = extentcatalog.FilterExtent(filter)
- >>> for i in range(1, 100, 2):
- ... extent.add(i, None)
- ...
- >>> alt_set = BTreeAPI.TreeSet()
- >>> alt_set.update(range(0, 166, 33)) # return value is unimportant here
- 6
- >>> sorted(alt_set)
- [0, 33, 66, 99, 132, 165]
- >>> sorted(extent & alt_set)
- [33, 99]
- >>> sorted(alt_set & extent)
- [33, 99]
- >>> sorted(extent.intersection(alt_set))
- [33, 99]
- >>> original = set(extent)
- >>> union_matches = original.copy()
- >>> union_matches.update(alt_set)
- >>> union_matches = sorted(union_matches)
- >>> sorted(alt_set | extent) == union_matches
- True
- >>> sorted(extent | alt_set) == union_matches
- True
- >>> sorted(extent.union(alt_set)) == union_matches
- True
- >>> sorted(alt_set - extent)
- [0, 66, 132, 165]
- >>> sorted(extent.rdifference(alt_set))
- [0, 66, 132, 165]
- >>> original.remove(33)
- >>> original.remove(99)
- >>> set(extent - alt_set) == original
- True
- >>> set(extent.difference(alt_set)) == original
- True
-
-
Self-populating extents
-----------------------
@@ -400,3 +460,12 @@
... return obj._p_jar.root()['components']
...
>>> zope.component.provideAdapter(getComponentLookup)
+
+
+.. [#cleanup] Unregister the objects of the previous tests from intid utility:
+
+ >>> intid = zope.component.getUtility(
+ ... zope.app.intid.interfaces.IIntIds, context=root)
+ >>> for doc_id in matches:
+ ... intid.unregister(intid.queryObject(doc_id))
+
Modified: zc.catalog/trunk/src/zc/catalog/interfaces.py
===================================================================
--- zc.catalog/trunk/src/zc/catalog/interfaces.py 2007-02-05 17:44:10 UTC (rev 72377)
+++ zc.catalog/trunk/src/zc/catalog/interfaces.py 2007-02-05 19:21:48 UTC (rev 72378)
@@ -23,16 +23,16 @@
from zc.catalog.i18n import _
import BTrees.Interfaces
+
class IExtent(interface.Interface):
+ """An extent represents the full set of objects indexed by a catalog.
+ It is useful for a variety of index operations and catalog queries.
+ """
__parent__ = interface.Attribute(
"""The catalog for which this is an extent; must be None before it is
set to a catalog""")
- def addable(uid, obj):
- """returns True or False, indicating whether the obj may be added to
- the extent"""
-
def add(uid, obj):
"""add uid to extent; raise ValueError if it is not addable.
@@ -88,6 +88,7 @@
def __contains__(uid):
"return boolean indicating if uid is in set"
+
class IFilterExtent(IExtent):
filter = interface.Attribute(
@@ -95,7 +96,11 @@
associated obj and should return a boolean True (is member of extent)
or False (is not member of extent).""")
+ def addable(uid, obj):
+ """returns True or False, indicating whether the obj may be added to
+ the extent"""
+
class ISelfPopulatingExtent(IExtent):
"""An extent that knows how to create it's own initial population."""
@@ -126,6 +131,7 @@
extent = interface.Attribute(
"""An IExtent of the objects cataloged""")
+
class IIndexValues(interface.Interface):
"""An index that allows introspection of the indexed values"""
@@ -175,6 +181,7 @@
IIndexValues.values(doc_id=id).
"""
+
class ISetIndex(interface.Interface):
def apply(query):
@@ -206,6 +213,7 @@
the extent that do not have any values in the index.
"""
+
class IValueIndex(interface.Interface):
def apply(query):
@@ -234,16 +242,19 @@
the extent that do not have any values in the index.
"""
+
class ICatalogValueIndex(zope.app.catalog.interfaces.IAttributeIndex,
zope.app.catalog.interfaces.ICatalogIndex):
"""Interface-based catalog value index
"""
+
class ICatalogSetIndex(zope.app.catalog.interfaces.IAttributeIndex,
zope.app.catalog.interfaces.ICatalogIndex):
"""Interface-based catalog set index
"""
+
class INormalizationWrapper(zope.index.interfaces.IInjection,
zope.index.interfaces.IIndexSearch,
zope.index.interfaces.IStatistics,
@@ -262,6 +273,7 @@
(each composite value normalized) or not (original value is
normalized)""")
+
class INormalizer(interface.Interface):
def value(value):
@@ -288,6 +300,7 @@
(_('day'), _('hour'), _('minute'), _('second'), _('microsecond')))])
# 0 1 2 3 4
+
class IDateTimeNormalizer(INormalizer):
resolution = schema.Choice(
vocabulary=resolution_vocabulary,
More information about the Checkins
mailing list