[Zope-CVS] CVS: Packages/zpkgtools/zpkgtools - app.py:1.36
include.py:1.30 loader.py:1.4
Fred L. Drake, Jr.
fred at zope.com
Fri Apr 23 15:56:43 EDT 2004
Update of /cvs-repository/Packages/zpkgtools/zpkgtools
In directory cvs.zope.org:/tmp/cvs-serv17724/zpkgtools
Modified Files:
app.py include.py loader.py
Log Message:
change the PACKAGE.cfg file to include three sections instead of two; the
<load> section is used to load files from external sources into the
collection, the <collection> section describes what files to include and
exclude from the component, and the <distribution> section describes what
to include in the distribution root
this required a lot of internal changes
=== Packages/zpkgtools/zpkgtools/app.py 1.35 => 1.36 ===
--- Packages/zpkgtools/zpkgtools/app.py:1.35 Tue Apr 20 15:01:30 2004
+++ Packages/zpkgtools/zpkgtools/app.py Fri Apr 23 15:56:12 2004
@@ -42,43 +42,52 @@
returned by `parse_args()`.
"""
self.logger = logging.getLogger(options.program)
- self.ip = None
self.options = options
- self.resource = locationmap.normalizeResourceId(options.resource)
- self.resource_type, self.resource_name = self.resource.split(":", 1)
- if not options.release_name:
- options.release_name = self.resource_name
- # Create a new directory for all temporary files to go in:
- self.tmpdir = tempfile.mkdtemp(prefix=options.program + "-")
- tempfile.tempdir = self.tmpdir
- if options.revision_tag:
- self.loader = loader.Loader(tag=options.revision_tag)
- else:
- self.loader = loader.Loader()
cf = config.Configuration()
cf.location_maps.extend(options.location_maps)
path = options.configfile
if path is None:
path = config.defaultConfigurationPath()
if os.path.exists(path):
+ self.logger.debug("loading configuration file %s", path)
cf.loadPath(path)
elif path:
+ self.logger.debug("loading configuration file %s", path)
cf.loadPath(path)
-
cf.finalize()
self.locations = cf.locations
+
+ # XXX Hack: This should be part of BuilderApplication
if options.include_support_code is None:
options.include_support_code = cf.include_support_code
+ def error(self, message, rc=1):
+ print >>sys.stderr, message
+ sys.exit(rc)
+
+
+class BuilderApplication(Application):
+
+ def __init__(self, options):
+ Application.__init__(self, options)
+ self.ip = None
+ self.resource = locationmap.normalizeResourceId(options.resource)
+ self.resource_type, self.resource_name = self.resource.split(":", 1)
+ if not options.release_name:
+ options.release_name = self.resource_name
+ # Create a new directory for all temporary files to go in:
+ self.tmpdir = tempfile.mkdtemp(prefix=options.program + "-")
+ tempfile.tempdir = self.tmpdir
+ if options.revision_tag:
+ self.loader = loader.Loader(tag=options.revision_tag)
+ else:
+ self.loader = loader.Loader()
+
if self.resource not in self.locations:
self.error("unknown resource: %s" % self.resource)
self.resource_url = self.locations[self.resource]
self.handled_resources = sets.Set()
- def error(self, message, rc=1):
- print >>sys.stderr, message
- sys.exit(rc)
-
def build_distribution(self):
"""Create the distribution tree.
@@ -92,7 +101,7 @@
# distribution; it's the former if there's an __init__.py in
# the source directory.
os.mkdir(self.destination)
- self.ip = include.InclusionProcessor(self.source, loader=self.loader)
+ self.ip = include.InclusionProcessor(self.source, self.loader)
self.ip.add_manifest(self.destination)
self.handled_resources.add(self.resource)
name = "build_%s_distribution" % self.resource_type
@@ -102,12 +111,14 @@
def build_package_distribution(self):
pkgname = self.metadata.name
pkgdest = os.path.join(self.destination, pkgname)
- spec, dist = include.load(self.source)
+ specs = include.load(self.source)
+ self.ip.addIncludes(self.source, specs.loads)
+ specs.collection.cook()
try:
- self.ip.createDistributionTree(pkgdest, spec)
+ self.ip.createDistributionTree(pkgdest, specs.collection)
except cvsloader.CvsLoadingError, e:
self.error(str(e))
- self.ip.addIncludes(self.destination, dist)
+ self.ip.addIncludes(self.destination, specs.distribution)
pkgdir = os.path.join(self.destination, pkgname)
pkginfo = package.loadPackageInfo(pkgname, pkgdir, pkgname)
setup_cfg = os.path.join(self.destination, "setup.cfg")
@@ -189,7 +200,10 @@
unhandled_resources.add(resource)
continue
#
- source = self.loader.load(self.locations[resource])
+ location = self.locations[resource]
+ self.logger.debug("loading resource %r from %s",
+ resource, location)
+ source = self.loader.load_mutable_copy(location)
self.handled_resources.add(resource)
deps = self.add_component(type, name, source)
if type == "package" and "." in name:
@@ -230,15 +244,17 @@
"""
destination = os.path.join(self.destination, name)
self.ip.add_manifest(destination)
- spec, dist = include.load(source)
+ specs = include.load(source)
+ self.ip.addIncludes(source, specs.loads)
+ specs.collection.cook()
if type == "package":
- self.add_package_component(name, destination, spec)
+ self.add_package_component(name, destination, specs.collection)
elif type == "collection":
- self.add_collection_component(name, destination, spec)
+ self.add_collection_component(name, destination, specs.collection)
if distribution:
- self.ip.addIncludes(self.destination, dist)
+ self.ip.addIncludes(self.destination, specs.distribution)
self.create_manifest(destination)
deps_file = os.path.join(source, "DEPENDENCIES.cfg")
@@ -299,7 +315,9 @@
def load_resource(self):
"""Load the primary resource and initialize internal metadata."""
- self.source = self.loader.load(self.resource_url)
+ self.logger.debug("loading resource %r from %s",
+ self.resource, self.resource_url)
+ self.source = self.loader.load_mutable_copy(self.resource_url)
self.load_metadata()
release_name = self.options.release_name
self.target_name = "%s-%s" % (release_name, self.options.version)
@@ -442,6 +460,8 @@
mod = sys.modules[name]
source = os.path.abspath(mod.__path__[0])
if source is None:
+ self.logger.debug("loading resource 'package:%s' from %s",
+ name, url)
source = self.loader.load(url)
tests_dir = os.path.join(source, "tests")
@@ -493,7 +513,7 @@
def run(self):
"""Run the application, using the other methods of the
- ``Application`` object.
+ ``BuilderApplication`` object.
"""
try:
try:
@@ -553,6 +573,8 @@
prog=prog,
usage="usage: %prog [options] resource",
version="%prog 0.1")
+
+ # "global" options:
parser.add_option(
"-C", "--configure", dest="configfile",
help="path or URL to the configuration file", metavar="FILE")
@@ -565,6 +587,8 @@
action="append", default=[],
help=("specify an additional location map to load before"
" maps specified in the configuration"), metavar="MAP")
+
+ # options specific to building a package:
parser.add_option(
"-n", "--name", dest="release_name",
help="base name of the distribution file", metavar="NAME")
@@ -581,6 +605,7 @@
parser.add_option(
"-v", dest="version",
help="version label for the new distribution")
+
options, args = parser.parse_args(argv[1:])
if len(args) != 1:
parser.error("wrong number of arguments")
@@ -612,7 +637,7 @@
return 2
try:
- app = Application(options)
+ app = BuilderApplication(options)
app.run()
except SystemExit, e:
return e.code
=== Packages/zpkgtools/zpkgtools/include.py 1.29 => 1.30 ===
--- Packages/zpkgtools/zpkgtools/include.py:1.29 Mon Apr 19 11:54:30 2004
+++ Packages/zpkgtools/zpkgtools/include.py Fri Apr 23 15:56:12 2004
@@ -22,6 +22,9 @@
that are copied. Any file with a name matching these patterns
will be ignored.
+ - `PACKAGE_CONF`: The name of the file that specifies how the
+ package is assembled.
+
"""
import fnmatch
@@ -35,7 +38,7 @@
from zpkgtools import Error
from zpkgtools import cfgparser
-from zpkgtools import cvsloader
+from zpkgtools import loader
from zpkgtools import publication
@@ -74,10 +77,10 @@
config = parser.load()
finally:
f.close()
- config.collection.excludes[package_conf] = package_conf
+ config.collection.add_exclusion(package_conf)
else:
config = schema.getConfiguration()
- return config.collection, config.distribution
+ return config
def filter_names(names):
@@ -93,7 +96,7 @@
return names
-def normalize_path(path, type):
+def normalize_path(path, type, group):
if ":" in path:
scheme, rest = urllib.splittype(path)
if len(scheme) == 1:
@@ -101,6 +104,9 @@
# 'cause that's not allowable:
raise InclusionSpecificationError(
"drive letters are not allowed in inclusions: %r" % path)
+ else:
+ raise InclusionSpecificationError(
+ "URLs are not allowed in inclusions")
np = posixpath.normpath(path)
if posixpath.isabs(np) or np[:1] == ".":
raise InclusionSpecificationError(
@@ -110,17 +116,17 @@
return np.replace("/", os.sep)
-def normalize_path_or_url(path, type):
+def normalize_path_or_url(path, type, group):
if ":" in path:
scheme, rest = urllib.splittype(path)
if len(scheme) != 1:
# should normalize the URL, but skip that for now
return path
- return normalize_path(path, type)
+ return normalize_path(path, type, group)
class SpecificationSchema(cfgparser.Schema):
- """Specialized schema that handles populating a pair of Specifications.
+ """Specialized schema that handles populating a set of Specifications.
"""
def __init__(self, source, filename):
@@ -129,8 +135,12 @@
def getConfiguration(self):
conf = cfgparser.SectionValue(None, None, None)
- conf.collection = self.collection = Specification(self.source)
- conf.distribution = self.distribution = Specification(self.source)
+ conf.loads = Specification(
+ self.source, self.filename, "load")
+ conf.collection = Specification(
+ self.source, self.filename, "collection")
+ conf.distribution = Specification(
+ self.source, self.filename, "distribution")
return conf
def startSection(self, parent, typename, name):
@@ -140,9 +150,11 @@
return parent.collection
elif typename == "distribution":
return parent.distribution
+ elif typename == "load":
+ return parent.loads
raise cfgparser.ConfigurationError("unknown section type: %s"
% typename)
-
+
def endSection(self, parent, typename, name, child):
pass
@@ -161,21 +173,18 @@
if not src:
raise InclusionSpecificationError("source information omitted",
self.filename)
- dest = normalize_path(dest, "destination")
- src = normalize_path_or_url(src, "source")
+ dest = normalize_path(dest, "destination", section.group)
+ if section.group == "load":
+ f = normalize_path_or_url
+ else:
+ f = normalize_path
+ src = f(src, "source", section.group)
if src == "-":
- if section is self.distribution:
- raise InclusionSpecificationError(
- "cannot exclude files from the distribution root",
- self.filename)
- path = os.path.join(self.source, dest)
- expansions = filter_names(glob.glob(path))
- if not expansions:
+ if section.group != "collection":
raise InclusionSpecificationError(
- "exclusion %r doesn't match any files" % dest,
+ "can only exclude files from the collection group",
self.filename)
- for fn in expansions:
- section.excludes[fn] = fn
+ section.add_exclusion(dest)
else:
section.includes[dest] = src
@@ -194,16 +203,7 @@
"""
- # XXX Needing to pass the source directory to the constructor is a
- # bit of a hack, ... A
- # better approach may be to have "raw" and "cooked" versions of
- # the specification object; the raw version would only have the
- # information loaded from a specification file, and the cooked
- # version would be (essentially) a list of directory creation and
- # file copy operations. The input source directory would be a
- # parameter to the "cook" operation.
-
- def __init__(self, source):
+ def __init__(self, source, filename, group):
"""Initialize the Specification object.
:Parameters:
@@ -213,9 +213,26 @@
"""
# The source directory is needed since globbing is performed
# to locate files if the spec includes wildcards.
- self.excludes = {}
+ self.excludes = []
self.includes = {}
self.source = source
+ self.filename = filename
+ self.group = group
+
+ def add_exclusion(self, path):
+ self.excludes.append(path)
+
+ def cook(self):
+ patterns = self.excludes
+ self.excludes = []
+ for pat in patterns:
+ path = os.path.join(self.source, pat)
+ expansions = filter_names(glob.glob(path))
+ if not expansions:
+ raise InclusionSpecificationError(
+ "exclusion %r doesn't match any files" % pat,
+ self.filename)
+ self.excludes.extend(expansions)
class InclusionProcessor:
@@ -225,15 +242,13 @@
the output tree.
"""
- def __init__(self, source, loader=None):
+ def __init__(self, source, loader):
if not os.path.exists(source):
raise InclusionError("source directory does not exist: %r"
% source)
self.source = os.path.abspath(source)
self.manifests = []
- if loader is None:
- loader = cvsloader.CvsLoader()
- self.cvs_loader = loader
+ self.loader = loader
def createDistributionTree(self, destination, spec=None):
"""Create the output tree according to `spec`.
@@ -248,9 +263,9 @@
"""
if spec is None:
- spec = Specification(self.source)
+ spec = Specification(self.source, None, "collection")
destination = os.path.abspath(destination)
- self.copyTree(spec.source, destination, spec.excludes)
+ self.copyTree(self.source, destination, spec.excludes)
self.addIncludes(destination, spec)
def copyTree(self, source, destination, excludes={}):
@@ -381,24 +396,18 @@
# This is what we want to create:
destdir = os.path.join(destdir, basename)
+ type = urllib.splittype(source)[0] or ''
+ if len(type) in (0, 1):
+ # figure it's a path ref, possibly w/ a Windows drive letter
+ source = os.path.join(self.source, source)
+ source = "file://" + urllib.pathname2url(source)
+ type = "file"
try:
- cvsurl = cvsloader.parse(source)
+ path = self.loader.load(source)
except ValueError:
- # not a cvs: or repository: URL
- type, rest = urllib.splittype(source)
- if type:
- # some sort of URL
- self.includeFromUrl(source, destdir)
- else:
- # local path; perhaps this join should be handled by
- # the Specification to avoid having to keep
- # self.source around?
- self.includeFromLocalTree(os.path.join(self.source, source),
- destdir)
- else:
- if isinstance(cvsurl, cvsloader.RepositoryUrl):
- raise InclusionError("can't load from repository: URL")
- self.includeFromCvs(cvsurl, destdir)
+ # not a supported URL type
+ raise InclusionError("cannot load from a %r URL" % type)
+ self.includeFromLocalTree(path, destdir)
def includeFromLocalTree(self, source, destination):
# Check for file-ness here since copyTree() doesn't handle
@@ -407,20 +416,3 @@
self.copy_file(source, destination)
else:
self.copyTree(source, destination)
-
- def includeFromUrl(self, source, destination):
- # XXX treat FTP URLs specially to get permission bits and directories?
- inf = urllib2.urlopen(source)
- try:
- outf = open(destination, "w")
- try:
- shutil.copyfileobj(inf, outf)
- finally:
- outf.close()
- self.add_output(destination)
- finally:
- inf.close()
-
- def includeFromCvs(self, cvsurl, destination):
- source = self.cvs_loader.load(cvsurl.getUrl())
- self.includeFromLocalTree(source, destination)
=== Packages/zpkgtools/zpkgtools/loader.py 1.3 => 1.4 ===
--- Packages/zpkgtools/zpkgtools/loader.py:1.3 Wed Apr 21 12:03:16 2004
+++ Packages/zpkgtools/zpkgtools/loader.py Fri Apr 23 15:56:12 2004
@@ -42,23 +42,46 @@
def __init__(self, tag=None):
self.tag = tag or None
- self.workdirs = {} # URL -> (directory, path, temporary)
+ self.workdirs = {} # URL -> (tmp.directory, path, istemporary)
self.cvsloader = None
- def add_working_dir(self, url, directory, path, temporary):
- self.workdirs[url] = (directory, path, temporary)
+ def add_working_dir(self, url, directory, path, istemporary):
+ self.workdirs[url] = (directory, path, istemporary)
def cleanup(self):
"""Remove all checkouts that are present."""
while self.workdirs:
- url, (directory, path, temporary) = self.workdirs.popitem()
- if temporary:
+ url, (directory, path, istemporary) = self.workdirs.popitem()
+ if istemporary:
if directory:
shutil.rmtree(directory)
else:
os.unlink(path)
+ def transform_url(self, url):
+ """Transform a URL to encode the tag passed to the constructor.
+
+ :param url: URL that may not be associated with a particular tag.
+
+ If `url` can be modified to encode the tag associated with the
+ loader, a modified URL that does so is returned. If not, the
+ original URL is returned unmodified.
+ """
+ if self.tag:
+ try:
+ parsed_url = cvsloader.parse(url)
+ except ValueError:
+ pass
+ else:
+ if not parsed_url.tag:
+ parsed_url.tag = self.tag
+ url = parsed_url.getUrl()
+ return url
+
def load(self, url):
+ url = self.transform_url(url)
+ if url in self.workdirs:
+ return self.workdirs[url][1]
if ":" in url and url.find(":") != 1:
type, rest = urllib.splittype(url)
# the replace() is to support svn+ssh: URLs
@@ -67,12 +90,48 @@
if method is None:
method = self.unknown_load
else:
- method = self.file_load
- return method(url)
+ raise ValueError("can only load from URLs, not path references")
+ path = method(url)
+ assert path == self.workdirs[url][1]
+ return path
- def file_load(self, path):
- """Load using a local path."""
- raise NotImplementedError("is this ever used?")
+ def load_mutable_copy(self, url):
+ """Load the resource referenced by `url` so the application
+ can modify it.
+
+ If the copy provided by the `load()` method isn't 'owned' by
+ the application (because it's a temporary copy), a copy will
+ be made and used instead.
+ """
+ url = self.transform_url(url)
+ self.load(url)
+ directory, path, istemporary = self.workdirs[url]
+ if not istemporary:
+ # We need a copy we can mutate and throw away later:
+ p = self.create_copy(url, path)
+ assert p != path
+ path = p
+ assert path == self.workdirs[url][1]
+ assert self.workdirs[url][2]
+ return path
+
+ def create_copy(self, url, path):
+ """Create a copy of the tree rooted at `path`.
+
+ The copy must be 'owned' by the application, and can be
+ mutated freely without affecting the original.
+ """
+ tmpdir = tempfile.mkdtemp(prefix="loader-")
+ # we have to normalize in case there's a trailing slash
+ path = os.path.normpath(path)
+ basename = os.path.basename(path)
+ filename = os.path.join(tmpdir, basename)
+ if os.path.isfile(path):
+ shutil.copy2(path, filename)
+ else:
+ shutil.copytree(path, filename, symlinks=False)
+ self.add_working_dir(url, tmpdir, filename, True)
+ return filename
def load_file(self, url):
# XXX This doesn't deal with file: URLs that Subversion uses,
@@ -89,10 +148,6 @@
def load_cvs(self, url):
if self.cvsloader is None:
self.cvsloader = cvsloader.CvsLoader()
- parsed_url = cvsloader.parse(url)
- if not parsed_url.tag:
- parsed_url.tag = self.tag
- url = parsed_url.getUrl()
# If we've already loaded this, use that copy. This doesn't
# consider fetching something with a different path that's
# represented by a previous load():
@@ -100,6 +155,7 @@
return self.workdirs[url][1]
tmp = tempfile.mkdtemp(prefix="loader-")
+ parsed_url = cvsloader.parse(url)
path = self.cvsloader.load(parsed_url, tmp)
self.add_working_dir(url, tmp, path, True)
return path
@@ -118,7 +174,7 @@
f = urllib2.urlopen(url)
fd, tmp = tempfile.mkstemp(prefix="loader-")
try:
- os.fwrite(fd, f.read())
+ os.write(fd, f.read())
except:
os.close(fd)
os.unlink(tmp)
More information about the Zope-CVS
mailing list