[Checkins] SVN: zc.async/trunk/ checkpoint: doc updates, CSS for sphinx files, tentative changes to be able to use the convenience functions to use the installed twisted reactor.
Gary Poster
gary at modernsongs.com
Thu Aug 14 22:35:04 EDT 2008
Log message for revision 89859:
checkpoint: doc updates, CSS for sphinx files, tentative changes to be able to use the convenience functions to use the installed twisted reactor.
Changed:
U zc.async/trunk/README.txt
U zc.async/trunk/setup.py
A zc.async/trunk/sphinx/.static/default.css
U zc.async/trunk/sphinx/index.txt
U zc.async/trunk/src/zc/async/QUICKSTART_1_VIRTUALENV.txt
U zc.async/trunk/src/zc/async/README.txt
U zc.async/trunk/src/zc/async/README_1.txt
U zc.async/trunk/src/zc/async/README_2.txt
U zc.async/trunk/src/zc/async/README_3.txt
U zc.async/trunk/src/zc/async/README_3a.txt
U zc.async/trunk/src/zc/async/README_3b.txt
U zc.async/trunk/src/zc/async/TODO.txt
U zc.async/trunk/src/zc/async/catastrophes.txt
U zc.async/trunk/src/zc/async/configure.py
U zc.async/trunk/src/zc/async/dispatcher.py
U zc.async/trunk/src/zc/async/subscribers.py
U zc.async/trunk/src/zc/async/z3.txt
-=-
Modified: zc.async/trunk/README.txt
===================================================================
--- zc.async/trunk/README.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/README.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -4,8 +4,8 @@
What is it?
-----------
-The ``zc.async`` package provides **a Python tool that schedules work across
-multiple processes and machines.**
+The ``zc.async`` package provides **an easy-to-use Python tool that schedules
+work persistently and reliably across multiple processes and machines.**
For instance...
Modified: zc.async/trunk/setup.py
===================================================================
--- zc.async/trunk/setup.py 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/setup.py 2008-08-15 02:35:04 UTC (rev 89859)
@@ -77,7 +77,8 @@
zip_safe=False,
author='Gary Poster',
author_email='gary at modernsongs.com',
- description='Schedules work across multiple processes and machines.',
+ description=
+ 'Schedule durable tasks across multiple processes and machines.',
long_description=text(
'README.txt',
"=======\nChanges\n=======\n\n",
Added: zc.async/trunk/sphinx/.static/default.css
===================================================================
--- zc.async/trunk/sphinx/.static/default.css (rev 0)
+++ zc.async/trunk/sphinx/.static/default.css 2008-08-15 02:35:04 UTC (rev 89859)
@@ -0,0 +1,871 @@
+/**
+ * Sphinx Doc Design
+ */
+
+body {
+ font-family: sans-serif;
+ font-size: 100%;
+ background-color: #68b89e;
+ color: #000;
+ margin: 0;
+ padding: 0;
+}
+
+/* :::: LAYOUT :::: */
+
+div.document {
+ background-color: #f2f2f2;
+}
+
+div.documentwrapper {
+ float: left;
+ width: 100%;
+}
+
+div.bodywrapper {
+ margin: 0 230px 0 0px;
+ border-right: 4px solid #4c4c4c;
+ border-top: 4px solid #4c4c4c;
+ border-bottom: 4px solid #4c4c4c;
+}
+
+div.body {
+ background-color: white;
+ padding: 0 20px 30px 20px;
+}
+
+div.sphinxsidebarwrapper {
+ padding: 10px 5px 0 10px;
+}
+
+div.sphinxsidebar {
+ float: right;
+ width: 230px;
+ margin-left: -100%;
+ font-size: 90%;
+}
+
+div.clearer {
+ clear: both;
+}
+
+div.footer {
+ color: #fff;
+ width: 100%;
+ padding: 20px 0;
+ text-align: center;
+ font-size: 75%;
+}
+
+div.footer a {
+ color: #fff;
+ text-decoration: underline;
+}
+
+div.related {
+ background-color: #f2f2f2;
+ color: #000;
+ width: 100%;
+ height: 30px;
+ line-height: 30px;
+ font-size: 90%;
+}
+
+div.related h3 {
+ display: none;
+}
+
+div.related ul {
+ margin: 0;
+ padding: 0 0 0 10px;
+ list-style: none;
+}
+
+div.related li {
+ display: inline;
+}
+
+div.related li.right {
+ float: right;
+ margin-right: 5px;
+}
+
+div.related a {
+ color: #666;
+}
+
+/* ::: TOC :::: */
+div.sphinxsidebar h3 {
+ font-family: "American Typewriter", Georgia, "Times New Roman", Times, serif;
+ color: #000;
+ font-size: 1.4em;
+ font-weight: normal;
+ margin: 0;
+ padding: 0;
+}
+
+div.sphinxsidebar h4 {
+ font-family: "American Typewriter", Georgia, "Times New Roman", Times, serif;
+ color: #d94365;
+ font-size: 1.3em;
+ font-weight: normal;
+ margin: 5px 0 0 0;
+ padding: 0;
+}
+
+div.sphinxsidebar p {
+ color: white;
+}
+
+div.sphinxsidebar p.topless {
+ margin: 5px 10px 10px 10px;
+}
+
+div.sphinxsidebar ul {
+ margin: 10px;
+ padding: 0;
+ list-style: none;
+ color: #000;
+}
+
+div.sphinxsidebar ul ul,
+div.sphinxsidebar ul.want-points {
+ margin-left: 20px;
+ list-style: square;
+}
+
+div.sphinxsidebar ul ul {
+ margin-top: 0;
+ margin-bottom: 0;
+}
+
+div.sphinxsidebar a {
+ color: #666;
+}
+
+div.sphinxsidebar form {
+ margin-top: 10px;
+}
+
+div.sphinxsidebar input {
+ border: 1px solid #98dbcc;
+ font-family: sans-serif;
+ font-size: 1em;
+}
+
+/* :::: MODULE CLOUD :::: */
+div.modulecloud {
+ margin: -5px 10px 5px 10px;
+ padding: 10px;
+ line-height: 160%;
+ border: 1px solid #cbe7e5;
+ background-color: #f2fbfd;
+}
+
+div.modulecloud a {
+ padding: 0 5px 0 5px;
+}
+
+/* :::: SEARCH :::: */
+ul.search {
+ margin: 10px 0 0 20px;
+ padding: 0;
+}
+
+ul.search li {
+ padding: 5px 0 5px 20px;
+ background-image: url(file.png);
+ background-repeat: no-repeat;
+ background-position: 0 7px;
+}
+
+ul.search li a {
+ font-weight: bold;
+}
+
+ul.search li div.context {
+ color: #888;
+ margin: 2px 0 0 30px;
+ text-align: left;
+}
+
+ul.keywordmatches li.goodmatch a {
+ font-weight: bold;
+}
+
+/* :::: COMMON FORM STYLES :::: */
+
+div.actions {
+ padding: 5px 10px 5px 10px;
+ border-top: 1px solid #cbe7e5;
+ border-bottom: 1px solid #cbe7e5;
+ background-color: #e0f6f4;
+}
+
+form dl {
+ color: #333;
+}
+
+form dt {
+ clear: both;
+ float: left;
+ min-width: 110px;
+ margin-right: 10px;
+ padding-top: 2px;
+}
+
+input#homepage {
+ display: none;
+}
+
+div.error {
+ margin: 5px 20px 0 0;
+ padding: 5px;
+ border: 1px solid #d00;
+ font-weight: bold;
+}
+
+/* :::: INLINE COMMENTS :::: */
+
+div.inlinecomments {
+ position: absolute;
+ right: 20px;
+}
+
+div.inlinecomments a.bubble {
+ display: block;
+ float: right;
+ background-image: url(style/comment.png);
+ background-repeat: no-repeat;
+ width: 25px;
+ height: 25px;
+ text-align: center;
+ padding-top: 3px;
+ font-size: 0.9em;
+ line-height: 14px;
+ font-weight: bold;
+ color: black;
+}
+
+div.inlinecomments a.bubble span {
+ display: none;
+}
+
+div.inlinecomments a.emptybubble {
+ background-image: url(style/nocomment.png);
+}
+
+div.inlinecomments a.bubble:hover {
+ background-image: url(style/hovercomment.png);
+ text-decoration: none;
+ color: #3ca0a4;
+}
+
+div.inlinecomments div.comments {
+ float: right;
+ margin: 25px 5px 0 0;
+ max-width: 50em;
+ min-width: 30em;
+ border: 1px solid #2eabb0;
+ background-color: #f2fbfd;
+ z-index: 150;
+}
+
+div#comments {
+ border: 1px solid #2eabb0;
+ margin-top: 20px;
+}
+
+div#comments div.nocomments {
+ padding: 10px;
+ font-weight: bold;
+}
+
+div.inlinecomments div.comments h3,
+div#comments h3 {
+ margin: 0;
+ padding: 0;
+ background-color: #2eabb0;
+ color: white;
+ border: none;
+ padding: 3px;
+}
+
+div.inlinecomments div.comments div.actions {
+ padding: 4px;
+ margin: 0;
+ border-top: none;
+}
+
+div#comments div.comment {
+ margin: 10px;
+ border: 1px solid #2eabb0;
+}
+
+div.inlinecomments div.comment h4,
+div.commentwindow div.comment h4,
+div#comments div.comment h4 {
+ margin: 10px 0 0 0;
+ background-color: #2eabb0;
+ color: white;
+ border: none;
+ padding: 1px 4px 1px 4px;
+}
+
+div#comments div.comment h4 {
+ margin: 0;
+}
+
+div#comments div.comment h4 a {
+ color: #d5f4f4;
+}
+
+div.inlinecomments div.comment div.text,
+div.commentwindow div.comment div.text,
+div#comments div.comment div.text {
+ margin: -5px 0 -5px 0;
+ padding: 0 10px 0 10px;
+}
+
+div.inlinecomments div.comment div.meta,
+div.commentwindow div.comment div.meta,
+div#comments div.comment div.meta {
+ text-align: right;
+ padding: 2px 10px 2px 0;
+ font-size: 95%;
+ color: #538893;
+ border-top: 1px solid #cbe7e5;
+ background-color: #e0f6f4;
+}
+
+div.commentwindow {
+ position: absolute;
+ width: 500px;
+ border: 1px solid #cbe7e5;
+ background-color: #f2fbfd;
+ display: none;
+ z-index: 130;
+}
+
+div.commentwindow h3 {
+ margin: 0;
+ background-color: #2eabb0;
+ color: white;
+ border: none;
+ padding: 5px;
+ font-size: 1.5em;
+ cursor: pointer;
+}
+
+div.commentwindow div.actions {
+ margin: 10px -10px 0 -10px;
+ padding: 4px 10px 4px 10px;
+ color: #538893;
+}
+
+div.commentwindow div.actions input {
+ border: 1px solid #2eabb0;
+ background-color: white;
+ color: #135355;
+ cursor: pointer;
+}
+
+div.commentwindow div.form {
+ padding: 0 10px 0 10px;
+}
+
+div.commentwindow div.form input,
+div.commentwindow div.form textarea {
+ border: 1px solid #3c9ea2;
+ background-color: white;
+ color: black;
+}
+
+div.commentwindow div.error {
+ margin: 10px 5px 10px 5px;
+ background-color: #fbe5dc;
+ display: none;
+}
+
+div.commentwindow div.form textarea {
+ width: 99%;
+}
+
+div.commentwindow div.preview {
+ margin: 10px 0 10px 0;
+ background-color: #70d0d4;
+ padding: 0 1px 1px 25px;
+}
+
+div.commentwindow div.preview h4 {
+ margin: 0 0 -5px -20px;
+ padding: 4px 0 0 4px;
+ color: white;
+ font-size: 1.3em;
+}
+
+div.commentwindow div.preview div.comment {
+ background-color: #f2fbfd;
+}
+
+div.commentwindow div.preview div.comment h4 {
+ margin: 10px 0 0 0!important;
+ padding: 1px 4px 1px 4px!important;
+ font-size: 1.2em;
+}
+
+/* :::: SUGGEST CHANGES :::: */
+div#suggest-changes-box input, div#suggest-changes-box textarea {
+ border: 1px solid #ccc;
+ background-color: white;
+ color: black;
+}
+
+div#suggest-changes-box textarea {
+ width: 99%;
+ height: 400px;
+}
+
+
+/* :::: PREVIEW :::: */
+div.preview {
+ background-image: url(style/preview.png);
+ padding: 0 20px 20px 20px;
+ margin-bottom: 30px;
+}
+
+
+/* :::: INDEX PAGE :::: */
+
+table.contentstable {
+ width: 90%;
+}
+
+table.contentstable p.biglink {
+ line-height: 150%;
+}
+
+a.biglink {
+ font-size: 1.3em;
+}
+
+span.linkdescr {
+ font-style: italic;
+ padding-top: 5px;
+ font-size: 90%;
+}
+
+/* :::: INDEX STYLES :::: */
+
+table.indextable td {
+ text-align: left;
+ vertical-align: top;
+}
+
+table.indextable dl, table.indextable dd {
+ margin-top: 0;
+ margin-bottom: 0;
+}
+
+table.indextable tr.pcap {
+ height: 10px;
+}
+
+table.indextable tr.cap {
+ margin-top: 10px;
+ background-color: #f2f2f2;
+}
+
+img.toggler {
+ margin-right: 3px;
+ margin-top: 3px;
+ cursor: pointer;
+}
+
+form.pfform {
+ margin: 10px 0 20px 0;
+}
+
+/* :::: GLOBAL STYLES :::: */
+
+.docwarning {
+ background-color: #ffe4e4;
+ padding: 10px;
+ margin: 0 -20px 0 -20px;
+ border-bottom: 1px solid #f66;
+}
+
+p.subhead {
+ font-weight: bold;
+ margin-top: 20px;
+}
+
+a {
+ color: #355f7c;
+ text-decoration: none;
+}
+
+a:hover {
+ text-decoration: underline;
+}
+
+div.body h1,
+div.body h2,
+div.body h3,
+div.body h4,
+div.body h5,
+div.body h6 {
+ font-family: "American Typewriter", Georgia, "Times New Roman", Times, serif;
+ background-color: #f2f2f2;
+ font-weight: normal;
+ color: #000;
+ border-bottom: 1px solid #ccc;
+ margin: 20px -20px 10px -20px;
+ padding: 3px 0 3px 10px;
+ clear: right;
+}
+
+div.body h1 { margin-top: 0; font-size: 200%;
+ background-color: #74c990;
+ color: #fff;
+}
+div.body h2 { font-size: 160%;
+ color: #d94164;
+}
+div.body h3 { font-size: 140%; }
+div.body h4 { font-size: 120%; }
+div.body h5 { font-size: 110%; }
+div.body h6 { font-size: 100%; }
+
+a.headerlink {
+ color: #c60f0f;
+ font-size: 0.8em;
+ padding: 0 4px 0 4px;
+ text-decoration: none;
+ visibility: hidden;
+}
+
+h1:hover > a.headerlink,
+h2:hover > a.headerlink,
+h3:hover > a.headerlink,
+h4:hover > a.headerlink,
+h5:hover > a.headerlink,
+h6:hover > a.headerlink,
+dt:hover > a.headerlink {
+ visibility: visible;
+}
+
+a.headerlink:hover {
+ background-color: #c60f0f;
+ color: white;
+}
+
+div.body p, div.body dd, div.body li {
+ text-align: left;
+ line-height: 130%;
+}
+
+div.body p.caption {
+ text-align: inherit;
+}
+
+div.body td {
+ text-align: left;
+}
+
+ul.fakelist {
+ list-style: none;
+ margin: 10px 0 10px 20px;
+ padding: 0;
+}
+
+.field-list ul {
+ padding-left: 1em;
+}
+
+.first {
+ margin-top: 0 !important;
+}
+
+/* "Footnotes" heading */
+p.rubric {
+ margin-top: 30px;
+ font-weight: bold;
+}
+
+/* "Topics" */
+
+div.topic {
+ background-color: #eee;
+ border: 1px solid #ccc;
+ padding: 0 7px 0 7px;
+ margin: 10px 0 10px 0;
+}
+
+p.topic-title {
+ font-size: 1.1em;
+ font-weight: bold;
+ margin-top: 10px;
+}
+
+/* Admonitions */
+
+div.admonition {
+ margin-top: 10px;
+ margin-bottom: 10px;
+ padding: 7px;
+}
+
+div.admonition dt {
+ font-weight: bold;
+}
+
+div.admonition dl {
+ margin-bottom: 0;
+}
+
+div.admonition p {
+ display: inline;
+}
+
+div.seealso {
+ background-color: #ffc;
+ border: 1px solid #ff6;
+}
+
+div.warning {
+ background-color: #ffe4e4;
+ border: 1px solid #f66;
+}
+
+div.note {
+ background-color: #eee;
+ border: 1px solid #ccc;
+}
+
+p.admonition-title {
+ margin: 0px 10px 5px 0px;
+ font-weight: bold;
+ display: inline;
+}
+
+p.admonition-title:after {
+ content: ":";
+}
+
+div.body p.centered {
+ text-align: center;
+ margin-top: 25px;
+}
+
+table.docutils {
+ border: 0;
+}
+
+table.docutils td, table.docutils th {
+ padding: 1px 8px 1px 0;
+ border-top: 0;
+ border-left: 0;
+ border-right: 0;
+ border-bottom: 1px solid #aaa;
+}
+
+table.field-list td, table.field-list th {
+ border: 0 !important;
+}
+
+table.footnote td, table.footnote th {
+ border: 0 !important;
+}
+
+.field-list ul {
+ margin: 0;
+ padding-left: 1em;
+}
+
+.field-list p {
+ margin: 0;
+}
+
+dl {
+ margin-bottom: 15px;
+ clear: both;
+}
+
+dd p {
+ margin-top: 0px;
+}
+
+dd ul, dd table {
+ margin-bottom: 10px;
+}
+
+dd {
+ margin-top: 3px;
+ margin-bottom: 10px;
+ margin-left: 30px;
+}
+
+.refcount {
+ color: #060;
+}
+
+dt:target,
+.highlight {
+ background-color: #fbe54e;
+}
+
+dl.glossary dt {
+ font-weight: bold;
+ font-size: 1.1em;
+}
+
+th {
+ text-align: left;
+ padding-right: 5px;
+}
+
+pre {
+ padding: 5px;
+ background-color: #efc;
+ color: #333;
+ border: 1px solid #ac9;
+ border-left: none;
+ border-right: none;
+ overflow: auto;
+}
+
+td.linenos pre {
+ padding: 5px 0px;
+ border: 0;
+ background-color: transparent;
+ color: #aaa;
+}
+
+table.highlighttable {
+ margin-left: 0.5em;
+}
+
+table.highlighttable td {
+ padding: 0 0.5em 0 0.5em;
+}
+
+tt {
+ background-color: #ecf0f3;
+ padding: 0 1px 0 1px;
+ font-size: 0.95em;
+}
+
+tt.descname {
+ background-color: transparent;
+ font-weight: bold;
+ font-size: 1.2em;
+}
+
+tt.descclassname {
+ background-color: transparent;
+}
+
+tt.xref, a tt {
+ background-color: transparent;
+ font-weight: bold;
+}
+
+.footnote:target { background-color: #ffa }
+
+h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt {
+ background-color: transparent;
+}
+
+.optional {
+ font-size: 1.3em;
+}
+
+.versionmodified {
+ font-style: italic;
+}
+
+form.comment {
+ margin: 0;
+ padding: 10px 30px 10px 30px;
+ background-color: #eee;
+}
+
+form.comment h3 {
+ background-color: #326591;
+ color: white;
+ margin: -10px -30px 10px -30px;
+ padding: 5px;
+ font-size: 1.4em;
+}
+
+form.comment input,
+form.comment textarea {
+ border: 1px solid #ccc;
+ padding: 2px;
+ font-family: sans-serif;
+ font-size: 100%;
+}
+
+form.comment input[type="text"] {
+ width: 240px;
+}
+
+form.comment textarea {
+ width: 100%;
+ height: 200px;
+ margin-bottom: 10px;
+}
+
+.system-message {
+ background-color: #fda;
+ padding: 5px;
+ border: 3px solid red;
+}
+
+/* :::: PRINT :::: */
+ at media print {
+ div.document,
+ div.documentwrapper,
+ div.bodywrapper {
+ margin: 0;
+ width : 100%;
+ }
+
+ div.sphinxsidebar,
+ div.related,
+ div.footer,
+ div#comments div.new-comment-box,
+ #top-link {
+ display: none;
+ }
+}
+
+div.sidebar {
+ float: right;
+ width: 27em;
+ text-align: left;
+ margin-left: 10px;
+ padding: 2px;
+ border: 2px solid #afafaf;
+}
+
+div.sidebar p {
+ text-align: left;
+ padding-right: 4px;
+ padding-left: 10px;
+}
+
+div.sidebar p.sidebar-title {
+ background-color: #f2f2f2;
+ padding: 2px 2px 2px 4px;
+ border-bottom: 1px solid #ccc;
+ font-family: "American Typewriter", Georgia, "Times New Roman", Times, serif;
+ background-color: #f2f2f2;
+ font-weight: normal;
+ color: #d94164;
+}
+
+div.sidebar p.sidebar-title tt {
+ background: transparent;
+}
\ No newline at end of file
Modified: zc.async/trunk/sphinx/index.txt
===================================================================
--- zc.async/trunk/sphinx/index.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/sphinx/index.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -1,56 +1,88 @@
-``zc.async``
-============
+|async|
+=======
What is it?
-----------
-The ``zc.async`` package provides **an easy-to-use Python tool that schedules
-work across multiple processes and machines.**
+The |async|_ package provides **an easy-to-use Python tool that schedules
+durable tasks across multiple processes and machines.**
For instance...
-- *Web apps*: maybe your web application lets users request the creation of a
- large PDF, or some other expensive task.
+- *Web apps*:
-- *Postponed work*: maybe you have a job that needs to be done at a certain time,
- not right now.
+ maybe your web application lets users request the creation of a large PDF, or
+ some other expensive task.
-- *Parallel processing*: maybe you have a long-running problem that can be made
- to complete faster by splitting it up into discrete parts, each performed in
- parallel, across multiple machines.
+- *Postponed work*:
-- *Serial processing*: maybe you want to decompose and serialize a job.
+ maybe you have a job that needs to be done at a certain time, not right now.
-Features include the following:
+- *Parallel processing*:
-- easy to use;
+ maybe you have a long-running problem that can be made to complete faster by
+ splitting it up into discrete parts, each performed in parallel, across
+ multiple machines.
-- flexible configuration;
+- *Serial processing*:
-- reliable;
+ maybe you want to decompose and serialize a job.
-- supports high availability;
+High-level features include the following.
-- good debugging tools;
+- **Easy to use.**
-- well-tested; and
+ At its simplest, put a function in a |async| queue and commit a
+ transaction. See the quick-starts for examples.
-- friendly to testing.
+- **Flexible configuration, changeable dynamically in production.**
-While developed as part of the Zope project, zc.async can be used stand-alone.
+ Add and remove worker processes on the fly, with configurable policy on how
+ to handle interrupts. Let processes decide how many of which tasks to
+ perform. Configuration for each process is stored in the database so no
+ restarts are needed and a change can happen for any process from any
+ database client.
+- **Reliable and fault tolerant, supporting high availability.**
+
+ Configurable policy lets |async| know when, how, and under what circumstances
+ to retry jobs that encounter problems. Multiple processes and machines can
+ be available to work on jobs, and a machine or process that suddenly dies
+ lets siblings decide what to do with incomplete jobs, with policy on a
+ per-job basis. The central ZODB_ database server can be replicated with
+ commercial tools (ZRS_) or open-source tools (RelStorage_ plus, for instance
+ PostgreSQL and slony; or `gocept.zeoraid`_).
+
+- **Good debugging tools.**
+
+ Exceptions generate persistent ``Failure`` objects (from the Twisted_
+ project) for analysis, and verbose log messages.
+
+- **Well-tested.**
+
+ The package has good automated tests and is in use in mission-critical
+ applications for large software deployments.
+
+- **Friendly to testing.**
+
+ The package exposes testing helpers for a variety of circumstances, to make
+ writing automated tests for zc.async-enabled software fairly painless.
+
+While developed as part of the Zope project, zc.async can be used stand-alone,
+as seen in the quick-starts and the majority of the tests.
+
How does it work?
-----------------
-The system uses the Zope Object Database (ZODB), a transactional, pickle-based
+The system uses the Zope Object Database (ZODB_), a transactional, pickle-based
Python object database, for communication and coordination among participating
processes.
-zc.async participants can each run in their own process, or share a process
+|async| participants can each run in their own process, or share a process
(run in threads) with other code.
-The Twisted framework supplies some code (failures and reactor implementations,
-primarily) and some concepts to the package.
+The Twisted_ framework supplies some code (failures and reactor
+implementations, primarily) and some concepts to the package.
Quick starts
------------
@@ -58,10 +90,13 @@
These quick-starts can help you get a feel for the package.
.. toctree::
- :maxdepth: 2
+ :maxdepth: 1
QUICKSTART_1_VIRTUALENV
+XXX QUICKSTART_ZC_BUILDOUT
+XXX QUICKSTART_GROK
+
Documentation
-------------
@@ -84,3 +119,18 @@
* :ref:`modindex`
* :ref:`search`
+
+
+.. |async| replace:: ``zc.async``
+
+.. _`async`: http://pypi.python.org/pypi/zc.async
+
+.. _Twisted: http://pypi.python.org/pypi/Twisted
+
+.. _ZODB: http://pypi.python.org/pypi/ZODB3
+
+.. _ZRS: http://www.zope.com/products/zope_replication_services.html
+
+.. _RelStorage: http://wiki.zope.org/ZODB/RelStorage
+
+.. _`gocept.zeoraid`: http://pypi.python.org/pypi/gocept.zeoraid
Modified: zc.async/trunk/src/zc/async/QUICKSTART_1_VIRTUALENV.txt
===================================================================
--- zc.async/trunk/src/zc/async/QUICKSTART_1_VIRTUALENV.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/QUICKSTART_1_VIRTUALENV.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -3,81 +3,105 @@
Quickstart with ``virtualenv``
==============================
+Prerequisites
+=============
+
+------------
Installation
-============
+------------
To start, install |virtualenv|_ and create a virtual environment for our
experiments.
-.. class:: handout
+.. sidebar:: ``virtualenv`` and ``zc.buildout``
- I prefer zc.buildout for production deployments, but virtualenv is very
- nice for quick experimentation.
+ I prefer |zc.buildout|_ for production deployments, but |virtualenv|_ is
+ very nice for quick experimentation. Another quick-start uses
+ |zc.buildout|_.
::
$ easy_install virtualenv
$ virtualenv quickstart
-Install zc.async in the virtual environment.
+Install |async|_ in the virtual environment.
::
$ cd quickstart/
$ ./bin/easy_install zc.async
+.. |zc.buildout| replace:: ``zc.buildout``
+
+.. _`zc.buildout`: http://pypi.python.org/pypi/zc.buildout
+
.. |virtualenv| replace:: ``virtualenv``
.. _virtualenv: http://pypi.python.org/pypi/virtualenv
+.. |async| replace:: ``zc.async``
+
+.. _`async`: http://pypi.python.org/pypi/zc.async
+
+------------
Dependencies
-============
+------------
+.. sidebar:: Example Dependencies
+
+ Here's an example listing of the site-packages brought in by a run of this
+ quick-start.
+
+ ::
+
+ $ ls lib/python2.5/site-packages/
+ Twisted-8.1.0-py2.5-macosx-10.5-i386.egg
+ ZConfig-2.5.1-py2.5.egg
+ ZODB3-3.8.1b5-py2.5-macosx-10.5-i386.egg
+ easy-install.pth
+ pytz-2008c-py2.5.egg
+ rwproperty-1.0-py2.5.egg
+ setuptools-0.6c8-py2.5.egg
+ setuptools.pth
+ uuid-1.30-py2.5.egg
+ zc.async-1.4.0-py2.5.egg
+ zc.dict-1.2.1-py2.5.egg
+ zc.queue-1.1-py2.5.egg
+ zc.twist-1.3-py2.5-macosx-10.5-i386.egg
+ zdaemon-2.0.2-py2.5.egg
+ zope.bforest-1.2-py2.5.egg
+ zope.component-3.4.0-py2.5.egg
+ zope.deferredimport-3.4.0-py2.5.egg
+ zope.deprecation-3.4.0-py2.5.egg
+ zope.event-3.4.0-py2.5.egg
+ zope.i18nmessageid-3.4.3-py2.5-macosx-10.5-i386.egg
+ zope.interface-3.4.1-py2.5-macosx-10.5-i386.egg
+ zope.minmax-1.1.0-py2.5.egg
+ zope.proxy-3.4.1-py2.5-macosx-10.5-i386.egg
+ zope.testing-3.6.0-py2.5.egg
+
This installed several packages.
-- the ZODB, an object database from the Zope project;
+- the ZODB_, an object database from the Zope project;
-- Twisted, a framework for networked applications;
+- Twisted_, a framework for networked applications;
- the component architecture from the Zope project;
- and a few smaller packages.
All of these, and zc.async, are distributed under BSD-like licenses such as
-LGPL and ZPL.
+LGPL and ZPL_.
-.. class:: handout
+.. _ZODB: http://pypi.python.org/pypi/ZODB3
- ::
+.. _Twisted: http://pypi.python.org/pypi/Twisted
- $ ls lib/python2.5/site-packages/
- Twisted-8.1.0-py2.5-macosx-10.5-i386.egg
- ZConfig-2.5.1-py2.5.egg
- ZODB3-3.8.1b5-py2.5-macosx-10.5-i386.egg
- easy-install.pth
- pytz-2008c-py2.5.egg
- rwproperty-1.0-py2.5.egg
- setuptools-0.6c8-py2.5.egg
- setuptools.pth
- uuid-1.30-py2.5.egg
- zc.async-1.4.0-py2.5.egg
- zc.dict-1.2.1-py2.5.egg
- zc.queue-1.1-py2.5.egg
- zc.twist-1.3-py2.5-macosx-10.5-i386.egg
- zdaemon-2.0.2-py2.5.egg
- zope.bforest-1.2-py2.5.egg
- zope.component-3.4.0-py2.5.egg
- zope.deferredimport-3.4.0-py2.5.egg
- zope.deprecation-3.4.0-py2.5.egg
- zope.event-3.4.0-py2.5.egg
- zope.i18nmessageid-3.4.3-py2.5-macosx-10.5-i386.egg
- zope.interface-3.4.1-py2.5-macosx-10.5-i386.egg
- zope.minmax-1.1.0-py2.5.egg
- zope.proxy-3.4.1-py2.5-macosx-10.5-i386.egg
- zope.testing-3.6.0-py2.5.egg
+.. _ZPL: http://en.wikipedia.org/wiki/Zope_Public_License
+----------
ZEO Server
-==========
+----------
zc.async relies on a distributed ZODB technology called ZEO ("Zope Enterprise
Objects") to distribute work. ZEO has a central database server to which client
@@ -90,8 +114,13 @@
That starts a database server, accessible on port 9999 of your local machine,
saving the data in the test.fs file.
+
+Starting |async|
+================
+
+--------
A Client
-========
+--------
Now let's start a Python with a client connection to the database server.
@@ -103,10 +132,11 @@
This will be our single client process.
You might have many, each connecting to the main database server, and each able
-to perform and/or request zc.async jobs.
+to perform and/or request |async| jobs.
+-------------------
Database Connection
-===================
+-------------------
Connect to the database.
@@ -127,17 +157,21 @@
>>> from ZODB.DB import DB
>>> db = DB(storage)
-Start zc.async: Basics
-======================
+-------------
+Start |async|
+-------------
+Basics
+------
+
Now we do some basic configuration. This first bit installs some default
adapters. You might not ever have to worry too much about them.
>>> import zc.async.configure
>>> zc.async.configure.base()
-Start zc.async: Policy
-======================
+Policy
+------
This second part is policy, and if you ever put zc.async in production, you'll
want to understand what's going on here. We'll talk about what's going on here
@@ -148,8 +182,12 @@
Now the system has a ``dispatcher`` polling for jobs every second.
+Using |async|
+=============
+
+---------
The Queue
-=========
+---------
The ``start`` function also installed a queue. To get zc.async to do work, you
put a job in a queue, and commit the transaction.
@@ -161,8 +199,9 @@
>>> import zc.async.interfaces
>>> q = zc.async.interfaces.IQueue(conn)
+-----
A Job
-=====
+-----
Let's put a job in our queue. This silly example will return the current time.
@@ -175,7 +214,7 @@
>>> j.status
u'pending-status'
-.. class:: handout
+.. sidebar:: A Silly Example
This is a silly example. Imagine instead that this was some really
long-running job. Maybe you have lots of these jobs coming in, and you need
@@ -186,16 +225,18 @@
Or maybe this is a silly example.
..
+-------------
A Transaction
-=============
+-------------
We have to commit the transaction for the dispatcher to see the job.
>>> import transaction
>>> transaction.commit()
+--------
A Result
-========
+--------
Now wait a second and then try this. "transaction.begin" will sync up our
database with database changes made elsewhere.
@@ -213,8 +254,9 @@
>>> j.status
u'completed-status'
+-----------
Another Job
-===========
+-----------
You can also make closures by passing in the job class explicitly. Generating
RSA keys is actually a reasonable real-world use case for something like this.
@@ -228,8 +270,8 @@
... 'key.pem', '1024']))
>> transaction.commit()
-Another Result
-==============
+We need to begin the transaction to see the result--which in this case is
+simply ``0``, indicating a successful UNIX process.
::
@@ -237,6 +279,11 @@
>> _ = transaction.begin()
>> j.result
0
+
+We can open the file to show the result.
+
+::
+
>> subprocess.call(['cat', 'key.pem'])
-----BEGIN RSA PRIVATE KEY-----
MIICXgIBAAKBgQCYAZW+HjDGJhRHnUlZZWqhrGOxU2K/RhssmcMs0JLnWI2cWmZ+
@@ -248,13 +295,41 @@
Running Your Own Code
=====================
+------------------------
+A Monte Carlo Simulation
+------------------------
+
+.. sidebar:: Another Silly Example
+
+ Monte Carlo simulations are generally better suited to other tasks in
+ real-world use; pi is typically calculated by `far more sophisticated
+ means`_.
+
We've now seen some simple examples from the standard library. But how do you
get your own work done? How can you debug it?
-Let's say we want to write a
+Let's say we want to write an approach a chestnut of a problem: use a `Monte
+Carlo simulation`_ (read "throwing darts and analyzing the results") to
+`calculate pi`_. This will use |async| much like the map-reduce approach
+of `Hadoop`_: we will want to distribute the work of running the simulation to
+multiple machines so the result can be done faster.
+.. note::
+
+ The ``zc.buildout`` quick-start instead uses |async| to generate PDFs
+ on the fly, if you'd like a different usage example.
+
+.. _`Monte Carlo simulation`: http://en.wikipedia.org/wiki/Monte_Carlo_method
+
+.. _`Hadoop`: http://hadoop.apache.org/core/
+
+.. _`calculate pi`: http://www.eveandersson.com/pi/monte-carlo-circle
+
+.. _`far more sophisticated means`: http://en.wikipedia.org/wiki/Computing_Ï€
+
+---------------------------------
Picklable Callables and Arguments
-=================================
+---------------------------------
You want a job to have a reference to your own callable, so the job will get
the work you define performed.
@@ -262,32 +337,89 @@
This reference, of the job to your callable, will need to be persisted in the
database.
+.. sidebar:: ZODB Persistence Rules
+
+ We don't need these now. But if you are really curious about ZODB
+ persistence rules, here's a start.
+
+ - Anything pickleable can be persisted. Module globals functions, can be
+ pickled (by name), for instance, and will come in handy for our examples.
+
+ - Custom classes should typically inherit from persistent.Persistent.
+ Instances of persistent.Persistent subclasses are each stored as a single
+ record in the database, and references to them are handled efficiently.
+
+ - Subclasses of persistent.Persistent produce instances that recognize when
+ their *direct* attributes have been written, and inform the database that
+ they are dirty, for the next committed transaction. Standard,
+ non-persistent-aware mutables such as lists, sets and dicts do *not* know
+ when they have been changed, and are best avoided, or at least not
+ mutated. A variety of persistent-aware replacements for these types are
+ available.
+
+ - Use the transaction module to commit and abort transactions in the ZODB.
+
+ More advanced concerns include the following.
+
+ - An optimistic implementation of MVCC_ in the ZODB means that you should be
+ careful reading (not writing) one persistent.Persistent instance (record)
+ and writing another persistent.Persistent instance with a value that
+ depends on the first, read value within a transaction.
+
+ - Concurrent transactions can produce write conflict errors. Transactions
+ are often automatically retried in |async| and other ZODB-based systems if
+ conflict errors are encountered. If this is not desired for a given
+ |async| job, it should be given a ``zc.async.job.NeverRetry`` retry
+ policy, as discussed elsewhere in this documentation.
+
+ For ZODB documentation see http://www.zope.org/Wikis/ZODB/guide/zodb.html
+
Because zc.async uses the ZODB for its persistence mechanism, the ZODB's
persistence rules are in effect.
Luckily, these are fairly simple.
-ZODB Persistence Rules
-======================
+For now, we'll stay as simple as it gets: if you use *module global functions*
+and *immutables*, and share software across instances, you'll be fine.
-- Anything pickleable can be persisted. Module globals, such as functions,
- can be pickled, for instance, and will come in handy for our examples.
+ZODB allows a lot more, if you're willing to follow a few more rules, but that
+one rule will get us moving for this quick-start.
-- Custom classes should typically inherit from persistent.Persistent.
- Instances of persistent.Persistent subclasses are each stored as a single
- record in the database, and references to them are handled efficiently.
+.. _MVCC: http://en.wikipedia.org/wiki/Multiversion_concurrency_control
-- Use the transacton module to commit and abort transactions in the ZODB.
+-----------
+Make a File
+-----------
-- For ZODB documentation see http://www.zope.org/Wikis/ZODB/guide/zodb.html
+Make a new Python file. Let's call it ``pi.py``.
-Make a File
-===========
+At the top, include the same code we used in the interpreter above.
-Make a new Python file. Let's call it ``example.py``.
+::
+ import ZEO.ClientStorage
+ import ZODB
+ import twisted.internet.reactor
+ twisted.internet.reactor.start()
+ def generateSample():
+ pass XXX
+
+ def processSamples(*sample_jobs):
+ pass XXX
+
+ if __name__ == '__main__':
+ storage = ZEO.ClientStorage.ClientStorage(
+ ('127.0.0.1', 9999))
+ db = ZODB.DB(storage)
+
+ import zc.async.configure
+ zc.async.configure.base()
+
+ zc.async.configure.start(
+ db, poll_interval=1, twisted=True)
+
XXX
===
Modified: zc.async/trunk/src/zc/async/README.txt
===================================================================
--- zc.async/trunk/src/zc/async/README.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/README.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -118,8 +118,7 @@
and providing threads and connections for the work to be done. The
dispatcher then asks the reactor to call itself again in a few seconds.
-Footnotes
-=========
+.. rubric:: Footnotes
.. [#async_history] The first generation, ``zasync``, had the following goals:
Modified: zc.async/trunk/src/zc/async/README_1.txt
===================================================================
--- zc.async/trunk/src/zc/async/README_1.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/README_1.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -797,8 +797,7 @@
.. _next section: :ref:`configuration-without-zope-3`
-Footnotes
-=========
+.. rubric:: Footnotes
.. [#usageSetUp] We set up the configuration for our usage examples here.
Modified: zc.async/trunk/src/zc/async/README_2.txt
===================================================================
--- zc.async/trunk/src/zc/async/README_2.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/README_2.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -518,8 +518,7 @@
>>> reactor.stop()
-Footnotes
-=========
+.. rubric:: Footnotes
.. [#specific_dependencies] More specifically, as of this writing,
these are the minimal egg dependencies (including indirect
@@ -727,6 +726,6 @@
>>> import zc.async.testing
>>> reactor = zc.async.testing.Reactor()
- >>> dispatcher.reactor = reactor
+ >>> dispatcher._reactor = reactor
>>> dispatcher.activate()
>>> reactor.start()
Modified: zc.async/trunk/src/zc/async/README_3.txt
===================================================================
--- zc.async/trunk/src/zc/async/README_3.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/README_3.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -51,8 +51,7 @@
README_3a
README_3b
-Footnotes
-=========
+.. rubric:: Footnotes
.. [#extras_require] The "[z3]" is an "extra", defined in zc.async's setup.py
in ``extras_require``. It pulls along zc.z3monitor and simplejson in
Modified: zc.async/trunk/src/zc/async/README_3a.txt
===================================================================
--- zc.async/trunk/src/zc/async/README_3a.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/README_3a.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -187,8 +187,7 @@
These instructions are very similar to the :ref:`two-database-set-up`.
-Footnotes
-=========
+.. rubric:: Footnotes
.. [#get_vals]
Modified: zc.async/trunk/src/zc/async/README_3b.txt
===================================================================
--- zc.async/trunk/src/zc/async/README_3b.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/README_3b.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -67,8 +67,7 @@
Hopefully zc.async will be an easy-to-configure, easy-to-use, and useful tool
for you! Good luck! [#shutdown]_
-Footnotes
-=========
+.. rubric:: Footnotes
.. [#process_multi]
Modified: zc.async/trunk/src/zc/async/TODO.txt
===================================================================
--- zc.async/trunk/src/zc/async/TODO.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/TODO.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -1,3 +1,8 @@
+- fix up tips so that it looks better
+- finish quickstart
+- write a zc.buildout quickstart
+- write a grok quickstart
+
Improvements
- queues should be pluggable like agent with filter
Modified: zc.async/trunk/src/zc/async/catastrophes.txt
===================================================================
--- zc.async/trunk/src/zc/async/catastrophes.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/catastrophes.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -852,9 +852,7 @@
jobs, getting the decisions from retry policies as described above. These
are demonstrated in the job.txt document.
-.. ......... ..
-.. Footnotes ..
-.. ......... ..
+.. rubric:: Footnotes
.. [#setUp]
Modified: zc.async/trunk/src/zc/async/configure.py
===================================================================
--- zc.async/trunk/src/zc/async/configure.py 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/configure.py 2008-08-15 02:35:04 UTC (rev 89859)
@@ -71,16 +71,22 @@
# this function installs a queue named '' (empty string), starts the
# dispatcher, and installs an agent named 'main', with default values.
# It is a convenience for quick starts.
-def start(db, poll_interval=5, db_name=None, agent_chooser=None, agent_size=3):
+def start(db, poll_interval=5, db_name=None, agent_chooser=None, agent_size=3,
+ twisted=False):
zope.component.provideAdapter(zc.async.queue.getDefaultQueue)
zope.component.provideAdapter(zc.async.queue.getDefaultQueue,
adapts=(ZODB.interfaces.IConnection,))
zope.component.provideHandler(
zc.async.subscribers.QueueInstaller(db_name=db_name))
+ if twisted:
+ zope.component.provideHandler(
+ zc.async.subscribers.TwistedDispatcherInstaller(
+ poll_interval=poll_interval))
+ else:
+ zope.component.provideHandler(
+ zc.async.subscribers.ThreadedDispatcherInstaller(
+ poll_interval=poll_interval))
zope.component.provideHandler(
- zc.async.subscribers.ThreadedDispatcherInstaller(
- poll_interval=poll_interval))
- zope.component.provideHandler(
zc.async.subscribers.AgentInstaller('main',
chooser=agent_chooser,
size=agent_size))
Modified: zc.async/trunk/src/zc/async/dispatcher.py
===================================================================
--- zc.async/trunk/src/zc/async/dispatcher.py 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/dispatcher.py 2008-08-15 02:35:04 UTC (rev 89859)
@@ -34,7 +34,6 @@
import zc.async.utils
import zc.async.interfaces
-
class PollInfo(dict):
key = None
@property
@@ -230,7 +229,7 @@
thread = None # this is just a placeholder that other code can use the
# way that zc.async.subscribers.ThreadedDispatcherInstaller.__call__ does.
- def __init__(self, db, reactor, poll_interval=5, uuid=None, jobs_size=200,
+ def __init__(self, db, reactor=None, poll_interval=5, uuid=None, jobs_size=200,
polls_size=400):
if uuid is None:
uuid = zope.component.getUtility(zc.async.interfaces.IUUID)
@@ -238,8 +237,7 @@
raise ValueError('dispatcher for this UUID is already registered')
_dispatchers[uuid] = self
self.db = db
- self.reactor = reactor # we may allow the ``reactor`` argument to be
- # None at some point, to default to the installed Twisted reactor.
+ self._reactor = reactor
self.poll_interval = poll_interval
self.UUID = uuid
# Let's talk about jobs_size and polls_size.
@@ -267,6 +265,15 @@
self.queues = {}
self.dead_pools = []
+ @property
+ def reactor(self):
+ res = self._reactor
+ if res is None:
+ # importing this the first time is kinda slow so we're lazy
+ import twisted.internet.reactor
+ res = self._reactor = twisted.internet.reactor
+ return res
+
def _getJob(self, agent):
identifier = (
'getting job for UUID %s from agent %s (oid %d) '
Modified: zc.async/trunk/src/zc/async/subscribers.py
===================================================================
--- zc.async/trunk/src/zc/async/subscribers.py 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/subscribers.py 2008-08-15 02:35:04 UTC (rev 89859)
@@ -133,6 +133,21 @@
threaded_dispatcher_installer = ThreadedDispatcherInstaller()
+class TwistedDispatcherInstaller(object):
+
+ def __init__(self, poll_interval=5):
+ self.poll_interval = poll_interval
+ # This IDatabaseOpenedEvent will be from zope.app.appsetup if that
+ # package is around
+ zope.component.adapter(zc.async.interfaces.IDatabaseOpenedEvent)(self)
+
+ def __call__(self, ev):
+ dispatcher = zc.async.dispatcher.Dispatcher(
+ ev.database, poll_interval=self.poll_interval)
+ dispatcher.activate(threaded=True)
+
+twisted_dispatcher_installer = TwistedDispatcherInstaller()
+
class AgentInstaller(object):
def __init__(self, agent_name, chooser=None, size=3, queue_names=None):
Modified: zc.async/trunk/src/zc/async/z3.txt
===================================================================
--- zc.async/trunk/src/zc/async/z3.txt 2008-08-15 01:49:03 UTC (rev 89858)
+++ zc.async/trunk/src/zc/async/z3.txt 2008-08-15 02:35:04 UTC (rev 89859)
@@ -92,6 +92,7 @@
>>> print zope.app.component.hooks.getSite()
None
+.. rubric:: Footnotes
.. [#zope3job_database_setup]
More information about the Checkins
mailing list