[Zope-CVS] CVS: Packages/TestScripts - summarizer.py:1.1

[Steve Alexander]

Update of /cvs-repository/Packages/TestScripts
In directory cvs.zope.org:/tmp/cvs-serv24889

Added Files:
	summarizer.py 
Log Message:
Added a script that looks in a pipermail archive and mails out to a
different list a summary of the last 24 hr's activities in the list
archive. This is useful to summarize the results of all the automated
tests that are run each day.


=== Added File Packages/TestScripts/summarizer.py ===
#! /usr/bin/env python2.3

##############################################################################
#
# Copyright (c) 2003 Zope Corporation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.0 (ZPL).  A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
"""Script to check pipermail archive for recent messages, and post a summary.

$Id: summarizer.py,v 1.1 2003/12/11 16:14:26 stevea Exp $
"""

import urllib2
import re
import datetime
import pprint
import StringIO
import rfc822
import smtplib
from email.MIMEText import MIMEText
from email.Utils import parseaddr

__metaclass__ = type

# Settings used by the script. You'll want to customize some of these.
archive_url_template = 'http://mail.zope.org/pipermail/zope-tests/%s-%s'
mailfrom = 'Zope tests summarizer <zopetests at z3u.com>'
mailto = 'Zope Coders <zope-coders at zope.org>'
smtpserver = 'mail.z3u.com'

months = ("January February March April May June July August September "
          "October November December").split()


def get_archive(year, month):
    """Returns a list of the URLs archived for the given year and month.

    If there is nothing at the appropriate URL for that year and month,
    returns an empty list.
    """
    if not (1 <= month <= 12):
        raise ValueError, month
    stem = archive_url_template % (year, months[month-1])
    url = '%s/date.html' % stem
    try:
        f = urllib2.urlopen(url)
    except urllib2.HTTPError, eee:
        if eee.code == 404:
            return []
        else:
            raise
    data = f.read()
    results = re.compile(r'(\d{6}.html)', re.M).findall(data)
    return ['%s/%s' % (stem, result) for result in results]


class Message:
    """Represents a single message, scraped from the mail archive."""

    def __init__(self, url, datetext, subject, fromaddr):
        self.url = url
        self.datetext = datetext
        self.date = datetime.datetime.utcfromtimestamp(
            rfc822.mktime_tz(rfc822.parsedate_tz(self.datetext))
            )
        self.fromaddr = fromaddr
        self.subject = subject

def get_message(url):
    """Returns a Message object from the message archived at the given URL."""
    f = urllib2.urlopen(url)
    data = f.read()

    # Although the data on the web has lower-case tag names, for some reason
    # these become upper-cased when retrieved using urllib2.

    # There should be only one date, between <I> tags.
    dates = re.compile(r'<I>([^<]*)</I>', re.M).findall(data)
    if len(dates) != 1:
        print "ERROR", dates
        if not dates:
            raise RuntimeError('Cannot find date')
    datetext = dates[0]

    # The subject and from-address should look like this:
    #   <H1>[Zope-tests] subject line</H1>  <B>from address</B>
    subjects = re.compile(r'<H1>\[Zope-tests\] ([^<]*)</H1>\s*'
                           '<B>([^>]*)</B>',
                          re.M).findall(data)
    if len(subjects) != 1:
        print "ERROR", subjects
        if subjects:
            subject, fromaddr = subjects[0]
        else:
            subject, fromaddr = 'ERROR IN TEST AGGREGATOR' * 2
    else:
        subject, fromaddr = subjects[0]
    return Message(url, datetext, subject, fromaddr)

def main():
    """Do the work!

    Get the list of URLs, get the appropriate messages, compose an email,
    send it to the mailing list.
    """
    # All dates used are naive dates (no explicit tz).
    now = datetime.datetime.utcnow()

    this_month_urls = get_archive(now.year, now.month)
    last_month_year, last_month = divmod((now.year * 12 + now.month) - 1, 12)
    last_month_urls = get_archive(last_month_year, last_month)

    # urls is a list of urls for this month an last month, most recent first.
    urls = last_month_urls + this_month_urls
    urls.reverse()

    one_day_ago = now - datetime.timedelta(days=1)

    # Get messages starting at the most recent message, and stopping when
    # we run out of messages or when we get a message that was posted more
    # than one day ago.
    messages = []
    for url in urls:
        message = get_message(url)
        if message.date < one_day_ago:
            break
        messages.append(message)

    out = StringIO.StringIO()

    print >>out, "Summary of messages to the zope-tests list."
    print >>out, "Period %s UTC to %s UTC." % (
        one_day_ago.ctime(), now.ctime())

    # Nicely format the number of messages, and a summary of whom they are
    # from.
    if len(messages) == 0:
        print >>out, "There were no messages."
    elif len(messages) == 1:
        print >>out, "There was 1 message, from %s" % messages[0].fromaddr
    else:
        print >>out, "There were %s messages:" % len(messages),
        msgcount = {}
        for message in messages:
            addr = message.fromaddr
            msgcount[addr] = msgcount.get(addr, 0) + 1
        fromaddrs = msgcount.keys()
        fromaddrs.sort()
        print >>out, ', '.join(
            ['%s from %s' % (msgcount[addr], addr) for addr in fromaddrs])+'.'

    print >>out
    print >>out

    # We want the messages to be oldest first, so reverse them.
    messages.reverse()

    for message in messages:
        print >>out, "Subject: %s" % message.subject
        print >>out, "From: %s" % message.fromaddr
        print >>out, "Date: %s" % message.datetext
        print >>out, "URL: %s" % message.url
        print >>out

    # Create a text/plain message.
    msg = MIMEText(out.getvalue())

    msg['Subject'] = 'summary of automated tests'
    msg['From'] = mailfrom
    msg['To'] = mailto

    fromname, fromaddr = parseaddr(mailfrom)
    toname, toaddr = parseaddr(mailto)

    s = smtplib.SMTP(smtpserver)
    s.sendmail(fromaddr, [toaddr], msg.as_string())
    s.close()

if __name__ == '__main__':
    main()