[Zope-Checkins] CVS: Zope/lib/python/ZServer/medusa/test - __init__.py:1.4 asyn_http_bench.py:1.5 max_sockets.py:1.6 test_11.py:1.5 test_lb.py:1.5 test_medusa.py:1.5 test_single_11.py:1.5 tests.txt:1.4

Fred L. Drake, Jr. fred@zope.com
Tue, 18 Mar 2003 16:16:50 -0500


Update of /cvs-repository/Zope/lib/python/ZServer/medusa/test
In directory cvs.zope.org:/tmp/cvs-serv23926

Added Files:
	__init__.py asyn_http_bench.py max_sockets.py test_11.py 
	test_lb.py test_medusa.py test_single_11.py tests.txt 
Log Message:
Move ZServer into new location, including configuration support from the
new-install-branch.


=== Zope/lib/python/ZServer/medusa/test/__init__.py 1.3 => 1.4 ===
--- /dev/null	Tue Mar 18 16:16:49 2003
+++ Zope/lib/python/ZServer/medusa/test/__init__.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,2 @@
+# make test appear as a package
+


=== Zope/lib/python/ZServer/medusa/test/asyn_http_bench.py 1.4 => 1.5 ===
--- /dev/null	Tue Mar 18 16:16:49 2003
+++ Zope/lib/python/ZServer/medusa/test/asyn_http_bench.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,98 @@
+#! /usr/local/bin/python1.4
+# -*- Mode: Python; tab-width: 4 -*-
+
+import asyncore
+import socket
+import string
+import sys
+
+def blurt (thing):
+    sys.stdout.write (thing)
+    sys.stdout.flush ()
+    
+total_sessions = 0
+
+class http_client (asyncore.dispatcher_with_send):
+    def __init__ (self, host='127.0.0.1', port=80, uri='/', num=10):
+        asyncore.dispatcher_with_send.__init__ (self)
+        self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+        self.host = host
+        self.port = port
+        self.uri = uri
+        self.num = num
+        self.bytes = 0
+        self.connect ((host, port))
+        
+    def log (self, *info):
+        pass
+        
+    def handle_connect (self):
+        self.connected = 1
+        #		blurt ('o')
+        self.send ('GET %s HTTP/1.0\r\n\r\n' % self.uri)
+        
+    def handle_read (self):
+    #		blurt ('.')
+        d = self.recv (8192)
+        self.bytes = self.bytes + len(d)
+        
+    def handle_close (self):
+        global total_sessions
+        #		blurt ('(%d)' % (self.bytes))
+        self.close()
+        total_sessions = total_sessions + 1
+        if self.num:
+            http_client (self.host, self.port, self.uri, self.num-1)
+            
+import time
+class timer:
+    def __init__ (self):
+        self.start = time.time()
+    def end (self):
+        return time.time() - self.start
+        
+from asyncore import socket_map, poll
+
+MAX = 0
+
+def loop (timeout=30.0):
+    global MAX
+    while socket_map:
+        if len(socket_map) > MAX:
+            MAX = len(socket_map)
+        poll (timeout)
+        
+if __name__ == '__main__':
+    if len(sys.argv) < 6:
+        print 'usage: %s <host> <port> <uri> <hits> <num_clients>' % sys.argv[0]
+    else:
+        [host, port, uri, hits, num] = sys.argv[1:]
+        hits = string.atoi (hits)
+        num = string.atoi (num)
+        port = string.atoi (port)
+        t = timer()
+        clients = map (lambda x: http_client (host, port, uri, hits-1), range(num))
+        #import profile
+        #profile.run ('loop')
+        loop()
+        total_time = t.end()
+        print (
+                '\n%d clients\n%d hits/client\n'
+                'total_hits:%d\n%.3f seconds\ntotal hits/sec:%.3f' % (
+                        num,
+                        hits,
+                        total_sessions,
+                        total_time,
+                        total_sessions / total_time
+                        )
+                )
+        print 'Max. number of concurrent sessions: %d' % (MAX)
+        
+        
+        # linux 2.x, talking to medusa
+        # 50 clients
+        # 1000 hits/client
+        # total_hits:50000
+        # 2255.858 seconds
+        # total hits/sec:22.165
+        # Max. number of concurrent sessions: 50


=== Zope/lib/python/ZServer/medusa/test/max_sockets.py 1.5 => 1.6 ===
--- /dev/null	Tue Mar 18 16:16:50 2003
+++ Zope/lib/python/ZServer/medusa/test/max_sockets.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,65 @@
+
+import socket
+import select
+
+# several factors here we might want to test:
+# 1) max we can create
+# 2) max we can bind
+# 3) max we can listen on
+# 4) max we can connect
+
+def max_server_sockets():
+    sl = []
+    while 1:
+        try:
+            s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+            s.bind (('',0))
+            s.listen(5)
+            sl.append (s)
+        except:
+            break
+    num = len(sl)
+    for s in sl:
+        s.close()
+    del sl
+    return num
+    
+def max_client_sockets():
+        # make a server socket
+    server = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+    server.bind (('', 9999))
+    server.listen (5)
+    sl = []
+    while 1:
+        try:
+            s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+            s.connect (('', 9999))
+            conn, addr = server.accept()
+            sl.append ((s,conn))
+        except:
+            break
+    num = len(sl)
+    for s,c in sl:
+        s.close()
+        c.close()
+    del sl
+    return num
+    
+def max_select_sockets():
+    sl = []
+    while 1:
+        try:
+            num = len(sl)
+            for i in range(1 + len(sl) * 0.05):
+                    # Increase exponentially.
+                s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+                s.bind (('',0))
+                s.listen(5)
+                sl.append (s)
+            select.select(sl,[],[],0)
+        except:
+            break
+    for s in sl:
+        s.close()
+    del sl
+    return num


=== Zope/lib/python/ZServer/medusa/test/test_11.py 1.4 => 1.5 ===
--- /dev/null	Tue Mar 18 16:16:50 2003
+++ Zope/lib/python/ZServer/medusa/test/test_11.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,110 @@
+# -*- Mode: Python; tab-width: 4 -*-
+
+import asyncore
+import asynchat
+import socket
+import string
+
+# get some performance figures for an HTTP/1.1 server.
+# use pipelining.
+
+class test_client (asynchat.async_chat):
+
+    ac_in_buffer_size = 16384
+    ac_out_buffer_size = 16384	
+    
+    total_in = 0
+    
+    concurrent = 0
+    max_concurrent = 0
+    
+    def __init__ (self, addr, chain):
+        asynchat.async_chat.__init__ (self)
+        self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
+        self.set_terminator ('\r\n\r\n')
+        self.connect (addr)
+        self.push (chain)
+        
+    def handle_connect (self):
+        test_client.concurrent = test_client.concurrent + 1
+        if (test_client.concurrent > test_client.max_concurrent):
+            test_client.max_concurrent = test_client.concurrent
+            
+    def handle_expt (self):
+        print 'unexpected FD_EXPT thrown.  closing()'
+        self.close()
+        
+    def close (self):
+        test_client.concurrent = test_client.concurrent - 1
+        asynchat.async_chat.close(self)
+        
+    def collect_incoming_data (self, data):
+        test_client.total_in = test_client.total_in + len(data)
+        
+    def found_terminator (self):
+        pass
+        
+    def log (self, *args):
+        pass
+        
+        
+import time
+
+class timer:
+    def __init__ (self):
+        self.start = time.time()
+        
+    def end (self):
+        return time.time() - self.start
+        
+def build_request_chain (num, host, request_size):
+    s = 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\n\r\n' % (request_size, host)
+    sl = [s] * (num-1)
+    sl.append (
+            'GET /test%d.html HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' % (
+                    request_size, host
+                    )
+            )
+    return string.join (sl, '')
+    
+if __name__ == '__main__':
+    import string
+    import sys
+    if len(sys.argv) != 6:
+        print 'usage: %s <host> <port> <request-size> <num-requests> <num-connections>\n' % sys.argv[0]
+    else:
+        host = sys.argv[1]
+        
+        ip = socket.gethostbyname (host)
+        
+        [port, request_size, num_requests, num_conns] = map (
+                string.atoi, sys.argv[2:]
+                )
+        
+        chain = build_request_chain (num_requests, host, request_size)
+        
+        t = timer()
+        for i in range (num_conns):
+            test_client ((host,port), chain)
+        asyncore.loop()
+        total_time = t.end()
+        
+        # ok, now do some numbers
+        total_bytes = test_client.total_in
+        num_trans = num_requests * num_conns
+        throughput = float (total_bytes) / total_time
+        trans_per_sec = num_trans / total_time
+        
+        sys.stderr.write ('total time: %.2f\n' % total_time)
+        sys.stderr.write ('number of transactions: %d\n' % num_trans)
+        sys.stderr.write ('total bytes sent: %d\n' % total_bytes)
+        sys.stderr.write ('total throughput (bytes/sec): %.2f\n' % throughput)
+        sys.stderr.write ('transactions/second: %.2f\n' % trans_per_sec)
+        sys.stderr.write ('max concurrent connections: %d\n' % test_client.max_concurrent)
+        
+        sys.stdout.write (
+                string.join (
+                        map (str, (num_conns, num_requests, request_size, throughput, trans_per_sec)),
+                        ','
+                        ) + '\n'
+                )


=== Zope/lib/python/ZServer/medusa/test/test_lb.py 1.4 => 1.5 ===
--- /dev/null	Tue Mar 18 16:16:50 2003
+++ Zope/lib/python/ZServer/medusa/test/test_lb.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,159 @@
+# -*- Mode: Python; tab-width: 4 -*-
+
+# Get a lower bound for Medusa performance with a simple async
+# client/server benchmark built on the async lib.  The idea is to test
+# all the underlying machinery [select, asyncore, asynchat, etc...] in
+# a context where there is virtually no processing of the data.
+
+import socket
+import select
+import sys
+
+# ==================================================
+# server
+# ==================================================
+
+import asyncore
+import asynchat
+
+class test_channel (asynchat.async_chat):
+
+    ac_in_buffer_size = 16384
+    ac_out_buffer_size = 16384	
+    
+    total_in = 0
+    
+    def __init__ (self, conn, addr):
+        asynchat.async_chat.__init__ (self, conn)
+        self.set_terminator ('\r\n\r\n')
+        self.buffer = ''
+        
+    def collect_incoming_data (self, data):
+        self.buffer = self.buffer + data
+        test_channel.total_in = test_channel.total_in + len(data)
+        
+    def found_terminator (self):
+            # we've gotten the data, now send it back
+        data = self.buffer
+        self.buffer = ''
+        self.push (data+'\r\n\r\n')
+        
+    def handle_close (self):
+        sys.stdout.write ('.'); sys.stdout.flush()
+        self.close()
+        
+    def log (self, *args):
+        pass
+        
+class test_server (asyncore.dispatcher):
+    def __init__ (self, addr):
+    
+        if type(addr) == type(''):
+            f = socket.AF_UNIX
+        else:
+            f = socket.AF_INET
+            
+        self.create_socket (f, socket.SOCK_STREAM)
+        self.bind (addr)
+        self.listen (5)
+        print 'server started on',addr
+        
+    def handle_accept (self):
+        conn, addr = self.accept()
+        test_channel (conn, addr)
+        
+        # ==================================================
+        # client
+        # ==================================================
+        
+        # pretty much the same behavior, except that we kick
+        # off the exchange and decide when to quit
+        
+class test_client (test_channel):
+
+    def __init__ (self, addr, packet, number):
+        if type(addr) == type(''):
+            f = socket.AF_UNIX
+        else:
+            f = socket.AF_INET
+            
+        asynchat.async_chat.__init__ (self)
+        self.create_socket (f, socket.SOCK_STREAM)
+        self.set_terminator ('\r\n\r\n')
+        self.buffer = ''
+        self.connect (addr)
+        self.push (packet + '\r\n\r\n')
+        self.number = number
+        self.count = 0
+        
+    def handle_connect (self):
+        pass
+        
+    def found_terminator (self):
+        self.count = self.count + 1
+        if self.count == self.number:
+            sys.stdout.write('.'); sys.stdout.flush()
+            self.close()
+        else:
+            test_channel.found_terminator (self)
+            
+import time
+
+class timer:
+    def __init__ (self):
+        self.start = time.time()
+        
+    def end (self):
+        return time.time() - self.start
+        
+if __name__ == '__main__':
+    import string
+    
+    if '--poll' in sys.argv:
+        sys.argv.remove ('--poll')
+        use_poll=1
+    else:
+        use_poll=0
+        
+    if len(sys.argv) == 1:
+        print 'usage: %s\n' \
+        '  (as a server) [--poll] -s <ip> <port>\n' \
+        '  (as a client) [--poll] -c <ip> <port> <packet-size> <num-packets> <num-connections>\n' % sys.argv[0]
+        sys.exit(0)
+    if sys.argv[1] == '-s':
+        s = test_server ((sys.argv[2], string.atoi (sys.argv[3])))
+        asyncore.loop(use_poll=use_poll)
+    elif sys.argv[1] == '-c':
+            # create the packet
+        packet = string.atoi(sys.argv[4]) * 'B'
+        host = sys.argv[2]
+        port = string.atoi (sys.argv[3])
+        num_packets = string.atoi (sys.argv[5])
+        num_conns = string.atoi (sys.argv[6])
+        
+        t = timer()
+        for i in range (num_conns):
+            test_client ((host,port), packet, num_packets)
+        asyncore.loop(use_poll=use_poll)
+        total_time = t.end()
+        
+        # ok, now do some numbers
+        bytes = test_client.total_in
+        num_trans = num_packets * num_conns
+        total_bytes = num_trans * len(packet)
+        throughput = float (total_bytes) / total_time
+        trans_per_sec = num_trans / total_time
+        
+        sys.stderr.write ('total time: %.2f\n' % total_time)
+        sys.stderr.write ( 'number of transactions: %d\n' % num_trans)
+        sys.stderr.write ( 'total bytes sent: %d\n' % total_bytes)
+        sys.stderr.write ( 'total throughput (bytes/sec): %.2f\n' % throughput)
+        sys.stderr.write ( ' [note, throughput is this amount in each direction]\n')
+        sys.stderr.write ( 'transactions/second: %.2f\n' % trans_per_sec)
+        
+        sys.stdout.write (
+                string.join (
+                        map (str, (num_conns, num_packets, len(packet), throughput, trans_per_sec)),
+                        ','
+                        ) + '\n'
+                )


=== Zope/lib/python/ZServer/medusa/test/test_medusa.py 1.4 => 1.5 ===
--- /dev/null	Tue Mar 18 16:16:50 2003
+++ Zope/lib/python/ZServer/medusa/test/test_medusa.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,51 @@
+# -*- Mode: Python; tab-width: 4 -*-
+
+import socket
+import string
+import time
+import http_date
+
+now = http_date.build_http_date (time.time())
+
+cache_request = string.joinfields (
+        ['GET / HTTP/1.0',
+         'If-Modified-Since: %s' % now,
+         ],
+        '\r\n'
+        ) + '\r\n\r\n'
+
+nocache_request = 'GET / HTTP/1.0\r\n\r\n'
+
+def get (request, host='', port=80):
+    s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+    s.connect (host, port)
+    s.send (request)
+    while 1:
+        d = s.recv (8192)
+        if not d:
+            break
+    s.close()
+    
+class timer:
+    def __init__ (self):
+        self.start = time.time()
+    def end (self):
+        return time.time() - self.start
+        
+def test_cache (n=1000):
+    t = timer()
+    for i in xrange (n):
+        get(cache_request)
+    end = t.end()
+    print 'cache: %d requests, %.2f seconds, %.2f hits/sec' % (n, end, n/end)
+    
+def test_nocache (n=1000):
+    t = timer()
+    for i in xrange (n):
+        get(nocache_request)
+    end = t.end()
+    print 'nocache: %d requests, %.2f seconds, %.2f hits/sec' % (n, end, n/end)
+    
+if __name__ == '__main__':
+    test_cache()
+    test_nocache()


=== Zope/lib/python/ZServer/medusa/test/test_single_11.py 1.4 => 1.5 ===
--- /dev/null	Tue Mar 18 16:16:50 2003
+++ Zope/lib/python/ZServer/medusa/test/test_single_11.py	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,53 @@
+# -*- Mode: Python; tab-width: 4 -*-
+
+# no-holds barred, test a single channel's pipelining speed
+
+import string
+import socket
+
+def build_request_chain (num, host, request_size):
+    s = 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\n\r\n' % (request_size, host)
+    sl = [s] * (num-1)
+    sl.append (
+            'GET /test%d.html HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' % (
+                    request_size, host
+                    )
+            )
+    return string.join (sl, '')
+    
+import time
+
+class timer:
+    def __init__ (self):
+        self.start = time.time()
+        
+    def end (self):
+        return time.time() - self.start
+        
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) != 5:
+        print 'usage: %s <host> <port> <request-size> <num-requests>' % (sys.argv[0])
+    else:
+        host = sys.argv[1]
+        [port, request_size, num_requests] = map (
+                string.atoi,
+                sys.argv[2:]
+                )
+        chain = build_request_chain (num_requests, host, request_size)
+        import socket
+        s = socket.socket (socket.AF_INET, socket.SOCK_STREAM)
+        s.connect ((host,port))
+        t = timer()
+        s.send (chain)
+        num_bytes = 0
+        while 1:
+            data = s.recv(16384)
+            if not data:
+                break
+            else:
+                num_bytes = num_bytes + len(data)
+        total_time = t.end()
+        print 'total bytes received: %d' % num_bytes
+        print 'total time: %.2f sec' % (total_time)
+        print 'transactions/sec: %.2f' % (num_requests/total_time)


=== Zope/lib/python/ZServer/medusa/test/tests.txt 1.3 => 1.4 ===
--- /dev/null	Tue Mar 18 16:16:50 2003
+++ Zope/lib/python/ZServer/medusa/test/tests.txt	Tue Mar 18 16:16:49 2003
@@ -0,0 +1,73 @@
+# server: linux, 486dx2/66
+# client: win95, cyrix 6x86 p166+
+# over ethernet.
+#
+# number of connections
+# |     number of requests per connection   
+# |        |      packet size
+# |        |         |    throughput (bytes/sec)
+# |        |         |       |      transactions/sec
+# |        |         |       |         |
+  1        50        64   3440.86    53.76
+  1       100        64   3422.45    53.47
+  1         1       256   5120.00    20.00
+  1        50       256  13763.44    53.76
+  1       100       256  13333.33    52.08
+  1         1      1024   6400.00     6.25
+  1        50      1024   6909.58     6.74
+  1       100      1024   6732.41     6.57
+  1         1      4096  14628.56     3.57
+  1        50      4096  17181.20     4.19
+  1       100      4096  16835.18     4.11
+  5         1        64   1882.35    29.41
+  5        50        64   3990.02    62.34
+  5       100        64   3907.20    61.05
+  5         1       256   5818.18    22.72
+  5        50       256  15533.98    60.67
+  5       100       256  15744.15    61.50
+  5         1      1024  15515.14    15.15
+  5        50      1024  23188.40    22.64
+  5       100      1024  23659.88    23.10
+  5         1      4096  28444.44     6.94
+  5        50      4096  34913.05     8.52
+  5       100      4096  35955.05     8.77
+ 10         1        64    191.04     2.98
+ 10        50        64   4045.51    63.21
+ 10       100        64   4045.51    63.21
+ 10         1       256    764.17     2.98
+ 10        50       256  15552.85    60.75
+ 10       100       256  15581.25    60.86
+ 10         1      1024   2959.53     2.89
+ 10        50      1024  25061.18    24.47
+ 10       100      1024  25498.00    24.90
+ 10         1      4096  11314.91     2.76
+ 10        50      4096  39002.09     9.52
+ 10       100      4096  38780.53     9.46
+ 15         1        64    277.45     4.33
+ 15        50        64   4067.79    63.55
+ 15       100        64   4083.36    63.80
+ 15         1       256    386.31     1.50
+ 15        50       256  15262.32    59.61
+ 15       100       256  15822.00    61.80
+ 15         1      1024   1528.35     1.49
+ 15        50      1024  27263.04    26.62
+ 15       100      1024  27800.90    27.14
+ 15         1      4096   6047.24     1.47
+ 15        50      4096  39695.05     9.69
+ 15       100      4096  37112.65     9.06
+ 20         1        64    977.09    15.26
+ 20        50        64   2538.67    39.66
+ 20       100        64   3377.30    52.77
+ 20         1       256    221.93     0.86
+ 20        50       256  10815.37    42.24
+ 20       100       256  15880.89    62.03
+ 20         1      1024    883.52     0.86
+ 20        50      1024  29315.77    28.62
+ 20       100      1024  29569.73    28.87
+ 20         1      4096   7892.10     1.92
+ 20        50      4096  40223.90     9.82
+ 20       100      4096  41325.73    10.08
+# 
+# There's a big gap in trans/sec between 256 and 1024 bytes, we should
+# probably stick a 512 in there.
+#