Edit File: test_multi_vs_thread.py
#! /usr/bin/env python # -*- coding: iso-8859-1 -*- # vi:ts=4:et # $Id: test_multi_vs_thread.py,v 1.16 2005/04/12 03:39:01 mfx Exp $ import os, sys, time from threading import Thread, RLock try: from cStringIO import StringIO except ImportError: from StringIO import StringIO import pycurl # We should ignore SIGPIPE when using pycurl.NOSIGNAL - see # the libcurl tutorial for more info. try: import signal from signal import SIGPIPE, SIG_IGN signal.signal(signal.SIGPIPE, signal.SIG_IGN) except ImportError: pass # The conclusion is: the multi interface is fastest! NUM_PAGES = 30 NUM_THREADS = 10 assert NUM_PAGES % NUM_THREADS == 0 ##URL = "http://pycurl.sourceforge.net/tests/testgetvars.php?%d" URL = "http://pycurl.sourceforge.net/tests/teststaticpage.html?%d" # # util # class Curl: def __init__(self, url): self.url = url self.body = StringIO() self.http_code = -1 # pycurl API calls self._curl = pycurl.Curl() self._curl.setopt(pycurl.URL, self.url) self._curl.setopt(pycurl.WRITEFUNCTION, self.body.write) self._curl.setopt(pycurl.NOSIGNAL, 1) def perform(self): self._curl.perform() def close(self): self.http_code = self._curl.getinfo(pycurl.HTTP_CODE) self._curl.close() def print_result(items): return # DO NOTHING # for c in items: data = c.body.getvalue() if 0: print "**********", c.url, "**********" print data elif 1: print "%-60s %3d %6d" % (c.url, c.http_code, len(data)) ### ### 1) multi ### def test_multi(): clock1 = time.time() # init handles = [] m = pycurl.CurlMulti() for i in range(NUM_PAGES): c = Curl(URL %i) m.add_handle(c._curl) handles.append(c) clock2 = time.time() # stir state machine into action while 1: ret, num_handles = m.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break # get data while num_handles: m.select(1.0) while 1: ret, num_handles = m.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break clock3 = time.time() # close handles for c in handles: c.close() m.close() clock4 = time.time() print "multi interface: %d pages: perform %5.2f secs, total %5.2f secs" % (NUM_PAGES, clock3 - clock2, clock4 - clock1) # print result print_result(handles) ### ### 2) thread ### class Test(Thread): def __init__(self, lock=None): Thread.__init__(self) self.lock = lock self.items = [] def run(self): if self.lock: self.lock.acquire() self.lock.release() for c in self.items: c.perform() def test_threads(lock=None): clock1 = time.time() # create and start threads, but block them if lock: lock.acquire() # init (FIXME - this is ugly) threads = [] handles = [] t = None for i in range(NUM_PAGES): if i % (NUM_PAGES / NUM_THREADS) == 0: t = Test(lock) if lock: t.start() threads.append(t) c = Curl(URL % i) t.items.append(c) handles.append(c) assert len(handles) == NUM_PAGES assert len(threads) == NUM_THREADS clock2 = time.time() # if lock: # release lock to let the blocked threads run lock.release() else: # start threads for t in threads: t.start() # wait for threads to finish for t in threads: t.join() clock3 = time.time() # close handles for c in handles: c.close() clock4 = time.time() if lock: print "thread interface [lock]: %d pages: perform %5.2f secs, total %5.2f secs" % (NUM_PAGES, clock3 - clock2, clock4 - clock1) else: print "thread interface: %d pages: perform %5.2f secs, total %5.2f secs" % (NUM_PAGES, clock3 - clock2, clock4 - clock1) # print result print_result(handles) ### ### 3) thread - threads grab curl objects on demand from a shared pool ### class TestPool(Thread): def __init__(self, lock, pool): Thread.__init__(self) self.lock = lock self.pool = pool def run(self): while 1: self.lock.acquire() c = None if self.pool: c = self.pool.pop() self.lock.release() if c is None: break c.perform() def test_thread_pool(lock): clock1 = time.time() # init handles = [] for i in range(NUM_PAGES): c = Curl(URL %i) handles.append(c) # create and start threads, but block them lock.acquire() threads = [] pool = handles[:] # shallow copy of the list, shared for pop() for i in range(NUM_THREADS): t = TestPool(lock, pool) t.start() threads.append(t) assert len(pool) == NUM_PAGES assert len(threads) == NUM_THREADS clock2 = time.time() # release lock to let the blocked threads run lock.release() # wait for threads to finish for t in threads: t.join() clock3 = time.time() # close handles for c in handles: c.close() clock4 = time.time() print "thread interface [pool]: %d pages: perform %5.2f secs, total %5.2f secs" % (NUM_PAGES, clock3 - clock2, clock4 - clock1) # print result print_result(handles) lock = RLock() if 1: test_multi() test_threads() test_threads(lock) test_thread_pool(lock) else: test_thread_pool(lock) test_threads(lock) test_threads() test_multi()
Back to File Manager