多线程网页爬虫 python 实现（二）

#!/usr/bin/env python

#coding=utf-8

import threading

import urllib

import re

import time

cur=0

last=0

totalcount=0

depth=0

t_mutex=threading.Condition() 

class Mycrawler:

    def __init__(self,crawlername,seeds,threadnum):

        self.crawlername=crawlername

        self.seeds=seeds

        self.crawqueue=CrawQueue()

        self.initQueue(self.seeds)

        self.threadnum=threadnum

        self.threadpools=[]

        self.logfile=file('log2.txt','w')

    def initQueue(self,seeds):

        if isinstance(seeds,str):

            self.crawqueue.push(seeds)

        elif isinstance(seeds,list):

            for seed in seeds:

                self.crawqueue.push(seed)

        global last

        global totalcount

        totalcount=self.crawqueue.getQueueCount()

        last=totalcount

    def crawling(self):

        global cur

        global depth

        global last

        global totalcount

        self.log(">>>Depth "+str(depth)+":\n")

        while self.crawqueue.getQueueCount()!=0:

            url=self.crawqueue.pop()

            self.log(url)

            if url==None:

                continue

            self.crawqueue.addToVisited(url)

            links=self.getLinks(url)

            if links==None:

                print 'None'

                self.crawqueue.failed.append(url)

                continue

            beforenum = self.crawqueue.getQueueCount()

            self.crawqueue.addLinks(links)

            afternum  = self.crawqueue.getQueueCount()

            totalcount+=afternum-beforenum

            cur+=1

            if cur==last:

                depth+=1

                self.log(">>>Depth "+str(depth)+":\n")

                last=totalcount

    def crawling2(self):

        global last

        global totalcount

        global depth

        self.log(">>>Depth "+str(depth)+":\n")

        totalcount=self.crawqueue.getQueueCount()

        last=totalcount

        while self.crawqueue.getQueueCount()!=0:

            for i in range(self.threadnum):

                url=self.crawqueue.pop()

                if url==None:

                    break

                crawthread=crawlerThread(url,i,self)

                self.threadpools.append(crawthread)

                crawthread.start()

            for i in range(len(self.threadpools)):

                crawthread=self.threadpools[i]

                crawthread.join(30)

    def log(self,content):

        self.logfile.write(content+"\n")

class crawlerThread(threading.Thread):

    def __init__(self,url,tid,mycrawler):

        threading.Thread.__init__(self)

        self.url=url

        self.tid=tid

        self.mycrawler=mycrawler

    def run(self):

        global t_mutex

        global cur

        global last

        global totalcount

        global depth

        t_mutex.acquire()

        self.mycrawler.log(self.url)

        t_mutex.release()

        links=self.getLinks(self.url)

        if links==None:

            t_mutex.acquire()

            self.mycrawler.crawqueue.addToVisited(self.url)

            self.mycrawler.crawqueue.addToFailed(self.url)

            t_mutex.release()

        else:

            t_mutex.acquire()

            self.mycrawler.crawqueue.addToVisited(self.url)

            beforenum=self.mycrawler.crawqueue.getQueueCount()

            self.mycrawler.crawqueue.addLinks(links)

            afternum =self.mycrawler.crawqueue.getQueueCount()

            totalcount+=afternum-beforenum

            t_mutex.release()

        t_mutex.acquire()

        cur+=1

        if cur==last:

            depth+=1

            self.mycrawler.log(">>>Depth "+str(depth)+":\n")

            last=totalcount

        t_mutex.release()

    def getLinks(self,url):

        try:

            page=urllib.urlopen(url)

            html=page.read()

            reg=r'"(http://.+?)"'

            regob=re.compile(reg,re.DOTALL)

            links=regob.findall(html)

            return links

        except:

            print 'Failed downloading and saving',url

            return None

class CrawQueue:

    def __init__(self):

        self.queue=[]

        self.visited=[]

        self.failed=[]

    def getQueue(self):

        return self.queue

    def getVisited(self):

        return self.visited

    def getFailed(self):

        return self.failed

    def push(self,url):

        if url!="" and url not in self.queue and url not in self.visited:

            self.queue.insert(0,url)

    def pop(self):

        if len(self.queue)==0:

            #print 'failed to pop: queue is empty'

            return None

        else:

            return self.queue.pop()

    def isEmpty(self):

        if len(self.queue)==0:

            return 1

        else:

            return 0

    def addToVisited(self,url):

        self.visited.append(url)

    def addToFailed(self,url):

        self.failed.append(url)

    def remove(self,url):

        self.queue.remove(url)

    def getVisitedCount(self):

        return len(self.visited)

    def getQueueCount(self):

        return len(self.queue)

    def addLinks(self,links):

        for link in links:

            self.push(link)

if __name__=="__main__":

    seeds="http://www.douban.com/"

    threadnum=int(raw_input("设置线程数:"))

    crawlername="小小爬虫"

    mycrawler=Mycrawler(crawlername,seeds,threadnum)

    mycrawler.crawling2()