python抓取新浪微博评论并分析

提示：扫一扫查出行
【扫一扫了解最新限行尾号】
复制提示

1，实现效果

watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQveGlhb2xhbnphbw==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" width="320" height="300" alt="">

watermark/2/text/aHR0cDovL2Jsb2cuY3Nkbi5uZXQveGlhb2xhbnphbw==/font/5a6L5L2T/fontsize/400/fill/I0JBQkFCMA==/dissolve/70/gravity/SouthEast" width="600" height="300" alt="">

2，数据库

3。主要步骤

1，输入账号password，模拟新浪微博登陆

2，抓取评论页的内容

3。用正則表達式过滤出username，评论时间和评论内容

4，将得到的内容存入数据库

5，用SQL语句实现其它功能：比如统计评论次数等

4，具体步骤

# -*- coding: utf-8 -*-

import requests

import base64

import re

import urllib

import rsa

import json

import binascii

import MySQLdb

class Userlogin:

    def userlogin(self,username,password,pagecount):

        session = requests.Session()

        url_prelogin = 'http://login.sina.com.cn/sso/prelogin.php?

entry=weibo&callback=sinaSSOController.preloginCallBack&su=&rsakt=mod&client=ssologin.js(v1.4.5)&_=1364875106625'

        url_login = 'http://login.sina.com.cn/sso/login.php?

client=ssologin.js(v1.4.5)'

        #get servertime,nonce, pubkey,rsakv

        resp = session.get(url_prelogin)

        json_data  = re.search('\((.*)\)', resp.content).group(1)

        data       = json.loads(json_data)

        servertime = data['servertime']

        nonce      = data['nonce']

        pubkey     = data['pubkey']

        rsakv      = data['rsakv']

        # calculate su

        su  = base64.b64encode(urllib.quote(username))

        #calculate sp

        rsaPublickey= int(pubkey,16)

        key = rsa.PublicKey(rsaPublickey,65537)

        message = str(servertime) +'\t' + str(nonce) + '\n' + str(password)

        sp = binascii.b2a_hex(rsa.encrypt(message,key))

        postdata = {

                            'entry': 'weibo',

                            'gateway': '1',

                            'from': '',

                            'savestate': '7',

                            'userticket': '1',

                            'ssosimplelogin': '1',

                            'vsnf': '1',

                            'vsnval': '',

                            'su': su,

                            'service': 'miniblog',

                            'servertime': servertime,

                            'nonce': nonce,

                            'pwencode': 'rsa2',

                            'sp': sp,

                            'encoding': 'UTF-8',

                           'url': 'http://weibo.com/ajaxlogin.php?framelogin=1&callback=parent.sinaSSOController.feedBackUrlCallBack',

                            'returntype': 'META',

                            'rsakv' : rsakv,

                            }

        resp = session.post(url_login,data=postdata)

        # print resp.headers

        login_url = re.findall('replace\(\'(.*)\'\)',resp.content)

        #

        respo = session.get(login_url[0])

        uid = re.findall('"uniqueid":"(\d+)",',respo.content)[0]

        url = "http://weibo.com/u/"+uid

        respo = session.get(url)

        # print respo.content #获取首页的内容html

#以上为成功登陆微博

        #获取数据库连接

        conn = MySQLdb.connect(host='localhost',user='root',passwd='root',db='weiboanalysis',charset='utf8')

        curs = conn.cursor()

        curs.execute('delete from outbox')

        myheaders={}

        myheaders['set-cookie'] = resp.headers['set-cookie']

        myheaders['Referer'] = 'http://weibo.com/comment/inbox?leftnav=1&wvr=5'

        # print myheaders

#下面是開始抓取信息

        for i in range(1,int(pagecount)+1):

            forwardUrl = """http://weibo.com/comment/inbox?

topnav=1&wvr=5&f=1&page=%d"""%i

            r = session.post(forwardUrl,headers=myheaders)

            page = r.content

            # print page

            #获取并过滤出用户名，存在pagename数组

            pagename = re.findall('<a\s*title=[^>]*usercard[^>]*>',page)

            for n in range(0,len(pagename)):

                pagename[n] = pagename[n].split('\\"')[1]

            #获取并过滤出评论时间，存在pagetime数组

            pagetime = re.findall('WB_time S_func2[^>]*>[^>]*>',page)

            for t in range(0,len(pagetime)):

                pagetime[t] = pagetime[t].split('>')[1].split('<')[0]

            #获取并过滤出评论内容。存在pagecont数组

            pagecont={}

            pagecontent = re.findall(r'<p class=\\\"detail\\(.*?

)<\\\/p>',page)

            for t in range(0,len(pagecontent)):

                a = pagecontent[t].split("<\/a>")

                b = a[len(a)-1]

                c = re.sub(r"<img(.*?)>",'[表情]',b) #去掉图片表情

                d = re.sub(r"<span(.*?)span>",'',c)

                pagecont[t] = re.sub(r"\\t|:|：",'',d)  #去掉最后的/t和最前的冒号

            for index in range(0,len(pagetime)):

                sql = """ insert into outbox(uname,time,text) values('%s','%s','%s')"""%(pagename[index],pagetime[index],pagecont[index])

                curs.execute(sql)

        conn.commit()

        curs.close()

        conn.close()

从数据库获取评论并分析：

# -*- encoding:utf-8 -*-

__author__ = 'lanzao'

import MySQLdb

class OutboxAnalysis:

    def getMost(self,num):<span style="white-space:pre">		</span>#查看评论最多的前num个人

        conn =  MySQLdb.connect(host='localhost',user='root',passwd='root',db='weiboanalysis',charset='utf8')

        curs = conn.cursor()

        sql="""

        select uid,uname,count(uname) as count

        from outbox

        group by uname

        order by count(uname) desc

        limit %d;

        """% int(num)

        curs.execute(sql)

        conn.commit()

        print "******************评论次数排行榜************************"

        for item in curs.fetchall():

            print item[1]+" ",str(item[2])+"次"

        print "*******************************************************"

        curs.close()

        conn.close()

    def getUser(self,user):<span style="white-space:pre">	</span>#查看某用户评论

        conn =  MySQLdb.connect(host='localhost',user='root',passwd='root',db='weiboanalysis',charset='utf8')

        curs = conn.cursor()

        curs.execute("""select * from outbox where uname='%s'"""%user)

        print "*****************************************"

        for item in curs.fetchall():

            print item[1]+"   ",item[2]+"   ",item[3]

        print "*****************************************"

        curs.close()

        conn.close()

程序入口：

# -*- encoding:utf-8 -*-

__author__ = 'lanzao'

from OutboxAnalysis import OutboxAnalysis

from UserLogin import Userlogin;

def menu():

    print"""

        选择你想要的功能：

        0,退出

        1，查询评论数最多的人

        2，查询某用户的全部评论

        3,登陆微博并抓取评论

    """

def menuChoice():

    choice = raw_input("输入你的选择（0/1/2/3）：")

    while choice != '0':

        if choice == '3':

            username = raw_input("输入新浪微博账号：")

            password = raw_input("输入密码：")

            pagecount = raw_input("输入想要抓取评论的页数：")

            o = Userlogin()

            o.userlogin(username=username,password=password,pagecount=pagecount)

            print "抓取完成"

            choice = raw_input("输入你的选择（0/1/2/3）：")

        elif choice == '1':

            num = raw_input("你想查看前几个人？请输入数字：")

            o = OutboxAnalysis()

            o.getMost(num)

            choice = raw_input("输入你的选择（0/1/2/3）：")

        elif choice == '2':

            name = raw_input("你想查看谁的评论：")

            o = OutboxAnalysis()

            o.getUser(name)

            choice = raw_input("输入你的选择（0/1/2/3）：")

        else:

            print """choice=%s"""%choice

            print "输入无效"

            choice = raw_input("输入你的选择（0/1/2/3）：")

menu()

menuChoice()