漫谈Pyspider网络爬虫的实践

import urllib
import urllib2
import lxml.html as HTML

class Login(object):

    def __init__(self, username, password, login_url, post_url_prefix):
        self.username = username
        self.password = password
        self.login_url = login_url
        self.post_url_prefix = post_url_prefix

    def login(self):
        post_url, post_data = self.getPostData()
        post_url = self.post_url_prefix + post_url
        req = urllib2.Request(url = post_url, data = post_data)
        resp = urllib2.urlopen(req)
        return True

    def getPostData(self):
        url = self.login_url.strip()
        if not re.match(r'^http://', url):
            return None, None
        req = urllib2.Request(url)
        resp = urllib2.urlopen(req)
        login_page = resp.read()
        doc = HTML.fromstring (login_page)
        post_url = doc.xpath("//form[@method='post' and @id='lsform']/@action")[0]
        cookietime = doc.xpath("//input[@name='cookietime' and @id='ls_cookietime']/@value")[0]
        username = self.username
        password = self.password
        post_data = urllib.urlencode({
            'fastloginfield' : 'username',
            'username' : username,
            'password' : password,
            'quickforward' : 'no',
            'handlekey' : 'ls',
            'cookietime' : cookietime,
        })
        return post_url, post_data

import os
import hashlib
import cookielib

LOGIN_URL = 'http://登录页面地址'
USER_NAME = '用户名'
PASSWORD = '密码'

HOST = '目标网页主域名'
REFERER = 'http://目标网页主域名/'
POST_URL_PREFIX = 'http://目标网页主域名/'

# !!! Notice !!!
# Tasks that share the same account MUST share the same cookies file
COOKIES_FILE = '/tmp/pyspider.%s.%s.cookies' % (HOST, hashlib.md5(USER_NAME).hexdigest())
COOKIES_DOMAIN = HOST

USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36'
HTTP_HEADERS = {
    'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Encoding' : 'gzip, deflate, sdch',
    'Accept-Language' : 'zh-CN,zh;q=0.8,en;q=0.6',
    'Connection' : 'keep-alive',
    'DNT' : '1',
    'Host' : HOST,
    'Referer' : REFERER,
    'User-Agent' : USER_AGENT,
}

def getCookies():
    cookiesJar = cookielib.MozillaCookieJar(COOKIES_FILE)
    if not os.path.isfile(COOKIES_FILE):
        cookiesJar.save()
    cookiesJar.load (COOKIES_FILE)
    cookieProcessor = urllib2.HTTPCookieProcessor(cookiesJar)
    cookieOpener = urllib2.build_opener(cookieProcessor, urllib2.HTTPHandler)
    for item in HTTP_HEADERS:
        cookieOpener.addheaders.append ((item ,HTTP_HEADERS[item]))
    urllib2.install_opener(cookieOpener)
    if len(cookiesJar) == 0:
        login = Login(USER_NAME, PASSWORD, LOGIN_URL, POST_URL_PREFIX)
        if login.login():
            cookiesJar.save()
        else:
            return None
    cookiesDict = {}
    for cookie in cookiesJar:
        if COOKIES_DOMAIN in cookie.domain:
            cookiesDict[cookie.name] = cookie.value
    return cookiesDict

cookies = getCookies()
self.crawl(url, cookies = cookies, callback=self.index_page)

response.doc('.ml.mlt.mtw.cl > li').items()

response.doc('.pti > .pdbt > .authi > em > span').attr('title')

import hashlib
import unicodedata
import mysql.connector
from mysql.connector import errorcode

class MySQLDB:
    
    username = '数据库用户名'
    password = '数据库密码'
    database = '数据库名'
    host = 'localhost'    #数据库主机地址
    connection = ''
    isconnect = True
    placeholder = '%s'
    
    def __init__(self):
        if self.isconnect:
            MySQLDB.connect(self)
            MySQLDB.initdb(self)
    
    def escape(self,string):
        return '`%s`' % string
    
    def connect(self):
        config = {
            'user':self.username,
            'password':self.password,
            'host':self.host
        }
        
        if self.database != None:
            config['database'] = self.database
    
        try:
            cnx = mysql.connector.connect(**config)
            self.connection = cnx
            return True
        except mysql.connector.Error as err:
            if (err.errno == errorcode.ER_ACCESS_DENIED_ERROR):
                print "The credentials you provided are not correct."
            elif (err.errno == errorcode.ER_BAD_DB_ERROR):
                print "The database you provided does not exist."
            else:
                print "Something went wrong: " , err
            return False
        
    def initdb(self):
        if self.connection == '':
            print "Please connect first"
            return False
        cursor = self.connection.cursor()
        
        # 创建表的定义
        sql = 'CREATE TABLE IF NOT EXISTS \
        table_name ( \
        id VARCHAR(64) PRIMARY KEY, \
        url TEXT, \
        title TEXT, \
        type TEXT, \
        thumb TEXT, \
        count INTEGER, \
        images TEXT, \
        tags TEXT, \
        post_time DATETIME \
        ) ENGINE=INNODB DEFAULT CHARSET=UTF8'
        try:
            cursor.execute(sql)
            self.connection.commit()
            return True
        except mysql.connector.Error as err:
            print ("An error occured: {}".format(err))
            return False
        
    def cleardb (self):
        if self.connection == '':
            print "Please connect first"
            return False
        cursor = self.connection.cursor()
        sql = 'DROP TABLE IF EXISTS table_name'
        try:
            cursor.execute(sql)
            self.connection.commit()
            return True
        except mysql.connector.Error as err:
            print ("An error occured: {}".format(err))
            return False
        
    def insert (self,**values):
        if self.connection == '':
            print "Please connect first"
            return False
        cursor = self.connection.cursor()
        # 插入数据
        sql = "insert into table_name (id, url, title, type, thumb, count, temperature, images, tags, post_time) values (%s,%s,%s,%s,%s,%s,%s,%s,%s) on duplicate key update id=VALUES(id), url=VALUES(url), title=VALUES(title), type=VALUES(type), thumb=VALUES(thumb), count=VALUES(count), images=VALUES(images), tags=VALUES(tags), post_time=VALUES(post_time)"
        title = unicodedata.normalize('NFKD', values['title']).encode('ascii','ignore')
        images = ", ".join('%s' % k for k in values['images'])
        params = (hashlib.md5(title + images).hexdigest(), values['url'], values['title'], values['type'], values['thumb'], values['count'], images, '', values['date'])
        try:
            cursor.execute(sql,params)
            self.connection.commit()
            return True
        except mysql.connector.Error as err:
            print ("An error occured: {}".format(err))
            return False
    
    def replace(self,tablename=None,**values):
        if self.connection == '':
            print "Please connect first"
            return False
    
        tablename = self.escape(tablename)
        if values:
            _keys = ", ".join(self.escape(k) for k in values)
            _values = ", ".join([self.placeholder, ] * len(values))
            sql_query = "REPLACE INTO %s (%s) VALUES (%s)" % (tablename, _keys, _values)
        else:
            sql_query = "REPLACE INTO %s DEFAULT VALUES" % tablename
                
        cur = self.connection.cursor()
        try:
            if values:
                cur.execute(sql_query, list(itervalues(values)))
            else:
                cur.execute(sql_query)
            self.connection.commit()
            return True
        except mysql.connector.Error as err:
            print ("An error occured: {}".format(err))
            return False

def on_result(self, result):
        db = MySQLDB()
        db.insert(**result)

class Handler(BaseHandler):
    crawl_config = {
        'headers': {
            'User-Agent': USER_AGENT,
        },
        'itag': 'v1'
    }