使用Beautifulsoup抓取91ud小程序

环境部署/依赖包安装

安装virtualenv

使用virtualenv为每个项目建立不同的/独立的Python环境，减少软件冲突。

安装方法

pip install -i https://pypi.douban.com/simple virtualenv

安装virtualenvwrapper

virtualenvwrapper 是一个建立在 virtualenv 上的工具，通过它可以方便的创建/激活/管理/销毁虚拟环境。

安装方法：

pip install -i https://pypi.douban.com/simple virtualenvwrapper

新建虚拟环境/安装软件

# 新建虚拟环境
mkvirtualenv --python=/usr/local/bin/python3 91ud-spider
# 切换虚拟环境
workon 91ud-spider

# 安装依赖包
pip install -i https://pypi.douban.com/simple requests
pip install -i https://pypi.douban.com/simple mysqlclient
pip install -i https://pypi.douban.com/simple beautifulsoup4

代码

# -*- coding: utf-8 -*-
__author__ = 'yunshu'
import requests
import os
import json
import time
import hashlib
import random
import MySQLdb

from bs4 import BeautifulSoup
from urllib import parse

'''
使用Beautifulsoup抓取91ud小程序
Beautifulsoup文档参考：http://www.jb51.net/article/65287.htm
python版本：3.6
'''
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
}

conn = MySQLdb.connect(host='localhost', user='root', passwd='123456', db='spiders', charset='utf8')
cursor = conn.cursor()


def spider(minpage=1, maxpage=2):
    content = []
    index = minpage*48
    for page in range(minpage, maxpage):
        if page == 1:
            request_url = 'http://www.91ud.com/app/'
        else:
            request_url = 'http://www.91ud.com/app/%d' % page
        r = requests.get(request_url, headers=headers)
        if r.status_code == 200:
            soup = BeautifulSoup(r.text, 'lxml')
            items = soup.find_all('li', attrs={'class':'item'})
            for item in items:
                index = index + 1
                detail_url = item.find('a', attrs={'class':'avatar'}).get('href')
                detail_url = parse.urljoin(r.url, detail_url)

                detail = get_detail(detail_url)
                detail['order'] = index

                insert_db(detail)

                content.append(detail)
        else:
            print(r.status_code)
            # r.raise_for_status()

        print('fetch %s' % request_url)

        time.sleep(random.randint(1,3))

    conn.close()

    with open('q1ud.json', 'w', encoding='utf-8') as fp:
        json.dump(content, fp=fp, indent=4, ensure_ascii=False)


def get_detail(url):
    detail = {}
    r = requests.get(url, headers=headers)
    if r.status_code == 200:
        soup = BeautifulSoup(r.text, 'lxml')
        title = soup.find('h1').get_text()
        tag_list = []
        avatar = soup.find('div', attrs={'class':'intro'}).find('img').get('src')
        tags = soup.find('div', attrs={'class':'tags'}).find_all('a')
        qrcode = soup.find('div', attrs={'class':'qrcode'}).find('img').get('src')
        category = soup.find('ul', attrs={'class':'info'}).find('a').get('href').strip('/')
        os_infos = soup.find('ul', attrs={'class':'info'}).find_all('strong')
        # 待过滤处理html，去掉超链接等
        desc = soup.find('div', attrs={'class':'description'}).find('p').prettify()
        os_info = os_infos[3].get_text()
        create_time = os_infos[1].get_text()
        for tag in tags:
            tag_list.append(tag.get_text())
        detail['title'] = title
        detail['avatar'] = avatar
        detail['tag_list'] = ','.join(tag_list)
        detail['qrcode'] = qrcode
        detail['category'] = category
        detail['create_time'] = create_time
        detail['os_info'] = os_info
        detail['desc'] = desc
        detail['url'] = r.url
        detail['url_object_id'] = get_md5(url)

        path = get_save_path('images')
        if download_image(detail['qrcode'], path):
            detail['local_image'] = path + '/' + qrcode.split('/')[-2] + '.jpg'
        else:
            detail['local_image'] = ''

        path = get_save_path('avatar')
        if download_image(detail['avatar'], path):
            detail['local_avatar'] = path + '/' + avatar.split('/')[-2] + '.jpg'
        else:
            detail['local_avatar'] = ''

    else:
        print('fetch url:%s error' % url)

    return detail


def get_save_path(path):
    subdir = (str(random.randint(1, 20)))
    if not os.path.exists(path + '/' + subdir):
        os.makedirs(path + '/' + subdir)

    return path + '/' + subdir


def insert_db(detail):
    sql = "insert into 91ud(title,avatar,local_image,local_avatar,tag_list,qrcode,category,os_info, `desc`, create_at, `order`,`url`,`url_object_id`) values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, %s,%s) ON DUPLICATE KEY UPDATE title=%s"
    vals = (detail['title'], detail['avatar'], detail['local_image'], detail['local_avatar'],
        detail['tag_list'], detail['qrcode'], detail['category'], detail['os_info'], detail['desc'], detail['create_time'],
        detail['order'],detail['url'], detail['url_object_id'],detail['title'])
    cursor.execute(sql, vals)

    conn.commit()


def download_image(url, path):
    print("download image %s" % url)
    get_file_name = lambda url: os.path.join(path, url.split('/')[-2] + '.jpg')

    headers['Referer'] = url
    response = requests.get(url, headers=headers)
    if response.status_code == 200:
        file_name = get_file_name(url)
        with open(file_name, 'wb') as f:
            f.write(response.content)

        return True
    else:
        print('fetch image %s error' % url)
        return False


def get_md5(url):
    # 如果是unicode字符串，则进行utf-8编码
    if isinstance(url, str):
        url = url.encode('utf-8')
    m = hashlib.md5()
    m.update(url)
    return m.hexdigest()


if __name__ == '__main__':
    spider(1, 2)

代码链接：https://github.com/pythondev-cn/pythonspiders/blob/master/91ud/91ud.py