【python】数据爬虫，抓取并分析豆瓣电影信息

url-lib-request模块获取HTML

spider.py
网页下载，链接提取，数据存储

http请求：
网址url，
http-header: (1)用户代理；(2)浏览器版本

网页下载用 urllib库。

# 将豆瓣电影top250的第一页下载

import urllib.request # 导入url-lib-request模块

# 设置网址
url = "https://movie.douban.com/top250"

# http头信息
header = {
    "User-Agent":"Mozilla/5.0(Windows NT 10.0;WOW64)"
    "AppleWebKit/537.36(KHTML, like Gecko)"
    "Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
}

# 构造一个Request请求对象
req = urllib.request.Request(url = url, headers = header)

# 向豆瓣服务器发送请求，获取到回复response
response = urllib.request.urlopen(req)

# 读取回复并使用UTF-8编码进行解码
html = response.read().decode("utf-8")

# 打印HTML
print(html)

打印满足条件的网址

import urllib.request # 导入url-lib-request模块

def download_html(url):
    # http头信息
    header = {
        "User-Agent":"Mozilla/5.0(Windows NT 10.0; WOW64)"
        "AppleWebKit/537.36(KHTML, like Gecko)"
        "Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
    }
    # 构造一个Request请求对象
    req = urllib.request.Request(url = url, headers = header)
    # 向豆瓣服务器发送请求，获取到回复response
    response = urllib.request.urlopen(req)
    # 读取回复并使用UTF-8编码进行解码
    html = response.read().decode("utf-8")

    return html

# 调用函数download_html，下载豆瓣top250第一页，保存至html
html = download_html("https://movie.douban.com/top250")

import re # 导入re正则库

# 电影介绍页网址的正则表达式
pattern = 'https://movie.douban.com/subject/[0-9]+/'

# 从html中提取全部满足电影介绍页的网址pattern的url
urls = re.findall(pattern, html)

urls = set(urls) # 通过set集合去重

# 打印出提取出的网址数量和具体的网址
print("urls count = %d" % (len(urls)))
for url in urls:
    print(url)

douban.txt 文件内容：

https://movie.douban.com/top250
https://movie.douban.com/top250?start=25
https://movie.douban.com/top250?start=50
https://movie.douban.com/top250?start=75
https://movie.douban.com/top250?start=100
https://movie.douban.com/top250?start=125
https://movie.douban.com/top250?start=150
https://movie.douban.com/top250?start=175
https://movie.douban.com/top250?start=200
https://movie.douban.com/top250?start=225

写入到文件中

import urllib.request # 导入url-lib-request模块
import re # 导入re正则库

def download_html(url):
    # http头信息
    header = {
        "User-Agent":"Mozilla/5.0(Windows NT 10.0; WOW64)"
        "AppleWebKit/537.36(KHTML, like Gecko)"
        "Chrome/76.0.3809.87 Safari/537.36 SLBrowser/6.0.1.8131"
    }
    # 构造一个Request请求对象
    req = urllib.request.Request(url = url, headers = header)
    # 向豆瓣服务器发送请求，获取到回复response
    response = urllib.request.urlopen(req)
    # 读取回复并使用UTF-8编码进行解码
    html = response.read().decode("utf-8")

    return html

# 调用函数download_html，下载豆瓣top250第一页，保存至html
html = download_html("https://movie.douban.com/top250")

# 电影介绍页网址提取的功能封装至函数extract_url中，
# 函数输入html，返回html中提取的url连接
def extract_url(html):
    # 电影介绍页网址的正则表达式
    pattern = 'https://movie.douban.com/subject/[0-9]+/'
    # 从html中提取全部满足电影介绍页的网址pattern的url
    urls = re.findall(pattern, html)
    return set(urls) # 通过set集合去重

# 以读取方式打开豆瓣电影列表文件
file = open('douban.txt', 'r')
# 然后以写模式打开movie.txt，用来存储提取出的电影介绍页网址
output = open('movie.txt', 'w')

# 将douban中的网址按行读入
lines = file.readlines()

for url in lines:
    # 遍历每一行url，通过strip函数去掉首尾空字符
    url = url.strip()
    print(url) # 打印url作为调试信息
    # 调用download_html获取url对应的html代码
    html = download_html(url)
    # 调用extract_url提取html中的电影介绍页链接网址
    urls = extract_url(html)
    for url in urls: # 遍历url集合
        output.write(url + '\n') # 将每个url都写入movie文件
# 最后关闭douban和movie两个文件
file.close()
output.close()

movie.txt 结果：

https://movie.douban.com/subject/1889243/
https://movie.douban.com/subject/1849031/
https://movie.douban.com/subject/1291561/
https://movie.douban.com/subject/6786002/
https://movie.douban.com/subject/20495023/
https://movie.douban.com/subject/1291841/
https://movie.douban.com/subject/3011091/
https://movie.douban.com/subject/1307914/
https://movie.douban.com/subject/1291549/
https://movie.douban.com/subject/1292213/
https://movie.douban.com/subject/1295124/
https://movie.douban.com/subject/1292063/
https://movie.douban.com/subject/2131459/
https://movie.douban.com/subject/1292052/
https://movie.douban.com/subject/1291546/
https://movie.douban.com/subject/1292064/
https://movie.douban.com/subject/25662329/
https://movie.douban.com/subject/1292001/
https://movie.douban.com/subject/3793023/
https://movie.douban.com/subject/1292720/
https://movie.douban.com/subject/1295644/
https://movie.douban.com/subject/5912992/
https://movie.douban.com/subject/1296141/
https://movie.douban.com/subject/3541415/
https://movie.douban.com/subject/1292722/
https://movie.douban.com/subject/1292365/
https://movie.douban.com/subject/1305487/
https://movie.douban.com/subject/1293182/
https://movie.douban.com/subject/21937452/
https://movie.douban.com/subject/1291560/
https://movie.douban.com/subject/1291552/
https://movie.douban.com/subject/1296736/
https://movie.douban.com/subject/3742360/
https://movie.douban.com/subject/1291572/
https://movie.douban.com/subject/1295038/
https://movie.douban.com/subject/1293172/
https://movie.douban.com/subject/1299398/
https://movie.douban.com/subject/30170448/
https://movie.douban.com/subject/1291858/
https://movie.douban.com/subject/26387939/
https://movie.douban.com/subject/1300267/
https://movie.douban.com/subject/3319755/
https://movie.douban.com/subject/26752088/
https://movie.douban.com/subject/2129039/
https://movie.douban.com/subject/1298624/
https://movie.douban.com/subject/1851857/
https://movie.douban.com/subject/1308807/
https://movie.douban.com/subject/1929463/
https://movie.douban.com/subject/25958717/
https://movie.douban.com/subject/1291583/
https://movie.douban.com/subject/1291548/
https://movie.douban.com/subject/1293839/
https://movie.douban.com/subject/27060077/
https://movie.douban.com/subject/3011235/
https://movie.douban.com/subject/1301753/
https://movie.douban.com/subject/21937445/
https://movie.douban.com/subject/1652587/
https://movie.douban.com/subject/1292849/
https://movie.douban.com/subject/1418019/
https://movie.douban.com/subject/1299131/
https://movie.douban.com/subject/26580232/
https://movie.douban.com/subject/1292402/
https://movie.douban.com/subject/1292220/
https://movie.douban.com/subject/1485260/
https://movie.douban.com/subject/1291818/
https://movie.douban.com/subject/1293350/
https://movie.douban.com/subject/1291543/
https://movie.douban.com/subject/3008247/
https://movie.douban.com/subject/1291571/
https://movie.douban.com/subject/1291828/
https://movie.douban.com/subject/1292000/
https://movie.douban.com/subject/1306029/
https://movie.douban.com/subject/1900841/
https://movie.douban.com/subject/1294408/
https://movie.douban.com/subject/1291843/
https://movie.douban.com/subject/1298070/
https://movie.douban.com/subject/1306249/
https://movie.douban.com/subject/11525673/
https://movie.douban.com/subject/1302425/
https://movie.douban.com/subject/1292343/
https://movie.douban.com/subject/6985810/
https://movie.douban.com/subject/1292224/
https://movie.douban.com/subject/1292434/
https://movie.douban.com/subject/1291544/
https://movie.douban.com/subject/2334904/
https://movie.douban.com/subject/3442220/
https://movie.douban.com/subject/1300299/
https://movie.douban.com/subject/1780330/
https://movie.douban.com/subject/1292656/
https://movie.douban.com/subject/5322596/
https://movie.douban.com/subject/1296996/
https://movie.douban.com/subject/1303021/
https://movie.douban.com/subject/1292223/
https://movie.douban.com/subject/1291832/
https://movie.douban.com/subject/1293544/
https://movie.douban.com/subject/1294371/
https://movie.douban.com/subject/1297192/
https://movie.douban.com/subject/1865703/
https://movie.douban.com/subject/1292262/
https://movie.douban.com/subject/1292679/
https://movie.douban.com/subject/3395373/
https://movie.douban.com/subject/1418834/
https://movie.douban.com/subject/1305164/
https://movie.douban.com/subject/1294639/
https://movie.douban.com/subject/21318488/
https://movie.douban.com/subject/10777687/
https://movie.douban.com/subject/2149806/
https://movie.douban.com/subject/1787291/
https://movie.douban.com/subject/10577869/
https://movie.douban.com/subject/1395091/
https://movie.douban.com/subject/11026735/
https://movie.douban.com/subject/4202302/
https://movie.douban.com/subject/1291990/
https://movie.douban.com/subject/27010768/
https://movie.douban.com/subject/1297447/
https://movie.douban.com/subject/1292337/
https://movie.douban.com/subject/1291999/
https://movie.douban.com/subject/1297359/
https://movie.douban.com/subject/1292370/
https://movie.douban.com/subject/1293359/
https://movie.douban.com/subject/26799731/
https://movie.douban.com/subject/1296339/
https://movie.douban.com/subject/1300992/
https://movie.douban.com/subject/1297630/
https://movie.douban.com/subject/1291875/
https://movie.douban.com/subject/4268598/
https://movie.douban.com/subject/1309055/
https://movie.douban.com/subject/1292215/
https://movie.douban.com/subject/6307447/
https://movie.douban.com/subject/1303037/
https://movie.douban.com/subject/10533913/
https://movie.douban.com/subject/10437779/
https://movie.douban.com/subject/3287562/
https://movie.douban.com/subject/25814707/
https://movie.douban.com/subject/26628357/
https://movie.douban.com/subject/1828115/
https://movie.douban.com/subject/25986180/
https://movie.douban.com/subject/25814705/
https://movie.douban.com/subject/2353023/
https://movie.douban.com/subject/1294240/
https://movie.douban.com/subject/1297518/
https://movie.douban.com/subject/1297052/
https://movie.douban.com/subject/1291579/
https://movie.douban.com/subject/5989818/
https://movie.douban.com/subject/1307315/
https://movie.douban.com/subject/1418200/
https://movie.douban.com/subject/1858711/
https://movie.douban.com/subject/3072124/
https://movie.douban.com/subject/1461403/
https://movie.douban.com/subject/1292274/
https://movie.douban.com/subject/21360417/
https://movie.douban.com/subject/1793929/
https://movie.douban.com/subject/1578507/
https://movie.douban.com/subject/24733428/
https://movie.douban.com/subject/10463953/
https://movie.douban.com/subject/1417598/
https://movie.douban.com/subject/1292208/
https://movie.douban.com/subject/1293181/
https://movie.douban.com/subject/1302467/
https://movie.douban.com/subject/1316510/
https://movie.douban.com/subject/27119724/
https://movie.douban.com/subject/26325320/
https://movie.douban.com/subject/4920389/
https://movie.douban.com/subject/4917726/
https://movie.douban.com/subject/1307856/
https://movie.douban.com/subject/25917973/
https://movie.douban.com/subject/1306861/
https://movie.douban.com/subject/1300374/
https://movie.douban.com/subject/26683290/
https://movie.douban.com/subject/26611804/
https://movie.douban.com/subject/1291557/
https://movie.douban.com/subject/1907966/
https://movie.douban.com/subject/1297574/
https://movie.douban.com/subject/1295399/
https://movie.douban.com/subject/1291545/
https://movie.douban.com/subject/1291822/
https://movie.douban.com/subject/1293460/
https://movie.douban.com/subject/1292329/
https://movie.douban.com/subject/1292226/
https://movie.douban.com/subject/1307106/
https://movie.douban.com/subject/1304447/
https://movie.douban.com/subject/1291585/
https://movie.douban.com/subject/1295409/
https://movie.douban.com/subject/2209573/
https://movie.douban.com/subject/1959195/
https://movie.douban.com/subject/1867345/
https://movie.douban.com/subject/10808442/
https://movie.douban.com/subject/25773932/
https://movie.douban.com/subject/24750126/
https://movie.douban.com/subject/3792799/
https://movie.douban.com/subject/1291578/
https://movie.douban.com/subject/27622447/
https://movie.douban.com/subject/1291992/
https://movie.douban.com/subject/1307811/
https://movie.douban.com/subject/27059130/
https://movie.douban.com/subject/25895901/
https://movie.douban.com/subject/6534248/
https://movie.douban.com/subject/1862151/
https://movie.douban.com/subject/1310177/
https://movie.douban.com/subject/3011051/
https://movie.douban.com/subject/1291879/
https://movie.douban.com/subject/1292401/
https://movie.douban.com/subject/3592854/
https://movie.douban.com/subject/4739952/
https://movie.douban.com/subject/1438652/
https://movie.douban.com/subject/1308857/
https://movie.douban.com/subject/1303394/
https://movie.douban.com/subject/1297880/
https://movie.douban.com/subject/25934014/
https://movie.douban.com/subject/1293908/
https://movie.douban.com/subject/5300054/
https://movie.douban.com/subject/25980443/
https://movie.douban.com/subject/1959877/
https://movie.douban.com/subject/1422283/
https://movie.douban.com/subject/3075287/
https://movie.douban.com/subject/1309163/
https://movie.douban.com/subject/4848115/
https://movie.douban.com/subject/1291844/
https://movie.douban.com/subject/1978709/
https://movie.douban.com/subject/34874432/
https://movie.douban.com/subject/1292287/
https://movie.douban.com/subject/1296909/
https://movie.douban.com/subject/1291870/
https://movie.douban.com/subject/1302827/
https://movie.douban.com/subject/1292328/
https://movie.douban.com/subject/1428175/
https://movie.douban.com/subject/2213597/
https://movie.douban.com/subject/1305690/
https://movie.douban.com/subject/1419936/
https://movie.douban.com/subject/1308767/
https://movie.douban.com/subject/26430107/
https://movie.douban.com/subject/1304141/
https://movie.douban.com/subject/2051007/
https://movie.douban.com/subject/25724855/
https://movie.douban.com/subject/3552028/
https://movie.douban.com/subject/1300117/
https://movie.douban.com/subject/1293323/
https://movie.douban.com/subject/25807345/
https://movie.douban.com/subject/2222996/
https://movie.douban.com/subject/25864085/
https://movie.douban.com/subject/1307394/
https://movie.douban.com/subject/26374197/
https://movie.douban.com/subject/1293318/
https://movie.douban.com/subject/1304102/
https://movie.douban.com/subject/1293964/
https://movie.douban.com/subject/26615208/
https://movie.douban.com/subject/26787574/
https://movie.douban.com/subject/4202982/
https://movie.douban.com/subject/1457217/
https://movie.douban.com/subject/2133323/