当前位置： 首页 > article >正文

16爬虫：使用requests和scrapy分别从链家获取二手房信息

article 2024/12/26 5:56:43

requests

import requests
import parsel
import time
from random import randint

# 写成方法的形式，方便翻页抓取
def spider(url, headers): # 爬虫方法
    response = requests.get(url=url, headers=headers)
    return response

def parse_response(response):# 解析响应
    response.encoding = "utf-8"
    selector = parsel.Selector(response.text)
    infos = selector.xpath('//div[@class="info clear"]')
    for info in infos:
        title = info.xpath('.//div[@class="title"]/a/text()').get()
        position = info.xpath('.//div[@class="flood"]//a[2]/text()').get()
        houseInfo = info.xpath('.//div[@class="address"]/div/text()').get()
        five = info.xpath('.//div[@class="tag"]/span[@class="five"]/text()').get()
        if five == None:
            five = '未知，需要联系户主或者中介'
        haskey = info.xpath('.//div[@class="tag"]/span[@class="haskey"]/text()').get()
        if haskey == None:
            haskey = '未知，需要联系户主或者中介'
        totalPrice = info.xpath('.//div[@class="priceInfo"]//span/text()').get() + '万'
        print('===================================================================')
        print('房子的形容词：' + title)
        print('房子的坐落位置：' + position)
        print('房子的基本信息：' + houseInfo)
        print('房本是否满五年：' + five)
        print('是否能够随时看房：' + haskey)
        print('房屋的总售价：' + totalPrice)

if __name__ == '__main__':
    headers = {
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
    }
    for i in range(1,30): # 实际有100页
        time.sleep(randint(1,5))
        url = "https://yt.lianjia.com/ershoufang/pg{}".format(i)
        response = spider(url, headers)
        parse_response(response)

scrapy

runner.py

from scrapy.cmdline import execute
# 在中间件中完成增加请求头
if __name__ == '__main__':
    execute("scrapy crawl secondHandHouse".split())

spider

import scrapy
from lianjia.items import LianjiaItem

class SecondhandhouseSpider(scrapy.Spider):
    name = "secondHandHouse"
    allowed_domains = ["lianjia.com"]
    # 假设我们爬取前30也的内容
    start_urls = []
    for i in range(1,5):
        url = "https://yt.lianjia.com/ershoufang/pg{}".format(i)
        start_urls.append(url) # 构建一个完整的url列表，里面存储我们所有需要爬取的url网址


    def parse(self, response,**kwargs):
        print(response.status)
        infos = response.xpath('//div[@class="info clear"]')
        for info in infos:
            title = info.xpath('.//div[@class="title"]/a/text()').extract_first()
            position = info.xpath('.//div[@class="flood"]//a[2]/text()').extract_first()
            houseInfo = info.xpath('.//div[@class="address"]/div/text()').extract_first()
            five = info.xpath('.//div[@class="tag"]/span[@class="five"]/text()').extract_first()
            if five == None:
                five = '未知，需要联系户主或者中介'
            haskey = info.xpath('.//div[@class="tag"]/span[@class="haskey"]/text()').extract_first()
            if haskey == None:
                haskey = '未知，需要联系户主或者中介'
            totalPrice = info.xpath('.//div[@class="priceInfo"]//span/text()').extract_first() + '万'
            print('===================================================================')
            print('房子的形容词：' + title)
            print('房子的坐落位置：' + position)
            print('房子的基本信息：' + houseInfo)
            print('房本是否满五年：' + five)
            print('是否能够随时看房：' + haskey)
            print('房屋的总售价：' + totalPrice)
        #     item = LianjiaItem()
        #     item['title'] = title
        #     item['position'] = position
        #     item['houseInfo'] = houseInfo
        #     item['five'] = five
        #     item['totalPrice'] = totalPrice
        #     yield item

settings.py

# Scrapy settings for lianjia project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = "lianjia"

SPIDER_MODULES = ["lianjia.spiders"]
NEWSPIDER_MODULE = "lianjia.spiders"


# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
LOG_LEVEL = "WARNING"

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
#    "Accept-Language": "en",
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    "lianjia.middlewares.LianjiaSpiderMiddleware": 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
# DOWNLOADER_MIDDLEWARES = {
#    "lianjia.middlewares.LianjiaDownloaderMiddleware": 543,
# }

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    "scrapy.extensions.telnet.TelnetConsole": None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   "lianjia.pipelines.LianjiaPipeline": 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = "httpcache"
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"

# Set settings whose default value is deprecated to a future-proof value
REQUEST_FINGERPRINTER_IMPLEMENTATION = "2.7"
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
FEED_EXPORT_ENCODING = "utf-8"

在scrapy中只修改了上述的内容，其他的组件代码保持不动。

结果


http://www.kler.cn/a/451750.html

相关文章：

  • Redis 安装部署[主从、哨兵、集群]（linux版）
  • 操作系统课程设计
  • vue 基础学习
  • Vue.js前端框架教程11：Vue监听器watch和watchEffect
  • QT-基础-1-Qt 中的字符串处理与常见数据类型
  • 高强度螺栓等级划分
  • 什么是微端游戏？微端应该选择什么配置的服务器？
  • 2024 Gartner 数据库魔力象限概要解读
  • js和html中，将Excel文件渲染在页面上
  • vue3封装而成的APP ，在版本更新后，页面显示空白
  • 2024年种子轮融资趋势：科技引领，消费降温
  • 【理解机器学习中的过拟合与欠拟合】
  • 前端Pako.js 压缩解压库 与 Java 的 zlib 压缩与解压 的互通实现
  • 关于科研中使用linux服务器的集锦
  • 指数函数与累积效应——数学模型
  • MLCC电容加直流偏压时，为什么容值会变低？
  • 虾皮开的很高，还有签字费。
  • VUE3——003、VUE 项目中的文件结构（index.html、main.ts、App.vue）
  • 细说STM32F407单片机轮询方式读写SPI FLASH W25Q16BV
  • ArrayList与LinkedList、Vector的区别
  • 力扣658. 找到 K 个最接近的元素
  • 【RAG实战】语言模型基础
  • WPS工具栏灰色怎么办
  • 工业“元宇宙化”：科技引领生产新潮流
  • 微软的AI转型故事
  • docker环境动态链接库找不到liblpsolve55.so