当前位置：首页 > article >正文

如何运用python爬虫爬取百度贴吧动态加载的图片？

article 2025/1/31 12:28:08

动态加载的图片通常是在页面加载后通过JavaScript异步请求获取的，而requests和BeautifulSoup只能获取页面的初始HTML内容，无法执行JavaScript代码。

要处理动态加载的图片，可以使用以下方法：

1. 使用Selenium

Selenium是一个用于自动化Web浏览器的工具，可以模拟用户浏览网页的行为，包括点击、滚动等操作，从而获取动态加载的内容。

安装依赖

bash复制

pip install selenium

示例代码

Python复制

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import os
import requests

# 初始化WebDriver
driver = webdriver.Chrome()

# 打开目标网页
url = 'https://tieba.baidu.com/p/1234567890'  # 替换为目标帖子的URL
driver.get(url)

# 等待页面加载
time.sleep(3)

# 滚动页面以加载更多图片
for _ in range(5):
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    time.sleep(2)

# 获取页面源码
html_content = driver.page_source

# 关闭浏览器
driver.quit()

# 解析HTML内容，提取图片URL
def extract_image_urls(html_content):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(html_content, 'html.parser')
    images = soup.find_all('img')  # 查找所有图片标签
    image_urls = []
    for img in images:
        img_url = img.get('src')  # 获取图片的src属性
        if img_url and img_url.startswith('http'):  # 确保是完整的URL
            image_urls.append(img_url)
    return image_urls

# 下载并保存图片
def download_images(image_urls, save_dir='images'):
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)  # 创建保存图片的文件夹
    for i, img_url in enumerate(image_urls):
        img_name = os.path.basename(img_url)  # 从URL中提取文件名
        save_path = os.path.join(save_dir, img_name)
        if os.path.exists(save_path):
            print(f"{img_name} already exists. Skipping...")
            continue
        try:
            response = requests.get(img_url, timeout=10)
            response.raise_for_status()  # 确保请求成功
            with open(save_path, 'wb') as img_file:
                img_file.write(response.content)  # 保存图片
            print(f"Downloaded {img_name}")
        except requests.RequestException as e:
            print(f"Failed to download {img_url}. Error: {e}")
        time.sleep(1)  # 随机延时，避免被封禁

# 主函数
def main():
    image_urls = extract_image_urls(html_content)
    download_images(image_urls)

if __name__ == "__main__":
    main()

2. 使用Playwright

Playwright是一个用于自动化Web浏览器的工具，支持多种浏览器（如Chrome、Firefox、Safari等），可以模拟用户浏览网页的行为，从而获取动态加载的内容。

安装依赖

bash复制

pip install playwright
playwright install

示例代码

Python复制

from playwright.sync_api import sync_playwright
import os
import requests

# 使用Playwright获取动态加载的图片
def get_dynamic_images(url, save_dir='images'):
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)  # 创建保存图片的文件夹

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)  # 启动浏览器
        page = browser.new_page()  # 打开新页面
        page.goto(url)  # 打开目标网页

        # 等待页面加载
        page.wait_for_load_state('networkidle')

        # 滚动页面以加载更多图片
        for _ in range(5):
            page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
            page.wait_for_load_state('networkidle')

        # 获取所有图片
        images = page.query_selector_all('img')
        image_urls = [img.get_attribute('src') for img in images if img.get_attribute('src')]

        # 下载并保存图片
        for i, img_url in enumerate(image_urls):
            img_name = os.path.basename(img_url)  # 从URL中提取文件名
            save_path = os.path.join(save_dir, img_name)
            if os.path.exists(save_path):
                print(f"{img_name} already exists. Skipping...")
                continue
            try:
                response = requests.get(img_url, timeout=10)
                response.raise_for_status()  # 确保请求成功
                with open(save_path, 'wb') as img_file:
                    img_file.write(response.content)  # 保存图片
                print(f"Downloaded {img_name}")
            except requests.RequestException as e:
                print(f"Failed to download {img_url}. Error: {e}")
            time.sleep(1)  # 随机延时，避免被封禁

        browser.close()  # 关闭浏览器

# 主函数
def main():
    url = 'https://tieba.baidu.com/p/1234567890'  # 替换为目标帖子的URL
    get_dynamic_images(url)

if __name__ == "__main__":
    main()