python 下载网页

 

import requests

from bs4 import BeautifulSoup

import os

from urllib.parse import urljoin, urlparse

import time

def download_webpage_with_images(url, folder_name):

    """

    下载网页及其包含的图片到本地

    """

    # 创建保存文件夹

    if not os.path.exists(folder_name):

        os.makedirs(folder_name)

   

    # 获取网页内容

    headers = {

        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'

    }

   

    try:

        response = requests.get(url, headers=headers)

        response.raise_for_status()

        response.encoding = response.apparent_encoding

       

        # 解析HTML

        soup = BeautifulSoup(response.text, 'html.parser')

       

        # 保存网页HTML文件

        html_file_path = os.path.join(folder_name, "index.html")

        with open(html_file_path, 'w', encoding='utf-8') as f:

            f.write(response.text)

        print(f"网页HTML已保存到: {html_file_path}")

       

        # 查找所有图片标签

        img_tags = soup.find_all('img')

        print(f"找到 {len(img_tags)} 张图片")

       

        # 下载图片

        for i, img_tag in enumerate(img_tags):

            img_url = img_tag.get('src')

            if not img_url:

                continue

               

            # 处理相对URL

            img_url = urljoin(url, img_url)

           

            try:

                # 下载图片

                img_response = requests.get(img_url, headers=headers)

                img_response.raise_for_status()

               

                # 获取图片文件名

                parsed_url = urlparse(img_url)

                img_filename = os.path.basename(parsed_url.path)

                if not img_filename or '.' not in img_filename:

                    img_filename = f"image_{i}.jpg"

               

                # 保存图片

                img_path = os.path.join(folder_name, img_filename)

                with open(img_path, 'wb') as f:

                    f.write(img_response.content)

                print(f"已下载图片: {img_filename}")

               

                # 更新HTML中的图片路径为相对路径

                img_tag['src'] = img_filename

               

                # 添加延时避免请求过快

                time.sleep(0.1)

               

            except Exception as e:

                print(f"下载图片失败 {img_url}: {str(e)}")

       

        # 保存更新后的HTML（图片路径已修改为本地路径）

        with open(html_file_path, 'w', encoding='utf-8') as f:

            f.write(str(soup))

        print(f"更新后的HTML已保存到: {html_file_path}")

       

        print(f"完成！网页和图片已保存到文件夹: {folder_name}")

       

    except Exception as e:

        print(f"处理网页时出错: {str(e)}")

def main():

    # 示例URL，请替换为实际要下载的网页URL

    url = input("请输入要下载的网页URL: ").strip()

    if not url:

        url = "https://httpbin.org/html"  # 默认示例URL

   

    folder_name = input("请输入保存文件夹名称(默认为webpage_output): ").strip()

    if not folder_name:

        folder_name = "webpage_output"

   

    print(f"开始下载网页: {url}")

    download_webpage_with_images(url, folder_name)

if __name__ == "__main__":

    main()