爬虫日常练习

1.爬取电子出版社的生活类书本数据

通过ajax查找实现

# -- coding: utf-8 --
# 目标：生活板块的书籍书名、价格和作者
import json
import jsonpath
import requests

headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
           'cookie':'acw_tc1a0c639f17292171657431353e00d3333404ced367ab5ea36be81ca90c3609;JSESSIONID=A8143E0428FC4DA3171E8560C47966E8'}

# response = requests.get('https://www.ptpress.com.cn/recommendBook/getRecommendBookListForPortal?bookTagId=d5cbb56d-09ef-41f5-9110-ced741048f5f',headers=headers)
# dict_data = json.loads(response.content)
# print(jsonpath.jsonpath(dict_data,'$..bookName'))

2.爬取详细信息

# 目标：书籍书名、价格和作者
# 1.获取书名id的ajax
# 2.根据id获取单本书籍信息的ajax
# 3.根据data得到想要的内容
import requests  # 导入请求库，用于发送 HTTP 请求
import json  # 导入 JSON 库，用于处理 JSON 数据

# 定义获取推荐书籍列表的 URL
url = 'https://www.ptpress.com.cn/recommendBook/getRecommendBookListForPortal?bookTagId=d5cbb56d-09ef-41f5-9110-ced741048f5f'

# 设置请求头，模拟浏览器请求
headers = {
    'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
    'cookie': 'acw_tc1a0c639f17292171657431353e00d3333404ced367ab5ea36be81ca90c3609; JSESSIONID=A8143E0428FC4DA3171E8560C47966E8'
}

# 发送 GET 请求以获取推荐书籍列表
response = requests.get(url=url, headers=headers)
books = response.json()['data']

def output_to_console(books):
    # 输出标题
    print('书名\t作者\t价格')

    # 遍历返回的数据中的每本书
    for book in books:
        book_id = book['bookId']
        detail = get_book_details(book_id)  # 获取书籍详细信息
        if detail:
            # 提取书名、作者和折后价格
            book_name = detail['bookName']
            author = detail['author']
            discount_price = detail['discountPrice']
            print(f"{book_name}\t{author}\t{discount_price}")

def get_book_details(book_id):
    url = 'https://www.ptpress.com.cn/bookinfo/getBookDetailsById'
    params = {'bookId': book_id}
    response = requests.post(url=url, headers=headers, params=params)
    return response.json()['data'] if response.json().get('success') else None

# 调用函数输出书籍信息
output_to_console(books)

3.爬取当当网书本信息（静态网页练习）

# -- coding: utf-8 --
# 爬取目标：排名、书名、图片地址、作者、推荐指数、五星评分次数、价格
import requests
import time
from lxml import etree

headers = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
           'cookie':'ddscreen=2; dest_area=country_id%3D9000%26province_id%3D111%26city_id%20%3D0%26district_id%3D0%26town_id%3D0; __permanent_id=20241017154613694111150420084440082; __visit_id=20241017154613696429022245816885774; __out_refer=; __trace_id=20241017154613696426813165695783602'}

response = requests.get('http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-1',headers=headers)
num=1
while num<10:
    url = 'http://bang.dangdang.com/books/fivestars/01.00.00.00.00.00-recent30-0-0-1-'+str(num)
    response  = requests.get(url,headers=headers)
    # print(response.status_code)
    # print(response.text)
    time.sleep(3)
    html = etree.HTML(response.text)

    # el_list = html.xpath('/html/body/div[3]/div[3]/div[2]/ul/li') # 已经确定访问成功且能读取数据,静态网页换xpath定位方法
    el_list = html.xpath('//ul[@class="bang_list clearfix bang_list_mode"]/li')
    print(len(el_list))

    for li in el_list:
        paiming = li.xpath('./div[1]/text()')
        shuming = li.xpath('./div[3]/a/@title')
        tupiandizhi = li.xpath('./div[2]/a/img/@src')
        zuozhe = li.xpath('./div[5]/a/text()') if len(li.xpath('./div[5]/a/text()')) > 0 else "未知"
        tuijian = li.xpath('./div[4]/span[2]/text()')
        cishu = li.xpath('./div[7]/span/text()')
        jiage = li.xpath('./div[8]/p[1]/span[1]/text()')

        dict = {
            '排名': paiming,
            '书名': shuming,
            '图片地址': tupiandizhi,
            '作者': zuozhe,
            '推荐指数': tuijian,
            '五星评分次数': cishu,
            '价格': jiage
        }
        print(dict)

    num+=1