python爬取网站信息

jmao1234

网站：https://cacizt-pc.eastfair.com/actorList
大佬指导下如何爬取网页上公司的名称、展位号以及查看详情对应的网址。

果心豆腐酱

虽然简单，但是只有10个精币。。。

  

import requests
import json
# 创建一个 Session 对象
session = requests.Session ()
# 设置请求头
headers = {
"authority": "cacizt-pc.eastfair.com",
"accept": "application/json, text/plain, */*",
"accept-language": "zh-CN,zh;q=0.9",
"authorization": "PBB+hfqyCn5u/Asp+KqEaro7AhH4z/+bW0+9qxZHTV3GqTTtK0xlBOZ/bJUKrMMzpQGrxvbaa1O+XC9IM1Esf+sq+GwKEVbFVC5oHf4ckhDvF0COJwODWI/otfPxmvDr8/lzo80KxYaPNqn7kEkK/5d3CRmNDUlQBGo9iBXFyJm9IcXhTs3UJ3PSx/3pyao67h18E1SncYIPukUmWlVAsHzhxLGBP9tOeJdS6oZ4RYKXIeoSHtEZefeF/0Ne5PpFdZjPGeTCeSLhi0QVNZ+70jSDR0ZiOfQJ5NTNtnT9EOJMGGPZkoevYDS/Wd1LcvI5xPT8hle8hE3TgEdYGc2Z+P4TQ3pEta1RRke+boQK/qNYu1LJkzJtNYIDRsf1bshJFAf5AxjrLvNiDx1pKcwNfgT1w5M/Dvv5yMKwTbI5yGQ=",
"content-type": "application/json;charset=UTF-8",
"cookie": "CAC-huikan-Token=PBB+hfqyCn5u/Asp+KqEaro7AhH4z/+bW0+9qxZHTV3GqTTtK0xlBOZ/bJUKrMMzpQGrxvbaa1O+XC9IM1Esf+sq+GwKEVbFVC5oHf4ckhDvF0COJwODWI/otfPxmvDr8/lzo80KxYaPNqn7kEkK/5d3CRmNDUlQBGo9iBXFyJm9IcXhTs3UJ3PSx/3pyao67h18E1SncYIPukUmWlVAsHzhxLGBP9tOeJdS6oZ4RYKXIeoSHtEZefeF/0Ne5PpFdZjPGeTCeSLhi0QVNZ+70jSDR0ZiOfQJ5NTNtnT9EOJMGGPZkoevYDS/Wd1LcvI5xPT8hle8hE3TgEdYGc2Z+P4TQ3pEta1RRke+boQK/qNYu1LJkzJtNYIDRsf1bshJFAf5AxjrLvNiDx1pKcwNfgT1w5M/Dvv5yMKwTbI5yGQ=",
"origin": "https://cacizt-pc.eastfair.com",
"referer": "https://cacizt-pc.eastfair.com/actorList",
"sec-fetch-dest": "empty",
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-origin",
"user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"
}
# 封装请求的函数
def fetch_exhibitors_data (page_num):
# 请求数据
data = {
"pageNum": page_num,
"pageRows": 10,
"total": 2194,
"searchKeywords": "",
"pages": 220,
"tagIds": "",
"questionList": [],
"initial": "",
"showType": "ACTOR_LIST",
"exhibitorSort": "recommend_custom_letter_asc",
"languageType": "cn",
"exhibitionId": "b33d8960b7a245cb97146670283ac360",
"appId": "7e46f33c139246069a3c17c9f4847a25",
"terminal": "web",
"subjectType": "VISITOR"
}

# 发送POST请求
response = session.post (
"https://cacizt-pc.eastfair.com/visitor/v2/exhibitor/browse/all/list",
headers=headers,
json=data
)

return response
# 遍历请求，获取数据并存储
def get_all_exhibitors ():
exhibitors = []  # 用于存储所有展商的数据
total_pages = 220  # 总页数，可以根据实际的响应调整

for page_num in range (1, total_pages + 1):  # 遍历每一页
print (f"正在请求第 {page_num} 页的数据...")

response = fetch_exhibitors_data (page_num)

if response.status_code == 200:
try:
# 解析JSON响应
response_json = response.json ()
exhibitor_list = response_json[ ' data']['list']

# 遍历列表获取每个展商的信息
for item in exhibitor_list:
exhibition_booth_number = item[ ' exhibitionBoothNumber']
company_name = item[ ' name']
exhibitor_id = item[ ' id']

exhibitors.append ({
"exhibitionBoothNumber": exhibition_booth_number,
"companyName": company_name,
"id": exhibitor_id,
"url":f"https://cacizt-pc.eastfair.com/actorDetail/{exhibitor_id}"
})
except Exception as e:
print ("解析响应时出错:", e)
else:
print (f"请求失败，第 {page_num} 页，状态码: {response.status_code}")

return exhibitors
# 调用函数获取所有展商数据
exhibitors_data = get_all_exhibitors ()
# 输出所有展商的展位号和公司名
for exhibitor in exhibitors_data:
print (f"展位号: {exhibitor[ ' exhibitionBoothNumber']}, 公司名: {exhibitor['companyName']}, URL: {exhibitor['url']}")

import requests<br /> import json<br /> <br /> # 创建一个 Session 对象<br /> session = requests.Session()<br /> <br /> # 设置请求头<br /> headers = {<br />     "authority": "cacizt-pc.eastfair.com",<br />     "accept": "application/json, text/plain, */*",<br />     "accept-language": "zh-CN,zh;q=0.9",<br />     "authorization": "PBB+hfqyCn5u/Asp+KqEaro7AhH4z/+bW0+9qxZHTV3GqTTtK0xlBOZ/bJUKrMMzpQGrxvbaa1O+XC9IM1Esf+sq+GwKEVbFVC5oHf4ckhDvF0COJwODWI/otfPxmvDr8/lzo80KxYaPNqn7kEkK/5d3CRmNDUlQBGo9iBXFyJm9IcXhTs3UJ3PSx/3pyao67h18E1SncYIPukUmWlVAsHzhxLGBP9tOeJdS6oZ4RYKXIeoSHtEZefeF/0Ne5PpFdZjPGeTCeSLhi0QVNZ+70jSDR0ZiOfQJ5NTNtnT9EOJMGGPZkoevYDS/Wd1LcvI5xPT8hle8hE3TgEdYGc2Z+P4TQ3pEta1RRke+boQK/qNYu1LJkzJtNYIDRsf1bshJFAf5AxjrLvNiDx1pKcwNfgT1w5M/Dvv5yMKwTbI5yGQ=",<br />     "content-type": "application/json;charset=UTF-8",<br />     "cookie": "CAC-huikan-Token=PBB+hfqyCn5u/Asp+KqEaro7AhH4z/+bW0+9qxZHTV3GqTTtK0xlBOZ/bJUKrMMzpQGrxvbaa1O+XC9IM1Esf+sq+GwKEVbFVC5oHf4ckhDvF0COJwODWI/otfPxmvDr8/lzo80KxYaPNqn7kEkK/5d3CRmNDUlQBGo9iBXFyJm9IcXhTs3UJ3PSx/3pyao67h18E1SncYIPukUmWlVAsHzhxLGBP9tOeJdS6oZ4RYKXIeoSHtEZefeF/0Ne5PpFdZjPGeTCeSLhi0QVNZ+70jSDR0ZiOfQJ5NTNtnT9EOJMGGPZkoevYDS/Wd1LcvI5xPT8hle8hE3TgEdYGc2Z+P4TQ3pEta1RRke+boQK/qNYu1LJkzJtNYIDRsf1bshJFAf5AxjrLvNiDx1pKcwNfgT1w5M/Dvv5yMKwTbI5yGQ=",<br />     "origin": "https://cacizt-pc.eastfair.com",<br />     "referer": "https://cacizt-pc.eastfair.com/actorList",<br />     "sec-fetch-dest": "empty",<br />     "sec-fetch-mode": "cors",<br />     "sec-fetch-site": "same-origin",<br />     "user-agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1"<br /> }<br /> <br /> # 封装请求的函数<br /> def fetch_exhibitors_data(page_num):<br />     # 请求数据<br />     data = {<br />         "pageNum": page_num,<br />         "pageRows": 10,<br />         "total": 2194,<br />         "searchKeywords": "",<br />         "pages": 220,<br />         "tagIds": "",<br />         "questionList": [],<br />         "initial": "",<br />         "showType": "ACTOR_LIST",<br />         "exhibitorSort": "recommend_custom_letter_asc",<br />         "languageType": "cn",<br />         "exhibitionId": "b33d8960b7a245cb97146670283ac360",<br />         "appId": "7e46f33c139246069a3c17c9f4847a25",<br />         "terminal": "web",<br />         "subjectType": "VISITOR"<br />     }<br />     <br />     # 发送POST请求<br />     response = session.post(<br />         "https://cacizt-pc.eastfair.com/visitor/v2/exhibitor/browse/all/list",<br />         headers=headers,<br />         json=data<br />     )<br />     <br />     return response<br /> <br /> # 遍历请求，获取数据并存储<br /> def get_all_exhibitors():<br />     exhibitors = []  # 用于存储所有展商的数据<br />     total_pages = 220  # 总页数，可以根据实际的响应调整<br />     <br />     for page_num in range(1, total_pages + 1):  # 遍历每一页<br />         print(f"正在请求第 {page_num} 页的数据...")<br />         <br />         response = fetch_exhibitors_data(page_num)<br />         <br />         if response.status_code == 200:<br />             try:<br />                 # 解析JSON响应<br />                 response_json = response.json()<br />                 exhibitor_list = response_json['data']['list']<br />                 <br />                 # 遍历列表获取每个展商的信息<br />                 for item in exhibitor_list:<br />                     exhibition_booth_number = item['exhibitionBoothNumber']<br />                     company_name = item['name']<br />                     exhibitor_id = item['id']<br />                     <br />                     exhibitors.append({<br />                         "exhibitionBoothNumber": exhibition_booth_number,<br />                         "companyName": company_name,<br />                         "id": exhibitor_id,<br />                         "url":f"https://cacizt-pc.eastfair.com/actorDetail/{exhibitor_id}"<br />                     })<br />             except Exception as e:<br />                 print("解析响应时出错:", e)<br />         else:<br />             print(f"请求失败，第 {page_num} 页，状态码: {response.status_code}")<br />     <br />     return exhibitors<br /> <br /> # 调用函数获取所有展商数据<br /> exhibitors_data = get_all_exhibitors()<br /> <br /> # 输出所有展商的展位号和公司名<br /> for exhibitor in exhibitors_data:<br />     print(f"展位号: {exhibitor['exhibitionBoothNumber']}, 公司名: {exhibitor['companyName']}, URL: {exhibitor['url']}")

hujie789

要爬哪些内容 ？

jmao1234

hujie789 发表于 2025-3-20 22:18
要爬哪些内容 ？

爬取网页上公司的名称、展位号以及查看详情对应的网址

冰墩墩

易语言不香?

hujie789

jmao1234 发表于 2025-3-20 22:19
爬取网页上公司的名称、展位号以及查看详情对应的网址

默认的展位信息 还是 自定义搜索的呢

jmao1234

hujie789 发表于 2025-3-20 22:24
默认的展位信息 还是 自定义搜索的呢

默认的展位信息，不需要搜索

jmao1234

冰墩墩 发表于 2025-3-20 22:23
易语言不香?

易语言怎么实现了？ 问题也不会

易有易用

要是网页上的先看看网页源码有没有数据，有就正则取出来好了，没有就看是不是有接口获取这些数据，抓接口就行了

jmao1234

易有易用 发表于 2025-3-20 22:34
要是网页上的先看看网页源码有没有数据，有就正则取出来好了，没有就看是不是有接口获取这些数据，抓接口就 ...

关键就是获取不到这些数据