从网页获取信息，并使用pandas和bs4写入.xls文件

Question

我是Python编程的初学者。我在python中使用bs4模块练习网页抓取。

我从网页中提取了一些字段，但是当我尝试将它们写入.xls文件时，除了标题之外，.xls文件仍为空。请告诉我在哪里做错了，如果可能的话，建议做什么。

from bs4 import BeautifulSoup as bs
import pandas as pd

res = requests.get('https://rwbj.com.au/find-an-agent.html')
soup = bs(res.content, 'lxml')

data = soup.find_all("div",{"class":"fluidgrid-cell fluidgrid-cell-2"})

records = []
name =[]
phone =[]
email=[]
title=[]
location=[]
for item in data:
    name = item.find('h3',class_='heading').text.strip()
    phone = item.find('a',class_='text text-link text-small').text.strip()
    email = item.find('a',class_='text text-link text-small')['href']
    title = item.find('div',class_='text text-small').text.strip()
    location = item.find('div',class_='text text-small').text.strip()

    records.append({'Names': name, 'Title': title, 'Email': email, 'Phone': phone, 'Location': location})

df = pd.DataFrame(records,columns=['Names','Title','Phone','Email','Location'])
df=df.drop_duplicates()
df.to_excel(r'C:\Users\laptop\Desktop\R&W.xls', sheet_name='MyData2', index = False, header=True)

Answer 1

如果您不想使用selenium，那么您可以使相同的帖子请求网页制作。这将为您提供xml响应，您可以使用Beautifulsoup解析它以获得所需的输出。

我们可以使用检查工具中的网络选项卡来获取正在进行的请求以及此请求的表单数据。

接下来，我们必须使用python-requests发出相同的请求并解析输出。

import requests
from bs4 import BeautifulSoup
import pandas as pd
number_of_agents_required=20 # they only have 20 on the site
payload={
'act':'act_fgxml',
'15[offset]':0,
'15[perpage]':number_of_agents_required,
'require':0,
'fgpid':15,
'ajax':1
}
records=[]
r=requests.post('https://www.rwbj.com.au/find-an-agent.html',data=payload)
soup=BeautifulSoup(r.text,'lxml')
for row in soup.find_all('row'):
    name=row.find('name').text
    title=row.position.text.replace('&','&')
    email=row.email.text
    phone=row.phone.text
    location=row.office.text
    records.append([name,title,email,phone,location])
df=pd.DataFrame(records,columns=['Names','Title','Phone','Email','Location'])
df.to_excel('R&W.xls', sheet_name='MyData2', index = False, header=True)

输出：

Answer 2

您可以使用像selenium这样的方法来允许javascript呈现内容。然后，您可以获取page_source以继续使用脚本。我故意保留你的脚本，只添加了等待内容的新行。

您可以运行selenium headless或切换为使用HTMLSession。

from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import pandas as pd

d = webdriver.Chrome()
d.get('https://rwbj.com.au/find-an-agent.html')

WebDriverWait(d,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "h3")))

soup = bs(d.page_source, 'lxml')
d.quit()
data = soup.find_all("div",{"class":"fluidgrid-cell fluidgrid-cell-2"})

records = []
name =[]
phone =[]
email=[]
title=[]
location=[]
for item in data:
    name = item.find('h3',class_='heading').text.strip()
    phone = item.find('a',class_='text text-link text-small').text.strip()
    email = item.find('a',class_='text text-link text-small')['href']
    title = item.find('div',class_='text text-small').text.strip()
    location = item.find('div',class_='text text-small').text.strip()
    records.append({'Names': name, 'Title': title, 'Email': email, 'Phone': phone, 'Location': location})

df = pd.DataFrame(records,columns=['Names','Title','Phone','Email','Location'])
print(df)

---

我可能会考虑，取决于是否所有项目都存在于每个人，例如：

from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import pandas as pd

options = Options()
options.headless = True

d = webdriver.Chrome(options = options) 
d.get('https://rwbj.com.au/find-an-agent.html')

WebDriverWait(d,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "h3")))

soup = bs(d.page_source, 'lxml')
d.quit()
names = [item.text for item in soup.select('h3')]
titles = [item.text for item in soup.select('h3 ~ div:nth-of-type(1)')]
tels = [item.text for item in soup.select('h3 + a')]
emails = [item['href'] for item in soup.select('h3 ~ a:nth-of-type(2)')]
locations = [item.text for item in soup.select('h3 ~ div:nth-of-type(2)')]      
records = list(zip(names, titles, tels, emails, positions))
df = pd.DataFrame(records,columns=['Names','Title','Phone','Email','Location'])
print(df)