pyhton2.7+selenium3.1+chromedriver 爬虫问题

295552891

  

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0
from selenium.webdriver.support import expected_conditions as EC # available since 2.26.0
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.remote.webelement import WebElement
from time import sleep
import traceback

chrome_options = Options ()
chrome_options.add_argument ( ' --headless')
# chrome_options.add_argument ( ' --disable-gpu')

driver = webdriver.Chrome ("C:\Program Files\Python2\chromedriver.exe",chrome_options=chrome_options)
driver.maximize_window ()        
# driver.get ("https://www.baidu.com")
driver.get ("https://www.nextbet.com/en/sports")
print (driver.title)
# 繁体字
html = driver.execute_script ("return document.documentElement.outerHTML")

try:
element = WebDriverWait (driver, 360).until (EC.presence_of_element_located ( (By.CLASS_NAME, "event-header-title")))
#driver .find_element_by_id ("").
eles = driver.find_elements_by_class_name ("event-header-title");
for ele in eles:
temp = ele.get_attribute ("outerHTML")
except Exception,e:  
traceback.print_exc ()

html = driver.execute_script ("return document.documentElement.outerHTML")

sleep (3)

from selenium import webdriver<br /> from selenium.webdriver.common.by import By<br /> from selenium.webdriver.support.ui import WebDriverWait # available since 2.4.0<br /> from selenium.webdriver.support import expected_conditions as EC # available since 2.26.0<br /> from selenium.webdriver.chrome.options import Options<br /> from selenium.webdriver.remote.webelement import WebElement<br /> from time import sleep<br /> import traceback <br /> <br /> chrome_options = Options()<br /> chrome_options.add_argument('--headless')<br /> # chrome_options.add_argument('--disable-gpu')<br />     <br /> driver = webdriver.Chrome("C:\Program Files\Python2\chromedriver.exe",chrome_options=chrome_options)<br /> <br /> driver.maximize_window()        <br /> <br /> # driver.get("https://www.baidu.com") <br /> driver.get("https://www.nextbet.com/en/sports")<br /> print(driver.title)<br /> # 繁体字<br /> html = driver.execute_script("return document.documentElement.outerHTML")<br /> <br /> <br /> try:<br />     element = WebDriverWait(driver, 360).until(EC.presence_of_element_located((By.CLASS_NAME, "event-header-title")))<br />    #driver.find_element_by_id("").<br />     eles = driver.find_elements_by_class_name("event-header-title");<br />     for ele in eles:<br />         temp = ele.get_attribute("outerHTML")<br /> except Exception,e:  <br />     traceback.print_exc()<br /> <br /> html = driver.execute_script("return document.documentElement.outerHTML")<br /> <br /> sleep(3)

如代码展示，如果去掉 "--headless" 可以正常抓取到dom并且进行操作，反之虽然有等待".event-header-title"加载完成的语句，并且执行到下一步抓取到n个eles集合，但是遍历获取html属性的时候报异常，是chromedriver的和windows下chrome浏览器版本的问题吗？

swpy

可能是chromedriver版本问题

295552891

swpy 发表于 2019-2-19 17:25
可能是chromedriver版本问题

嗯，我再试试

zjl1515987

upupupupup

zpython

异常捕获那个e就是原因，猜测应该是某条元素没有那个attribute属性吧

zpython

可能是，但是更多可能是页面加载未完成，报的什么异常

Soul2018

或许也有可能是元素未加载完全

越越

此类问题加qq 8128697 有偿解决

negen

看看具体的异常信息，可能是获取到的元素为空强行取值造成的