import asyncio
from requests_html import HTMLSession
url = 'http://www.xiaohuar.com/hua/'
session = HTMLSession( browser_args=[
'--no-sand',
'--disable-infobars'
'--user-agent=Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36'
],headless=False)
res = session.request(url=url,method='GET')
script = """
() => {
return {
width: document.documentElement.clientWidth,
height: document.documentElement.clientHeight,
deviceScaleFactor: window.devicePixelRatio,
}
}
"""
try:
res.html.render(keep_page = True)
async def main():
await res.html.page.waitFor(1000)
await res.html.page.setViewport({'width': 1366, 'height': 768})
url_list = await res.html.page.xpath('//div[@class="img"]/a')
for url in url_list:
url_link = await (await url.getProperty('href')).jsonValue()
print(url_link)
asyncio.get_event_loop().run_until_complete(main())
except Exception as e:
print(e)
finally:
session.close()
本站文章如无特殊说明,均为本站原创,如若转载,请注明出处:request-html 简单爬虫 - Python技术站