'Running synchronous code within asynchronous code using AsyncHTMLSession object
Ok, after some digging around I was able to almost complete the project As of now rendering the page results in NonType object.
Here's the code:
from bs4 import BeautifulSoup as bs
from requests_html import AsyncHTMLSession
import asyncio
#packs all session tasks in a list
def get_session_tasks(session, urls):
tasks = []
for url in urls:
tasks.append(session.get(url))
return tasks
#packs all render tasks in a list
def get_rendered_tasks(responses):
tasks = []
for response in responses:
tasks.append(response.html.arender(timeout=0, sleep=2, send_cookies_session=response.cookies))
return tasks
# retrieve CSRF Token and login
async def login(session):
login = await session.get(login_url)
soup = bs(login.content, "lxml")
cdata_payload["__RequestVerificationToken"] = soup.find_all("input", type="hidden")[1]["value"]
_ = session.post(login_url, data=cdata_payload, headers=headers)
return session
# asynchronously get responses
async def get_responses(session):
tasks = get_session_tasks(session, create_urls(suffix_list))
responses = await asyncio.gather(*tasks)
return responses
# asynchronously get rendered responses
async def render_pages(responses):
tasks = get_rendered_tasks(responses)
rendered_responses = await asyncio.gather(*tasks)
return rendered_responses
# starting a session
with AsyncHTMLSession() as session:
loop = asyncio.get_event_loop()
login_coroutine = login(session)
login_session = loop.run_until_complete(login_coroutine)
response_coroutine = get_responses(login_session)
responses = loop.run_until_complete(response_coroutine)
render_coroutine = render_pages(responses)
rendered = loop.run_until_complete(render_coroutine)
parse(rendered) # parse is synchronous
terminal print responses: [<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>] terminal print rendered: [None, None, None, None]
obviously parse returns an error: NoneType object has no attribute....
Would appreciate your help as to what am I doing wrong.
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
