'Running synchronous code within asynchronous code using AsyncHTMLSession object

Ok, after some digging around I was able to almost complete the project As of now rendering the page results in NonType object.

Here's the code:

from bs4 import BeautifulSoup as bs
from requests_html import AsyncHTMLSession
import asyncio


#packs all session tasks in a list
def get_session_tasks(session, urls):

    tasks = []

    for url in urls:
        tasks.append(session.get(url))
    return tasks

#packs all render tasks in a list
def get_rendered_tasks(responses):

    tasks = []
    for response in responses:
        tasks.append(response.html.arender(timeout=0, sleep=2, send_cookies_session=response.cookies))
    return tasks

# retrieve CSRF Token and login
async def login(session):

    login = await session.get(login_url)
    soup = bs(login.content, "lxml")
    cdata_payload["__RequestVerificationToken"] = soup.find_all("input", type="hidden")[1]["value"]
    _ = session.post(login_url, data=cdata_payload, headers=headers)
    return session

# asynchronously get responses
async def get_responses(session):

    tasks = get_session_tasks(session, create_urls(suffix_list))
    responses = await asyncio.gather(*tasks)

    return responses

# asynchronously get rendered responses
async def render_pages(responses):

    tasks = get_rendered_tasks(responses)
    rendered_responses = await asyncio.gather(*tasks)
    return rendered_responses

# starting a session
with AsyncHTMLSession() as session:

    loop = asyncio.get_event_loop()

    login_coroutine = login(session)
    login_session = loop.run_until_complete(login_coroutine)

    response_coroutine = get_responses(login_session)
    responses = loop.run_until_complete(response_coroutine)

    render_coroutine = render_pages(responses)
    rendered = loop.run_until_complete(render_coroutine)
    parse(rendered) # parse is synchronous

terminal print responses: [<Response [200]>, <Response [200]>, <Response [200]>, <Response [200]>] terminal print rendered: [None, None, None, None]

obviously parse returns an error: NoneType object has no attribute....

Would appreciate your help as to what am I doing wrong.



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source