Click on close element using selenium

Question

I am new to selenium and web development. I am working on a project to take screenshots from the websites from web.archive.org.

Here is a link to the sample webpage. I am trying to click on the close button (on the top right of this page) before capturing the screenshots. I am not sure what kind of element is the close button and I was unsuccessful in my attempts.

Here is the element I am attempting to click from selenium:

Here is the corresponding HTML

<a id="wm-tb-close" href="#close" style="top:-2px;" title="Close the toolbar"><span class="iconochive-remove-circle" style="color:#888888;font-size:240%;"></span></a>

Here is my code:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By

options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--start-maximized')
options.add_argument('--disable-dev-shm-usage')
options.binary_location = "<path-to-local-dir>/google-chrome/opt/google/chrome/google-chrome"
driver = webdriver.Chrome(options=options)
driver.get('https://web.archive.org/web/20220315011343/https://stackoverflow.com/')

# My attempts at closing the wayback toolbar:
# driver.find_element(By.LINK_TEXT, 'close').click()    # Attempt 1
# driver.find_element(By.ID, 'wm-tb-close').click() # Attempt 2
# driver.find_element_by_xpath("a[@title='Close the toolbar']").click() # Attempt 3

# Capture full webpage screenshot (with scrolling)
original_size = driver.get_window_size()
# required_width = driver.execute_script('return document.body.parentNode.scrollWidth')
required_height = driver.execute_script('return document.body.parentNode.scrollHeight')
driver.set_window_size(1920, max(required_height, 1080))
driver.find_element(By.TAG_NAME, 'body').screenshot('webpage_screenshot.png')  # avoids scrollbar
driver.set_window_size(original_size['width'], original_size['height'])  # reset to defaults

driver.quit()

When I make an attempt to click on the close button using the three techniques (shown in the code), I receive the following error:

selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element

Answer 1

The element you are trying to click is inside the SHADOW-ROOT, to access such elements you need to use some special techniques.
Also you need to use Expected Conditions explicit waits to let the elements loaded before accessing them.
This should work:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
options = Options()
options.add_argument('--headless')
options.add_argument('--no-sandbox')
options.add_argument('--start-maximized')
options.add_argument('--disable-dev-shm-usage')
options.binary_location = "<path-to-local-dir>/google-chrome/opt/google/chrome/google-chrome"
driver = webdriver.Chrome(options=options)
wait = WebDriverWait(driver, 20)

driver.get('https://web.archive.org/web/20220315011343/https://stackoverflow.com/')
#locate the shadow root element
root_element = wait.until(EC.presence_of_element_located((By.ID, "wm-ipp-base")))
shadow_root = driver.execute_script('return arguments[0].shadowRoot', root_element)
#access the close button inside the shadow root
shadow_root.find_element_by_css_selector("span.iconochive-remove-circle").click()