Selenium can't locate an element that is on web page

Question

Parts of code that work: I enter the USGS site and go to the latest earthquakes page. Now what I want to do is download the csv file of latest earthquakes. The download button is at the end of the page and when I run the script it returns selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element. Any ideas how to fix this are welcome. Thanks in advance

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service

ser = Service('C:\\chromedriver.exe')
opt = webdriver.ChromeOptions()

web = webdriver.Chrome(service=ser, options=opt)
web.get('https://www.usgs.gov/')

web.find_element(by=By.CSS_SELECTOR, value= '#block-tools > div > p > a:nth-child(1)').click()
web.maximize_window()

web.find_element(by=By.CSS_SELECTOR, value= 'body > usgs-root > div > usgs-list > cdk-virtual-scroll-viewport > div.cdk-virtual-scroll-content-wrapper > usgs-download-button > div > button > span.mat-button-wrapper').click()

downloadCsv = web.find_element(by=By.CSS_SELECTOR, value='#mat-dialog-1 > usgs-download-options > div.mat-dialog-content > ul > li:nth-child(2) > a')
downloadCsv.click()

Answer 1

you need to wait for the element to load, with a explicit wait or you can add an implicit wait web.implicitly_wait(10) you can use sleep(10) to test if it is the problem

see https://selenium-python.readthedocs.io/waits.html'

also, you need to switch to the new tab

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

# used this lib to update my chrome driver automatically
from webdriver_manager.chrome import ChromeDriverManager

opt = webdriver.ChromeOptions()

web = webdriver.Chrome(ChromeDriverManager().install(), options=opt)
web.get('https://www.usgs.gov/')


try:
    web.maximize_window()
    element = WebDriverWait(web, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, '#block-tools > div > p > a:nth-child(1)'))
    )
    element.click()

    # after click you need to switch to the new tab
    web.switch_to.window(web.window_handles[1])

    element = WebDriverWait(web, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, 'body > usgs-root > div > usgs-list > cdk-virtual-scroll-viewport > div.cdk-virtual-scroll-content-wrapper > usgs-download-button > div > button > span.mat-button-wrapper'))
    )
    element.click()

    # your css selector did not work so I changed it

    element = WebDriverWait(web, 10).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, 'usgs-download-options > div.mat-dialog-content > ul > li:nth-child(2) > a'))
    )
    element.click()
finally:
    web.quit()

P.S. If your whole selenium is just for this I think you can use a much more simple and faster script to download the CSV direct, as the CSV as a direct download link,, except i missing something

import csv
import requests

CSV_URL = 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/2.5_day.csv'

with requests.Session() as s:
    download = s.get(CSV_URL)

    decoded_content = download.content.decode('utf-8')

    cr = csv.reader(decoded_content.splitlines(), delimiter=',')
    my_list = list(cr)
    for row in my_list:
        print(row)

#             or write to file
    local_filename = CSV_URL.split('/')[-1]

    with open(local_filename, 'wb') as f:
        for chunk in download.iter_content(chunk_size=8192):
            # If you have chunk encoded response uncomment if
            # and set chunk_size parameter to None.
            # if chunk:
            f.write(chunk)