'Selenium don't get all elements located by xpath or class name
I have to scrape a large list of matches on this page 1 clicking the show more button, it always worked but today I wanted to scrape all available results and only get the first 1005 results despite there are 1036 elements matched with the same class when I run document.getElementsByClassName('event__match--twoLine') in the browser, I can't find what is the problem. Does selenium have a limit of how many elements can it store?
This is what my code looks like: The function recives two parameters which are the start and end date, after that it searches the elements that match. I want all the matches in this case from the 25 of September to today, but only get the matches from September 30
def get_range(start_d, end_d):
print("\nSearching for coincidences...")
is_the_lsat_in_range = True
while is_the_lsat_in_range:
current_year = year
all_matches = []
# Get all the matches available on the page
matches = WebDriverWait(driver, 5).until(ec.presence_of_all_elements_located((
By.CLASS_NAME, 'event__match--twoLine'
)))
for element in matches:
code = element.get_attribute("id")
date = element.find_element_by_xpath(".//div[@class='event__time']").text
date_list_format = date.split('.')
element_moth = int(date_list_format[1])
# is it the first one?
if matches.index(element) == 0:
last_verified_match_month = element_moth
# Verify which is the match year
if last_verified_match_month >= element_moth:
# Current year
pass
else:
current_year = current_year - 1
# Verify that it is in the range of dates
match_date = datetime.datetime(day=int(date_list_format[0]), month=int(date_list_format[1]),
year=current_year)
# Save the match month
if matches.index(element) != 0:
last_verified_match_month = element_moth
if match_date > end_d:
pass
elif start_d <= match_date <= end_d:
all_matches.append(Match(date, code[4:]))
else:
is_the_lsat_in_range = False
break
if is_the_lsat_in_range:
# search and click show more results button
try:
show_more = WebDriverWait(driver, 5).until(ec.element_to_be_clickable((
By.XPATH, "//a[@class='event__more event__more--static']"
)))
driver.execute_script("arguments[0].scrollIntoView();", show_more)
driver.execute_script("arguments[0].click();", show_more)
except TimeoutException:
print("End of the page")
return all_matches
else:
return all_matches
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
