'dataframe to save csv: not accumulating the records only saving the last dataframe group records
dataframe question in web scraping data group
example:the first loop-eg:5 records, second loop-eg:3 records
when I did my below code, the csv file was saved the second loop group without stacking from the first loop group records. please advise me.
df=[]
for i in range(1,3,1):
state=driver.find_element(By.XPATH, "//select[@name='state']")
stateDD=Select(state)
stateDD.select_by_index(i)
time.sleep(3)
print(f'Getting page, {i}')
soup = BeautifulSoup(driver.page_source,'lxml')
data=[]
for card in soup.select('#resultscontainer > ul > li > div.repcontent'):
state = i
company = card.select_one('h3.company_title').text
#print(company)
address = card.select_one('div.address').text
address2 = card.select_one('div.address2').text
try:
phone = card.select('span')[0].text
except IndexError:
phone = ""
try:
fax = card.select('span')[1].text
except IndexError:
fax = ""
data.append({
'state': state,
'company':company,
'address':address,
'address2':address2,
'phone':phone,
'fax':fax
})
#print(data)
df = pd.DataFrame(data)
#print(df.head())
df.to_csv('D:/92Python/Output/gastite.csv', index=False)
driver.close
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
