'How to testing scrapy?
I have been racking my brains for 3 days on how to test this piece of code, there are very few examples on this topic. For example, how to test. that the parser collects data? and saves them
class ExchangerSpider(scrapy.Spider):
name = "exchanger"
start_urls = ['https://bank.gov.ua/control/uk/curmetal/detail/currency?period=daily']
def parse(self, response):
responses = response.xpath('//*[@id="exchangeRates"]//tbody//tr')
for row in responses:
currency = row.xpath("td[4]//text()").extract()
official_rate = row.xpath("td[5]//text()").extract()
alphabetic_code = row.xpath("td[2]//text()").extract()
yield {
"digital_code": row.xpath("td[1]//text()").extract(),
"alphabetic_code": [line.strip() for line in alphabetic_code],
"number_of_currency_units": row.xpath("td[3]//text()").extract(),
"currency_name": [line.strip() for line in currency],
"official_rate": official_rate}
and
class ExchangerPipeline:
def open_spider(self, spider):
self.connection = ConnectDB.set_connection()
self.exchanger = ExchangeCurrency()
def process_item(self, item, spider):
for key,value in item.items():
val_str = ""
if key == 'digital_code':
self.exchanger.digital_code = int(val_str.join(value))
elif key == 'alphabetic_code':
self.exchanger.alphabetic_code = val_str.join(value)
elif key == 'number_of_currency_units':
self.exchanger.number_of_currency_units = int(val_str.join(value))
elif key == 'currency_name':
self.exchanger.currency_name = val_str.join(value)
elif key == 'official_rate':
for i in value:
self.exchanger.official_rate = float(i.replace(',', '.'))
self.exchanger.save()
return item
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
