How to fix 302 redirect Scrapy?

Question

I am trying to scrape https://howlongtobeat.com, but I keep getting 302 redirects. I found that the website is using ajax from the network monitor.

My code:

class HltbSpider(scrapy.Spider):
    name = 'hltb'
    def start_requests(self):
        for i in list(range(1,2)):
            
            url = f'https://howlongtobeat.com/search_results?page={i}'
            payload = "queryString=&t=games&sorthead=popular&sortd=0&plat=&length_type=main&length_min=&length_max=&v=&f=&g=&detail=&randomize=0"
            headers = {
                "content-type":"application/x-www-form-urlencoded",
                "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.67 Mobile Safari/537.36"
            }

            yield scrapy.Request(url, meta = {'dont_redirect': True,'handle_httpstatus_list': [302]}, method='POST', body=payload, headers=headers, callback=self.parse)

           

    def parse(self, response):
        cards = response.css('div[class="search_list_details"]')

        for card in cards: 
            game_name = card.css('a[class=text_white]::attr(title)').get()
            game_dict =  {"Game_name":game_name}
           
            yield game_dict

It was working previously when all of the sudden it stopped working and I kept getting the 302 redirects. What seems to be the issue?

Answer 1

• You don't need jQuery
• When using classes make always sure to reference to a target parent - that way it will not mater if you have one or hundreds of such elements on your page
• Echo your price and currency in a parent wrapper data-* attributes, not in JS.
• Most of the other issues are mentioned in the comments below your Question

Here's a remake of your Item amount calculator in pure JavaScript:

const EL = (sel, parent) => (parent||document).querySelector(sel);
const ELS = (sel, parent) => (parent||document).querySelectorAll(sel);

const calcItemInit = (EL_item) => {
  // Get elements
  const ELS_ckb = ELS(".wc-pao-addon-checkbox", EL_item);
  const ELS_sel = ELS(".wc-pao-addon-select", EL_item);
  const EL_amount = EL(".woocommerce-Price-amount", EL_item);
  const EL_currency = EL(".woocommerce-Price-currencySymbol", EL_item);
  
  // Get data
  const price = parseFloat(EL_item.dataset.price);
  const currency = EL_item.dataset.currency;
  
  // Calc
  const calc = () => {
    let amount = price || 0;
    
    // Add checkboxes values
    ELS_ckb.forEach(el => {
      if (el.checked) amount += parseFloat(el.value);
    });
    
    // Add select values
    ELS_sel.forEach(el => {
      amount += parseFloat(el.value || 0);
    });
    
    // Set Amount!
    EL_amount.textContent = amount.toFixed(2);
    EL_currency.textContent = currency;
  };
  
  // Events:
  ELS_ckb.forEach(el => el.addEventListener("change", calc));
  ELS_sel.forEach(el => el.addEventListener("change", calc));
  
  // Init:
  calc();
};

ELS(".calc").forEach(calcItemInit);

<div class="calc" data-currency="€" data-price="12.00">
  <div>Item: <b>JS Book Of The Year</b></div>
  <div>Price: <b>12.00 €</b></div>
  <div>Extras:
    <label><input type="checkbox" class="wc-pao-addon-checkbox" value="3.99"> + Book cover</label>
    <label><input type="checkbox" class="wc-pao-addon-checkbox" value="14.00"> + All pages</label>
    <label>
      <select class="wc-pao-addon-select">
        <option value="29.99" selected>One year old</option>
        <option value="10.25">Two years old</option>
        <option value="10.15">Ten years old</option>
      </select>
    </label>
  </div>
  <div>
    Final price: 
    <span class="woocommerce-Price-amount"></span>
    <span class="woocommerce-Price-currencySymbol"></span>
  </div>
</div>