PHP: Curl is responding slow in loop

Question

I have the following function I am calling in a loop

function post( $url, $data ) {
    $ch = curl_init();
    curl_setopt( $ch, CURLOPT_URL, $url );
    curl_setopt( $ch, CURLOPT_RETURNTRANSFER, true );
    curl_setopt( $ch, CURLOPT_CUSTOMREQUEST, 'POST' );
    curl_setopt($ch, CURLOPT_ENCODING, 'gzip');
    curl_setopt( $ch, CURLOPT_HTTPHEADER, [
        'Accept'             => 'text/html, */*; q=0.01',
        'Accept-Language'    => 'en-US,en;q=0.9,ur;q=0.8,zh-CN;q=0.7,zh;q=0.6',
        'Cache-Control'      => 'no-cache',
        'Connection'         => 'keep-alive',
        'Content-Type'       => 'application/x-www-form-urlencoded; charset=UTF-8',
        'DNT'                => '1',
        'Origin'             => 'https://dps.psx.com.pk',
        'Pragma'             => 'no-cache',
        'Referer'            => 'https://dps.psx.com.pk/historical',
        'Sec-Fetch-Dest'     => 'empty',
        'Sec-Fetch-Mode'     => 'cors',
        'Sec-Fetch-Site'     => 'same-origin',
        'User-Agent'         => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
        'X-Requested-With'   => 'XMLHttpRequest',
        'sec-ch-ua'          => '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
        'sec-ch-ua-mobile'   => '?0',
        'sec-ch-ua-platform' => '"macOS"',
        'Accept-Encoding'    => 'gzip',
    ] );
//  curl_setopt( $ch, CURLOPT_COOKIE,
//      'dis-request-id=a349ed4a96a83953fd19740ba4bc2de6; dis-timestamp=2022-03-28T04:00:24-07:00; dis-remote-addr=39.48.180.94' );
    //curl_setopt( $ch, CURLOPT_POSTFIELDS, 'month=1&year=2022&symbol=OGDC' );
    curl_setopt( $ch, CURLOPT_POSTFIELDS, $data );    
    $response = curl_exec( $ch );    
    curl_close( $ch );    
    return $response;
}

When I call in loop like below, it takes forever

for($x = 0; $x < 200;$x++) {
     $month = 1;
     $year=2022;
     $symbol = "OGDC";
     $data = [];
     $data[] = 'month=' . $month;
     $data[] = 'year=' . $year;
     $data[] = 'symbol=' . strtoupper( $symbol );
     $query = implode( '&', $data );
     $html = post( 'https://dps.psx.com.pk/historical', $query );
}

Answer 1

As you seem to be fetching historical data which is unlikely to change, how about caching that data so it will read from the file system if it's recently been fetched from the remote source.

I've slightly modified your post function (first 2 lines) and integrated a new cachef function. The configuration will cache the response for 1 week (604800 seconds).

I've also added some output to show progress with echo '. '.str_repeat(',',4096); and flush(); Hope this helps.

for ($x = 0; $x < 200; $x++) {
  $month = 1;
  $year = 2022;
  $symbol = 'OGDC';
  $data = [];
  $data[] = 'month=' . $month;
  $data[] = 'year=' . $year;
  $data[] = 'symbol=' . strtoupper($symbol);
  $query = implode('&', $data);
  $args = ['https://dps.psx.com.pk/historical', $query];
  $html = cachef('post', $args, 604800);
  echo '. '.str_repeat(',',4096);
  flush();
}




/**
 * Caches the output of any function for a given amount of time
 *
 * @param string function name $func
 * @param array of function arguments $args
 * @param integer number of seconds to cache the function response $seconds 604800 (one week)
 * @param boolean whether to treat the output of the function as JSON $json true
 * @param string relative or path of cached function results (with trailing slash) $cache_dir ./cache
 * @return mixed
 */
function cachef($func, $args, $seconds = 604800, $json = true, $cache_dir = './cache/') {
  // Caches for x seconds ($seconds) the result ($result) of any function ($func) in directory ($cache_dir)
  if (!is_dir($cache_dir)) {
    mkdir($cache_dir, 0755, true);
  }
  $request_hash = md5(json_encode(array($func, $args)));
  $request_file = $cache_dir . $request_hash . '.json';
  $run = false;
  if (file_exists($request_file)) {

    if (filemtime($request_file) < time() - $seconds) {
      $run = true;
    }
  } else {
    $run = true;
  }
  if ($run) {
    $result = $func($args);
    if ($json) {
      file_put_contents($request_file, json_encode($result));
    } else {
      file_put_contents($request_file, $result);
    }
  } else {
    if ($json) {
      $result = json_decode(file_get_contents($request_file), true);
    } else {
      $result = file_get_contents($request_file);
    }
  }
  return ($result);
}

function post($args) {

  $url = $args[0];
  $data = $args[1];

  $ch = curl_init();
  curl_setopt($ch, CURLOPT_URL, $url);
  curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
  curl_setopt($ch, CURLOPT_CUSTOMREQUEST, 'POST');
  curl_setopt($ch, CURLOPT_ENCODING, 'gzip');
  curl_setopt($ch, CURLOPT_HTTPHEADER, [
    'Accept'             => 'text/html, */*; q=0.01',
    'Accept-Language'    => 'en-US,en;q=0.9,ur;q=0.8,zh-CN;q=0.7,zh;q=0.6',
    'Cache-Control'      => 'no-cache',
    'Connection'         => 'keep-alive',
    'Content-Type'       => 'application/x-www-form-urlencoded; charset=UTF-8',
    'DNT'                => '1',
    'Origin'             => 'https://dps.psx.com.pk',
    'Pragma'             => 'no-cache',
    'Referer'            => 'https://dps.psx.com.pk/historical',
    'Sec-Fetch-Dest'     => 'empty',
    'Sec-Fetch-Mode'     => 'cors',
    'Sec-Fetch-Site'     => 'same-origin',
    'User-Agent'         => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.75 Safari/537.36',
    'X-Requested-With'   => 'XMLHttpRequest',
    'sec-ch-ua'          => '" Not A;Brand";v="99", "Chromium";v="100", "Google Chrome";v="100"',
    'sec-ch-ua-mobile'   => '?0',
    'sec-ch-ua-platform' => '"macOS"',
    'Accept-Encoding'    => 'gzip',
  ]);

  curl_setopt($ch, CURLOPT_POSTFIELDS, $data);
  $response = curl_exec($ch);
  curl_close($ch);
  return $response;
}