'script & sheet timing out when trying to print large arrays in google script

Background

I have a function that makes a REST API call using UrlFetchApp in Google Scripts.

But the response only returns 2000 records at a time. If there are more records, there is, in the response, a key called nextRecordsUrl, which contains the endpoint and parameters needed to get the next batch of records.

I use a do...while loop to iterate through, pushing the records into a predesignated array, make the next api call. And when it reaches the last batch of records, it exists the do-while loop, then prints (not sure if that's the right term here) the entire to a Google Sheet.

The code

It looks like this:

function getCampaignAssociations() {
  clearPage('CampaignAssociations');
  var query = '?q=select+CampaignMember.FirstName,CampaignMember.LastName,CampaignMember.LeadId,CampaignMember.ContactId,CampaignMember.Name,CampaignMember.CampaignId,CampaignMember.SystemModstamp,CampaignMember.Email+from+CampaignMember+ORDER+BY+Email ASC,SystemModstamp+ASC';

  try {
    var arrCampAssociation = getInfoByQuery(query);

    if (arrCampAssociation.records.length < 1) {
      throw 'there are no records in this query';
    }

    var campaignAssoc = [];
    do {
      Logger.log(arrCampAssociation.nextRecordsUrl);

      for (var i in arrCampAssociation.records) {
        let data = arrCampAssociation.records[i];
        let createDate = Utilities.formatDate(new Date(data.SystemModstamp), "GMT", "dd-MM-YYYY");
        let a1 = "$A" + (parseInt(i) + 2);
        let nameFormula = '=IFERROR(INDEX(Campaigns,MATCH(' + a1 + ',Campaigns!$A$2:A,0),2),"")';
        let typeFormula = '=IFERROR(INDEX(Campaigns,MATCH(' + a1 + ',Campaigns!$A$2:A,0),3),"")';

        campaignAssoc.push([data.CampaignId, nameFormula, typeFormula, data.Email, data.FirstName, data.LastName, data.LeadId, data.ContactId, createDate]);
      }

      var arrCampAssociation = getQueryWithFullEndPoint(arrCampAssociation.nextRecordsUrl);
      
    } while (arrCampAssociation.nextRecordsUrl != null && arrCampAssociation.nextRecordsUrl != undefined);

      let endRow = campAssocSheet.getLastRow(), 
       endColumn = campAssocSheet.getLastColumn(), 
       nameRange = campAssocSheet.getRange(2, 1, endRow, endColumn),
       destRange = campAssocSheet.getRange(2, 1, campaignAssoc.length, campaignAssoc[0].length);
      destRange.setValues(campaignAssoc);
      sheet.setNamedRange('CampaignAssociation', nameRange);


    } catch (e) {
      Logger.log(e);
      Logger.log(arrCampAssociation);
      Logger.log(campaignAssoc);
      Logger.log(i);
    }
  }

Issue

Everything works nicely until it comes to printing the array campaignAssoc to the Google Sheet.

See screenshot of the log below. It contains the endpoint for the next both. Notice the timestamp between the earlier logs and the timestamp between the last endPoint and the log where it timed out.

enter image description here

It seems to me that the issue is that when it comes to the printing of the data, it's having issues. If that's the case, have I overloaded the array? There are a total of over 36400 records.

Second attempt

I've tried resetting the array at each loop and printing the array to Google sheet. This is just 2000 records at each attempt and I've definitely done more rows at 1 time, but that didn't help.

Here's the code for that attempt.

function getCampaignAssociations() {
  clearPage('CampaignAssociations');
  var query = '?q=select+CampaignMember.FirstName,CampaignMember.LastName,CampaignMember.LeadId,CampaignMember.ContactId,CampaignMember.Name,CampaignMember.CampaignId,CampaignMember.SystemModstamp,CampaignMember.Email+from+CampaignMember+ORDER+BY+Email ASC,SystemModstamp+ASC';

  try {
    var arrCampAssociation = getInfoByQuery(query);

    if (arrCampAssociation.records.length < 1) {
      throw 'there are no records in this query';
    }

    
    do {
      Logger.log(arrCampAssociation.nextRecordsUrl);
      var campaignAssoc = [];
      for (var i in arrCampAssociation.records) {
        let data = arrCampAssociation.records[i];
        let createDate = Utilities.formatDate(new Date(data.SystemModstamp), "GMT", "dd-MM-YYYY");
        let a1 = "$A" + (parseInt(i) + 2);
        let nameFormula = '=IFERROR(INDEX(Campaigns,MATCH(' + a1 + ',Campaigns!$A$2:A,0),2),"")';
        let typeFormula = '=IFERROR(INDEX(Campaigns,MATCH(' + a1 + ',Campaigns!$A$2:A,0),3),"")';

        campaignAssoc.push([data.CampaignId, nameFormula, typeFormula, data.Email, data.FirstName, data.LastName, data.LeadId, data.ContactId, createDate]);
        
      }
      let lastRow = campAssocSheet.getLastRow()+1;
      campAssocSheet.getRange(lastRow,1,campaignAssoc.length,campaignAssoc[0].length).setValues(campaignAssoc);
      var arrCampAssociation = getQueryWithFullEndPoint(arrCampAssociation.nextRecordsUrl);
      
    } while (arrCampAssociation.nextRecordsUrl != null && arrCampAssociation.nextRecordsUrl != undefined);

      let endRow = campAssocSheet.getLastRow(), 
       endColumn = campAssocSheet.getLastColumn(), 
       nameRange = campAssocSheet.getRange(2, 1, endRow, endColumn);

      sheet.setNamedRange('CampaignAssociation', nameRange);


    } catch (e) {
      Logger.log(e);
      Logger.log(arrCampAssociation);
      Logger.log(campaignAssoc);
      Logger.log(i);
    }
  }

So here, each loop took a lot longer. Instead of being 1-2 seconds between each loop, it took 45 seconds to a minute between each and timed out after the 4th loop. See the log below: enter image description here

How do I fix this?



Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source