'NodeJS Gunzip "Error: invalid distance too far back" for larger files

I'm trying to process some audit data using MongoDB Atlas Log API which returns a .gz file. This works for very small files (<100kb uncompressed / <10kb compressed) but anything larger and "Error: invalid distance too far back" error returned:

Any help very much appreciated

import AxiosDigestAuth from '@mhoc/axios-digest-auth';
const zlib = require("zlib");

exports = async function() {
  var testCol = context.services.get('Cluster0').db('test').collection('test');
  
  const digestAuth = new AxiosDigestAuth({
    username: 'xxx',
    password: 'xxx'
  });
  
  async function test (callback) { 
    var buffer = [];
    
    const response = await digestAuth.request({
        headers: { 'Accept-Encoding':'gzip','Content-Encoding': 'gzip'},
        method: "GET",
        url: "https://cloud.mongodb.com/api/atlas/v1.0/groups/5ea0476502794826ee77a9f9/clusters/xxx/logs/mongodb.gz?startDate=1645541400",
        responseType: 'stream',
        decompress: true
    });
  
    var gunzip = zlib.createGunzip();            
    response.data.pipe(gunzip);

    gunzip.on('data', function(data) {
        // decompression chunk ready, add it to the buffer
        buffer.push(data.toString())

    }).on("end", function() {
        // response and decompression complete, join the buffer and return
        callback(null, buffer.join("")); 

    }).on("error", function(e) {
        callback(e);
    })   
    
  }
  
  test(function(err, data) {
    if(err) {
      console.log(err)
    } else {
      testCol.insertOne({"payload":data})
    }
  })
  
};


Sources

This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.

Source: Stack Overflow

Solution Source