'NodeJS Gunzip "Error: invalid distance too far back" for larger files
I'm trying to process some audit data using MongoDB Atlas Log API which returns a .gz file. This works for very small files (<100kb uncompressed / <10kb compressed) but anything larger and "Error: invalid distance too far back" error returned:
Any help very much appreciated
import AxiosDigestAuth from '@mhoc/axios-digest-auth';
const zlib = require("zlib");
exports = async function() {
var testCol = context.services.get('Cluster0').db('test').collection('test');
const digestAuth = new AxiosDigestAuth({
username: 'xxx',
password: 'xxx'
});
async function test (callback) {
var buffer = [];
const response = await digestAuth.request({
headers: { 'Accept-Encoding':'gzip','Content-Encoding': 'gzip'},
method: "GET",
url: "https://cloud.mongodb.com/api/atlas/v1.0/groups/5ea0476502794826ee77a9f9/clusters/xxx/logs/mongodb.gz?startDate=1645541400",
responseType: 'stream',
decompress: true
});
var gunzip = zlib.createGunzip();
response.data.pipe(gunzip);
gunzip.on('data', function(data) {
// decompression chunk ready, add it to the buffer
buffer.push(data.toString())
}).on("end", function() {
// response and decompression complete, join the buffer and return
callback(null, buffer.join(""));
}).on("error", function(e) {
callback(e);
})
}
test(function(err, data) {
if(err) {
console.log(err)
} else {
testCol.insertOne({"payload":data})
}
})
};
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
| Solution | Source |
|---|
