Nodejs Async/Await not working on a file stream function

Question

Currently, I am trying to parse a csv file with {Lat, Lng, Mag}. Also, I am creating Polygons to use the magnitude. If the point {Lat, Lng} is in the Polygon, then the Magnitude will be added to the Polygon Magnitude. The function works as described, but the data is computed very slowly and the response does not contain the magnitude calculated. Below is my code:

const express = require('express');
const router = express.Router();
const csv = require('csv-parser')
const fs = require('fs');
const { exit } = require('process');
const { resolveObjectURL } = require('buffer');


/*
Things to parse:
1.) offense count
2.) latitude
3.) longitude
*/

function getPolygons() {
    let Count = 16
    let ParentPolygon = [
        {lat: 30.118713, lng: -95.812735 }, // Top Left
        {lat: 30.118713, lng: -95.034621 }, // Top Right
        {lat: 29.49764, lng:  -95.034621}, // Bottom Right
        {lat: 29.49764, lng: -95.812735 } // Bottom Left
    ]

    length = Math.abs(30.11873 - 29.49764)
    width = Math.abs(-95.812735 + 95.034621)

    length_cut = length/Math.sqrt(Count)
    width_cut = width/Math.sqrt(Count)

    let result = []
    let point = ParentPolygon[0] // Start at top left of Parent Polygon
    for(let j = 0; j < Math.sqrt(Count); j++){
        let FindNewChild = true;
        let FirstChildBottomLeft = {lat: 0, lng: 0}
        for(let i = 0; i < Math.sqrt(Count); i++ ){
            
            let ChildPolygon = [
                {lat: point.lat, lng: point.lng  }, //Top left
                {lat: point.lat, lng: point.lng + width_cut}, //Top Right
                {lat: point.lat - length_cut, lng: point.lng + width_cut},  //Bottom Right
                {lat: point.lat - length_cut, lng: point.lng} //Bottom Left
            ]
            if(FindNewChild){
                FirstChildBottomLeft = ChildPolygon[3]
                FindNewChild = false;
            }

            if(i === Math.sqrt(Count) - 1){
                point = FirstChildBottomLeft
                FindNewChild = true;
            } else
            {
                point = ChildPolygon[1]
            }       

            result.push({mag: 0, square: ChildPolygon})
        }

    }

    return result;


}
/*
                    {
                        {
                            magnitude: 0,
                            square: [
                                {lat: 30, lng: 45},
                                {lat: 30, lng: 55},
                                {lat: 40, lng: 45},
                                {lat: 40, lng: 55}
                            ]
                        },
                        {
                            magnitude: 0,
                            square: {
                                {lat: 30, lng: 45},
                                {lat: 30, lng: 55},
                                {lat: 40, lng: 45},
                                {lat: 40, lng: 55}
                            }
                        },
                        {
                            magnitude: 0,
                            square: {
                                {lat: 30, lng: 45},
                                {lat: 30, lng: 55},
                                {lat: 40, lng: 45},
                                {lat: 40, lng: 55}
                            }
                        }


                    }


                */


async function getParsedData() {
    let results = await getPolygons();
    let file = '../server/datafiles/NIBRSPublicViewJan-Mar22.csv' // hard coded file & file directory

    await fs.createReadStream(file)
        .pipe(csv({mapValues: ({value}) => parseFloat(value)}))
        .on('data', (data) => {
            if(data.lat && data.lng && data.mag) { // check if a latitude and longitude exist
            
                let point = ({lng: parseFloat(data.lat), lat: parseFloat(data.lng), mag: parseFloat(data.mag)})
                results.forEach(polygon => {
                    let TopLeft = polygon.square[0]
                    let BottomRight = polygon.square[2]

                    //Something is wrong with this statement
                    
                    if(point.lat < TopLeft.lat && point.lng > TopLeft.lng){
                        if(point.lat > BottomRight.lat && point.lng < BottomRight.lng){
                            //console.log("Top Left:" + TopLeft.lat + ", " + TopLeft.lng, "Point: "+ point.lat + ", " + point.lng)
                            //console.log("Bottom Right:" + BottomRight.lat + ", " + BottomRight.lng, "Point: "+ point.lat + ", " + point.lng)
                            //console.log("point Magnitude: ", point.mag, "Polygon mag: ", polygon.mag)

                            polygon.mag += point.mag
                        } 
                    }
                })

            }
        })
        .on('end', () => {
        console.log("What should go to Front-end", results); // check to see if the results are correct
       
    }); 

    return results;
}

router.get('/', async (req, res) => {
    getParsedData()
    .then(results => {
        console.log("To Front-end" , results)
        res.json(results)
    })
})

module.exports = router;

EDIT: The result should look like this:

[
  { mag: 0, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 279, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 880, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 429, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 8, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 5428, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 8605, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 58, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 1224, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 19229, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 17830, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 756, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 0, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 2260, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 2873, square: [ [Object], [Object], [Object], [Object] ] },
  { mag: 1923, square: [ [Object], [Object], [Object], [Object] ] }
]

Instead I am getting a result with all magnitudes set to 0

Answer 1

There are a number of issues here, but the main issue is that await fs.createReadStream(...) does not do anything useful. await only does something useful when you're awaiting a promise and fs.createReadStream() returns a stream, not a promise.

So, nothing in getParsedData() actually communicates back to the caller when the stream is done.

Since streams in nodejs are event based, they don't directly support promises. You could either make getParsedData() be callback-based so you pass in a callback and call that callback from the end event when the stream is done or you can wrap it in a promise (my preference) and you can manually resolve or reject the promise when the stream finishes.

Here's how you can make getParsedData() return a promise that resolves/rejects when the stream is done:

function getParsedData() {
    return new Promise((resolve, reject) => {
        const results = getPolygons();
        const file = '../server/datafiles/NIBRSPublicViewJan-Mar22.csv' // hard coded file & file directory

        fs.createReadStream(file)
            .pipe(csv({ mapValues: ({ value }) => parseFloat(value) }))
            .on('data', (data) => {
                if (data.lat && data.lng && data.mag) { // check if a latitude and longitude exist

                    let point = ({ lng: parseFloat(data.lat), lat: parseFloat(data.lng), mag: parseFloat(data.mag) })
                    results.forEach(polygon => {
                        const TopLeft = polygon.square[0]
                        const BottomRight = polygon.square[2]

                        //Something is wrong with this statement

                        if (point.lat < TopLeft.lat && point.lng > TopLeft.lng) {
                            if (point.lat > BottomRight.lat && point.lng < BottomRight.lng) {
                                //console.log("Top Left:" + TopLeft.lat + ", " + TopLeft.lng, "Point: "+ point.lat + ", " + point.lng)
                                //console.log("Bottom Right:" + BottomRight.lat + ", " + BottomRight.lng, "Point: "+ point.lat + ", " + point.lng)
                                //console.log("point Magnitude: ", point.mag, "Polygon mag: ", polygon.mag)

                                polygon.mag += point.mag
                            }
                        }
                    })

                }
            }).on('end', () => {
                console.log("What should go to Front-end", results); // check to see if the results are correct
                resolve(results);
            }).on('error', reject);
    });
}

router.get('/', async (req, res) => {
    getParsedData().then(results => {
        console.log("To Front-end", results)
        res.json(results)
    }).catch(err => {
        console.log(err);
        res.sendStatus(500);
    });
});

Changes to note:

1. We've added a promise inside of getParsedData() that lets us resolve or reject that promise based on the stream events.
2. Inside of the end event handler, we resolve the promise and resolved it with the desired results.
3. Inside of the error event handler, we reject the promise.
4. There is no point in using await as in await getPolygons() because getPolygons() is purely synchronous.
5. There's no reason for getParsedData()to beasyncas we're just returning a manually constructed promise and not usingawait` in it.
6. A number of local variables can be const instead of let as they are not modified.
7. An error handler is added on the stream to detect errors.
8. In your route handler, you need to also have a .catch() handler so your route sends a response if there's an error.