'How do I build one dynamic JSON object using data from two different async functions?
I need to build one JSON object with data that I scrape from a website in two different async functions, below is my current code:
const cheerio = require("cheerio");
const axios = require("axios");
var jsonObject = {};
async function scrapeSite(){
const result = await axios.get("https://tradingeconomics.com/country-list/corporate-tax-rate");
const $ = cheerio.load(result.data);
$("#ctl00_ContentPlaceHolder1_ctl01_UpdatePanel1 > div > div > table > tbody > tr > td:nth-child(3)").each((index, element) => {
console.log($(element).text());
});
$("#ctl00_ContentPlaceHolder1_ctl01_UpdatePanel1 > div > div > table > tbody > tr > td:nth-child(1) > a").each((index, element) => {
console.log($(element).text().trim());
});
return "Done";
}
console.log(jsonObject);
async function scrapeSiteYield(){
const yieldResult = await axios.get("https://tradingeconomics.com/forecast/government-bond-10y");
const $ = cheerio.load(yieldResult.data);
$("#p").each((index, element) => {
console.log("Yield: "+$(element).text().trim());
});
$("#aspnetForm > div.container > div > div > div:nth-child(4) > div > table > tbody > tr > td.datatable-item-first").each((index, element) => {
console.log("Major: "+$(element).text().trim());
});
$("#aspnetForm > div.container > div > div > div:nth-child(8) > div > table > tbody > tr > td.datatable-item-first").each((index, element) => {
console.log("Europe: "+$(element).text().trim());
});
$("#aspnetForm > div.container > div > div > div:nth-child(12) > div > table > tbody > tr > td.datatable-item-first").each((index, element) => {
console.log("America: "+$(element).text().trim());
});
$("#aspnetForm > div.container > div > div > div:nth-child(16) > div > table > tbody > tr > td.datatable-item-first").each((index, element) => {
console.log("Asia: "+$(element).text().trim());
});
$("#aspnetForm > div.container > div > div > div:nth-child(20) > div > table > tbody > tr > td.datatable-item-first").each((index, element) => {
console.log("Australia: "+$(element).text().trim());
});
$("#aspnetForm > div.container > div > div > div:nth-child(24) > div > table > tbody > tr > td.datatable-item-first").each((index, element) => {
console.log("Africa: "+$(element).text().trim());
});
return "Done";
}
scrapeSite();
scrapeSiteYield();
Below is the JSON Object I require:
{
[
{
"Country":"South Africa",
"Taxrate": 15,
"Yield" : 0.55
},
{
"Country":"Australia",
"Taxrate": 20,
"Yield" : 0.6506
}
]
}
- Country is being returned in both async functions.
- Taxrate is being returned in the first async function.
- Yield is being returned in the second async function.
Solution 1:[1]
I edited your code a little and now it does what you need. You can check in the online IDE. The only point is that Yield is not available everywhere.
Code:
const cheerio = require("cheerio");
const axios = require("axios");
async function scrapeSite() {
const results = [];
return new Promise(async (resolve) => {
const result = await axios.get("https://tradingeconomics.com/country-list/corporate-tax-rate");
const $ = cheerio.load(result.data);
$("#ctl00_ContentPlaceHolder1_ctl01_UpdatePanel1 tr").each((index, element) => {
results[index] = {
Taxrate: $(element).find("td:nth-child(3)").text(),
Country: $(element).find("td:nth-child(1) > a").text().trim(),
};
});
resolve(results);
});
}
async function scrapeSiteYield(results) {
return new Promise(async (resolve) => {
const yieldResult = await axios.get("https://tradingeconomics.com/forecast/government-bond-10y");
const $ = cheerio.load(yieldResult.data);
$(".table tr").each((index, element) => {
let country = $(element).find(".datatable-item-first a").text().trim();
if (country.includes("UK")) country = "United Kingdom";
if (country.includes("US")) country = "United States";
if (country.includes("15Y")) country = "Peru";
results.forEach((el) => {
if (country.includes(el.Country)) {
el.Yield = $(element).find("#p").text().trim();
}
});
});
resolve(results);
});
}
scrapeSite().then(scrapeSiteYield).then(console.log);
Output:
[
{ "Taxrate": "20", "Country": "Afghanistan" },
{ "Taxrate": "15", "Country": "Albania" },
{ "Taxrate": "26", "Country": "Algeria" },
{ "Taxrate": "30", "Country": "Angola" },
{ "Taxrate": "30", "Country": "Argentina" },
{ "Taxrate": "18", "Country": "Armenia" },
{ "Taxrate": "25", "Country": "Aruba" },
{ "Taxrate": "30", "Country": "Australia", "Yield": "3.4910" },
{ "Taxrate": "25", "Country": "Austria", "Yield": "1.5030" },
{ "Taxrate": "20", "Country": "Azerbaijan" },
{ "Taxrate": "0", "Country": "Bahamas" },
{ "Taxrate": "0", "Country": "Bahrain" },
... other results
]
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
Solution | Source |
---|---|
Solution 1 | Mikhail Zub |