'Remove Duplicate objects from JSON Array
I'm trying to create a series of lists in HTML of the form:
<div>
<h3>Math K</h3>
<li>Counting & Cardinality</li>
<li>Geometry</li>
</div>
<div>
<h3>Math 1</h3>
<li>Counting & Cardinality</li>
<li>Orders of Operation</li>
</div>
<div>
<h3>Math 2</h3>
<li>Geometry</li>
</div>
My original though was to create an array and push that into the <div> element on the page with $("#divid").append(array). I created an array that looks like this:
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
And I need to remove the duplicates so that something like this remains:
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
I've tried installing underscore.js and using ._uniq but that only seems to work when a single key:value pair appears in the object. I can't seem to get it to work across multiple keys.
When I try something like:
var uniqueStandards = _.uniq(standardsList, function(item, key, Domain){
return item.Domain;
});
I only get the first three unique values (one per grade). But I need all the unique values across both grade and domain. Is there a simple way to feed both keys to the _.uniq function?
Ultimately, I need a list with the each unique grade as the header and the unique domains as the list items to pass into an HTML page.
Solution 1:[1]
I know there are many answers already but the best one that worked for me for a complex json structure is:
var arr = [{ "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RWCW", "desc": "WEST", "code": "RWCW", "level": 0, "save": "RWCW : WEST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RCSW", "desc": "SOUTHWEST", "code": "RCSW", "level": 0, "save": "RCSW : SOUTHWEST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RCNW", "desc": "MIDWEST", "code": "RCNW", "level": 0, "save": "RCNW : MIDWEST", "attribute1": "", "attribute2": "" }, { "State": "RSCW", "desc": "SOUTHEAST", "code": "RSCW", "level": 0, "save": "RSCW : SOUTHEAST", "attribute1": "", "attribute2": "" }, { "State": "RECW", "desc": "NORTHEAST", "code": "RECW", "level": 0, "save": "RECW : NORTHEAST", "attribute1": "", "attribute2": "" }];
var clean = arr.filter((arr, index, self) =>
index === self.findIndex((t) => (t.save === arr.save && t.State === arr.State)))
console.log(clean);
You can try this directly to chrome browser console and edit according to your need.
I hope this helps someone.
Solution 2:[2]
function arrUnique(arr) {
var cleaned = [];
arr.forEach(function(itm) {
var unique = true;
cleaned.forEach(function(itm2) {
if (_.isEqual(itm, itm2)) unique = false;
});
if (unique) cleaned.push(itm);
});
return cleaned;
}
var standardsList = arrUnique(standardsList);
This will return
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
Which is exactly what you asked for ?
Solution 3:[3]
I needed to do some de-duping of JSON objects so I stumbled across this page. However, I went with the short ES6 solution (no need for external libs), running this in Chrome Dev Tools Snippets:
const data = [ /* any list of objects */ ];
const set = new Set(data.map(item => JSON.stringify(item)));
const dedup = [...set].map(item => JSON.parse(item));
console.log(`Removed ${data.length - dedup.length} elements`);
console.log(dedup);
Solution 4:[4]
Reviving an old question, but I wanted to post an iteration on @adeneo's answer. That answer is completely general, but for this use case it could be more efficient (it's slow on my machine with an array of a few thousand objects). If you know the specific properties of the objects you need to compare, just compare them directly:
var sl = standardsList;
var out = [];
for (var i = 0, l = sl.length; i < l; i++) {
var unique = true;
for (var j = 0, k = out.length; j < k; j++) {
if ((sl[i].Grade === out[j].Grade) && (sl[i].Domain === out[j].Domain)) {
unique = false;
}
}
if (unique) {
out.push(sl[i]);
}
}
console.log(sl.length); // 10
console.log(out.length); // 5
Solution 5:[5]
Javascript solution for your case:
console.log(unique(standardsList));
function unique(obj){
var uniques=[];
var stringify={};
for(var i=0;i<obj.length;i++){
var keys=Object.keys(obj[i]);
keys.sort(function(a,b) {return a-b});
var str='';
for(var j=0;j<keys.length;j++){
str+= JSON.stringify(keys[j]);
str+= JSON.stringify(obj[i][keys[j]]);
}
if(!stringify.hasOwnProperty(str)){
uniques.push(obj[i]);
stringify[str]=true;
}
}
return uniques;
}
Solution 6:[6]
**The following method does the way you want. It filters the array based on all properties values. **
var standardsList = [
{ "Grade": "Math K", "Domain": "Counting & Cardinality" },
{ "Grade": "Math K", "Domain": "Counting & Cardinality" },
{ "Grade": "Math K", "Domain": "Counting & Cardinality" },
{ "Grade": "Math K", "Domain": "Counting & Cardinality" },
{ "Grade": "Math K", "Domain": "Geometry" },
{ "Grade": "Math 1", "Domain": "Counting & Cardinality" },
{ "Grade": "Math 1", "Domain": "Counting & Cardinality" },
{ "Grade": "Math 1", "Domain": "Orders of Operation" },
{ "Grade": "Math 2", "Domain": "Geometry" },
{ "Grade": "Math 2", "Domain": "Geometry" }
];
const removeDupliactes = (values) => {
let concatArray = values.map(eachValue => {
return Object.values(eachValue).join('')
})
let filterValues = values.filter((value, index) => {
return concatArray.indexOf(concatArray[index]) === index
})
return filterValues
}
removeDupliactes(standardsList)
Results this
[{Grade: "Math K", Domain: "Counting & Cardinality"}
{Grade: "Math K", Domain: "Geometry"}
{Grade: "Math 1", Domain: "Counting & Cardinality"}
{Grade: "Math 1", Domain: "Orders of Operation"}
{Grade: "Math 2", Domain: "Geometry"}]
Solution 7:[7]
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
standardsList = standardsList.filter((li, idx, self) => self.map(itm => itm.Grade+itm.Domain).indexOf(li.Grade+li.Domain) === idx)
document.write(JSON.stringify(standardsList))
here is a functional way of doing it that is much easier
standardsList = standardsList.filter((li, idx, self) => self.map(itm => iem.Grade+itm.domain).indexOf(li.Grade+li.domain) === idx)
Solution 8:[8]
The following works for me:
_.uniq(standardsList, JSON.stringify)
This would probably be slow for very long lists, however.
Solution 9:[9]
Use Map to remove the duplicates. (For new readers)
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
var grades = new Map();
standardsList.forEach( function( item ) {
grades.set(JSON.stringify(item), item);
});
console.log( [...grades.values()]);
/*
[
{ Grade: 'Math K', Domain: 'Counting & Cardinality' },
{ Grade: 'Math K', Domain: 'Geometry' },
{ Grade: 'Math 1', Domain: 'Counting & Cardinality' },
{ Grade: 'Math 1', Domain: 'Orders of Operation' },
{ Grade: 'Math 2', Domain: 'Geometry' }
]
*/
Solution 10:[10]
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
function uniqurArray(array){
var a = array.concat();
for(var i=0; i<a.length; i++) {
for(var j=i+1; j<a.length; j++) {
if(a[i].Grade === a[j].Grade){
a.splice(j--, 1);
}
}
}
return a;
}
uniqurArray(standardsList) // put this js in console and you get uniq object in array
Solution 11:[11]
I found here every answer is so expensive to compute, So we need an answer that is:
- Efficient
- Short & Elegant
- And easy to compute.
By using map
let standardsList = [{ "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Geometry" }, { "Grade": "Math 1", "Domain": "Counting & Cardinality" }, { "Grade": "Math 1", "Domain": "Counting & Cardinality" }, { "Grade": "Math 1", "Domain": "Orders of Operation" }, { "Grade": "Math 2", "Domain": "Geometry" }, { "Grade": "Math 2", "Domain": "Geometry" }],
map = {};
for (let list of standardsList) {
map[Object.values(list).join('')] = list;
}
console.log('Using Map', Object.values(map));
By using set
let standardsList = [{ "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Counting & Cardinality" }, { "Grade": "Math K", "Domain": "Geometry" }, { "Grade": "Math 1", "Domain": "Counting & Cardinality" }, { "Grade": "Math 1", "Domain": "Counting & Cardinality" }, { "Grade": "Math 1", "Domain": "Orders of Operation" }, { "Grade": "Math 2", "Domain": "Geometry" }, { "Grade": "Math 2", "Domain": "Geometry" }],
set = new Set, results = [];
for (let list of standardsList) {
let id = Object.values(list).join('');
if (!set.has(id)) {
set.add(id);
results.push(list);
}
}
console.log('Using Set', results);
Solution 12:[12]
You can use lodash, download here (4.17.15)
Example code:
var object = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 2 }];
_.uniqWith(object, _.isEqual);
// => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]
Solution 13:[13]
There are two main ways to do this very concisely with Underscore out of the box. The first was already mentioned by cookie monster in a comment directly under the question:
_.uniq(standardsList, item => item.Grade + item.Domain)
With direct string concatenation as shown above, problematic edge cases could potentially occur in situations like the following:
var standardsList = [
{Grade: 'abcdef', Domain: 'ghi'},
{Grade: 'abc', Domain: 'defghi'}
];
since both produce the combined string 'abcdefghi' despite them not being equivalent. The simplest solution to this is to interject a substring that you know never occurs as a suffix of the Grade (or as a prefix of the Domain), for example assuming that ': ' is such a substring:
_.uniq(standardsList, item => item.Grade + ': ' + item.Domain)
The second was already mentioned in the answer by dpmemcry. Here is the version that is compatible with Underscore:
_.uniq(standardsList, _.isEqual);
The difference is that the first solution only takes the fields into account that you specifically select, while the second takes all fields into account and even performs a recursive comparison. In some cases, this may be exactly what you want, while in other cases, it might be too expensive.
With a bit more effort, i.e., by writing a custom helper function, you can achieve a compromise between these solutions where you can select the fields that are taken into account without having to write out a concatenation expression every time. Here is one possible way to write that helper function:
function byFields(fields, separator) {
return function(item) {
return _.map(fields, _.propertyOf(item)).join(separator);
};
}
and here is how you use it:
_.uniq(standardsList, byFields(['Grade', 'Domain'], ': '));
With this latter approach, you also have the option to perform selective deep comparisons by passing array paths instead of strings in the list of fields. This lets you have deep property lookups without forcing you to recurse all the way through each object. For example, suppose that the Domain property of each item is a nested object, like this:
var standardsList = [
{Grade: "Math K", Domain: {id: 1, name: "Counting & Cardinality"}},
// ...
];
Then you could use the above helper function like this, without modification:
_.uniq(standardsList, byFields['Grade', ['Domain', 'id']], ': '));
Solution 14:[14]
I came across a solution to this online a few months ago but can't remember the source. Credits to the author. The solution works as below.
// Example use case
const students = [
{
"m_name": "Jason",
"m_id": "1",
"age": 18,
"subject": "Biology"
},
{
"m_name": "Jason",
"m_id": "1",
"age": 18,
"subject": "Chemistry"
},
{
"m_name": "Sameer",
"m_id": "2",
"age": 18,
"subject": "English"
},
{
"m_name": "Sameer",
"m_id": "2",
"age": 18,
"subject": "History"
},
{
"m_name": "Philip",
"m_id": "3",
"age": 18,
"subject": "Drama"
}
]
// Function to merge by student name (or ID)
const groupBy = key => array =>
array.reduce((objectByKeyValue, obj) => {
const value = obj[key];
objectByKeyValue[value] = (objectByKeyValue[value] || []).concat(obj);
return objectByKeyValue;
}, {});
// Replace with the key name
const groupBy_StudentName = groupBy('m_name') // or groupBy('m_id')
// Now pass your array name here
const grouped = groupBy_StudentName(students)
// Check your answer
console.log(grouped)
// Output
{
"Jason": [
{
m_name: 'Jason',
m_id: '1',
age: 18,
subject: 'Biology'
},
{
m_name: 'Jason',
m_id: '1',
age: 18,
subject: 'Chemistry'
}
],
"Sameer": [
{
m_name: 'Sameer',
m_id: '2',
age: 18,
subject: 'English'
},
{
m_name: 'Sameer',
m_id: '2',
age: 18,
subject: 'History'
}
],
"Philip": [
{
m_name: 'Philip',
m_id: '3',
age: 18,
subject: 'Drama'
}
]
}
Solution 15:[15]
Use this Pseudocode
var standardsList = [
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Counting & Cardinality"},
{"Grade": "Math K", "Domain": "Geometry"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Counting & Cardinality"},
{"Grade": "Math 1", "Domain": "Orders of Operation"},
{"Grade": "Math 2", "Domain": "Geometry"},
{"Grade": "Math 2", "Domain": "Geometry"}
];
var newArr =[]
for(var i in standardsList){
newArr.push(JSON.stringify(standardsList[i]))
}
var obj = {};
newArr= newArr.filter((item)=>{
return obj.hasOwnProperty(item) ? false : (obj[item] = true);
})
standardsList.length = 0
for(var i in newArr){
standardsList.push(JSON.parse(newArr[i]))
}
console.log(standardsList)
I have choose a sample array similar to yours. Its easier to compare objects once you stringfy them. Then you just have to compare strings.
Solution 16:[16]
Here's a short one-liner with es6!
const nums = [
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC9233F2015",
"AC9233F2015",
"AC9233F2015",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E2",
"AC8818E2",
"AC8818E2",
"AC8818E2",
"AC9233F2015",
"AC9233F2015",
"AC9233F2015",
"AC9233F2015",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E2",
"AC8818E2",
"AC9233F2015",
"AC9233F2015",
"AC8818E1",
"AC8818E1",
"AC8818E1",
"AC8818E2",
"AC8818E2",
"AC8818E2",
"AC8818E2",
"ACB098F25",
"ACB098F25",
"ACB098F25",
"ACB098F25",
"AC8818E2",
"AC8818E2",
"AC8818E1",
"AC8818E1",
"AC8818E1",
]
Set is a new data object introduced in ES6. Because Set only lets you store unique values. When you pass in an array, it will remove any duplicate values.
export const $uniquenums = [...new Set(nums)].sort();
Solution 17:[17]
var standardsList = [
{ Grade: "Math K", Domain: "Counting & Cardinality" },
{ Grade: "Math K", Domain: "Counting & Cardinality" },
{ Grade: "Math K", Domain: "Counting & Cardinality" },
{ Grade: "Math K", Domain: "Counting & Cardinality" },
{ Grade: "Math K", Domain: "Geometry" },
{ Grade: "Math 1", Domain: "Counting & Cardinality" },
{ Grade: "Math 1", Domain: "Counting & Cardinality" },
{ Grade: "Math 1", Domain: "Orders of Operation" },
{ Grade: "Math 2", Domain: "Geometry" },
{ Grade: "Math 2", Domain: "Geometry" },
];
let fUArr = standardsList.filter(
(li, idx, self) =>
self
.map((itm) =>
Object.values(itm)
.reduce((r, c) => r.concat(c), '')
)
.indexOf(
Object.values(li)
.reduce((r, c) => r.concat(c), '')
) === idx
);
console.log(fUArr);
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
