Remove Duplicates from JavaScript Array

"Smart" but naïve way

uniqueArray = myArray.filter(function(item, pos) {
    return myArray.indexOf(item) == pos;
})

Basically, we iterate over the array and, for each element, check if the first position of this element in the array is equal to the current position. Obviously, these two positions are different for duplicate elements.

Using the 3rd ("this array") parameter of the filter callback we can avoid a closure of the array variable:

uniqueArray = myArray.filter(function(item, pos, self) {
    return self.indexOf(item) == pos;
})

Although concise, this algorithm is not particularly efficient for large arrays (quadratic time).

Hashtables to the rescue

function uniq(a) {
    var seen = {};
    return a.filter(function(item) {
        return seen.hasOwnProperty(item) ? false : (seen[item] = true);
    });
}

This is how it's usually done. The idea is to place each element in a hashtable and then check for its presence instantly. This gives us linear time, but has at least two drawbacks:

• since hash keys can only be strings in Javascript, this code doesn't distinguish numbers and "numeric strings". That is, uniq([1,"1"]) will return just [1]
• for the same reason, all objects will be considered equal: uniq([{foo:1},{foo:2}]) will return just [{foo:1}].

That said, if your arrays contain only primitives and you don't care about types (e.g. it's always numbers), this solution is optimal.

The best from two worlds

An universal solution combines both approaches: it uses hash lookups for primitives and linear search for objects.

function uniq(a) {
    var prims = {"boolean":{}, "number":{}, "string":{}}, objs = [];

    return a.filter(function(item) {
        var type = typeof item;
        if(type in prims)
            return prims[type].hasOwnProperty(item) ? false : (prims[type][item] = true);
        else
            return objs.indexOf(item) >= 0 ? false : objs.push(item);
    });
}

sort | uniq

Another option is to sort the array first, and then remove each element equal to the preceding one:

function uniq(a) {
    return a.sort().filter(function(item, pos) {
        return !pos || item != a[pos - 1];
    })
}

Again, this doesn't work with objects (because all objects are equal for sort). Additionally, this method silently changes the original array as a side effect - not good!

Unique by...

Sometimes it's desired to uniquify a list based on some criteria other than just equality, for example, to filter out objects that are different, but share some property. This can be done elegantly by passing a callback, usually called a "key". key() is applied to each element, and elements with equal "keys" are removed. Since key is expected to return a primitive, hash table will work fine here:

function uniqBy(a, key) {
    var seen = {};
    return a.filter(function(item) {
        var k = key(item);
        return seen.hasOwnProperty(k) ? false : (seen[k] = true);
    })
}

A particularly useful key() is JSON.stringify which helps to filter out objects with identical structure:

a = [[1,2,3], [4,5,6], [1,2,3]]
b = uniqBy(a, JSON.stringify)
console.log(b) // [[1,2,3], [4,5,6]]

Use Underscore.js

It's a library with a host of functions for manipulating arrays.

It's the tie to go along with jQuery's tux, and Backbone.js's suspenders.

_.uniq

_.uniq(array, [isSorted], [iterator]) Alias: unique
Produces a duplicate-free version of the array, using === to test object equality. If you know in advance that the array is sorted, passing true for isSorted will run a much faster algorithm. If you want to compute unique items based on a transformation, pass an iterator function.

Example

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];

alert(_.uniq(names, false));

Note: Lo-Dash (an underscore competitor) also offers a comparable .uniq implementation.

Got tired of seeing all bad examples with for-loops or jQuery. Javascript has the perfect tools for this nowadays: sort, map and reduce.

Uniq reduce while keeping existing order

var names = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];

var uniq = names.reduce(function(a,b){
    if (a.indexOf(b) < 0 ) a.push(b);
    return a;
  },[]);

console.log(uniq, names) // [ 'Mike', 'Matt', 'Nancy', 'Adam', 'Jenny', 'Carl' ]

// one liner
return names.reduce(function(a,b){if(a.indexOf(b)<0)a.push(b);return a;},[]);

Faster uniq with sorting

There are probably faster ways but this one is pretty decent.

var uniq = names.slice() // slice makes copy of array before sorting it
  .sort(function(a,b){
    return a - b;
  })
  .reduce(function(a,b){
    if (a.slice(-1)[0] !== b) a.push(b); // slice(-1)[0] means last item in array without removing it (like .pop())
    return a;
  },[]); // this empty array becomes the starting value for a

// one liner
return names.slice().sort(function(a,b){return a - b}).reduce(function(a,b){if (a.slice(-1)[0] !== b) a.push(b);return a;},[]);

You can always try putting it into an object, and then iterating through its keys:

a = ["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"];
b = {};
for (var i = 0; i < a.length; i++) {
    b[a[i]] = a[i];
}
c = [];
for (var key in b) {
    c.push(key);
}
//c:
//["Mike", "Matt", "Nancy", "Adam", "Jenny", "Carl"]

A single line version using array filter and indexOf functions:

arr = arr.filter (function (v, i, a) { return a.indexOf (v) == i });

The top answers have complexity of O(n²), but this can be done with just O(n) by using an object as a hash:

function getDistinctArray(arr) {
    var dups = {};
    return arr.filter(function(el) {
        var hash = el.valueOf();
        var isDup = dups[hash];
        dups[hash] = true;
        return !isDup;
    });
}

This will work for strings, numbers, and dates. If your array contains complex objects (ie, they have to be compared with ===), the above solution won't work. You can get an O(n) implementation for objects by setting a flag on the object itself:

function getDistinctObjArray(arr) {
    var distinctArr = arr.filter(function(el) {
        var isDup = el.inArray;
        el.inArray = true;
        return !isDup;
    });
    distinctArr.forEach(function(el) {
        delete el.inArray;
    });
    return distinctArr;
}

The following is more than 80% faster than the jQuery method listed (see tests below). It is an answer from a similar question a few years ago, if I come across the person who originally proposed it I will post credit. Pure JS.

var temp = {};
  for (var i = 0; i < array.length; i++)
  temp[array[i]] = true;
  var r = [];
  for (var k in temp)
  r.push(k);
  return r;

My Test Case comparison: http://jsperf.com/remove-duplicate-array-tests

You could also use the Array.unique() method from the JavaScript Lab library – or steal an idea from there.

However, the code there isn’t very well written, since it declares the unique() method as a property of the Array prototype, thus adding it to every Array, breaking the for...in functionality (because a for...in loop will always iterate over the unique variable, too).

$(document).ready(function() {

    var arr1=["dog","dog","fish","cat","cat","fish","apple","orange"]

    var arr2=["cat","fish","mango","apple"]

    var uniquevalue=[];
    var seconduniquevalue=[];
    var finalarray=[];

    $.each(arr1,function(key,value){

       if($.inArray (value,uniquevalue) === -1)
       {
           uniquevalue.push(value)

       }

    });

     $.each(arr2,function(key,value){

       if($.inArray (value,seconduniquevalue) === -1)
       {
           seconduniquevalue.push(value)

       }

    });

    $.each(uniquevalue,function(ikey,ivalue){

        $.each(seconduniquevalue,function(ukey,uvalue){

            if( ivalue == uvalue)

            {
                finalarray.push(ivalue);
            }   

        });

    });
    alert(finalarray);
});

A slight modification of thg435's excellent answer to use a custom comparator:

function contains(array,obj) {
    for(var i =0;i<array.length;i++) {
        if(isEqual(array[i],obj))return true;
    }
    return false;
}
//comparator
function isEqual(obj1,obj2) {
    if(obj1.name==obj2.name) return true;
    return false;
}
function removeDuplicates(ary) {
    var arr = [];
    return ary.filter(function(x) {
        return !contains(arr,x) && arr.push(x);
    });
}

This is probably one of the fastest way to remove permanently the duplicates from an array 10x times faster than the most functions here.& 78x faster in safari

function toUnique(a,b,c){               //array,placeholder,placeholder
 b=a.length;while(c=--b)while(c--)a[b]!==a[c]||a.splice(c,1)
}

1. Test: http://jsperf.com/wgu
2. Demo: http://jsfiddle.net/46S7g/
3. More: http://stackoverflow.com/a/25082874/2450730

if you can't read the code above ask, read a javascript book or here are some explainations about shorter code. http://stackoverflow.com/a/21353032/2450730

Here is another approach using jQuery,

function uniqueArray(array){
  if ($.isArray(array)){
    var dupes = {}; var len, i;
    for (i=0,len=array.length;i<len;i++){
      var test = array[i].toString();
      if (dupes[test]) { array.splice(i,1); len--; i--; } else { dupes[test] = true; }
    }
  } 
  else {
    if (window.console) console.log('Not passing an array to uniqueArray, returning whatever you sent it - not filtered!');
      return(array);
  }
  return(array);
}

Author: William Skidmore

function removeDuplicates(inputArray) {
            var outputArray=new Array();

            if(inputArray.length>0){
                jQuery.each(inputArray, function(index, value) {
                    if(jQuery.inArray(value, outputArray) == -1){
                        outputArray.push(value);
                    }
                });
            }           
            return outputArray;
        }

If you don't want to include a whole library, you can use this one off to add a method that any array can use:

Array.prototype.uniq = function uniq() {
  return this.reduce(function(accum, cur) { 
    if (accum.indexOf(cur) === -1) accum.push(cur); 
    return accum; 
  }, [] );
}

["Mike","Matt","Nancy","Adam","Jenny","Nancy","Carl"].uniq()

If you're creating the array yourself, you can save yourself a loop and the extra unique filter by doing the check as you're inserting the data;

var values = [];
$.each(collection, function() {
    var x = $(this).value;
    if (!$.inArray(x, values)) {
        values.push(x);
    }
});