我怎样才能从数组中删除重复的项目（复杂对象）

Question

在每个文档中，

records 是一个包含许多重复对象的数组。

并且在buy_items中也有很多重复项。

如何清理重复项？

原始文件:

{
  "_id": "0005d116qwwewdq82a1b84f148fa6027d429f3e",
  "records": [
    {
      "DATE": new Date("1996-02-08T08:00:00+0800"),
      "buy_items": [
        "5210 ",
        "5210 ",
        "5210 "
      ]
    },
    {
      "DATE": new Date("1996-02-08T08:00:00+0800"),
      "buy_items": [
        "5210 ",
        "5210 ",
        "5210 "
      ]
    }
    {
      "DATE": new Date("2012-12-08T08:00:00+0800"),
      "buy_items": [
        "5210 ",
        "1234 ",
        " "
      ]
    }        
    ]
}

预期输出：

{
  "_id": "0005d116qwwewdq82a1b84f148fa6027d429f3e",
  "records": [
    {
      "DATE": new Date("1996-02-08T08:00:00+0800"),
      "buy_items": [
        "5210 "
      ]
    },
    {
      "DATE": new Date("2012-12-08T08:00:00+0800"),
      "buy_items": [
        "5210 ",
        "1234 ",
        " "
      ]
    }    
    ]
}

使用 Michaels 解决方案，输出可能如下所示

{
  "_id": "0005d116qwwewdq82a1b84f148fa6027d429f3e",
  "records": [
    "date": new Date("1996-02-08T08:00:00+0800"),
      "buy_items": [
        "5210 "
        "1234 ",
        " "
      ]
    ]
}

Answer 1

您可以使用 aggregation framework

删除重复的对象

db.collection.aggregate(
    [
        { $unwind: "$records" }, 
        { $unwind: "$records.buy_items" }, 
        { $group: { "_id": {id: "$_id", date: "$records.DATE" }, buy_items: { $addToSet: "$records.buy_items" }}}, 
        { $group: {"_id": "$_id.id", records: { $push: {"date": "$_id.date", "buy_items": "$buy_items" }}}}, { $sort: { "records.0.date": 1 }} ,
        { $out: "collection" }
    ]
)

$out 运算符可让您将聚合结果写入指定集合或替换现有集合。

使用 "Bulk" 操作更好

var bulk = bulk = db.collection.initializeOrderedBulkOp(),
    count = 0;

db.collection.aggregate([
    { "$unwind": "$records" }, 
    { "$project": { 
        "date": "$records.DATE", 
        "buy_items": { "$setIntersection": "$records.buy_items" }
    }}, 
    { "$unwind": "$buy_items" }, 
    { "$group": { 
        "_id": { "id": "$_id", "date": "$date" }, 
        "buy_items": { "$addToSet": "$buy_items" }
    }},
    { "$group": { 
        "_id": "$_id.id", 
        "records": { "$push": { 
            "date": "$_id.date", 
            "buy_items": "$buy_items" 
        }}
    }}
]).forEach(function(doc) { 
       bulk.find({"_id": doc._id}).updateOne({
       "$set": { "records": doc.records }
       }); 
       count++; 
       if (count % 500 == 0) {   
           bulk.execute();    
           bulk = db.collection.initializeOrderedBulkOp(); 
       } 
})

if (count % 500 != 0)
    bulk.execute();

结果：

{
    "_id" : "0005d116qwwewdq82a1b84f148fa6027d429f3e",
    "records" : [
            {
                    "date" : ISODate("2012-12-08T00:00:00Z"),
                    "buy_items" : [
                            " ",
                            "1234 ",
                            "5210 "
                    ]
            },
            {
                    "date" : ISODate("1996-02-08T00:00:00Z"),
                    "buy_items" : [
                            "5210 "
                    ]
            }
    ]
}

Answer 2

如果您想更新当前的 collection 而不创建新的 collection 并删除以前的 collection。我试过了，但是这样做你应该运行两个不同的更新命令。

首先将 records 更新为 distinct，如下所示：

db.collectionName.update({},{"$set":{"records":db.collectionName.distinct('records')}})

buy_items 和 distinct 的第二次更新，如下所示：

db.collectionName.update({},{"$set":{"records.0.buy_items":db.collectionName.distinct('records.buy_items')}})

如果您想避免两次更新查询，请按照 Michael 的回答。

Answer 3

您可以尝试使用 forEach() method of the find() 游标遍历每个文档属性，检查唯一性并过滤不同的值，如下所示：

db.collection.find().forEach(function(doc){
    var records = [], seen = {};
    doc.records.forEach(function (item){
         var uniqueBuyItems = item["buy_items"].filter(function(i, pos) {
            return item["buy_items"].indexOf(i) == pos;
         });
         item["buy_items"] = uniqueBuyItems;
         if (JSON.stringify(item["buy_items"]) !== JSON.stringify(seen["buy_items"])) {
            records.push(item);
            seen["buy_items"] = item["buy_items"];
         }         
    }); 
    doc.records = records;
    db.collection.save(doc);
})

我怎样才能从数组中删除重复的项目（复杂对象）

How could I remove the duplicated items(complex object) from array

mongodb

aggregation-framework