如何使用 fs 和批量请求读取多个 json 文件

Question

我在我的 React 应用程序中使用 elasticsearch 搜索引擎，我在后端读取一个文件，正如您在代码中看到的那样，它运行良好，但现在我想读取三个不同的 JSON 文件使用“fs”包和批量请求到三个不同的索引，你能帮我吗？

代码：

// Start reading the json file
fs.readFile("DocRes.json", { encoding: "utf-8" }, function (err, data) {
  if (err) {
    throw err;
  }

  // Build up a giant bulk request for elasticsearch.
  bulk_request = data.split("\n").reduce(function (bulk_request, line) {
    var obj, ncar;

    try {
      obj = JSON.parse(line);
    } catch (e) {
      console.log("Done reading 1");
      return bulk_request;
    }

    // Rework the data slightly
    ncar = {
      id: obj.id,
      name: obj.name,
      summary: obj.summary,
      image: obj.image,
      approvetool: obj.approvetool,
      num: obj.num,
      date: obj.date,
    };

    bulk_request.push({
      index: { _index: "ncar_index", _type: "ncar", _id: ncar.id },
    });
    bulk_request.push(ncar);
    return bulk_request;
  }, []);

  // A little voodoo to simulate synchronous insert
  var busy = false;
  var callback = function (err, resp) {
    if (err) {
      console.log(err);
    }

    busy = false;
  };

  // Recursively whittle away at bulk_request, 1000 at a time.
  var perhaps_insert = function () {
    if (!busy) {
      busy = true;
      client.bulk(
        {
          body: bulk_request.slice(0, 1000),
        },
        callback
      );
      bulk_request = bulk_request.slice(1000);
      console.log(bulk_request.length);
    }

    if (bulk_request.length > 0) {
      setTimeout(perhaps_insert, 100);
    } else {
      console.log("Inserted all records.");
    }
  };

  perhaps_insert();
});

Answer 1

您可以为每个读取的文件创建多个承诺并将其提供给弹性搜索bulk_request。

const fsPromises = require('fs').promises, 
      files = ['filename1', 'filename1'],
      response = [];

const fetchFile = async (filename) => {
      return new Promise((resolve, reject) => {
       const path = path.join(__dirname, filename);
       try {
       const data = await fsPromises.readFile(path)); // make sure path is correct
       resolve(data);
       } catch (e) {
         reject(e)
       }

    });

files.forEach((fileName) => results.push(fetchFile()));

Promise.all(results).then(data => console.log(data)).catch(e => console.log(e));

}

从所有 promise 中获取数据后，将其传递给弹性搜索。

如何使用 fs 和批量请求读取多个 json 文件

How to read multiple json file using fs and bulk request

javascript

node.js

fs

bulk