如何在节点js中向后读取大型json文件

Question

一种从上到下逐行读取文件的简单方法：

const { once } = require('events');
const { createReadStream } = require('fs');
const { createInterface } = require('readline');

let storeProducts = async () => {
    try {
        const filename = path.join(__dirname, "bulk-101638307937.jsonl");
        const rl = createInterface({
          input: createReadStream(filename),
          crlfDelay: Infinity
        });
    
        rl.on('line', (line) => {
          // Process the line.
          console.log({line: JSON.parse(line)});
        });
    
        await once(rl, 'close');
    
        console.log('File processed.');
      } catch (err) {
        console.error(err);
    }
}

这很好用。但是我需要逐行向后读取一个大的jsonl文件。我试过。它向后读取文件但不是逐行读取。任何帮助将不胜感激。

示例 jsonl

{"id": 1, "product": "A"}
{"variant_id": 1, "product": "A"}
{"id": 2, "product": "B"}
{"variant_id": 2, "product": "B"}
{"id": 3, "product": "C"}
{"variant_id": 3, "product": "C"}
{"id": 4, "product": "D"}
{"variant_id": 4, "product": "D"}
{"variant_id": 5, "product": "D"}

Answer 1

库 fs-reverse 应该可以解决问题：

const fsR = require('fs-reverse');
const filename = './sample.jsonl';

const readStream = fsR(filename, {})

readStream.on('data', (line) => {
  if(line) { // have this check to make sure empty lines are not parsed
    console.log(JSON.parse(line))
  }
})

如何在节点js中向后读取大型json文件

How to read large jsonl file backwards in nodejs

large-files

node.js