保存到文件导致异步功能

Question

我得到了一个简单的异步函数，我从 URL 中“抓取”网站。一切正常，但现在我想将结果保存到我的 txt 文件中。

我尝试做简单的数组，在其中我能够推送每个结果也有错误；

现在我遇到了一个问题，我应该在哪里写入文件。

我试着把它放到一个单独的函数中，然后在我的异步函数中执行 await 函数，但是我总是先触发写入文件的函数。

有完整代码

const https = require("https");
const fs = require("fs");
const readline = require("readline");
const path = require("path");


let urls = [];
let results = [];

(async function readUrls() {
  const fileStream = fs.createReadStream("urls.txt");

  const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity,
  });

  for await (let line of rl) {
    urls.push(line);
  }
  for await (let url of urls) {
    https
      .get(url, (res) => {
        const {
          statusCode
        } = res;
        const contentType = res.headers["content-type"];

        let error;
        if (statusCode !== 200) {
          error = new Error("Request Failed.\n" + `Status Code: ${statusCode}`);
        }
        if (error) {
          const firstPath = url.split("/")[7];
          //there is array
          results.push(firstPath);
          //--------------
          console.error("data : " + firstPath + " - " + " nothing found");
          res.resume();
          return;
        }
        res.setEncoding("utf8");
        let rawData = "";
        res.on("data", (chunk) => {
          rawData += chunk;
        });
        (async () => {
          await res.on("end", () => {
            try {
              const parsedData = JSON.parse(rawData);
              const parsedResult = parsedData["data"]["id"] + " - " + parsedData["data"]["price"];
              //there is array
              results.push(parsedResult);
              //--------------
              console.log("data : " + parsedData["data"]["id"] + " - " + parsedData["data"]["price"]);
            } catch (e) {
              console.error(e.message);
            }
          });
        })();
      })
      .on("error", (e) => {
        console.error(`Got error: ${e.message}`);
      });
  }
})();

我有一个简单的函数可以写入文件

fs.writeFile('result.txt', results, +(new Date()), function (err) {
    if (err) {
        console.log("Error occurred", err);
    }
    console.log("File write successfull");
});

我试着做点什么

async function secondFunction(){
  await firstFunction();
  // wait for firstFunction...
};

我想达到什么目的？我想从我的文本文件中抓取每个 url 并获取 ID 和价格（这很简单 JSON 对浏览器的响应没有 html - 它有效）最后我想把所有的东西都保存到文本文件中。

Answer 1

我制作了一个使用 node-fetch 调用 url 的代码版本。我更喜欢这个，因为它类似于可以在网络上使用的东西

要使用它，您应该安装它： npm install node-fetch

    const fetch = require("node-fetch"); // I prefer to use node-fetch for my calls
    const fs = require("fs");
    const readline = require("readline");
    const path = require("path");


    let urls = [];
    let results = [];

    (async function readUrls() {
      const fileStream = fs.createReadStream("urls.txt");

      const rl = readline.createInterface({
        input: fileStream,
        crlfDelay: Infinity,
      });

      for await (let line of rl) {
        urls.push(line);
      }
      // Make the calls one after the other
      for (let url of urls) {
        try {
          // We can call the urls with node-fetch and await the response
          const res = await fetch(url);
          const { status } = res;
          let error;
          if (status !== 200)
            error = new Error("Request Failed.\n" + `Status Code: ${statusCode}`);
          if (error) {
            const firstPath = url.split('/')[7];
            results.push(firstPath);
            console.error("data : " + firstPath + " - " + " nothing found");
            // As we are inside a loop here, we use continue instead of return
            continue;
          }
          try {
            // Here we try to take the response as json
            const parsedData = await res.json();
            const parsedResult = parsedData["data"]["id"] + " - " + parsedData["data"]["price"];
            //there is array
            results.push(parsedResult);
            //--------------
            console.log(`Data: ${parsedResult}`);
          } catch (e) {
            // In case we can't get the response as json we log the error
            console.error(e.message);
          }
        } catch (httpError) {
          //This is for when the call to fetch fails for some reason
          console.error(httpError.message);
        }  
      }
      // Here we join the results to a string so that we can save it properly to the file
      const resultAsText = results.join("\n");
      // Then after all the urls are processed we can write them to a file
      fs.writeFile('result.txt', resultAsText, 'utf8', function (err) {
        if (err) {
          console.log("Error occurred", err);
        } else {
          console.log("File write successfull");
        }
      });
    })();

保存到文件导致异步功能

Save to file results in async function

javascript

fs

node.js

async-await