检查所有要完成的请求

Question

我目前正在开发一个异步获取多个 URI 的抓取工具。我的问题是，我想等待处理所有响应，但我不知道如何找出每个响应的处理时间。

带有 "Done!" 的最后一个日志甚至在响应返回之前就已经运行。

感谢帮助泽维尔 :)

const rp = require('request-promise');
const cheerio = require('cheerio');

//The function
function getData(arr) {


    // Using loop to fetch every adress
    for (let i = 0; i < arr.length; i++) {



        //Setting my options
    let opt = {
        method: 'GET',
        //Inserting index into url to get different site every loop
        uri: `https://example.de/${i}`,
        headers: {
            Referer: 'example.de',
        },
        resolveWithFullResponse: true,
        //Loading jquery/cherrio
        transform: function (body) {
            return cheerio.load(body);
        }
    };

    //running the request
    rp(opt)
    .then(($) => {
        console.log('Success fetching Data of : ' + arr[i])
        console.log('Now formatting!')


        //formatting my data ...


    })

    .catch((err => {
        //err handling
        console.error(err.message)
        console.info(`Retrying in ${errDelay / 1000}sec`)
        setTimeout(() => {
            rp(opt)
        }, errDelay);
    }))



}

//This should run after every request is formatted but currently it runs before the requests are done BUT IT DOESN'T
console.log('Done!')


}

getData()```

Answer 1

Promise.all就是你想要的。

Promise.all(arr.map((_,i)=>{
    //Setting my options
    let opt = {
        method: 'GET',
        //Inserting index into url to get different site every loop
        uri: `https://example.de/${i}`,
        headers: {
            Referer: 'example.de',
        },
        resolveWithFullResponse: true,
        //Loading jquery/cherrio
        transform: function (body) {
            return cheerio.load(body);
        }
    };

    //running the request
    return rp(opt)
    .then(($) => {
        console.log('Success fetching Data of : ' + arr[i])
        console.log('Now formatting!')


        //formatting my data ...

    })
    .catch((err => {
        //err handling
        console.error(err.message)
        console.info(`Retrying in ${errDelay / 1000}sec`)
        setTimeout(() => {
            rp(opt)
        }, errDelay);
    }))
}).then(()=> console.log('done'))

Answer 2

您可以使用 async await 和 try catch 块来处理错误。

MDN documentation on async await

async function getData(arr) {
  for (let i = 0; i < arr.length; i++) {
    let opt = {
      method: 'GET',
      uri: `https://example.de/${i}`,
      headers: {
        Referer: 'example.de',
      },
      resolveWithFullResponse: true,
      transform: function (body) {
        return cheerio.load(body);
      }
    };

    try {
      const response = await rp(opt);
      console.log('Success fetching Data of : ' + arr[i])
      console.log('Now formatting!')
    }
    catch (e) {
      console.error(err.message)
      console.info(`Retrying in ${errDelay / 1000}sec`)
      setTimeout(() => {
      rp(opt)
      }, errDelay);
    }
  }
}

检查所有要完成的请求

Check for all requests to be done

javascript

node.js

asynchronous

request-promise