回调函数 Node.js

Question

我最近正在研究 nodejs（并使用 nightmare.js）来解析网站，但在回调函数和显示 returned 结果方面遇到了问题。我试图在另一个函数中调用一个单独的函数，但似乎 return 没有任何结果。它们都 return 未定义。非常感谢对此的任何帮助。

 function getDetails(loadURL, callback){
    nightmare.goto(loadURL)
        .wait(2000)
        .evaluate(function(){
            var gigs = [];
            $('.hidden-xs .used-vehicle').each(function(){
                item = {}
                item["year"] = $(this).attr('data-year')
                item["make"] = $(this).attr('data-make')
                item["model"] = $(this).attr('data-model')
                item["body"] = $(this).attr('data-body')
                item["color"] = $(this).attr('data-ext-color')
                item["trim"] = $(this).attr('data-trim')
                item["mileage"] = $(this).attr('data-mileage')
                item["transmission"] = $(this).attr('data-transmission')
                item["vin"] = $(this).find(".vehicle-overview").attr('id')
                item["title"] = $(this).find(".vehicle-overview h2 a").text()
                item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
                item["price"] = $(this).find(".vehicle-content .price").text()
                gigs.push(item)
            })
            return gigs
        })
        .end()
        .then(function(result){
            var returnString = '';
            for(gig in result){
                returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
            }
            callback(returnString)
        })  
}

    // We will need to get the total amount of pages that we need to parse
    function getInventory(sURL, callback){
        nightmare.goto(sURL)
            .wait(2000)
            .evaluate(function(){
                totals = [];
                items = {}
                totalCars = $('.total-found .count').text()
                carsOnPage = $('.hidden-xs .used-vehicle').size()
                items['carTotal'] = totalCars
                items['onPage'] = carsOnPage
                var pageCalc = (totalCars / carsOnPage)
                items['tPages'] = Math.ceil(pageCalc)
                totals.push(items)
                return totals
            })
            .end()
            .then(function(result){
                var totalCars = '';
                var totalPages = '';
                for (item in result){
                    totalPages = result[item].tPages
                    totalCars = result[item].carTotal               
                }
                counter = 0;
                newURL = '';
                returnDetails = '';
                for (i =0; i < totalPages; i++){
                    if (i == 0){
                        newURL = sURL;
                    } else {
                        counter = i + 1;
                        newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
                    }
                    //console.log(newURL)
                    getINV = getDetails(newURL, function(returnString){
                        callback(returnString)
                    })
                    returnDetails = returnDetails + getINV
                }
                callback(returnDetails)
            })
    }

    getInventory(startURL, function(result){
        console.log(result)
    })

Answer 1

尝试使用回调而不是 return

function getDetails(loadURL, callback){
    nightmare.goto(loadURL)
        .wait(2000)
        .evaluate(function(callback){
            var gigs = [];
            $('.hidden-xs .used-vehicle').each(function(){
                item = {}
                item["year"] = $(this).attr('data-year')
                item["make"] = $(this).attr('data-make')
                item["model"] = $(this).attr('data-model')
                item["body"] = $(this).attr('data-body')
                item["color"] = $(this).attr('data-ext-color')
                item["trim"] = $(this).attr('data-trim')
                item["mileage"] = $(this).attr('data-mileage')
                item["transmission"] = $(this).attr('data-transmission')
                item["vin"] = $(this).find(".vehicle-overview").attr('id')
                item["title"] = $(this).find(".vehicle-overview h2 a").text()
                item["link"] = $(this).find(".vehicle-overview h2 a").attr('href')
                item["price"] = $(this).find(".vehicle-content .price").text()
                gigs.push(item)
            })
            callback(gigs)
        })
        .end()
        .then(function(result){
            var returnString = '';
            for(gig in result){
                returnString = returnString + result[gig].title + " " + result[gig].link + " " + result[gig].year + " " + result[gig].make + " " + result[gig].model + " " + result[gig].body + " " + result[gig].color + " " + result[gig].trim + " " + result[gig].transmission + " " + result[gig].vin + " " + result[gig].price + "\n"
            }
            callback(returnString)
        })  
}

    // We will need to get the total amount of pages that we need to parse
    function getInventory(sURL, callback){
        nightmare.goto(sURL)
            .wait(2000)
            .evaluate(function(){
                totals = [];
                items = {}
                totalCars = $('.total-found .count').text()
                carsOnPage = $('.hidden-xs .used-vehicle').size()
                items['carTotal'] = totalCars
                items['onPage'] = carsOnPage
                var pageCalc = (totalCars / carsOnPage)
                items['tPages'] = Math.ceil(pageCalc)
                totals.push(items)
                return totals
            })
            .end()
            .then(function(result){
                var totalCars = '';
                var totalPages = '';
                for (item in result){
                    totalPages = result[item].tPages
                    totalCars = result[item].carTotal               
                }
                counter = 0;
                newURL = '';
                returnDetails = '';
                for (i =0; i < totalPages; i++){
                    if (i == 0){
                        newURL = sURL;
                    } else {
                        counter = i + 1;
                        newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
                    }
                    //console.log(newURL)
                    getINV = getDetails(newURL, function(returnString){
                        callback(returnString)
                    })
                    returnDetails = returnDetails + getINV
                }
                callback(returnDetails)
            })
    }

    getInventory(startURL, function(result){
        console.log(result)
    })

Answer 2

也许以下会起作用，将循环更改为 reduce 和 map，取出 jQuery 并进行一些小改动。

最重要的是：

从 html 元素获取属性 return 是一个字符串，您应该将其转换为数字。
摆脱了回调并拥有 return promises 的功能。

代码如下：

const getDetails = loadURL =>
nightmare.goto(loadURL)//return promise here
.wait(2000)
.evaluate(
  ()=>
    Array.from(document.querySelectorAll('.hidden-xs .used-vehicle'))
    .reduce(
      (all,item)=>
        all.concat(
          [
            element.getAttribute('data-year'),
            element.getAttribute('data-make'),
            element.getAttribute('data-model'),
            element.getAttribute('data-body'),
            element.getAttribute('data-ext-color'),
            element.getAttribute('data-trim'),
            element.getAttribute('data-mileage'),
            element.getAttribute('data-transmission'),
            element.querySelector(".vehicle-overview").getAttribute('id'),
            element.querySelector(".vehicle-overview h2 a").innerText,
            element.querySelector(".vehicle-overview h2 a").getAttribute('href'),
            element.querySelector(".vehicle-content .price").innerText
          ].join(" ")
        ),
      []//the all array
    )
);

// We will need to get the total amount of pages that we need to parse
const getInventory = sURL =>
nightmare.goto(sURL)
  .wait(2000)
  .evaluate(
    ()=> {
      //there is only one item here, not sure why you push it into totals
      //  and call it items
      const item = {}
      //getAttribute returns a string, parse it to number
      totalCars = parseInt(document.querySelector('.total-found .count').innerText,10);
      carsOnPage = document.querySelectorAll('.hidden-xs .used-vehicle').length;
      item['carTotal'] = totalCars
      item['onPage'] = carsOnPage
      var pageCalc = (totalCars / carsOnPage)
      item['tPages'] = Math.ceil(pageCalc)
      return item;  
    }
  )
  .then(
    totalItem =>{
      var totalCars = '';
      var totalPages = '';
      totalPages = totalItem.tPages
      totalCars = totalItem.carTotal
      newURL = '';
      returnDetails = '';
      return Array.from(new Array(totalPages),(_,index)=>index+1)
      .reduce(
        (p,counter)=>
          p.then(
            results=>{
              if (counter === 1) {
                newURL = sURL;
              } else {
                newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
              }
              return getDetails(newURL)
              .then(
                result=>results.concat(result)
              );
            }
          ),
          Promise.resolve([])
      );
    }
  );

getInventory(startURL)
.then(
result=>
  console.log(result)
).catch(
err=>
  console.warn("Something went wrong:",err)
);

Answer 3

我不会费心告诉你你不应该将回调与这样的承诺混合在一起。但让我们现在看看问题所在。

案例一

你也检查错误怎么样？也许您的脚本正在抛出错误。我可以看到您在 .then 上调用回调，但在 .catch 上没有调用任何回调。也许那时永远不会得到任何数据。

案例二

让我们检查一下您的功能。您每次都在呼叫 .end。你也每次都创建新的噩梦实例吗？

在getInventory函数上，你不应该调用.end。在 getDetails 函数上，您不应调用 .end。它结束了噩梦实例，你正在丢失你的数据。

完成所有功能和工作后，请致电 nightmare.end()。要正确执行此操作，您需要了解更多有关下面的 Promises 检查案例 3 的信息。

案例三

了解承诺的工作原理。在下面的行中，您永远不会等待函数完成。

getINV = getDetails(newURL, function(returnString){
 callback(returnString)
})

您应该等待承诺完成。另外，确保 nightmare 没有尝试同时浏览两个链接。

所以继续学习 Promises 和异步等待的东西。

我将如何解决您的代码？

我会使用 Promise.all、.map 和其他一些新东西。以下是为您完成的一些示例代码，请勿直接复制粘贴或运行代码，请尝试理解它与您的代码不同的原因以及它可能产生的结果。

const pLimit = require("promise-limit")(2);

function getDetails(loadURL) {
  return nightmare
    .goto(loadURL)
    .wait(2000)
    .evaluate(() => {
      const gigs = [];
      $(".hidden-xs .used-vehicle").each(function() {
        item = {};
        item["year"] = $(this).attr("data-year");
        item["make"] = $(this).attr("data-make");
        item["model"] = $(this).attr("data-model");
        item["body"] = $(this).attr("data-body");
        item["color"] = $(this).attr("data-ext-color");
        item["trim"] = $(this).attr("data-trim");
        item["mileage"] = $(this).attr("data-mileage");
        item["transmission"] = $(this).attr("data-transmission");
        item["vin"] = $(this)
          .find(".vehicle-overview")
          .attr("id");
        item["title"] = $(this)
          .find(".vehicle-overview h2 a")
          .text();
        item["link"] = $(this)
          .find(".vehicle-overview h2 a")
          .attr("href");
        item["price"] = $(this)
          .find(".vehicle-content .price")
          .text();
        gigs.push(item);
      });
      return gigs;
    })
    .then(result => {
      let returnString = "";
      for (gig in result) {
        returnString =
          `${returnString +
result[gig].title} ${result[gig].link} ${result[gig].year} ${result[gig].make} ${result[gig].model} ${result[gig].body} ${result[gig].color} ${result[gig].trim} ${result[gig].transmission} ${result[gig].vin} ${result[gig].price}\n`;
      }
      return returnString;
    })
    .catch(error => {
      throw new Error(error);
    });
}

// We will need to get the total amount of pages that we need to parse
function getInventory(sURL) {
  return nightmare
    .goto(sURL)
    .wait(2000)
    .evaluate(() => {
      totals = [];
      items = {};
      totalCars = $(".total-found .count").text();
      carsOnPage = $(".hidden-xs .used-vehicle").size();
      items["carTotal"] = totalCars;
      items["onPage"] = carsOnPage;
      const pageCalc = totalCars / carsOnPage;
      items["tPages"] = Math.ceil(pageCalc);
      totals.push(items);
      return totals;
    })
    .then(result => {
      let totalCars = "";
      let totalPages = "";
      for (item in result) {
        totalPages = result[item].tPages;
        totalCars = result[item].carTotal;
      }
      counter = 0;
      newURL = "";
      urls = [];
      returnDetails = [];
      for (i = 0; i < totalPages; i++) {
        if (i == 0) {
          newURL = sURL;
        } else {
          counter = i + 1;
          newURL =
            `${sURL}#action=im_ajax_call&perform=get_results&_post_id=5&page=${counter}&show_all_filters=false`;
        }
        // push to the url array
        // use .map for cleaner code
        urls.push(newURL);
      }
      // return a new promise with concurrency limit
      return Promise.all(
        urls.map(url => {
          return limit(() => getDetails(newURL));
        })
      );
    })
    .catch(error => {
      throw new Error(error);
    });
}

getInventory(startURL)
  .then(result => {
    console.log(result);
  })
  .catch(error => {
    console.err(error);
  });

资源：

Answer 4

学习异步函数的承诺和回调是一项艰巨的任务。我能够让它与以下一起工作。谢谢大家的帮助和指导。每个答案都让我陷入了一个不同的兔子洞，最终找到了解决方案。

    function getInventory(sURL){    
    nightmare.goto(sURL)
        .wait(2000)
        .evaluate(function(){
            totals = [];
            items = {}
            totalCars = $('.total-found .count').text()
            carsOnPage = $('.hidden-xs .used-vehicle').size()
            items['carTotal'] = totalCars
            items['onPage'] = carsOnPage
            var pageCalc = (totalCars / carsOnPage)
            items['tPages'] = Math.ceil(pageCalc)
            totals.push(items)
            return totals
        })
        .then(result => {
            var totalCars = '';
            var totalPages = '';
            for (item in result){
                totalPages = result[item].tPages
                totalCars = result[item].carTotal               
            }
            counter = 0;
            let links = [];
            let returnLinks = '';
            newURL = '';            
            for (i = 0; i < totalPages; i++){
                if (i == 0){
                    newURL = sURL;
                } else {
                    counter = i + 1;
                    newURL = sURL + "#action=im_ajax_call&perform=get_results&_post_id=5&page=" + counter + "&show_all_filters=false";
                }
                links.push(newURL);
            }
            return links;
        })
        .then(results => {
            var arrayLinks = results;
            arrayLinks.reduce(function(accumulator, url){
                return accumulator.then(function(newResults){
                    return nightmare.goto(url)
                        .wait(5000)
                        .evaluate(() => {
                            const gigs = [];
                            $(".hidden-xs .used-vehicle").each(function() {
                                item = {};
                                item["year"] = $(this).attr("data-year");
                                item["make"] = $(this).attr("data-make");
                                item["model"] = $(this).attr("data-model");
                                item["body"] = $(this).attr("data-body");
                                item["color"] = $(this).attr("data-ext-color");
                                item["trim"] = $(this).attr("data-trim");
                                item["mileage"] = $(this).attr("data-mileage");
                                item["transmission"] = $(this).attr("data-transmission");
                                item["vin"] = $(this).find(".vehicle-overview").attr("id");
                                item["title"] = $(this).find(".vehicle-overview h2 a").text();
                                item["link"] = $(this).find(".vehicle-overview h2 a").attr("href");
                                item["price"] = $(this).find(".vehicle-content .price").text();
                                gigs.push(item);
                            });
                            return gigs;
                        })
                        .then(detail => {                       
                            for (gig in detail) {
                                try {
                                    var carVin = detail[gig].vin;
                                    var carTitle = detail[gig].title;
                                    var carDescrip = detail[gig].year + " " + detail[gig].make + " " + detail[gig].model;
                                    var carURL = detail[gig].link;
                                    var carMake = detail[gig].make;
                                    var carModel = detail[gig].model;
                                    var carYear = detail[gig].year;
                                    var carMileageFull = detail[gig].mileage;
                                    var carMileage = carMileageFull.replace(',', '');
                                    var carTransmission = detail[gig].transmission;
                                    var carBody = detail[gig].body;
                                    var carPriceFull = detail[gig].price;
                                    var carPriceFull = carPriceFull.replace('$', '');               
                                    var carPriceFull = carPriceFull.replace('*', '');               
                                    var carPriceFull = carPriceFull.replace(',', '');               
                                    var carPrice = carPriceFull.trim();
                                    var dealerAddress = "{addr1: '"+ addressFull.addr1 +"', city: '"+ addressFull.city +"', region: '"+ addressFull.region +"', postal_code: '"+ addressFull.postal_code +"', country: '"+ addressFull.country +"'}";
                                    var dealerLat = latLongFull.latitude;
                                    var dealerLong = latLongFull.longitude;
                                    var carColor = detail[gig].color;

                                    arrSetup = [carVin, carTitle, carDescrip, carURL, carMake, carModel, carYear, carMileage, 'MI', '', '', 'AUTOMATIC', 'GASOLINE', 'OTHER', 'Other', carVin, 'OTHER', carPrice + " USD", dealerAddress, carColor, carPrice + " USD", 'AVAILABLE', 'USED', dealerLat, dealerLong];
                                    newResults.push(arrSetup);
                                }
                                catch(error){
                                    returnString += error;
                                }
                            }
                            return newResults;

                        })
                        .catch(error => {
                            throw new Error(error);
                        });

                });             
            }, Promise.resolve([]))
                .then(function(finalCall){

                    /*
                        We need to get the 3rd image on every vdp in the array. We will need to create a loop, go to the page, get the image and properly insert it into the proper array index
                    */

                    finalCall.reduce(function(accumulator, resultArray){
                        return accumulator.then(function(finalResults){
                            var vdp = resultArray[3];
                            return nightmare.goto(vdp)
                                .wait(500)
                                .evaluate(() => {
                                    var thirdIMG = $('.gallery-thumbs .owl-item:nth-of-type(3) img').attr('src');
                                    return thirdIMG;
                                })
                                .then(imgResult => {
                                    // 9
                                    resultArray.splice(9, 1, imgResult);
                                    console.log(resultArray);
                                    finalResults.push(resultArray);
                                    return finalResults;
                                })
                                .catch(error => {
                                    throw new Error(error);
                                });

                        });
                    }, Promise.resolve([]))
                        .then(finalInsert => {
                            const csvWriter = createCsvWriter({
                            header: ["vehicle_id", "title", "description", "url", "make", "model", "year", "mileage.value", "mileage.unit", "image[0].url", "image[0].tag[0]", "transmission", "fuel_type", "body_style", "drivetrain", "vin", "condition", "price", "address", "exterior_color", "sale_price", "availability", "state_of_vehicle", "latitude", "longitude"],
                                path: 'test.csv'
                            });

                            var records = finalInsert;
                            console.log(records)
                            csvWriter.writeRecords(records)
                            .then(() => {
                                nightmare.end();
                                console.log('...Done');
                            });
                        })
                });
        })      
        .catch(function(error){
            return error;
        })
}

    getInventory(startURL, function(response){
        try {
            console.log("This is the response" + response);
        }
        catch(error){
            console.log(error)
        }
    });

回调函数 Node.js

Callback function Node JS

javascript

node.js

nightmare

案例一

案例二

案例三