Node.JS 同时下载数百个文件

Question

我正在尝试同时下载超过 100 个文件。但是当我执行下载功能时，我的 macbook 在 windows 中冻结（无法执行新任务）也没有下载（但不冻结）并且在两种情况下都没有下载进度（空闲网络）。

这是我的下载模块：

var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');

router.get('/', function (req, res, next) {
    links.find({dlStatus: false}, function (err, docs) {

        if (err) {
            console.log(err);
            res.end();
        } else if (!docs) {
            console.log('No incomplete downloads!');
            res.end();
        } else {
            for (var i = 0; i < docs.length; i++) {

                //todo scraping


                var video = youtubedl(docs[i].url, [], {cwd: __dirname});

                // Will be called when the download starts.
                video.on('info', function (info) {
                    console.log('Download started');
                    console.log(info);
                });

                video.pipe(fs.createWriteStream('./downloads/' + docs[i].id + '-' + i + '.mp4'));
                video.on('complete', function complete(info) {
                    links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
                        if (err)console.log(err);
                        else console.log('Download completed!')
                    });
                });
            }
        }
    });
});

module.exports = router;

现在有人可以帮我吗？我正在使用 this 模块下载文件。

Answer 1

在这种情况下，解决方案是使用 async。

以这种方式尝试......与 async.each()

var express = require('express');
var router = express.Router();
var fs = require('fs');
var youtubedl = require('youtube-dl');
var links = require('../models/Links');
var async = require('async')

router.get('/', function (req, res, next) {
    links.find({dlStatus: false}, function (err, docs) {

        if (err) {
            console.log(err);
            res.end();
        } else if (!docs) {
            console.log('No incomplete downloads!');
            res.end();
        } else {
            async.each(docs,function(doc,cb){
              var video = youtubedl(doc.url, [], {cwd: __dirname});

              // Will be called when the download starts.
              video.on('info', function (info) {
                  console.log('Download started');
                  console.log(info);
              });

              video.pipe(fs.createWriteStream('./downloads/' + docs.id + '-' + i + '.mp4'));
              video.on('complete', function complete(info) {
                  links.findOneAndUpdate({url: info.webpage_url}, {dlStatus: true}, function (err, doc) {
                      if (err){
                        console.log(err);
                        cb(err);
                      }
                      else {
                        console.log('Download completed!');
                        cb()
                      }
                  });
              });

            },function(err){
              if(err)
                return console.log(err);
              console.log("Every thing is done,Here!!");
            })
        }
    });
});

module.exports = router;

您也可以使用 async.eachLimits().

批量处理每件事

Node.JS 同时下载数百个文件

Node.JS downloading hundreds of files simultaneously

node.js

express

youtube-dl