Got Error: Stream yields empty buffer when resizing images from S3 with local node service
Got Error: Stream yields empty buffer when resizing images from S3 with local node service
我有一个非常相似的方法:Resizing image with nodeJs and AWS. And I still get the same error when I run my service locally on Windows 10. It works on lambda cloud flawless. I found this discussion about how limited memory or limited timeout could throw an stderr
in gm
: Stream yields empty buffer error when processing large image files using gm。此外,这个讨论还提到了如何
有限的内存将使您的 lambda 服务易受攻击且不健壮:。但是,我 运行 在本地使用大量空闲内存连接此节点服务,我不应该一直 运行 进入此 "limited memory" 陷阱。我认为这个问题来自 gm
。这是我的代码:
async.forEachOf(_sizesArray, function(value, key, callback) {
async.waterfall([
function download(next) {
s3.getObject({
Bucket: srcBucket,
Key: srcKey
}, next);
},
function convert(response, next) {
console.log(response.Body);
gm(response.Body,srcKey).antialias(true).density(
300).toBuffer('JPG', function(err,
buffer) {
if (err) {
console.log(err + "\n\nfrom convert\n");
next(err);
} else {
next(null, buffer);
}
});
},
function process(response, next) {
gm(response).size(function(err, size) {
var scalingFactor = Math.min(
_sizesArray[key].width /
size.width, _sizesArray[
key].width / size.height
);
var width = scalingFactor *
size.width;
var height = scalingFactor *
size.height;
var index = key;
this.resize(width, height).toBuffer(
'JPG', function(err,
buffer) {
if (err) {
console.log(err+"\n\nfrom process\n");
next(err);
} else {
next(null,buffer,key);
}
});
});
},
function upload(data, index, next) {
s3.putObject({
Bucket: dstBucket,
Key: myPath + "/" + fileName.slice(0, -4) +
_sizesArray[index].suffix +
".jpg",
Body: data,
ContentType: 'JPG'
}, next);
}
], function(err, result) {
if (err) {
console.error(err);
}
console.log("End of step " + key);
callback();
});
}, function(err) {
if (err) {
console.error('Unable to resize ' + srcBucket +
'/' + srcKey + ' and upload to ' + dstBucket +
myPath + '/' + ' due to an error: ' + err);
} else {
console.log('Successfully resized ' + srcBucket +
' and uploaded to ' + dstBucket + '/' + myPath + '/');
}
cb(myPath+"/"+fileName);
});
更具体地说,我的async.waterfall()
函数将在执行函数convert(res,next)
时退出,而toBuffer()
将抛出我在问题标题中提到的错误。
所以我的原始代码适用于 lambda。我无法在我的本地节点上运行它的原因是我没有正确安装 gm
。(缺少 convert
二进制文件)为了解决这个问题,我重新安装了 gm
个来自 GraphicsMagick Download 的二进制文件并将其更新到最新版本。在安装向导中,检查类似 "associate related files" 的内容。然后直接导入为:
var gm = require('gm');
而不是:
var gm = require('gm').subClass({
imageMagick: true
});
我有一个非常相似的方法:Resizing image with nodeJs and AWS. And I still get the same error when I run my service locally on Windows 10. It works on lambda cloud flawless. I found this discussion about how limited memory or limited timeout could throw an stderr
in gm
: Stream yields empty buffer error when processing large image files using gm。此外,这个讨论还提到了如何
有限的内存将使您的 lambda 服务易受攻击且不健壮:gm
。这是我的代码:
async.forEachOf(_sizesArray, function(value, key, callback) {
async.waterfall([
function download(next) {
s3.getObject({
Bucket: srcBucket,
Key: srcKey
}, next);
},
function convert(response, next) {
console.log(response.Body);
gm(response.Body,srcKey).antialias(true).density(
300).toBuffer('JPG', function(err,
buffer) {
if (err) {
console.log(err + "\n\nfrom convert\n");
next(err);
} else {
next(null, buffer);
}
});
},
function process(response, next) {
gm(response).size(function(err, size) {
var scalingFactor = Math.min(
_sizesArray[key].width /
size.width, _sizesArray[
key].width / size.height
);
var width = scalingFactor *
size.width;
var height = scalingFactor *
size.height;
var index = key;
this.resize(width, height).toBuffer(
'JPG', function(err,
buffer) {
if (err) {
console.log(err+"\n\nfrom process\n");
next(err);
} else {
next(null,buffer,key);
}
});
});
},
function upload(data, index, next) {
s3.putObject({
Bucket: dstBucket,
Key: myPath + "/" + fileName.slice(0, -4) +
_sizesArray[index].suffix +
".jpg",
Body: data,
ContentType: 'JPG'
}, next);
}
], function(err, result) {
if (err) {
console.error(err);
}
console.log("End of step " + key);
callback();
});
}, function(err) {
if (err) {
console.error('Unable to resize ' + srcBucket +
'/' + srcKey + ' and upload to ' + dstBucket +
myPath + '/' + ' due to an error: ' + err);
} else {
console.log('Successfully resized ' + srcBucket +
' and uploaded to ' + dstBucket + '/' + myPath + '/');
}
cb(myPath+"/"+fileName);
});
更具体地说,我的async.waterfall()
函数将在执行函数convert(res,next)
时退出,而toBuffer()
将抛出我在问题标题中提到的错误。
所以我的原始代码适用于 lambda。我无法在我的本地节点上运行它的原因是我没有正确安装 gm
。(缺少 convert
二进制文件)为了解决这个问题,我重新安装了 gm
个来自 GraphicsMagick Download 的二进制文件并将其更新到最新版本。在安装向导中,检查类似 "associate related files" 的内容。然后直接导入为:
var gm = require('gm');
而不是:
var gm = require('gm').subClass({
imageMagick: true
});