如何加快获取 javascript 数组的数据
How to speed up getting data for javascript array
我想获取20000个文件的创建日期,存入数组
完成的总时间是 35 分钟,相当长的时间。 (Image Processing Time)
有没有办法以更快的处理时间创建数组?
当前获取文件创建日期数组的逻辑是否有问题?
①‖数组声明:var arr = [];
②‖我使用下面的代码获取文件创建日期:
var fs = global.get('fs');
// Get file creation date time
msg.createdDate = fs.statSync(msg.pathFName).birthtime;
return msg;
③‖将创建日期存储在数组中。
我的流程:
Image Flow
[
{
"id": "de157360.11d49",
"type": "tab",
"label": "フロー 1",
"disabled": false,
"info": ""
},
{
"id": "6a8db36.f55e14c",
"type": "inject",
"z": "de157360.11d49",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "str",
"x": 750,
"y": 160,
"wires": [
[
"4907d69b.07f738"
]
]
},
{
"id": "4907d69b.07f738",
"type": "change",
"z": "de157360.11d49",
"name": "",
"rules": [
{
"t": "set",
"p": "pathFiles",
"pt": "msg",
"to": "D:\Data\",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 910,
"y": 160,
"wires": [
[
"d68da128.cd6d9"
]
]
},
{
"id": "d68da128.cd6d9",
"type": "fs-ops-dir",
"z": "de157360.11d49",
"name": "",
"path": "pathFiles",
"pathType": "msg",
"filter": "*",
"filterType": "str",
"dir": "files",
"dirType": "msg",
"x": 1080,
"y": 160,
"wires": [
[
"764296f2.688338"
]
]
},
{
"id": "764296f2.688338",
"type": "change",
"z": "de157360.11d49",
"name": "f=0",
"rules": [
{
"t": "set",
"p": "f",
"pt": "msg",
"to": "0",
"tot": "num"
},
{
"t": "set",
"p": "arrObjFiles",
"pt": "msg",
"to": "[]",
"tot": "json"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1210,
"y": 160,
"wires": [
[
"ab664988.c25da8",
"232fdda0.b11e22"
]
]
},
{
"id": "1912092c.fbab77",
"type": "change",
"z": "de157360.11d49",
"name": "f++",
"rules": [
{
"t": "set",
"p": "f",
"pt": "msg",
"to": "$.f + 1\t",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1650,
"y": 60,
"wires": [
[
"ab664988.c25da8"
]
]
},
{
"id": "ab664988.c25da8",
"type": "switch",
"z": "de157360.11d49",
"name": "u<number of files",
"property": "f",
"propertyType": "msg",
"rules": [
{
"t": "lt",
"v": "files.length",
"vt": "msg"
},
{
"t": "else"
}
],
"checkall": "true",
"repair": false,
"outputs": 2,
"x": 1410,
"y": 160,
"wires": [
[
"15154524.a6828b"
],
[
"c7625457.8ccb18"
]
]
},
{
"id": "f8030294.f6334",
"type": "function",
"z": "de157360.11d49",
"name": "Get file creation date time",
"func": "var fs = global.get('fs');\n// Get file creation date time\nmsg.createdDate = fs.statSync(msg.pathFName).birthtime;\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"x": 1830,
"y": 160,
"wires": [
[
"59879c16.6c7564"
]
]
},
{
"id": "59879c16.6c7564",
"type": "change",
"z": "de157360.11d49",
"name": "Storage file creation date time",
"rules": [
{
"t": "set",
"p": "objFiles",
"pt": "msg",
"to": "{}",
"tot": "json"
},
{
"t": "set",
"p": "objFiles",
"pt": "msg",
"to": "$merge([$.objFiles,{'fileName': $.fileName,'createdDate': $.createdDate}])",
"tot": "jsonata"
},
{
"t": "set",
"p": "arrObjFiles",
"pt": "msg",
"to": "$append(arrObjFiles, [objFiles])",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 2090,
"y": 160,
"wires": [
[
"1912092c.fbab77"
]
]
},
{
"id": "15154524.a6828b",
"type": "change",
"z": "de157360.11d49",
"name": "",
"rules": [
{
"t": "set",
"p": "fileName",
"pt": "msg",
"to": "$.files[$$.f]",
"tot": "jsonata"
},
{
"t": "set",
"p": "pathFName",
"pt": "msg",
"to": "pathFiles & fileName",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1620,
"y": 160,
"wires": [
[
"f8030294.f6334"
]
]
},
{
"id": "c7625457.8ccb18",
"type": "debug",
"z": "de157360.11d49",
"name": "END",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "arrObjFiles",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 1590,
"y": 220,
"wires": []
},
{
"id": "232fdda0.b11e22",
"type": "debug",
"z": "de157360.11d49",
"name": "START",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "files.length",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 1380,
"y": 220,
"wires": []
}
]
您正在使用 fs.statSync
这是一个同步函数,这意味着每次调用它时,所有代码的执行都会停止,直到该函数完成。考虑使用 fs.stat
(异步版本),映射文件路径数组,并使用 Promise.all.
使用fs.stat
(异步版本)功能,您可以一次启动多个文件的调用,这样整体上发生得更快(因为可以一次加载多个文件而无需等待超级慢的)
这是我的意思的示例,您可以在浏览器中 运行:
const arr = ["hello", "i", "am", "here"];
// This is an example of a function that returns a promise.
// This function returns a promise that resolves after `seconds` amount of seconds, returning the value of `word` to uppercase
const sleepAndUppercase = (seconds, word) => new Promise(
(resolve) => setTimeout(
() => resolve(word.toUpperCase()), seconds*1000
)
);
// This wraps the array of promises
Promise.all(
// this converts the array of words into an array of promises generated by the sleepAndUppercase function defined above
arr.map((word, i) => sleepAndUppercase(i, word))
// the next line waits until all the promies are done, and uses the result as the variable listOfWords,
// which is an array of the words capitalized
).then((listOfWords) => {
// we can log the results
console.log(listOfWords);
// we can create variables based off of those results
const sentence = listOfWords.join(" ");
console.log(sentence);
});
console.log("Promises started")
现在让它更接近你的问题:
在这个例子中,每次调用 fs.stat
需要 5 秒来解决,但所有 4 个结果一起出来的速度比 5 * 4 秒快得多,因为多个调用一次开始。
// This object is just here so that this snippet makes sense on Whosebug.
// Do not copy this object into your code.
const fs = {
stat: (pathFName) => {
const birthtime = Date.now() - Math.floor(Math.random() * 100000000);
return new Promise((resolve, reject) => {
setTimeout(() => resolve({birthtime}), 5000)
})
}
}
// start timer
let timer = 0;
const timerInterval = setInterval(() => document.querySelector("#ptimer").innerText = `${timer += 0.02}s`, 20);
const msgs = [
{pathFName: "foo.txt"},
{pathFName: "bar.png"},
{pathFName: "baz.html"},
{pathFName: "idk.jpg"},
{pathFName: "pepe.sql"},
];
(async () => {
const msgsWithTimeStamps = await Promise.all(
msgs.map(async (msg) => {
const {birthtime} = await fs.stat(msg.pathFName);
return {...msg, birthtime}
})
)
console.log(msgsWithTimeStamps);
// stop timer
clearInterval(timerInterval);
})()
<p id="ptimer">0s</p>
我想获取20000个文件的创建日期,存入数组
完成的总时间是 35 分钟,相当长的时间。 (Image Processing Time)
有没有办法以更快的处理时间创建数组?
当前获取文件创建日期数组的逻辑是否有问题?
①‖数组声明:var arr = [];
②‖我使用下面的代码获取文件创建日期:
var fs = global.get('fs');
// Get file creation date time
msg.createdDate = fs.statSync(msg.pathFName).birthtime;
return msg;
③‖将创建日期存储在数组中。
我的流程:
Image Flow
[
{
"id": "de157360.11d49",
"type": "tab",
"label": "フロー 1",
"disabled": false,
"info": ""
},
{
"id": "6a8db36.f55e14c",
"type": "inject",
"z": "de157360.11d49",
"name": "",
"props": [
{
"p": "payload"
},
{
"p": "topic",
"vt": "str"
}
],
"repeat": "",
"crontab": "",
"once": false,
"onceDelay": 0.1,
"topic": "",
"payload": "",
"payloadType": "str",
"x": 750,
"y": 160,
"wires": [
[
"4907d69b.07f738"
]
]
},
{
"id": "4907d69b.07f738",
"type": "change",
"z": "de157360.11d49",
"name": "",
"rules": [
{
"t": "set",
"p": "pathFiles",
"pt": "msg",
"to": "D:\Data\",
"tot": "str"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 910,
"y": 160,
"wires": [
[
"d68da128.cd6d9"
]
]
},
{
"id": "d68da128.cd6d9",
"type": "fs-ops-dir",
"z": "de157360.11d49",
"name": "",
"path": "pathFiles",
"pathType": "msg",
"filter": "*",
"filterType": "str",
"dir": "files",
"dirType": "msg",
"x": 1080,
"y": 160,
"wires": [
[
"764296f2.688338"
]
]
},
{
"id": "764296f2.688338",
"type": "change",
"z": "de157360.11d49",
"name": "f=0",
"rules": [
{
"t": "set",
"p": "f",
"pt": "msg",
"to": "0",
"tot": "num"
},
{
"t": "set",
"p": "arrObjFiles",
"pt": "msg",
"to": "[]",
"tot": "json"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1210,
"y": 160,
"wires": [
[
"ab664988.c25da8",
"232fdda0.b11e22"
]
]
},
{
"id": "1912092c.fbab77",
"type": "change",
"z": "de157360.11d49",
"name": "f++",
"rules": [
{
"t": "set",
"p": "f",
"pt": "msg",
"to": "$.f + 1\t",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1650,
"y": 60,
"wires": [
[
"ab664988.c25da8"
]
]
},
{
"id": "ab664988.c25da8",
"type": "switch",
"z": "de157360.11d49",
"name": "u<number of files",
"property": "f",
"propertyType": "msg",
"rules": [
{
"t": "lt",
"v": "files.length",
"vt": "msg"
},
{
"t": "else"
}
],
"checkall": "true",
"repair": false,
"outputs": 2,
"x": 1410,
"y": 160,
"wires": [
[
"15154524.a6828b"
],
[
"c7625457.8ccb18"
]
]
},
{
"id": "f8030294.f6334",
"type": "function",
"z": "de157360.11d49",
"name": "Get file creation date time",
"func": "var fs = global.get('fs');\n// Get file creation date time\nmsg.createdDate = fs.statSync(msg.pathFName).birthtime;\nreturn msg;",
"outputs": 1,
"noerr": 0,
"initialize": "",
"finalize": "",
"x": 1830,
"y": 160,
"wires": [
[
"59879c16.6c7564"
]
]
},
{
"id": "59879c16.6c7564",
"type": "change",
"z": "de157360.11d49",
"name": "Storage file creation date time",
"rules": [
{
"t": "set",
"p": "objFiles",
"pt": "msg",
"to": "{}",
"tot": "json"
},
{
"t": "set",
"p": "objFiles",
"pt": "msg",
"to": "$merge([$.objFiles,{'fileName': $.fileName,'createdDate': $.createdDate}])",
"tot": "jsonata"
},
{
"t": "set",
"p": "arrObjFiles",
"pt": "msg",
"to": "$append(arrObjFiles, [objFiles])",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 2090,
"y": 160,
"wires": [
[
"1912092c.fbab77"
]
]
},
{
"id": "15154524.a6828b",
"type": "change",
"z": "de157360.11d49",
"name": "",
"rules": [
{
"t": "set",
"p": "fileName",
"pt": "msg",
"to": "$.files[$$.f]",
"tot": "jsonata"
},
{
"t": "set",
"p": "pathFName",
"pt": "msg",
"to": "pathFiles & fileName",
"tot": "jsonata"
}
],
"action": "",
"property": "",
"from": "",
"to": "",
"reg": false,
"x": 1620,
"y": 160,
"wires": [
[
"f8030294.f6334"
]
]
},
{
"id": "c7625457.8ccb18",
"type": "debug",
"z": "de157360.11d49",
"name": "END",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "arrObjFiles",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 1590,
"y": 220,
"wires": []
},
{
"id": "232fdda0.b11e22",
"type": "debug",
"z": "de157360.11d49",
"name": "START",
"active": true,
"tosidebar": true,
"console": false,
"tostatus": false,
"complete": "files.length",
"targetType": "msg",
"statusVal": "",
"statusType": "auto",
"x": 1380,
"y": 220,
"wires": []
}
]
您正在使用 fs.statSync
这是一个同步函数,这意味着每次调用它时,所有代码的执行都会停止,直到该函数完成。考虑使用 fs.stat
(异步版本),映射文件路径数组,并使用 Promise.all.
使用fs.stat
(异步版本)功能,您可以一次启动多个文件的调用,这样整体上发生得更快(因为可以一次加载多个文件而无需等待超级慢的)
这是我的意思的示例,您可以在浏览器中 运行:
const arr = ["hello", "i", "am", "here"];
// This is an example of a function that returns a promise.
// This function returns a promise that resolves after `seconds` amount of seconds, returning the value of `word` to uppercase
const sleepAndUppercase = (seconds, word) => new Promise(
(resolve) => setTimeout(
() => resolve(word.toUpperCase()), seconds*1000
)
);
// This wraps the array of promises
Promise.all(
// this converts the array of words into an array of promises generated by the sleepAndUppercase function defined above
arr.map((word, i) => sleepAndUppercase(i, word))
// the next line waits until all the promies are done, and uses the result as the variable listOfWords,
// which is an array of the words capitalized
).then((listOfWords) => {
// we can log the results
console.log(listOfWords);
// we can create variables based off of those results
const sentence = listOfWords.join(" ");
console.log(sentence);
});
console.log("Promises started")
现在让它更接近你的问题:
在这个例子中,每次调用 fs.stat
需要 5 秒来解决,但所有 4 个结果一起出来的速度比 5 * 4 秒快得多,因为多个调用一次开始。
// This object is just here so that this snippet makes sense on Whosebug.
// Do not copy this object into your code.
const fs = {
stat: (pathFName) => {
const birthtime = Date.now() - Math.floor(Math.random() * 100000000);
return new Promise((resolve, reject) => {
setTimeout(() => resolve({birthtime}), 5000)
})
}
}
// start timer
let timer = 0;
const timerInterval = setInterval(() => document.querySelector("#ptimer").innerText = `${timer += 0.02}s`, 20);
const msgs = [
{pathFName: "foo.txt"},
{pathFName: "bar.png"},
{pathFName: "baz.html"},
{pathFName: "idk.jpg"},
{pathFName: "pepe.sql"},
];
(async () => {
const msgsWithTimeStamps = await Promise.all(
msgs.map(async (msg) => {
const {birthtime} = await fs.stat(msg.pathFName);
return {...msg, birthtime}
})
)
console.log(msgsWithTimeStamps);
// stop timer
clearInterval(timerInterval);
})()
<p id="ptimer">0s</p>