将数据解析到数据库时连接 EADDRINUSE 18.62.228.13:443
connect EADDRINUSE 18.62.228.13:443 when parsing data to the db
我正在从网络上抓取一些数据，并希望将所有数据推送到我的数据库中。
我在 thread_workers 的帮助下用 node.js 完成了它。
每个线程都从另一个 url 抓取数据，当抓取完成后，我得到了我需要的所有对象 post。
在每个线程结束废弃数据时，我得到一个至少包含 40000 个对象的数组，
我需要 post 当我开始 post 它到数据库时，它会抛出这个错误 connect EADDRINUSE 18.62.228.13:443 完全异常：
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 168
},
method: 'post'
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: '/m/insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalalalal'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 168,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST lalalala HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 168\r\n' +
'Host: alalalala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'search.findmanual.guru',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: ,
[Symbol(kCapture)]: false
}
}
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 167
},
method: 'post',
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: 'insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalallalaa'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 167,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 167\r\n' +
'Host: lallala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'sllalallaau',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'https://lalalalal',
[Symbol(kCapture)]: false
}
}
这是我的线程函数：
async function parseData(url) {
const {data} = await axios.get(url ).catch(console.log)
const $ = cheerio.load(data);
const result = [];
const obj = {};
const header = $('h1.Uheader');
obj.brand = $(header[0]).text();
const category = $('div.cathead');
const categoryArray = [];
for (let i = 0; i < category.length; i++) {
categoryArray.push({"href": $(category[i]).children("a").attr('href'), "text":
$(category[i]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim()})
}
for (let i = 0; i < categoryArray.length; i++) {
try {
obj.category = categoryArray[i].text;
const dataTwo = await axios.get("https://www.lalalala.com" +
categoryArray[i].href).catch()
const cher = cheerio.load(dataTwo.data);
const aTag = cher('div.col-sm-2.mname')
for (let j = 0 ; j < aTag.length; j++) {
obj.url = "https://www.lalalala.com" + $(aTag[j]).children("a").attr('href');
obj.title = obj.brand + " " + categoryArray[i].text + " " + $(aTag[j]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim();
//result.push(obj)
axios.post("https://lalalalala", obj)
.then(data => console.log("ok " + j))
.catch(e => {
console.log(e)
});
}
} catch (e) {
}
}
parentPort.postMessage({message : "done"});
}
有人可以向我解释为什么会这样吗？我该如何解决？
因为（根据您的评论 ）您的应用程序正在执行大量 POST 请求，因此限制 并发 POST 可以执行的请求。否则，可能有（10 的）数千个 POST 请求同时执行，这可能会导致问题（不仅在本地，而且在请求发送到的服务器上）。
错误connect EADDRINUSE意味着本地机器上没有更多可用的 TCP 端口（到远程服务器的 TCP 连接也需要一个本地 TCP 端口，称为“临时端口”，因为它只是在请求期间使用中），因为有太多待处理的请求。
一种可能的修复方法，也是您所说的有效方法，是等待每个 POST 请求完成，然后再继续代码的其余部分：
await axios.post(…)
如果这也没有帮助，您将不得不研究可以配置为限制并发数量的作业队列 requests/promises/jobs/...)。
我正在从网络上抓取一些数据，并希望将所有数据推送到我的数据库中。
我在 thread_workers 的帮助下用 node.js 完成了它。
每个线程都从另一个 url 抓取数据，当抓取完成后，我得到了我需要的所有对象 post。
在每个线程结束废弃数据时，我得到一个至少包含 40000 个对象的数组，
我需要 post 当我开始 post 它到数据库时，它会抛出这个错误 connect EADDRINUSE 18.62.228.13:443 完全异常：
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 168
},
method: 'post'
},
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: '/m/insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalalalal'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 168,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST lalalala HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 168\r\n' +
'Host: alalalala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'search.findmanual.guru',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: ,
[Symbol(kCapture)]: false
}
}
AxiosError: connect EADDRINUSE 18.62.228.13:443
at TCPConnectWrap.afterConnect [as oncomplete] (node:net:1161:16) {
port: 443,
address: '18.62.228.13',
syscall: 'connect',
code: 'EADDRINUSE',
errno: -4091,
config: {
transitional: {
silentJSONParsing: true,
forcedJSONParsing: true,
clarifyTimeoutError: false
},
adapter: [Function: httpAdapter],
transformRequest: [ [Function: transformRequest] ],
transformResponse: [ [Function: transformResponse] ],
timeout: 0,
xsrfCookieName: 'XSRF-TOKEN',
xsrfHeaderName: 'X-XSRF-TOKEN',
maxContentLength: -1,
maxBodyLength: -1,
env: { FormData: [Function] },
validateStatus: [Function: validateStatus],
headers: {
Accept: 'application/json, text/plain, */*',
'Content-Type': 'application/json',
'User-Agent': 'axios/0.27.2',
'Content-Length': 167
},
method: 'post',
request: <ref *1> Writable {
_writableState: WritableState {
objectMode: false,
highWaterMark: 16384,
finalCalled: false,
needDrain: false,
ending: false,
ended: false,
finished: false,
destroyed: false,
decodeStrings: true,
defaultEncoding: 'utf8',
length: 0,
writing: false,
corked: 0,
sync: true,
bufferProcessing: false,
onwrite: [Function: bound onwrite],
writecb: null,
writelen: 0,
afterWriteTickInfo: null,
buffered: [],
bufferedIndex: 0,
allBuffers: true,
allNoop: true,
pendingcb: 0,
constructed: true,
prefinished: false,
errorEmitted: false,
emitClose: true,
autoDestroy: true,
errored: null,
closed: false,
closeEmitted: false,
[Symbol(kOnFinished)]: []
},
_events: [Object: null prototype] {
response: [Function: handleResponse],
error: [Function: handleRequestError],
socket: [Function: handleRequestSocket]
},
_eventsCount: 3,
_maxListeners: undefined,
_options: {
maxRedirects: 21,
maxBodyLength: 10485760,
protocol: 'https:',
path: 'insert',
method: 'POST',
headers: [Object],
agent: undefined,
agents: [Object],
auth: undefined,
hostname: 'lalalala',
port: null,
nativeProtocols: [Object],
pathname: 'lalallalaa'
},
_ended: false,
_ending: true,
_redirectCount: 0,
_redirects: [],
_requestBodyLength: 167,
_requestBodyBuffers: [ [Object] ],
_onNativeResponse: [Function (anonymous)],
_currentRequest: ClientRequest {
_events: [Object: null prototype],
_eventsCount: 7,
_maxListeners: undefined,
outputData: [],
outputSize: 0,
writable: true,
destroyed: false,
_last: true,
chunkedEncoding: false,
shouldKeepAlive: false,
maxRequestsOnConnectionReached: false,
_defaultKeepAlive: true,
useChunkedEncodingByDefault: true,
sendDate: false,
_removedConnection: false,
_removedContLen: false,
_removedTE: false,
_contentLength: null,
_hasBody: true,
_trailer: '',
finished: false,
_headerSent: true,
_closed: false,
socket: [TLSSocket],
_header: 'POST HTTP/1.1\r\n' +
'Accept: application/json, text/plain, */*\r\n' +
'Content-Type: application/json\r\n' +
'User-Agent: axios/0.27.2\r\n' +
'Content-Length: 167\r\n' +
'Host: lallala\r\n' +
'Connection: close\r\n' +
'\r\n',
_keepAliveTimeout: 0,
_onPendingData: [Function: nop],
agent: [Agent],
socketPath: undefined,
method: 'POST',
maxHeaderSize: undefined,
insecureHTTPParser: undefined,
path: '/manual/search/insert',
_ended: false,
res: null,
aborted: false,
timeoutCb: null,
upgradeOrConnect: false,
parser: null,
maxHeadersCount: null,
reusedSocket: false,
host: 'sllalallaau',
protocol: 'https:',
_redirectable: [Circular *1],
[Symbol(kCapture)]: false,
[Symbol(kNeedDrain)]: false,
[Symbol(corked)]: 0,
[Symbol(kOutHeaders)]: [Object: null prototype]
},
_currentUrl: 'https://lalalalal',
[Symbol(kCapture)]: false
}
}
这是我的线程函数：
async function parseData(url) {
const {data} = await axios.get(url ).catch(console.log)
const $ = cheerio.load(data);
const result = [];
const obj = {};
const header = $('h1.Uheader');
obj.brand = $(header[0]).text();
const category = $('div.cathead');
const categoryArray = [];
for (let i = 0; i < category.length; i++) {
categoryArray.push({"href": $(category[i]).children("a").attr('href'), "text":
$(category[i]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim()})
}
for (let i = 0; i < categoryArray.length; i++) {
try {
obj.category = categoryArray[i].text;
const dataTwo = await axios.get("https://www.lalalala.com" +
categoryArray[i].href).catch()
const cher = cheerio.load(dataTwo.data);
const aTag = cher('div.col-sm-2.mname')
for (let j = 0 ; j < aTag.length; j++) {
obj.url = "https://www.lalalala.com" + $(aTag[j]).children("a").attr('href');
obj.title = obj.brand + " " + categoryArray[i].text + " " + $(aTag[j]).children("a").text().replace(/[^a-zA-Z0-9 ]/g, '').trim();
//result.push(obj)
axios.post("https://lalalalala", obj)
.then(data => console.log("ok " + j))
.catch(e => {
console.log(e)
});
}
} catch (e) {
}
}
parentPort.postMessage({message : "done"});
}
有人可以向我解释为什么会这样吗？我该如何解决？
因为（根据您的评论
错误connect EADDRINUSE意味着本地机器上没有更多可用的 TCP 端口（到远程服务器的 TCP 连接也需要一个本地 TCP 端口，称为“临时端口”，因为它只是在请求期间使用中），因为有太多待处理的请求。
一种可能的修复方法，也是您所说的有效方法，是等待每个 POST 请求完成，然后再继续代码的其余部分：
await axios.post(…)
如果这也没有帮助，您将不得不研究可以配置为限制并发数量的作业队列 requests/promises/jobs/...)。