NJS-024：OracleDB 中的内存分配失败 - Nodejs

Question

我正在尝试运行使用 OracleDB 和 Nodejs 进行查询以获取 UI 中填充的视图，但我得到 NJS-024：内存分配失败 错误。有人可以帮我吗？该视图总共包含 120 列，当我在 SQL Developer 中查询该视图时，它工作得很好。

ConnectionPool.js:

var path = require('path');
var oracledb = require('oracledb');
var poolMap = {};

var logger = require(path.join(global.root + '/app/util/logger.js'))();

function createPool(poolName, config, callback) {
    oracledb.createPool(
        config,
        function(err, p) {
            if (err){
                logger.error(err);
                return;
            }

            poolMap[poolName] = p;

            callback(poolMap[poolName]);
        }
    );
}

function getPool(poolName) {
    return poolMap[poolName];
}

module.exports = {
    createPool: createPool,
    getPool: getPool
};

这是我的池属性：

var pool;
oracledb.prefetchRows = 10000;
oracledb.maxRows = 400000;

var poolAttrs = {
    user: dbcfg.username,
    password: dbcfg.password,
    connectString: dbcfg.connectionString,
    connectionClass : 'Report API',
    poolMin : 3,
    poolMax : 10,
    poolIncrement: 2,
    poolTimeout : 600 //seconds
};

connectionPool.createPool("Reports", poolAttrs, function(connPool){
    pool = connPool;
    logger.info("Pool created by reports.");
});

这是我的代码：

router.post('/report/', jsonParser, function (req, res) {
    var data = req.body,
        startRow = data.startRow,
        numRows = data.numRows,
        sortCol = data.sortCol,
        sortDir = data.sortDir;

    var countQuery = 'SELECT COUNT(*) ' +
        'FROM this_view ' ;

    var query = 'SELECT * ' +
        'FROM this_view' ;

    var seg,
        orderBy,
        offset;

    orderBy = ' ORDER BY UPPER(' + sortCol + ') ' + sortDir;
    offset = ' OFFSET ' + startRow + ' ROWS FETCH NEXT ' + numRows + ' ROWS ONLY';

    query += orderBy;
    query += offset;

    logger.info("Begin: " + (new Date().toString()));

    async.parallel({
        rows: function (callback) {
        pool.getConnection(function (err, connection) {
            logger.info("Begin Connection: " + (new Date().toString()));
            if (err) {
                logger.error(err.message);
                return;
            }

            logger.info("Begin execute: " + (new Date().toString()));

            connection.execute(
                query,
                {},
                {
                    resultSet: true,
                    prefetchRows: 1000
                },
                function (err, results) {
                    logger.info("End execute: " + (new Date().toString()));
                    var rowsProcessed = 0;
                    var startTime;
                    if (err) {
                        logger.error(err.message);
                        callback("Something broke in the first thing");
                        doRelease(connection);
                        return;
                    }
                    var procJson = [];

                    function fetchRowsFromRS(connection, resultSet, numRows) {
                        resultSet.getRows(
                            numRows,  // get this many rows
                            function (err, rows) {
                                if (err) {
                                    console.error(err);
                                    doClose(connection, resultSet); // always close the result set
                                } else if (rows.length >= 0) {
                                    /**
                                     *  For each row in the result, pushes a new object to the rows array
                                     *  In each new object, the key is assigned and the result row value set
                                     */
                                    for (var i = 0; i < rows.length; i++) {
                                        procJson.push({});
                                        console.log(procJson);
                                        for (var j = 0; j < resultSet.metaData.length; j++) {
                                            procJson[i][resultSet.metaData[j].name.toLowerCase()] = rows[i][j];
                                        }
                                    }

                                    //TODO: Add null handling
                                    logger.info("Send JSON: " + (new Date().toString()));
                                    logger.info("JSON Sent: " + (new Date().toString()));
                                    if (rows.length === numRows) // might be more rows
                                        fetchRowsFromRS(connection, resultSet, numRows);
                                    else
                                        doClose(connection, resultSet); // always close the result set
                                } else { // no rows
                                    doClose(connection, resultSet); // always close the result set
                                }
                            });
                    }
                    fetchRowsFromRS(connection, result.resultSet, numRows);
                    callback(null, procJson);
                });
        });
    },
        totalRows: function (callback) {
            pool.getConnection(function (err, connection) {
                logger.info("Begin Connection: " + (new Date().toString()));
                if (err) {
                    logger.error(err.message);
                    return;
                }

                logger.info("Begin execute: " + (new Date().toString()));

                connection.execute(
                    countQuery,
                    function (err, result) {
                        logger.info("End execute: " + (new Date().toString()));
                        if (err) {
                            logger.error(err.message);
                            callback("Something broke");
                            doRelease(connection);
                            return;
                        }

                        logger.info("Send JSON: " + (new Date().toString()));
                        console.log(result.rows);
                        callback(null, result.rows[0][0]);
                        logger.info("JSON Sent: " + (new Date().toString()));

                        doRelease(connection);
                    });
            });
        }
    }, function(err, result){
        if(err){
            logger.error(err);
        }

        res.send(result);
    });
});

如果 rows.length >=0 并且如果查询 returns 0 个结果，我明白了。

Answer 1

您的 Node.js 服务器有多少内存？您将 maxRows 设置得非常高，并一次性获取所有数据。这可能会导致您运行内存不足。通常，关键是平衡往返行程（您想要减少）与内存使用量（随着往返行程的减少而增加。

您需要利用 ResultSet API，它允许您以更小的块流式传输读取一致的数据视图。看看这个想法：https://jsao.io/2015/07/an-overview-of-result-sets-in-the-nodejs-driver/

与其在 Node.js 服务器中缓冲数据（这会导致同样的问题），不如将其流式传输到 http 请求。

最后，但也许最重要的是，请注意您的代码当前对 SQL 注入开放。用户通过 req.body 提供的值不可信。它们必须要么使用绑定变量进行绑定，要么使用 dbms_assert.

之类的东西进行清理

只能绑定值（如 numRows）。必须清理标识符（如 sortCol）。您可能想要在 Node.js 中进行清理，所以这里有一个非常基本的检查应该会有所帮助。

您可以创建一个 "assert" 模块：

function simpleSqlName(name) {
  if (name.length > 30) {
    throw new Error('Not simple SQL');
  }

  // Fairly generic, but effective. Would need to be adjusted to accommodate quoted identifiers,
  // schemas, etc.
  if (!/^[a-zA-Z0-9#_$]+$/.test(name)) {
    throw new Error('Not simple SQL');
  }

  return name;
}

module.exports.simpleSqlName = simpleSqlName;

function validSortOrder(order) {
  if (order !== 'desc' && order !== 'asc') {
    throw new Error('Not valid sort order');
  }

  return order;
}

module.exports.validSortOrder = validSortOrder;

那么你的代码看起来更像这样（注意我同时使用了断言模块和绑定变量）：

let assert = require('assert.js');  

router.post('/report/', jsonParser, function (req, res) {
    var data = req.body,
        startRow = data.startRow,
        numRows = data.numRows,
        sortCol = assert.simpleSqlName(data.sortCol),
        sortDir = assert.validSortOrder(data.sortDir);

    var countQuery = 'SELECT COUNT(*) ' +
        'FROM this_view ' ;

    var query = 'SELECT * ' +
        'FROM this_view' ;

    var seg,
        orderBy,
        offset;

    orderBy = ' ORDER BY UPPER(' + sortCol + ') ' + sortDir;
    offset = ' OFFSET :start_row ROWS FETCH NEXT :num_rows ROWS ONLY';

    query += orderBy;
    query += offset;

    logger.info("Begin: " + (new Date().toString()));

    async.parallel({
        rows: function (callback) {
            pool.getConnection(function (err, connection) {
                logger.info("Begin Connection: " + (new Date().toString()));
                if (err) {
                    logger.error(err.message);
                    return;
                }

                logger.info("Begin execute: " + (new Date().toString()));

                connection.execute(
                    query,
                    {
                      start_row: startRow,
                      num_rows: numRows
                    },
                    function (err, result) {
                        logger.info("End execute: " + (new Date().toString()));
                        if (err) {
                            logger.error(err.message);
                            callback("Something broke in the first thing");
                            doRelease(connection);
                            return;
                        }
                        console.log(result.rows);

                        var procJson = [];

                        /**
                         *  For each row in the result, pushes a new object to the rows array
                         *  In each new object, the key is assigned and the result row value set
                         */
                        for (var i = 0; i < result.rows.length; i++) {
                            procJson.push({});
                            for (var j = 0; j < result.metaData.length; j++) {
                                procJson[i][result.metaData[j].name.toLowerCase()] = result.rows[i][j];
                            }
                        }

                        logger.info("Send JSON: " + (new Date().toString()));
                        callback(null, procJson);
                        logger.info("JSON Sent: " + (new Date().toString()));

                        doRelease(connection);
                    });
            });
        },
        totalRows: function (callback) {
            pool.getConnection(function (err, connection) {
                logger.info("Begin Connection: " + (new Date().toString()));
                if (err) {
                    logger.error(err.message);
                    return;
                }

                logger.info("Begin execute: " + (new Date().toString()));

                connection.execute(
                    countQuery,
                    function (err, result) {
                        logger.info("End execute: " + (new Date().toString()));
                        if (err) {
                            logger.error(err.message);
                            callback("Something broke");
                            doRelease(connection);
                            return;
                        }

                        logger.info("Send JSON: " + (new Date().toString()));
                        console.log(result.rows);
                        callback(null, result.rows[0][0]);
                        logger.info("JSON Sent: " + (new Date().toString()));

                        doRelease(connection);
                    });
            });
        }
    }, function(err, result){
        if(err){
            logger.error(err);
        }

        res.send(result);
    });
});

在此处了解有关绑定变量的更多信息：https://github.com/oracle/node-oracledb/blob/master/doc/api.md#bind

此外，请查看我最近发表的演讲中的幻灯片。你可能会从中得到一些东西... https://www.dropbox.com/s/2rhnu74z2y21gsy/Tips%20and%20Tricks%20for%20Getting%20Started%20with%20the%20Oracle%20Database%20Driver%20for%20Node.pdf?dl=0

NJS-024：OracleDB 中的内存分配失败 - Nodejs

NJS-024: memory allocation failed in OracleDB - Nodejs

javascript

node.js

oracle

node-oracledb