如何通过节点 oracle-db 更新或插入数百万行?
How to update or insert millions of rows via node oracle-db?
我正在为一个问题而苦恼 - 如何使用节点 oracle-db 驱动程序插入或更新大量数据(数千或数百万行)?
关键是我可以在 resultSet (handling result set) 的帮助下 select 大量数据...但是我必须连续执行一些操作,然后更新或插入一个新行。这就是问题所在——我不知道如何尽快完成。
谁能帮我提点建议?谢谢
I can assure you that these actions can't be done in db.
实际上,有很多不同的方法可以在需要时通过 SQL 和 PL/SQL 在数据库中完成。人们通常希望使用他们喜欢的语言table,在这种情况下可能 JavaScript,但是如果数据不必在层之间来回传输,性能会好得多。
这里有一个单独的例子 SQL...当然,这可以通过虚拟列来完成,但它应该能说明这一点。
假设我们有以下 tables:
create table things (
id number not null,
val1 number not null,
val2 number not null,
constraint things_pk primary key (id)
);
insert into things (id, val1, val2) values (1, 1, 2);
insert into things (id, val1, val2) values (2, 2, 2);
insert into things (id, val1, val2) values (3, 5, 5);
-- Will hold the sum of things val1 and val2
create table thing_sums (
thing_id number,
sum number
);
alter table thing_sums
add constraint thing_sums_fk1
foreign key (thing_id)
references things (id);
现在,最简单和最有效的方法是通过 SQL:
insert into thing_sums (
thing_id,
sum
)
select id,
val1 + val2
from things
where id not in (
select thing_id
from thing_sums
);
这是另一个仅通过 PL/SQL 执行相同操作的示例,它可以提供更多控制。
begin
-- This cursor for loop will bulk collect (reduces context switching between
-- SQL and PL/SQL engines) implictly.
for thing_rec in (
select *
from things
where id not in(
select thing_id
from thing_sums
)
)
loop
-- Logic in this loop could be endlessly complex. I'm inserting the values
-- within the loop but this logic could be modified to store data in arrays
-- and then insert with forall (another bulk operation) after the loop.
insert into thing_sums(
thing_id,
sum
) values (
thing_rec.id,
thing_rec.val1 + thing_rec.val2
);
end loop;
end;
其中任何一个都可以从 Node.js 驱动程序中调用。但是,假设您需要从驱动程序执行此操作(也许您正在摄取数据库中不存在的数据)。下面是一个示例,演示了从使用批量处理而不是逐行操作的驱动程序调用 PL/SQL。由于减少了往返行程,这要快得多。
我从博客中提取了这个 post 我正在研究所以 table 定义有点不同:
create table things (
id number not null,
name varchar2(50),
constraint things_pk primary key (id)
);
这是 JavaScript:
var oracledb = require('oracledb');
var async = require('async');
var config = require('./dbconfig');
var things = [];
var idx;
function getThings(count) {
var things = [];
for (idx = 0; idx < count; idx += 1) {
things[idx] = {
id: idx,
name: "Thing number " + idx
};
}
return things;
}
things = getThings(500);
oracledb.getConnection(config, function(err, conn) {
var ids = [];
var names = [];
var start = Date.now();
if (err) {throw err;}
// We need to break up the array of JavaScript objects into arrays that
// work with node-oracledb bindings.
for (idx = 0; idx < things.length; idx += 1) {
ids.push(things[idx].id);
names.push(things[idx].name);
}
conn.execute(
` declare
type number_aat is table of number
index by pls_integer;
type varchar2_aat is table of varchar2(50)
index by pls_integer;
l_ids number_aat := :ids;
l_names varchar2_aat := :names;
begin
forall x in l_ids.first .. l_ids.last
insert into things (id, name) values (l_ids(x), l_names(x));
end;`,
{
ids: {
type: oracledb.NUMBER,
dir: oracledb.BIND_IN,
val: ids
},
names: {
type: oracledb.STRING,
dir: oracledb.BIND_IN,
val: names
}
},
{
autoCommit: true
},
function(err) {
if (err) {console.log(err); return;}
console.log('Success. Inserted ' + things.length + ' rows in ' + (Date.now() - start) + ' ms.');
}
);
});
希望对您有所帮助! :)
我正在为一个问题而苦恼 - 如何使用节点 oracle-db 驱动程序插入或更新大量数据(数千或数百万行)?
关键是我可以在 resultSet (handling result set) 的帮助下 select 大量数据...但是我必须连续执行一些操作,然后更新或插入一个新行。这就是问题所在——我不知道如何尽快完成。
谁能帮我提点建议?谢谢
I can assure you that these actions can't be done in db.
实际上,有很多不同的方法可以在需要时通过 SQL 和 PL/SQL 在数据库中完成。人们通常希望使用他们喜欢的语言table,在这种情况下可能 JavaScript,但是如果数据不必在层之间来回传输,性能会好得多。
这里有一个单独的例子 SQL...当然,这可以通过虚拟列来完成,但它应该能说明这一点。
假设我们有以下 tables:
create table things (
id number not null,
val1 number not null,
val2 number not null,
constraint things_pk primary key (id)
);
insert into things (id, val1, val2) values (1, 1, 2);
insert into things (id, val1, val2) values (2, 2, 2);
insert into things (id, val1, val2) values (3, 5, 5);
-- Will hold the sum of things val1 and val2
create table thing_sums (
thing_id number,
sum number
);
alter table thing_sums
add constraint thing_sums_fk1
foreign key (thing_id)
references things (id);
现在,最简单和最有效的方法是通过 SQL:
insert into thing_sums (
thing_id,
sum
)
select id,
val1 + val2
from things
where id not in (
select thing_id
from thing_sums
);
这是另一个仅通过 PL/SQL 执行相同操作的示例,它可以提供更多控制。
begin
-- This cursor for loop will bulk collect (reduces context switching between
-- SQL and PL/SQL engines) implictly.
for thing_rec in (
select *
from things
where id not in(
select thing_id
from thing_sums
)
)
loop
-- Logic in this loop could be endlessly complex. I'm inserting the values
-- within the loop but this logic could be modified to store data in arrays
-- and then insert with forall (another bulk operation) after the loop.
insert into thing_sums(
thing_id,
sum
) values (
thing_rec.id,
thing_rec.val1 + thing_rec.val2
);
end loop;
end;
其中任何一个都可以从 Node.js 驱动程序中调用。但是,假设您需要从驱动程序执行此操作(也许您正在摄取数据库中不存在的数据)。下面是一个示例,演示了从使用批量处理而不是逐行操作的驱动程序调用 PL/SQL。由于减少了往返行程,这要快得多。
我从博客中提取了这个 post 我正在研究所以 table 定义有点不同:
create table things (
id number not null,
name varchar2(50),
constraint things_pk primary key (id)
);
这是 JavaScript:
var oracledb = require('oracledb');
var async = require('async');
var config = require('./dbconfig');
var things = [];
var idx;
function getThings(count) {
var things = [];
for (idx = 0; idx < count; idx += 1) {
things[idx] = {
id: idx,
name: "Thing number " + idx
};
}
return things;
}
things = getThings(500);
oracledb.getConnection(config, function(err, conn) {
var ids = [];
var names = [];
var start = Date.now();
if (err) {throw err;}
// We need to break up the array of JavaScript objects into arrays that
// work with node-oracledb bindings.
for (idx = 0; idx < things.length; idx += 1) {
ids.push(things[idx].id);
names.push(things[idx].name);
}
conn.execute(
` declare
type number_aat is table of number
index by pls_integer;
type varchar2_aat is table of varchar2(50)
index by pls_integer;
l_ids number_aat := :ids;
l_names varchar2_aat := :names;
begin
forall x in l_ids.first .. l_ids.last
insert into things (id, name) values (l_ids(x), l_names(x));
end;`,
{
ids: {
type: oracledb.NUMBER,
dir: oracledb.BIND_IN,
val: ids
},
names: {
type: oracledb.STRING,
dir: oracledb.BIND_IN,
val: names
}
},
{
autoCommit: true
},
function(err) {
if (err) {console.log(err); return;}
console.log('Success. Inserted ' + things.length + ' rows in ' + (Date.now() - start) + ' ms.');
}
);
});
希望对您有所帮助! :)