如何在 bigquery 中删除 array_agg

Question

我创建了一个包含重复记录的新 table。我正在尝试找到最有效的方法来删除重复记录，因为这将是运行在具有数百万条记录的 table 上。如果您使用多个嵌套的 CTE，那么您的数据结构是在内存中完成处理还是在有大量数据时写入临时 tables 是否重要。

create or replace table t1.cte4 as
WITH t1 AS (
  SELECT 1 as id,'eren' AS last_name UNION ALL
  SELECT 1 as id,'yilmaz' AS last_name UNION ALL
  SELECT 1 as id,'kaya' AS last_name UNION ALL
  SELECT 1 as id,'kaya' AS last_name UNION ALL
  SELECT 2 as id,'smith' AS last_name UNION ALL
  SELECT 2 as id,'jones' AS last_name UNION ALL
  SELECT 2 as id,'jones' AS last_name UNION ALL
  SELECT 2 as id,'jones' AS last_name UNION ALL
  SELECT 2 as id,'brown' AS last_name
)
SELECT id,ARRAY_AGG(STRUCT(last_name)) AS last_name_rec
FROM t1
GROUP BY id;

我可以按如下方式删除重复项。

QUERY 1 How to dedup the concat_struct ?
select id, 
STRING_AGG( distinct ln.last_name ,'~') as concat_string,
ARRAY_AGG(STRUCT( ln.last_name )) as concat_struct
from `t1.cte4`, unnest(last_name_rec) ln
group by id;

QUERY 1

QUERY 2 Is there a better way then this to dedup?
select distinct id, 
TO_JSON_STRING(ARRAY_AGG(ln.last_name) OVER (PARTITION BY id)) json_string
from `t1.cte4`, unnest(last_name_rec) ln
group by id,
ln.last_name;

QUERY 2

如何将它从 table 中分离出来，而不是使用 CTE。这不会去重。

select id,  ARRAY_AGG(STRUCT( ln.last_name )) as concat_struct 
from t1.cte4, 
unnest(last_name_rec) ln group by id;

我做不到。

select id,  ARRAY_AGG(distinct STRUCT( ln.last_name )) as concat_struct from t1.cte4, 
unnest(last_name_rec) ln group by id;

Answer 1

更新：在重复数据删除之前分解结构，然后将其组合回去：

select id, ARRAY_AGG(STRUCT(last_name)) as concat_struct 
from (
  select id, ln.last_name
  from cte4, unnest(last_name_rec) ln
  group by id, ln.last_name 
) d
group by id

（基于对 table 定义的不必要更改的原始答案如下）

只需使用array_agg(distinct ...):

WITH t1 AS (
  SELECT 1 as id,'eren' AS last_name UNION ALL
  SELECT 1 as id,'yilmaz' AS last_name UNION ALL
  SELECT 1 as id,'kaya' AS last_name UNION ALL
  SELECT 1 as id,'kaya' AS last_name UNION ALL
  SELECT 2 as id,'smith' AS last_name UNION ALL
  SELECT 2 as id,'jones' AS last_name UNION ALL
  SELECT 2 as id,'jones' AS last_name UNION ALL
  SELECT 2 as id,'jones' AS last_name UNION ALL
  SELECT 2 as id,'brown' AS last_name
)
SELECT id,ARRAY_AGG(distinct last_name) AS last_name_rec
FROM t1
GROUP BY id;

如何在 bigquery 中删除 array_agg

How to dedup array_agg in bigquery

sql

duplicates

google-bigquery

array-agg