如何在这个 drop nulls 函数中避免 collect() ?提高性能的(其他)方法可能是什么?
How can collect() be avoided in this drop nulls function ? What could be (additional) way(s) to increase performance?
我有一个函数可以用来计算每列中所有不同的值。我有一个非常大的数据集,其中有时包含没有数据的列。然后我删除这些列和 return 打印语句告诉我哪些列已被删除。我的数据大小将来可能会增加,因此我想避免使用 collect(),因为我不想收集到驱动程序。在这种情况下我该如何避免呢?你能想到对这个功能有什么改进吗?建议/示例非常感谢!
def dropNullColumns(df):
# A set of all the null values you can encounter
null_set = {"none", "null" , "nan"}
# Iterate over each column in the DF
for col in df.columns:
# Get the distinct values of the column
unique_val = df.select(col).distinct().collect()[0][0]
# See whether the unique value is only none/nan or null
if str(unique_val).lower() in null_set:
print("Dropping " + col + " because of all null values.")
df = df.drop(col)
return(df)
df = dropNullColumns(df)
这套功能合不合适?
您稍后可以过滤百分比 == 1.0
def check_percent_na(sdf: DataFrame, col_name: str, customized_na_list=['', "null", "nan"]) -> Tuple:
total_num: int = sdf.count()
if dict(sdf.dtypes)[col_name] == "string":
na_num: int = sdf.filter((sf.col(col_name).isNull()) | (sf.col(col_name).isin(customized_na_list))).count()
else:
na_num: int = sdf.filter(sf.col(col_name).isNull()).count()
return col_name, na_num/total_num
def check_percent_na_dataframe(sdf: DataFrame) -> List:
return list(map(lambda x: check_percent_na(sdf=sdf, col_name=x), sdf.columns))
我贴的功能其实有很多缺点。不仅如此,它还使用了 collect()。它也不是很健壮,会丢弃实际上不应该丢弃的列。请考虑以下方法:
rows = [(None, 18, None, None),
(1, None, None, None),
(1, 9, 4.0, None),
(None, 0, 0., None)]
schema = "a: int, b: int, c: float, d:int"
df = spark.createDataFrame(data=rows, schema=schema)
def get_null_column_names(df):
column_names = []
for col_name in df.columns:
min_ = df.select(F.min(col_name)).first()[0]
max_ = df.select(F.max(col_name)).first()[0]
if min_ is None and max_ is None:
column_names.append(col_name)
return column_names
null_columns = get_null_column_names(df)
def drop_column(null_columns, df):
for column_ in drop_these_columns:
df = df.drop(column_)
return df
df = drop_column(null_columns, df)
df.show()
产生以下输出:
我有一个函数可以用来计算每列中所有不同的值。我有一个非常大的数据集,其中有时包含没有数据的列。然后我删除这些列和 return 打印语句告诉我哪些列已被删除。我的数据大小将来可能会增加,因此我想避免使用 collect(),因为我不想收集到驱动程序。在这种情况下我该如何避免呢?你能想到对这个功能有什么改进吗?建议/示例非常感谢!
def dropNullColumns(df):
# A set of all the null values you can encounter
null_set = {"none", "null" , "nan"}
# Iterate over each column in the DF
for col in df.columns:
# Get the distinct values of the column
unique_val = df.select(col).distinct().collect()[0][0]
# See whether the unique value is only none/nan or null
if str(unique_val).lower() in null_set:
print("Dropping " + col + " because of all null values.")
df = df.drop(col)
return(df)
df = dropNullColumns(df)
这套功能合不合适?
您稍后可以过滤百分比 == 1.0
def check_percent_na(sdf: DataFrame, col_name: str, customized_na_list=['', "null", "nan"]) -> Tuple:
total_num: int = sdf.count()
if dict(sdf.dtypes)[col_name] == "string":
na_num: int = sdf.filter((sf.col(col_name).isNull()) | (sf.col(col_name).isin(customized_na_list))).count()
else:
na_num: int = sdf.filter(sf.col(col_name).isNull()).count()
return col_name, na_num/total_num
def check_percent_na_dataframe(sdf: DataFrame) -> List:
return list(map(lambda x: check_percent_na(sdf=sdf, col_name=x), sdf.columns))
我贴的功能其实有很多缺点。不仅如此,它还使用了 collect()。它也不是很健壮,会丢弃实际上不应该丢弃的列。请考虑以下方法:
rows = [(None, 18, None, None),
(1, None, None, None),
(1, 9, 4.0, None),
(None, 0, 0., None)]
schema = "a: int, b: int, c: float, d:int"
df = spark.createDataFrame(data=rows, schema=schema)
def get_null_column_names(df):
column_names = []
for col_name in df.columns:
min_ = df.select(F.min(col_name)).first()[0]
max_ = df.select(F.max(col_name)).first()[0]
if min_ is None and max_ is None:
column_names.append(col_name)
return column_names
null_columns = get_null_column_names(df)
def drop_column(null_columns, df):
for column_ in drop_these_columns:
df = df.drop(column_)
return df
df = drop_column(null_columns, df)
df.show()
产生以下输出: