如何在分层类别结构中按值对 pandas 中的数据框进行排序
How to sort dataframe in pandas by value in hierarchical category structure
我在 pandas 中有一个数据框。
pd.DataFrame({
"category": ["Transport", "Transport : Car", "Transport : Train", "Household", "Household : Utilities", "Household : Utilities : Water", "Household : Utilities : Electric", "Household : Cleaning", "Household : Cleaning : Bathroom", "Household : Cleaning : Kitchen", "Household : Rent", "Living", "Living : Other", "Living : Food", "Living : Something", "Living : Anitsomething"],
"amount": [5000, 4900, 100, 1100, 600, 400, 200, 100, 75, 25, 400, 250, 150, 100, 1000, -1000]
})
类别和子类别由冒号分隔。
我正在尝试按数量（绝对值）降序对这个数据框进行排序。同时尊重层次分组。 IE。排序后的结果应该是
Transport 5000
Transport : Car 4900
Transport : Train 100
Household 1600
Household : Utilities 600
Household : Utilities : Water 400
Household : Utilities : Electric 200
Household : Rent 400
Living 250
Living : Something 1000
Living : Antisomething -1000
Living : Other 150
Living : Food 100
我可以以非常低效的方式递归执行此操作。超慢但有效。
def sort_hierachical(self, full_df, name_column, sort_column, parent="", level=0):
result_df = pd.DataFrame(columns=full_df.columns)
part_df = full_df.loc[(full_df[name_column].str.count(':') == level) & (full_df[name_column].str.startswith(parent)), :]
part_df['abs'] = part_df[sort_column].abs()
part_df = part_df.sort_values('abs', ascending=False)
for _, row in part_df.iterrows():
category = row[name_column]
row_df = pd.DataFrame(columns = full_df.columns).append(row)
child_rows = self.sort_hierachical(full_df, name_column, sort_column, category, level+1)
if not child_rows.empty:
result_df = pd.concat([result_df, row_df], sort=False)
result_df = pd.concat([result_df, child_rows], sort=False)
else:
result_df = pd.concat([result_df, row_df], sort=False)
return result_df
df = self.sort_hierachical(df, "category", "amount")
我的问题：在 pandas 中是否有一种性能良好的方法来做这样的事情。某种分组排序或多索引技巧??
能够解决这个具有挑战性的问题的人会得到善缘:)
编辑：
这几乎可以工作...但是 -1000、1000 打乱了排序顺序。
def _sort_tree_df(self, df, tree_column, sort_column):
sort_key = sort_column + '_abs'
df[sort_key] = df[sort_column].abs()
df.index = pd.MultiIndex.from_frame(df[tree_column].str.split(":").apply(lambda x: [y.strip() for y in x]).apply(pd.Series))
sort_columns = [df[tree_column].values]
sort_columns.append(df[sort_key].values)
for x in range(df.index.nlevels, 0, -1):
group_lvl = list(range(0, x))
sort_columns.append(df.groupby(level=group_lvl)[sort_key].transform('max').values)
sort_indexes = np.lexsort(sort_columns)
df_sorted = df.iloc[sort_indexes[::-1]]
df_sorted.reset_index(drop=True, inplace=True)
df_sorted = df_sorted.drop(sort_key, axis=1)
return df_sorted
编辑2：
好的，我想我已经成功了。我仍然很困惑 lexsort 是如何工作的。我通过有根据的反复试验完成了这项工作。如果您理解它，请随时解释它。也欢迎 post 更好的方法。
def _sort_tree_df(self, df, tree_column, sort_column, delimeter=':'):
df.index = pd.MultiIndex.from_frame(df[tree_column].str.split(delimeter).apply(lambda x: [y.strip() for y in x]).apply(pd.Series))
sort_columns = [df[tree_column].values]
sort_columns.append(df[sort_column].abs().values)
for x in range(df.index.nlevels, 0, -1):
group_lvl = list(range(0, x))
sort_columns.append(df.groupby(level=group_lvl)[sort_column].transform('sum').abs().values)
sort_indexes = np.lexsort(sort_columns)
df_sorted = df.iloc[sort_indexes[::-1]]
df_sorted.reset_index(drop=True, inplace=True)
return df_sorted
编辑3 :
实际上这并不总是正确排序:(
编辑4
问题是我需要一种方法使 transform('sum') 仅适用于 level = x-1
的项目
例如：
df['level'] = df[tree_column].str.count(':')
sorting_by = df.groupby(level=group_lvl)[sort_column].transform('sum' if 'level' = x-1).abs().values
或
sorting_by = df.groupby(level=group_lvl).loc['level' = x-1: sort_column].transform('sum').abs().values
两者均无效
有人知道如何在多索引 df 上进行这样的条件转换吗？
我不确定我是否完全理解了这个问题，但我认为您应该将列拆分为子类别，然后根据您想要的层次结构进行值排序。像下面这样的东西可能会完成这项工作。
使用以下命令创建新列：
for _, row in df.iterrows():
for item, col in zip(row.category.split(':'), ['cat', 'sub_cat', 'sub_sub_cat']):
df.loc[_, col] = item
然后对它们进行排序
df.sort_values(['cat', 'sub_cat', 'sub_sub_cat', 'amount'])
category amount cat sub_cat sub_sub_cat
3 Household 1100 Household NaN NaN
7 Household : Cleaning 100 Household Cleaning NaN
8 Household : Cleaning : Bathroom 75 Household Cleaning Bathroom
9 Household : Cleaning : Kitchen 25 Household Cleaning Kitchen
10 Household : Rent 400 Household Rent NaN
4 Household : Utilities 600 Household Utilities NaN
6 Household : Utilities : Electric 200 Household Utilities Electric
5 Household : Utilities : Water 400 Household Utilities Water
11 Living 250 Living NaN NaN
15 Living : Anitsomething -1000 Living Anitsomething NaN
13 Living : Food 100 Living Food NaN
12 Living : Other 150 Living Other NaN
14 Living : Something 1000 Living Something NaN
0 Transport 5000 Transport NaN NaN
1 Transport : Car 4900 Transport Car NaN
2 Transport : Train 100 Transport Train Na
好的，我花了一些时间才弄明白，但现在我很确定这行得通。也比递归方法快得多。
def _sort_tree_df(self, df, tree_column, sort_column, delimeter=':'):
df=df.copy()
parts = df[tree_column].str.split(delimeter).apply(lambda x: [y.strip() for y in x]).apply(pd.Series)
for i, column in enumerate(parts.columns):
df[column] = parts[column]
sort_columns = [df[tree_column].values]
sort_columns.append(df[sort_column].abs().values)
df['level'] = df[tree_column].str.count(':')
for x in range(len(parts.columns), 0, -1):
group_columns = list(range(0, x))
sorting_by = df.copy()
sorting_by.loc[sorting_by['level'] != x-1, sort_column] = np.nan
sorting_by = sorting_by.groupby(group_columns)[sort_column].transform('sum').abs().values
sort_columns.append(sorting_by)
sort_indexes = np.lexsort(sort_columns)
df_sorted = df.iloc[sort_indexes[::-1]]
df_sorted.reset_index(drop=True, inplace=True)
df.drop([column for column in parts.columns], inplace=True, axis=1)
df.drop('level', inplace=True, axis=1)
return df_sorted
我在 pandas 中有一个数据框。
pd.DataFrame({
"category": ["Transport", "Transport : Car", "Transport : Train", "Household", "Household : Utilities", "Household : Utilities : Water", "Household : Utilities : Electric", "Household : Cleaning", "Household : Cleaning : Bathroom", "Household : Cleaning : Kitchen", "Household : Rent", "Living", "Living : Other", "Living : Food", "Living : Something", "Living : Anitsomething"],
"amount": [5000, 4900, 100, 1100, 600, 400, 200, 100, 75, 25, 400, 250, 150, 100, 1000, -1000]
})
类别和子类别由冒号分隔。
我正在尝试按数量（绝对值）降序对这个数据框进行排序。同时尊重层次分组。 IE。排序后的结果应该是
Transport 5000
Transport : Car 4900
Transport : Train 100
Household 1600
Household : Utilities 600
Household : Utilities : Water 400
Household : Utilities : Electric 200
Household : Rent 400
Living 250
Living : Something 1000
Living : Antisomething -1000
Living : Other 150
Living : Food 100
我可以以非常低效的方式递归执行此操作。超慢但有效。
def sort_hierachical(self, full_df, name_column, sort_column, parent="", level=0):
result_df = pd.DataFrame(columns=full_df.columns)
part_df = full_df.loc[(full_df[name_column].str.count(':') == level) & (full_df[name_column].str.startswith(parent)), :]
part_df['abs'] = part_df[sort_column].abs()
part_df = part_df.sort_values('abs', ascending=False)
for _, row in part_df.iterrows():
category = row[name_column]
row_df = pd.DataFrame(columns = full_df.columns).append(row)
child_rows = self.sort_hierachical(full_df, name_column, sort_column, category, level+1)
if not child_rows.empty:
result_df = pd.concat([result_df, row_df], sort=False)
result_df = pd.concat([result_df, child_rows], sort=False)
else:
result_df = pd.concat([result_df, row_df], sort=False)
return result_df
df = self.sort_hierachical(df, "category", "amount")
我的问题：在 pandas 中是否有一种性能良好的方法来做这样的事情。某种分组排序或多索引技巧??
能够解决这个具有挑战性的问题的人会得到善缘:)
编辑：
这几乎可以工作...但是 -1000、1000 打乱了排序顺序。
def _sort_tree_df(self, df, tree_column, sort_column):
sort_key = sort_column + '_abs'
df[sort_key] = df[sort_column].abs()
df.index = pd.MultiIndex.from_frame(df[tree_column].str.split(":").apply(lambda x: [y.strip() for y in x]).apply(pd.Series))
sort_columns = [df[tree_column].values]
sort_columns.append(df[sort_key].values)
for x in range(df.index.nlevels, 0, -1):
group_lvl = list(range(0, x))
sort_columns.append(df.groupby(level=group_lvl)[sort_key].transform('max').values)
sort_indexes = np.lexsort(sort_columns)
df_sorted = df.iloc[sort_indexes[::-1]]
df_sorted.reset_index(drop=True, inplace=True)
df_sorted = df_sorted.drop(sort_key, axis=1)
return df_sorted
编辑2：
好的，我想我已经成功了。我仍然很困惑 lexsort 是如何工作的。我通过有根据的反复试验完成了这项工作。如果您理解它，请随时解释它。也欢迎 post 更好的方法。
def _sort_tree_df(self, df, tree_column, sort_column, delimeter=':'):
df.index = pd.MultiIndex.from_frame(df[tree_column].str.split(delimeter).apply(lambda x: [y.strip() for y in x]).apply(pd.Series))
sort_columns = [df[tree_column].values]
sort_columns.append(df[sort_column].abs().values)
for x in range(df.index.nlevels, 0, -1):
group_lvl = list(range(0, x))
sort_columns.append(df.groupby(level=group_lvl)[sort_column].transform('sum').abs().values)
sort_indexes = np.lexsort(sort_columns)
df_sorted = df.iloc[sort_indexes[::-1]]
df_sorted.reset_index(drop=True, inplace=True)
return df_sorted
编辑3 : 实际上这并不总是正确排序:(
编辑4 问题是我需要一种方法使 transform('sum') 仅适用于 level = x-1
的项目例如：
df['level'] = df[tree_column].str.count(':')
sorting_by = df.groupby(level=group_lvl)[sort_column].transform('sum' if 'level' = x-1).abs().values
或
sorting_by = df.groupby(level=group_lvl).loc['level' = x-1: sort_column].transform('sum').abs().values
两者均无效
有人知道如何在多索引 df 上进行这样的条件转换吗？
我不确定我是否完全理解了这个问题，但我认为您应该将列拆分为子类别，然后根据您想要的层次结构进行值排序。像下面这样的东西可能会完成这项工作。
使用以下命令创建新列：
for _, row in df.iterrows():
for item, col in zip(row.category.split(':'), ['cat', 'sub_cat', 'sub_sub_cat']):
df.loc[_, col] = item
然后对它们进行排序
df.sort_values(['cat', 'sub_cat', 'sub_sub_cat', 'amount'])
category amount cat sub_cat sub_sub_cat
3 Household 1100 Household NaN NaN
7 Household : Cleaning 100 Household Cleaning NaN
8 Household : Cleaning : Bathroom 75 Household Cleaning Bathroom
9 Household : Cleaning : Kitchen 25 Household Cleaning Kitchen
10 Household : Rent 400 Household Rent NaN
4 Household : Utilities 600 Household Utilities NaN
6 Household : Utilities : Electric 200 Household Utilities Electric
5 Household : Utilities : Water 400 Household Utilities Water
11 Living 250 Living NaN NaN
15 Living : Anitsomething -1000 Living Anitsomething NaN
13 Living : Food 100 Living Food NaN
12 Living : Other 150 Living Other NaN
14 Living : Something 1000 Living Something NaN
0 Transport 5000 Transport NaN NaN
1 Transport : Car 4900 Transport Car NaN
2 Transport : Train 100 Transport Train Na
好的，我花了一些时间才弄明白，但现在我很确定这行得通。也比递归方法快得多。
def _sort_tree_df(self, df, tree_column, sort_column, delimeter=':'):
df=df.copy()
parts = df[tree_column].str.split(delimeter).apply(lambda x: [y.strip() for y in x]).apply(pd.Series)
for i, column in enumerate(parts.columns):
df[column] = parts[column]
sort_columns = [df[tree_column].values]
sort_columns.append(df[sort_column].abs().values)
df['level'] = df[tree_column].str.count(':')
for x in range(len(parts.columns), 0, -1):
group_columns = list(range(0, x))
sorting_by = df.copy()
sorting_by.loc[sorting_by['level'] != x-1, sort_column] = np.nan
sorting_by = sorting_by.groupby(group_columns)[sort_column].transform('sum').abs().values
sort_columns.append(sorting_by)
sort_indexes = np.lexsort(sort_columns)
df_sorted = df.iloc[sort_indexes[::-1]]
df_sorted.reset_index(drop=True, inplace=True)
df.drop([column for column in parts.columns], inplace=True, axis=1)
df.drop('level', inplace=True, axis=1)
return df_sorted