如何解析 df 中的 json 列，我们使用选定的键附加新列
how to parse a json column in a df where we append new column using selected keys
嗨，我是 python 和 R 的初学者。我有一个简短的问题：
#I have a data frame that looks like this:
# Import pandas library
import pandas as pd
# initialize list of lists
data = [['BarackObama', {'cap': {'english': 0.014543680863094452, 'universal': 0.005213309669283597},
'categories': {'content': 0.13252377443365895, 'friend': 0.27037007428252813,
'network': 0.07904647486470226, 'sentiment': 0.13142975907620189,
'temporal': 0.0560116435619808, 'user': 0.2120791504162319},
'display_scores': {'content': 0.7, 'english': 1.1, 'friend': 1.4, 'network': 0.4,
'sentiment': 0.7, 'temporal': 0.3, 'universal': 0.6, 'user': 1.1},
'scores': {'english': 0.22180647190550215, 'universal': 0.11116719108518804},
'user': {'id_str': '813286', 'screen_name': 'BarackObama'}}],
['realDonaldTrump', {'cap': {'english': 0.0014187924969112314, 'universal': 0.0018655051726169808},
'categories': {'content': 0.062020196630026815, 'friend': 0.19869669732913162,
'network': 0.05312993020038088, 'sentiment': 0.05985886859558471,
'temporal': 0.07924665710801207, 'user': 0.037517839108884524},
'display_scores': {'content': 0.3, 'english': 0.2, 'friend': 1.0, 'network': 0.3,
'sentiment': 0.3, 'temporal': 0.4, 'universal': 0.2, 'user': 0.2},
'scores': {'english': 0.03265990956683609, 'universal': 0.032398754737074244},
'user': {'id_str': '25073877', 'screen_name': 'realDonaldTrump'}}]]
# Create the pandas DataFrame
df = pd.DataFrame(data, columns=['Name', 'botScore'])
# print dataframe.
print(df)
# Name botScore
#0 BarackObama {'cap': {'english': 0.014543680863094452, 'uni...
#1 realDonaldTrump {'cap': {'english': 0.0014187924969112314, 'un...
所以我怎么能有这样的东西，我从数据框中 json 的 display_score 部分选择键和值并将它们附加到现有数据框？
# data-wrangling part using the display_scores key in json column....
# print(df)
# Name botScore english friend sentiment
#0 BarackObama {'cap':... 1.1 1.4 0.7
#1 realDonaldTrump {'cap':... 0.3 1.0 0.3
非常感谢您的帮助！我查看了几篇过去的帖子，但无法使用他们的方法解决我的问题：
首先，修复 data
- 将每个列表中位置
0 的名称添加到位置 1 的 dict
- 将
list of lists 转换为 list of dicts
for x in data:
x[1]['name'] = x[0]
data2 = [x[1] for x in data]
处理字典列表
- 使用 flatten package
- 我将在此处包含特定功能
def flatten_json(nested_json: dict, exclude: list=[''], sep='_') -> dict:
"""
Flatten a list of nested dicts.
"""
out = dict()
def flatten(x: (list, dict, str), name: str='', exclude=exclude):
if type(x) is dict:
for a in x:
if a not in exclude:
flatten(x[a], f'{name}{a}{sep}')
elif type(x) is list:
i = 0
for a in x:
flatten(a, f'{name}{i}{sep}')
i += 1
else:
out[name[:-1]] = x
flatten(nested_json)
return out
df = pd.DataFrame([flatten_json(x) for x in data2])
print(df)
cap_english cap_universal categories_content categories_friend categories_network categories_sentiment categories_temporal categories_user display_scores_content display_scores_english display_scores_friend display_scores_network display_scores_sentiment display_scores_temporal display_scores_universal display_scores_user scores_english scores_universal user_id_str user_screen_name name
0 0.014544 0.005213 0.132524 0.270370 0.079046 0.131430 0.056012 0.212079 0.7 1.1 1.4 0.4 0.7 0.3 0.6 1.1 0.221806 0.111167 813286 BarackObama BarackObama
1 0.001419 0.001866 0.062020 0.198697 0.053130 0.059859 0.079247 0.037518 0.3 0.2 1.0 0.3 0.3 0.4 0.2 0.2 0.032660 0.032399 25073877 realDonaldTrump realDonaldTrump
其他资源：
您可以使用 python 的 defaultdict 整理 botScore 列中的所有条目 'display_scores' 并连接回主数据框：
from collections import defaultdict
d = defaultdict(list)
#iterate through the display_scores key
#and append to dictionary
for ent in df.botScore:
for k, v in ent['display_scores'].items():
d[k].append(v)
d
defaultdict(list,
{'content': [0.7, 0.3],
'english': [1.1, 0.2],
'friend': [1.4, 1.0],
'network': [0.4, 0.3],
'sentiment': [0.7, 0.3],
'temporal': [0.3, 0.4],
'universal': [0.6, 0.2],
'user': [1.1, 0.2]})
#combine main dataframe with the dictionary
pd.concat([df,pd.DataFrame(d)],axis=1)
Name botScore content english friend network sentiment temporal universal user
0 BarackObama {'cap': {'english': 0.014543680863094452, 'uni... 0.7 1.1 1.4 0.4 0.7 0.3 0.6 1.1
1 realDonaldTrump {'cap': {'english': 0.0014187924969112314, 'un... 0.3 0.2 1.0 0.3 0.3 0.4 0.2 0.2
嗨，我是 python 和 R 的初学者。我有一个简短的问题：
#I have a data frame that looks like this:
# Import pandas library
import pandas as pd
# initialize list of lists
data = [['BarackObama', {'cap': {'english': 0.014543680863094452, 'universal': 0.005213309669283597},
'categories': {'content': 0.13252377443365895, 'friend': 0.27037007428252813,
'network': 0.07904647486470226, 'sentiment': 0.13142975907620189,
'temporal': 0.0560116435619808, 'user': 0.2120791504162319},
'display_scores': {'content': 0.7, 'english': 1.1, 'friend': 1.4, 'network': 0.4,
'sentiment': 0.7, 'temporal': 0.3, 'universal': 0.6, 'user': 1.1},
'scores': {'english': 0.22180647190550215, 'universal': 0.11116719108518804},
'user': {'id_str': '813286', 'screen_name': 'BarackObama'}}],
['realDonaldTrump', {'cap': {'english': 0.0014187924969112314, 'universal': 0.0018655051726169808},
'categories': {'content': 0.062020196630026815, 'friend': 0.19869669732913162,
'network': 0.05312993020038088, 'sentiment': 0.05985886859558471,
'temporal': 0.07924665710801207, 'user': 0.037517839108884524},
'display_scores': {'content': 0.3, 'english': 0.2, 'friend': 1.0, 'network': 0.3,
'sentiment': 0.3, 'temporal': 0.4, 'universal': 0.2, 'user': 0.2},
'scores': {'english': 0.03265990956683609, 'universal': 0.032398754737074244},
'user': {'id_str': '25073877', 'screen_name': 'realDonaldTrump'}}]]
# Create the pandas DataFrame
df = pd.DataFrame(data, columns=['Name', 'botScore'])
# print dataframe.
print(df)
# Name botScore
#0 BarackObama {'cap': {'english': 0.014543680863094452, 'uni...
#1 realDonaldTrump {'cap': {'english': 0.0014187924969112314, 'un...
所以我怎么能有这样的东西，我从数据框中 json 的 display_score 部分选择键和值并将它们附加到现有数据框？
# data-wrangling part using the display_scores key in json column....
# print(df)
# Name botScore english friend sentiment
#0 BarackObama {'cap':... 1.1 1.4 0.7
#1 realDonaldTrump {'cap':... 0.3 1.0 0.3
非常感谢您的帮助！我查看了几篇过去的帖子，但无法使用他们的方法解决我的问题：
首先，修复 data
- 将每个列表中位置
0的名称添加到位置1 的 - 将
list of lists转换为list of dicts
dict
for x in data:
x[1]['name'] = x[0]
data2 = [x[1] for x in data]
处理字典列表
- 使用 flatten package
- 我将在此处包含特定功能
def flatten_json(nested_json: dict, exclude: list=[''], sep='_') -> dict:
"""
Flatten a list of nested dicts.
"""
out = dict()
def flatten(x: (list, dict, str), name: str='', exclude=exclude):
if type(x) is dict:
for a in x:
if a not in exclude:
flatten(x[a], f'{name}{a}{sep}')
elif type(x) is list:
i = 0
for a in x:
flatten(a, f'{name}{i}{sep}')
i += 1
else:
out[name[:-1]] = x
flatten(nested_json)
return out
df = pd.DataFrame([flatten_json(x) for x in data2])
print(df)
cap_english cap_universal categories_content categories_friend categories_network categories_sentiment categories_temporal categories_user display_scores_content display_scores_english display_scores_friend display_scores_network display_scores_sentiment display_scores_temporal display_scores_universal display_scores_user scores_english scores_universal user_id_str user_screen_name name
0 0.014544 0.005213 0.132524 0.270370 0.079046 0.131430 0.056012 0.212079 0.7 1.1 1.4 0.4 0.7 0.3 0.6 1.1 0.221806 0.111167 813286 BarackObama BarackObama
1 0.001419 0.001866 0.062020 0.198697 0.053130 0.059859 0.079247 0.037518 0.3 0.2 1.0 0.3 0.3 0.4 0.2 0.2 0.032660 0.032399 25073877 realDonaldTrump realDonaldTrump
其他资源：
您可以使用 python 的 defaultdict 整理 botScore 列中的所有条目 'display_scores' 并连接回主数据框：
from collections import defaultdict
d = defaultdict(list)
#iterate through the display_scores key
#and append to dictionary
for ent in df.botScore:
for k, v in ent['display_scores'].items():
d[k].append(v)
d
defaultdict(list,
{'content': [0.7, 0.3],
'english': [1.1, 0.2],
'friend': [1.4, 1.0],
'network': [0.4, 0.3],
'sentiment': [0.7, 0.3],
'temporal': [0.3, 0.4],
'universal': [0.6, 0.2],
'user': [1.1, 0.2]})
#combine main dataframe with the dictionary
pd.concat([df,pd.DataFrame(d)],axis=1)
Name botScore content english friend network sentiment temporal universal user
0 BarackObama {'cap': {'english': 0.014543680863094452, 'uni... 0.7 1.1 1.4 0.4 0.7 0.3 0.6 1.1
1 realDonaldTrump {'cap': {'english': 0.0014187924969112314, 'un... 0.3 0.2 1.0 0.3 0.3 0.4 0.2 0.2