Django-使用 ManyToManyField 查询的嵌套模型中的重复查询
Django- Duplicated queries in nested models querying with ManyToManyField
如何删除屏幕截图中的重复查询？
我有以下两个模型，
class Genre(MPTTModel):
name = models.CharField(max_length=50, unique=True)
parent = TreeForeignKey('self', on_delete=models.CASCADE, null=True,
blank=True, related_name='children')
def __str__(self):
return self.name
class Game(models.Model):
name = models.CharField(max_length=50)
genre = models.ManyToManyField(Genre, blank=True, related_name='games')
def __str__(self):
return self.name
并且有序列化器和视图，
class GameSerializer(serializers.ModelSerializer):
class Meta:
model = Game
exclude = ['genre', ]
class GenreGameSerializer(serializers.ModelSerializer):
children = RecursiveField(many=True)
games = GameSerializer(many=True,)
class Meta:
model = Genre
fields = ['id', 'name', 'children', 'games']
class GamesByGenreAPI(APIView):
queryset = Genre.objects.root_nodes()
serializer_class = GenreGameSerializer
def get(self, request, *args, **kwargs):
ser = GenreGameSerializer(data=Genre.objects.root_nodes()
.prefetch_related('children__children', 'games'), many=True)
if ser.is_valid():
pass
return Response(ser.data)
所以基本上序列化时填充的模型看起来像这样
结果如我所料，但每种类型都有 n 个重复查询。我该如何解决？谢谢..
here is a paste https://pastebin.com/xfRdBaF4 with all code, if you want to reproduce the issue.
Also add path('games/', GamesByGenreAPI.as_view()), in urls.py which is omitted in paste.
更新
尝试记录查询以检查它是否与调试工具栏有关，但不是，查询是重复的。这是屏幕截图。
以下是我克服多重查询的方法。
from collections import defaultdict
from rest_framework.serializers import SerializerMethodField
class GameSerializer(serializers.ModelSerializer):
class Meta:
model = Game
exclude = ['genre', ]
class GenreGameSerializer(serializers.ModelSerializer):
children = SerializerMethodField(source='get_children')
games = GameSerializer(many=True)
class Meta:
model = Genre
fields = ['id', 'name', 'games']
def get_children(self, obj):
# get genre childrens from context and pass it to same serializer
# no extra queries are done, since we alredy have the instances
children = self.context['children'].get(obj.id, [])
serializer = GenreGameSerializer(children, many=True, context=self.context)
return serializer.data
class GamesByGenreAPI(APIView):
queryset = Genre.objects.root_nodes()
serializer_class = GenreGameSerializer
def get(self, request, *args, **kwargs):
# gather genres from queryset class attribute and prefetch games
genres = self.get_queryset().prefetch_related('games')
# gather all descendants of root nodes and prefetch games
genre_descendants = genres.get_descendants().prefetch_related('games')
# create a dictionary with key parent and value list of children
# this will not require extra queries
children_dict = defaultdict(list)
for descendant in descendants:
children_dict[descendant.parent_id].append(descendant)
# add the dictionary as context for serializer
context = self.get_serializer_context()
context['children'] = children_dict
# send the context to serializer
ser = GenreGameSerializer(data=genres, context=context, many=True)
return Response(ser.data)
GamesByGenreAPI class 可以写得更好，通过覆盖 self.get_queryset() 和 self.get_serializer_context() 但我试图将它保留在一种方法中以便更好地理解。
prefetch_related 仅适用于根级树，因为它仅在该查询中指定。通过使用 RecursiveField 生成的新查询获得的后续子项没有这些相关对象 预取 。也许您可以覆盖 RecursiveField 中的查询，但我认为它对找到的 childrens 的每个属性运行一个单独的新查询。
好吧，如果您想将查询减少到只有 3 个，则需要从模型中获取所有数据并以递归方式手动构建输出数组。
这是一个非常肮脏的代码，你失去了“Django 的魔力”，正如评论中所说，我认为在大多数情况下这是浪费时间。
class GamesByGenreAPI(APIView):
def get(self, request, *args, **kwargs):
games = {}
for g in Game.objects.all():
games[g.pk] = {
'id': g.id,
'name': g.name
}
TreeGame = Game.genre.through
tree_game = {}
for tg in TreeGame.objects.all():
if tg.genre_id not in tree_game:
tree_game[tg.genre_id] = []
tree_game[tg.genre_id].append(tg.game_id)
childrens = {}
roots = []
for g in Genre.objects.all():
if g.level == 0:
roots.append(g)
else:
if g.parent_id not in childrens:
childrens[g.parent_id] = []
childrens[g.parent_id].append(g)
def _get_data_from_tree_branch(game):
branch_data = {
'id': game.pk,
'name': game.name
}
if game.pk in childrens:
# Move up if you need a children array in every response
branch_data['children'] = []
for c in childrens[game.pk]:
branch_data['children'].append(
_get_data_from_tree_branch(c)
)
if game.pk in tree_game:
# Move up if you need a games array in every response
branch_data['games'] = []
for rel in tree_game[game.pk]:
branch_data['games'].append(games[rel])
return branch_data
data = []
for g in roots:
data.append(_get_data_from_tree_branch(g))
return Response(data)
根据调试工具栏的输出，我假设您的流派模型中有两层嵌套（根，第 1 层）。不知道Level 1有没有children，也就是有Level 2的流派，因为我看不到查询结果（但这与当前问题无关）
根级别流派是 (1, 4, 7)，级别 1 是 (2, 3, 5, 6, 8, 9)。
预取适用于这些查找 prefetch_related("children__children") 因为查询被分组在两个单独的查询中，这是应该的。
与根级别类型 (prefetch_related("games")) 相关的游戏的下一个查询也被预取。这是调试工具栏输出中的第四个查询。
如您所见，接下来的查询是在单独的查询中获取每个级别 1 流派的游戏，我认为这是从序列化程序字段触发的，因为视图中没有指定查找，可以预取那些记录。添加另一个针对这些记录的预取查找应该可以解决问题。
ser = GenreGameSerializer(data=Genre.objects.root_nodes()
.prefetch_related(
'children__children',
'games'
# prefetching games for Level 1 genres
'children__games'),
many=True)
请注意，如果有更多的嵌套流派，则应为每个嵌套级别应用相同的逻辑。例如，如果有 2 级流派，那么您应该预取这些流派的相关游戏：
ser = GenreGameSerializer(data=Genre.objects.root_nodes()
.prefetch_related(
'children__children',
'games'
'children__games',
'children__children__games'),
many=True)
您错过了 prefetch_related() 中的 children__games 关系。如果您替换
它将起作用
prefetch_related('children__children', 'games')
和
prefetch_related('children__children', 'children__games', 'games')
Here is request list at Django debug panel
如何删除屏幕截图中的重复查询？
我有以下两个模型，
class Genre(MPTTModel):
name = models.CharField(max_length=50, unique=True)
parent = TreeForeignKey('self', on_delete=models.CASCADE, null=True,
blank=True, related_name='children')
def __str__(self):
return self.name
class Game(models.Model):
name = models.CharField(max_length=50)
genre = models.ManyToManyField(Genre, blank=True, related_name='games')
def __str__(self):
return self.name
并且有序列化器和视图，
class GameSerializer(serializers.ModelSerializer):
class Meta:
model = Game
exclude = ['genre', ]
class GenreGameSerializer(serializers.ModelSerializer):
children = RecursiveField(many=True)
games = GameSerializer(many=True,)
class Meta:
model = Genre
fields = ['id', 'name', 'children', 'games']
class GamesByGenreAPI(APIView):
queryset = Genre.objects.root_nodes()
serializer_class = GenreGameSerializer
def get(self, request, *args, **kwargs):
ser = GenreGameSerializer(data=Genre.objects.root_nodes()
.prefetch_related('children__children', 'games'), many=True)
if ser.is_valid():
pass
return Response(ser.data)
所以基本上序列化时填充的模型看起来像这样
结果如我所料，但每种类型都有 n 个重复查询。我该如何解决？谢谢..
here is a paste https://pastebin.com/xfRdBaF4 with all code, if you want to reproduce the issue.
Also addpath('games/', GamesByGenreAPI.as_view()),in urls.py which is omitted in paste.
更新
尝试记录查询以检查它是否与调试工具栏有关，但不是，查询是重复的。这是屏幕截图。
以下是我克服多重查询的方法。
from collections import defaultdict
from rest_framework.serializers import SerializerMethodField
class GameSerializer(serializers.ModelSerializer):
class Meta:
model = Game
exclude = ['genre', ]
class GenreGameSerializer(serializers.ModelSerializer):
children = SerializerMethodField(source='get_children')
games = GameSerializer(many=True)
class Meta:
model = Genre
fields = ['id', 'name', 'games']
def get_children(self, obj):
# get genre childrens from context and pass it to same serializer
# no extra queries are done, since we alredy have the instances
children = self.context['children'].get(obj.id, [])
serializer = GenreGameSerializer(children, many=True, context=self.context)
return serializer.data
class GamesByGenreAPI(APIView):
queryset = Genre.objects.root_nodes()
serializer_class = GenreGameSerializer
def get(self, request, *args, **kwargs):
# gather genres from queryset class attribute and prefetch games
genres = self.get_queryset().prefetch_related('games')
# gather all descendants of root nodes and prefetch games
genre_descendants = genres.get_descendants().prefetch_related('games')
# create a dictionary with key parent and value list of children
# this will not require extra queries
children_dict = defaultdict(list)
for descendant in descendants:
children_dict[descendant.parent_id].append(descendant)
# add the dictionary as context for serializer
context = self.get_serializer_context()
context['children'] = children_dict
# send the context to serializer
ser = GenreGameSerializer(data=genres, context=context, many=True)
return Response(ser.data)
GamesByGenreAPI class 可以写得更好，通过覆盖 self.get_queryset() 和 self.get_serializer_context() 但我试图将它保留在一种方法中以便更好地理解。
prefetch_related 仅适用于根级树，因为它仅在该查询中指定。通过使用 RecursiveField 生成的新查询获得的后续子项没有这些相关对象 预取 。也许您可以覆盖 RecursiveField 中的查询，但我认为它对找到的 childrens 的每个属性运行一个单独的新查询。
好吧，如果您想将查询减少到只有 3 个，则需要从模型中获取所有数据并以递归方式手动构建输出数组。
这是一个非常肮脏的代码，你失去了“Django 的魔力”，正如评论中所说，我认为在大多数情况下这是浪费时间。
class GamesByGenreAPI(APIView):
def get(self, request, *args, **kwargs):
games = {}
for g in Game.objects.all():
games[g.pk] = {
'id': g.id,
'name': g.name
}
TreeGame = Game.genre.through
tree_game = {}
for tg in TreeGame.objects.all():
if tg.genre_id not in tree_game:
tree_game[tg.genre_id] = []
tree_game[tg.genre_id].append(tg.game_id)
childrens = {}
roots = []
for g in Genre.objects.all():
if g.level == 0:
roots.append(g)
else:
if g.parent_id not in childrens:
childrens[g.parent_id] = []
childrens[g.parent_id].append(g)
def _get_data_from_tree_branch(game):
branch_data = {
'id': game.pk,
'name': game.name
}
if game.pk in childrens:
# Move up if you need a children array in every response
branch_data['children'] = []
for c in childrens[game.pk]:
branch_data['children'].append(
_get_data_from_tree_branch(c)
)
if game.pk in tree_game:
# Move up if you need a games array in every response
branch_data['games'] = []
for rel in tree_game[game.pk]:
branch_data['games'].append(games[rel])
return branch_data
data = []
for g in roots:
data.append(_get_data_from_tree_branch(g))
return Response(data)
根据调试工具栏的输出，我假设您的流派模型中有两层嵌套（根，第 1 层）。不知道Level 1有没有children，也就是有Level 2的流派，因为我看不到查询结果（但这与当前问题无关）
根级别流派是 (1, 4, 7)，级别 1 是 (2, 3, 5, 6, 8, 9)。
预取适用于这些查找 prefetch_related("children__children") 因为查询被分组在两个单独的查询中，这是应该的。
与根级别类型 (prefetch_related("games")) 相关的游戏的下一个查询也被预取。这是调试工具栏输出中的第四个查询。
如您所见，接下来的查询是在单独的查询中获取每个级别 1 流派的游戏，我认为这是从序列化程序字段触发的，因为视图中没有指定查找，可以预取那些记录。添加另一个针对这些记录的预取查找应该可以解决问题。
ser = GenreGameSerializer(data=Genre.objects.root_nodes()
.prefetch_related(
'children__children',
'games'
# prefetching games for Level 1 genres
'children__games'),
many=True)
请注意，如果有更多的嵌套流派，则应为每个嵌套级别应用相同的逻辑。例如，如果有 2 级流派，那么您应该预取这些流派的相关游戏：
ser = GenreGameSerializer(data=Genre.objects.root_nodes()
.prefetch_related(
'children__children',
'games'
'children__games',
'children__children__games'),
many=True)
您错过了 prefetch_related() 中的 children__games 关系。如果您替换
prefetch_related('children__children', 'games')
和
prefetch_related('children__children', 'children__games', 'games')
Here is request list at Django debug panel