Django Inline for ManyToMany 生成重复查询
Django Inline for ManyToMany generate duplicate queries
我的 Django 管理员遇到了一些主要的性能问题。根据我有多少内联，有很多重复查询。
models.py
class Setting(models.Model):
name = models.CharField(max_length=50, unique=True)
class Meta:
ordering = ('name',)
def __str__(self):
return self.name
class DisplayedGroup(models.Model):
name = models.CharField(max_length=30, unique=True)
position = models.PositiveSmallIntegerField(default=100)
class Meta:
ordering = ('priority',)
def __str__(self):
return self.name
class Machine(models.Model):
name = models.CharField(max_length=20, unique=True)
settings = models.ManyToManyField(
Setting, through='Arrangement', blank=True
)
class Meta:
ordering = ('name',)
def __str__(self):
return self.name
class Arrangement(models.Model):
machine = models.ForeignKey(Machine, on_delete=models.CASCADE)
setting = models.ForeignKey(Setting, on_delete=models.CASCADE)
displayed_group = models.ForeignKey(
DisplayedGroup, on_delete=models.PROTECT,
default=1)
priority = models.PositiveSmallIntegerField(
default=100,
help_text='Smallest number will be displayed first'
)
class Meta:
ordering = ('priority',)
unique_together = (("machine", "setting"),)
admin.py
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 1
class MachineAdmin(admin.ModelAdmin):
inlines = (ArrangementInline,)
如果我在内联表单中添加了 3 个设置和 1 个额外的设置，我将有大约 10 个重复查询
SELECT "corps_setting"."id", "corps_setting"."name", "corps_setting"."user_id", "corps_setting"."tagged", "corps_setting"."created", "corps_setting"."modified" FROM "corps_setting" ORDER BY "corps_setting"."name" ASC
- Duplicated 5 times
SELECT "corps_displayedgroup"."id", "corps_displayedgroup"."name", "corps_displayedgroup"."color", "corps_displayedgroup"."priority", "corps_displayedgroup"."created", "corps_displayedgroup"."modified" FROM "corps_displayedgroup" ORDER BY "corps_displayedgroup"."priority" ASC
- Duplicated 5 times.
有人可以告诉我我做错了什么吗？我花了 3 天时间试图自己解决问题，但运气不佳。
当我有大约 50 个机器内联设置时，问题会变得更糟，我将有大约 100 个查询。
Here is the screenshot
EDIT 2020:
Check out the answer by @isobolev below who's taken this answer and improved on it to make it more generic. :)
这在 Django 中几乎是正常的行为 - 它不会为您进行优化，但它为您提供了自己进行优化的合适工具。不要担心，100 个查询并不是真正需要立即修复的大问题（我在一页上看到 16k 个查询）。但是如果你的数据量会迅速增加，那么处理它当然是明智的。
您将配备的主要武器是查询集方法 select_related() 和 prefetch_related()。真的没有必要深入研究它们，因为它们有很好的文档记录 here，但只是一个通用的指针：
当您查询的对象只有一个相关对象（FK 或 one2one）时使用 select_related()
当你查询的对象有多个相关对象（FK或M2M的另一端）时使用prefetch_related()
以及如何在 Django 管理中使用它们？初级，我亲爱的华生。覆盖管理页面方法 get_queryset(self, request) 所以它看起来像这样：
from django.contrib import admin
class SomeRandomAdmin(admin.ModelAdmin):
def get_queryset(self, request):
return super().get_queryset(request).select_related('field1', 'field2').prefetch_related('field3')
编辑： 阅读了您的评论后，我意识到我对您问题的初步解释是绝对错误的。对于您的问题，我也有多种解决方案，这里是：
我大部分时间使用并推荐的简单方法：只需将 Django 默认的 select 小部件替换为 raw_id_field 小部件，并且不进行任何查询。只需在内联管理中设置 raw_id_fields = ('setting', 'displayed_group') 即可。
但是，如果你不想摆脱 select 框，我可以提供一些半黑的代码来解决这个问题，但是相当冗长而且不是很漂亮的。这个想法是覆盖创建表单的表单集并为表单集中的这些字段指定选项，以便它们只从数据库中查询一次。
开始了：
from django import forms
from django.contrib import admin
from app.models import Arrangement, Machine, Setting, DisplayedGroup
class ChoicesFormSet(forms.BaseInlineFormSet):
setting_choices = list(Setting.objects.values_list('id', 'name'))
displayed_group_choices = list(DisplayedGroup.objects.values_list('id', 'name'))
def _construct_form(self, i, **kwargs):
kwargs['setting_choices'] = self.setting_choices
kwargs['displayed_group_choices'] = self.displayed_group_choices
return super()._construct_form(i, **kwargs)
class ArrangementInlineForm(forms.ModelForm):
class Meta:
model = Arrangement
exclude = ()
def __init__(self, *args, **kwargs):
setting_choices = kwargs.pop('setting_choices', [((), ())])
displayed_group_choices = kwargs.pop('displayed_group_choices', [((), ())])
super().__init__(*args, **kwargs)
# This ensures that you can still save the form without setting all 50 (see extra value) inline values.
# When you save, the field value is checked against the "initial" value
# of a field and you only get a validation error if you've changed any of the initial values.
self.fields['setting'].choices = [('-', '---')] + setting_choices
self.fields['setting'].initial = self.fields['setting'].choices[0][0]
self.fields['setting'].empty_values = (self.fields['setting'].choices[0][0],)
self.fields['displayed_group'].choices = displayed_group_choices
self.fields['displayed_group'].initial = self.fields['displayed_group'].choices[0][0]
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 50
form = ArrangementInlineForm
formset = ChoicesFormSet
def get_queryset(self, request):
return super().get_queryset(request).select_related('setting')
class MachineAdmin(admin.ModelAdmin):
inlines = (ArrangementInline,)
admin.site.register(Machine, MachineAdmin)
如果您发现可以改进的地方或有任何疑问，请告诉我。
我根据@makaveli 的回答组装了一个通用解决方案，似乎没有评论中提到的问题：
class CachingModelChoicesFormSet(forms.BaseInlineFormSet):
"""
Used to avoid duplicate DB queries by caching choices and passing them all the forms.
To be used in conjunction with `CachingModelChoicesForm`.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
sample_form = self._construct_form(0)
self.cached_choices = {}
try:
model_choice_fields = sample_form.model_choice_fields
except AttributeError:
pass
else:
for field_name in model_choice_fields:
if field_name in sample_form.fields and not isinstance(
sample_form.fields[field_name].widget, forms.HiddenInput):
self.cached_choices[field_name] = [c for c in sample_form.fields[field_name].choices]
def get_form_kwargs(self, index):
kwargs = super().get_form_kwargs(index)
kwargs['cached_choices'] = self.cached_choices
return kwargs
class CachingModelChoicesForm(forms.ModelForm):
"""
Gets cached choices from `CachingModelChoicesFormSet` and uses them in model choice fields in order to reduce
number of DB queries when used in admin inlines.
"""
@property
def model_choice_fields(self):
return [fn for fn, f in self.fields.items()
if isinstance(f, (forms.ModelChoiceField, forms.ModelMultipleChoiceField,))]
def __init__(self, *args, **kwargs):
cached_choices = kwargs.pop('cached_choices', {})
super().__init__(*args, **kwargs)
for field_name, choices in cached_choices.items():
if choices is not None and field_name in self.fields:
self.fields[field_name].choices = choices
您需要做的就是从 CachingModelChoicesForm 子class 您的模型并在内联中使用 CachingModelChoicesFormSet class:
class ArrangementInlineForm(CachingModelChoicesForm):
class Meta:
model = Arrangement
exclude = ()
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 50
form = ArrangementInlineForm
formset = CachingModelChoicesFormSet
如今，（感谢 that question), BaseFormset receives a form_kwargs attribute。
已接受答案中的 ChoicesFormSet 代码可以稍微修改如下：
class ChoicesFormSet(forms.BaseInlineFormSet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
setting_choices = list(Setting.objects.values_list('id', 'name'))
displayed_group_choices = list(DisplayedGroup.objects.values_list('id', 'name'))
self.form_kwargs['setting_choices'] = self.setting_choices
self.form_kwargs['displayed_group_choices'] = self.displayed_group_choices
其余代码保持不变，如已接受的答案中所述：
class ArrangementInlineForm(forms.ModelForm):
class Meta:
model = Arrangement
exclude = ()
def __init__(self, *args, **kwargs):
setting_choices = kwargs.pop('setting_choices', [((), ())])
displayed_group_choices = kwargs.pop('displayed_group_choices', [((), ())])
super().__init__(*args, **kwargs)
# This ensures that you can still save the form without setting all 50 (see extra value) inline values.
# When you save, the field value is checked against the "initial" value
# of a field and you only get a validation error if you've changed any of the initial values.
self.fields['setting'].choices = [('-', '---')] + setting_choices
self.fields['setting'].initial = self.fields['setting'].choices[0][0]
self.fields['setting'].empty_values = (self.fields['setting'].choices[0][0],)
self.fields['displayed_group'].choices = displayed_group_choices
self.fields['displayed_group'].initial = self.fields['displayed_group'].choices[0][0]
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 50
form = ArrangementInlineForm
formset = ChoicesFormSet
def get_queryset(self, request):
return super().get_queryset(request).select_related('setting')
class MachineAdmin(admin.ModelAdmin):
inlines = (ArrangementInline,)
admin.site.register(Machine, MachineAdmin)
我的 Django 管理员遇到了一些主要的性能问题。根据我有多少内联，有很多重复查询。
models.py
class Setting(models.Model):
name = models.CharField(max_length=50, unique=True)
class Meta:
ordering = ('name',)
def __str__(self):
return self.name
class DisplayedGroup(models.Model):
name = models.CharField(max_length=30, unique=True)
position = models.PositiveSmallIntegerField(default=100)
class Meta:
ordering = ('priority',)
def __str__(self):
return self.name
class Machine(models.Model):
name = models.CharField(max_length=20, unique=True)
settings = models.ManyToManyField(
Setting, through='Arrangement', blank=True
)
class Meta:
ordering = ('name',)
def __str__(self):
return self.name
class Arrangement(models.Model):
machine = models.ForeignKey(Machine, on_delete=models.CASCADE)
setting = models.ForeignKey(Setting, on_delete=models.CASCADE)
displayed_group = models.ForeignKey(
DisplayedGroup, on_delete=models.PROTECT,
default=1)
priority = models.PositiveSmallIntegerField(
default=100,
help_text='Smallest number will be displayed first'
)
class Meta:
ordering = ('priority',)
unique_together = (("machine", "setting"),)
admin.py
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 1
class MachineAdmin(admin.ModelAdmin):
inlines = (ArrangementInline,)
如果我在内联表单中添加了 3 个设置和 1 个额外的设置，我将有大约 10 个重复查询
SELECT "corps_setting"."id", "corps_setting"."name", "corps_setting"."user_id", "corps_setting"."tagged", "corps_setting"."created", "corps_setting"."modified" FROM "corps_setting" ORDER BY "corps_setting"."name" ASC
- Duplicated 5 times
SELECT "corps_displayedgroup"."id", "corps_displayedgroup"."name", "corps_displayedgroup"."color", "corps_displayedgroup"."priority", "corps_displayedgroup"."created", "corps_displayedgroup"."modified" FROM "corps_displayedgroup" ORDER BY "corps_displayedgroup"."priority" ASC
- Duplicated 5 times.
有人可以告诉我我做错了什么吗？我花了 3 天时间试图自己解决问题，但运气不佳。
当我有大约 50 个机器内联设置时，问题会变得更糟，我将有大约 100 个查询。
Here is the screenshot
EDIT 2020:
Check out the answer by @isobolev below who's taken this answer and improved on it to make it more generic. :)
这在 Django 中几乎是正常的行为 - 它不会为您进行优化，但它为您提供了自己进行优化的合适工具。不要担心，100 个查询并不是真正需要立即修复的大问题（我在一页上看到 16k 个查询）。但是如果你的数据量会迅速增加，那么处理它当然是明智的。
您将配备的主要武器是查询集方法 select_related() 和 prefetch_related()。真的没有必要深入研究它们，因为它们有很好的文档记录 here，但只是一个通用的指针：
当您查询的对象只有一个相关对象（FK 或 one2one）时使用
select_related()当你查询的对象有多个相关对象（FK或M2M的另一端）时使用
prefetch_related()
以及如何在 Django 管理中使用它们？初级，我亲爱的华生。覆盖管理页面方法 get_queryset(self, request) 所以它看起来像这样：
from django.contrib import admin
class SomeRandomAdmin(admin.ModelAdmin):
def get_queryset(self, request):
return super().get_queryset(request).select_related('field1', 'field2').prefetch_related('field3')
编辑： 阅读了您的评论后，我意识到我对您问题的初步解释是绝对错误的。对于您的问题，我也有多种解决方案，这里是：
我大部分时间使用并推荐的简单方法：只需将 Django 默认的 select 小部件替换为
raw_id_field小部件，并且不进行任何查询。只需在内联管理中设置raw_id_fields = ('setting', 'displayed_group')即可。但是，如果你不想摆脱 select 框，我可以提供一些半黑的代码来解决这个问题，但是相当冗长而且不是很漂亮的。这个想法是覆盖创建表单的表单集并为表单集中的这些字段指定选项，以便它们只从数据库中查询一次。
开始了：
from django import forms
from django.contrib import admin
from app.models import Arrangement, Machine, Setting, DisplayedGroup
class ChoicesFormSet(forms.BaseInlineFormSet):
setting_choices = list(Setting.objects.values_list('id', 'name'))
displayed_group_choices = list(DisplayedGroup.objects.values_list('id', 'name'))
def _construct_form(self, i, **kwargs):
kwargs['setting_choices'] = self.setting_choices
kwargs['displayed_group_choices'] = self.displayed_group_choices
return super()._construct_form(i, **kwargs)
class ArrangementInlineForm(forms.ModelForm):
class Meta:
model = Arrangement
exclude = ()
def __init__(self, *args, **kwargs):
setting_choices = kwargs.pop('setting_choices', [((), ())])
displayed_group_choices = kwargs.pop('displayed_group_choices', [((), ())])
super().__init__(*args, **kwargs)
# This ensures that you can still save the form without setting all 50 (see extra value) inline values.
# When you save, the field value is checked against the "initial" value
# of a field and you only get a validation error if you've changed any of the initial values.
self.fields['setting'].choices = [('-', '---')] + setting_choices
self.fields['setting'].initial = self.fields['setting'].choices[0][0]
self.fields['setting'].empty_values = (self.fields['setting'].choices[0][0],)
self.fields['displayed_group'].choices = displayed_group_choices
self.fields['displayed_group'].initial = self.fields['displayed_group'].choices[0][0]
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 50
form = ArrangementInlineForm
formset = ChoicesFormSet
def get_queryset(self, request):
return super().get_queryset(request).select_related('setting')
class MachineAdmin(admin.ModelAdmin):
inlines = (ArrangementInline,)
admin.site.register(Machine, MachineAdmin)
如果您发现可以改进的地方或有任何疑问，请告诉我。
我根据@makaveli 的回答组装了一个通用解决方案，似乎没有评论中提到的问题：
class CachingModelChoicesFormSet(forms.BaseInlineFormSet):
"""
Used to avoid duplicate DB queries by caching choices and passing them all the forms.
To be used in conjunction with `CachingModelChoicesForm`.
"""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
sample_form = self._construct_form(0)
self.cached_choices = {}
try:
model_choice_fields = sample_form.model_choice_fields
except AttributeError:
pass
else:
for field_name in model_choice_fields:
if field_name in sample_form.fields and not isinstance(
sample_form.fields[field_name].widget, forms.HiddenInput):
self.cached_choices[field_name] = [c for c in sample_form.fields[field_name].choices]
def get_form_kwargs(self, index):
kwargs = super().get_form_kwargs(index)
kwargs['cached_choices'] = self.cached_choices
return kwargs
class CachingModelChoicesForm(forms.ModelForm):
"""
Gets cached choices from `CachingModelChoicesFormSet` and uses them in model choice fields in order to reduce
number of DB queries when used in admin inlines.
"""
@property
def model_choice_fields(self):
return [fn for fn, f in self.fields.items()
if isinstance(f, (forms.ModelChoiceField, forms.ModelMultipleChoiceField,))]
def __init__(self, *args, **kwargs):
cached_choices = kwargs.pop('cached_choices', {})
super().__init__(*args, **kwargs)
for field_name, choices in cached_choices.items():
if choices is not None and field_name in self.fields:
self.fields[field_name].choices = choices
您需要做的就是从 CachingModelChoicesForm 子class 您的模型并在内联中使用 CachingModelChoicesFormSet class:
class ArrangementInlineForm(CachingModelChoicesForm):
class Meta:
model = Arrangement
exclude = ()
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 50
form = ArrangementInlineForm
formset = CachingModelChoicesFormSet
如今，（感谢 that question), BaseFormset receives a form_kwargs attribute。
已接受答案中的 ChoicesFormSet 代码可以稍微修改如下：
class ChoicesFormSet(forms.BaseInlineFormSet):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
setting_choices = list(Setting.objects.values_list('id', 'name'))
displayed_group_choices = list(DisplayedGroup.objects.values_list('id', 'name'))
self.form_kwargs['setting_choices'] = self.setting_choices
self.form_kwargs['displayed_group_choices'] = self.displayed_group_choices
其余代码保持不变，如已接受的答案中所述：
class ArrangementInlineForm(forms.ModelForm):
class Meta:
model = Arrangement
exclude = ()
def __init__(self, *args, **kwargs):
setting_choices = kwargs.pop('setting_choices', [((), ())])
displayed_group_choices = kwargs.pop('displayed_group_choices', [((), ())])
super().__init__(*args, **kwargs)
# This ensures that you can still save the form without setting all 50 (see extra value) inline values.
# When you save, the field value is checked against the "initial" value
# of a field and you only get a validation error if you've changed any of the initial values.
self.fields['setting'].choices = [('-', '---')] + setting_choices
self.fields['setting'].initial = self.fields['setting'].choices[0][0]
self.fields['setting'].empty_values = (self.fields['setting'].choices[0][0],)
self.fields['displayed_group'].choices = displayed_group_choices
self.fields['displayed_group'].initial = self.fields['displayed_group'].choices[0][0]
class ArrangementInline(admin.TabularInline):
model = Arrangement
extra = 50
form = ArrangementInlineForm
formset = ChoicesFormSet
def get_queryset(self, request):
return super().get_queryset(request).select_related('setting')
class MachineAdmin(admin.ModelAdmin):
inlines = (ArrangementInline,)
admin.site.register(Machine, MachineAdmin)