Django celery 和 celery-beat 守护脚本错误
Django celery and celery-beat daemonizing script error
我将使用 celery 设置周期性任务,以尝试使用脚本(在官方网站上引用)在测试服务器中守护 django celery 进程:
所以,这是我的 celeryd 文件:
# Names of nodes to start
# most people will only start one node:
CELERYD_NODES="indicators"
# but you can also start multiple and configure settings
# for each in CELERYD_OPTS (see `celery multi --help` for examples):
#CELERYD_NODES="worker1 worker2 worker3"
# alternatively, you can specify the number of nodes to start:
#CELERYD_NODES=10
# Absolute or relative path to the 'celery' command:
#CELERY_BIN="/usr/local/bin/celery"
CELERY_BIN="/opt/pymis/envs/indicators/indicators_test/bin/celery"
# App instance to use
# comment out this line if you don't use an app
#CELERY_APP="sgiprocess"
# or fully qualified:
CELERY_APP="indicator.tasks:app"
# Where to chdir at start.
CELERYD_CHDIR="/opt/pymis/reps/indicator_repository/indicator_ms/"
# Extra command-line arguments to the worker
CELERYD_OPTS="--time-limit=300 --concurrency=8"
# %N will be replaced with the first part of the nodename.
CELERYD_LOG_FILE="/var/log/celery/worker1.log"
CELERYD_PID_FILE="/var/run/celery/worker1.pid"
# Workers should run as an unprivileged user.
# You need to create this user manually (or you can choose
# a user/group combination that already exists, e.g. nobody).
CELERYD_USER="celeryuser"
CELERYD_GROUP="mygroup"
# If enabled pid and log directories will be created if missing,
# and owned by the userid/group configured.
CELERY_CREATE_DIRS=1
这是我的 celerybeat 文件:
# Absolute or relative path to the 'celery' command:
#CELERY_BIN="/usr/local/bin/celery"
CELERY_BIN="/opt/pymis/envs/indicators/indicators_test/bin/celery"
# App instance to use
# comment out this line if you don't use an app
#CELERY_APP="sgiprocess"
# or fully qualified:
CELERY_APP="indicator.tasks:app"
# Where to chdir at start.
CELERYBEAT_CHDIR="/opt/pymis/reps/indicators_repository/indicator_ms/"
# Extra arguments to celerybeat
CELERYBEAT_OPTS="--schedule=/var/run/celery/celerybeat-schedule"
这是我的错误轨迹:
(indicators_test) user@server:/opt/pymis/reps/indicator_repository/indicator_ms$ sudo /etc/init.d/celeryd restart
celery init v10.1.
Using config script: /etc/default/celeryd
celery multi v4.0.2 (latentcall)
Traceback (most recent call last):
File "/usr/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/__main__.py", line 18, in <module>
main()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/__main__.py", line 14, in main
_main()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/celery.py", line 326, in main
cmd.execute_from_commandline(argv)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/celery.py", line 488, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 279, in execute_from_commandline
argv = self.setup_app_from_commandline(argv)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 481, in setup_app_from_commandline
self.app = self.find_app(app)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 503, in find_app
return find_app(app, symbol_by_name=self.symbol_by_name)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/app/utils.py", line 355, in find_app
sym = symbol_by_name(app, imp=imp)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 506, in symbol_by_name
return imports.symbol_by_name(name, imp=imp)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/kombu/utils/imports.py", line 56, in symbol_by_name
module = imp(module_name, package=package, **kwargs)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/importlib/__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 986, in _gcd_import
File "<frozen importlib._bootstrap>", line 969, in _find_and_load
File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 665, in exec_module
File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed
File "/opt/pymis/reps/indicator_repository/indicator_ms/indicator/tasks.py", line 7, in <module>
from .models import Value
File "/opt/pymis/reps/indicator_repository/indicator_ms/indicator/models.py", line 1, in <module>
from django.contrib.auth.models import User
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/contrib/auth/models.py", line 4, in <module>
from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/contrib/auth/base_user.py", line 52, in <module>
class AbstractBaseUser(models.Model):
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/db/models/base.py", line 105, in __new__
app_config = apps.get_containing_app_config(module)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/apps/registry.py", line 237, in get_containing_app_config
self.check_apps_ready()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/apps/registry.py", line 124, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
> Restarting node indicators@EMT-LIBREAPP: * Child terminated with exit code 1
FAILED
事情是,当我手动启动 celery 和 celerybeat 时:
celery -A config worker -l info
celery -A config beat
我没有收到任何错误,并且一切正常加载。
更新:这是我的 celery.py 文件
from __future__ import absolute_import
from celery import Celery
from django.conf import settings
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.test')
app = Celery('indicators')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
更新 2:这是我的目录树
indicator_ms (project root)
├── config
│ ├── settings
│ │ ├── __init__.py
│ │ ├── local.py
│ │ ├── test.py
│ │ └── production.py
│ ├── __init__.py
│ ├── celery.py
│ ├── urls.py
│ └── wsgi.py
│
└── indicators (app)
├── __init__.py
└── tasks.py
这是配置文件夹(不是设置文件夹)中的 init.py 文件
from __future__ import absolute_import
from .celery import app as celery_app
首先...您是否喜欢这个项目结构? :D
因为你刚开始吃芹菜,所以我要冒险说你可以移动celery.py
。
如果您的 celeryd
文件位于自定义路径中,您可以尝试导出您的设置模块。
# regular celeryd config stuff...
# projects settings module.
export DJANGO_SETTINGS_MODULE=my_project.settings.production
export PYTHONPATH=$PYTHONPATH:/path/to/my_project/
并在您的 celery.py
文件中使用 'indicator_celery.settings'
,而不是 config.settings.local
# celery.py file
from __future__ import absolute_import
import os
from celery import Celery
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'indicator_celery.settings')
from django.conf import settings # noqa
app = Celery('indicator_celery')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
也indicator_celery.__init__
文件应该是这样的:
# celery app init file
# -*- coding: utf-8 -*-
from __future__ import absolute_import
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app # noqa
最终目录树:
indicator_ms (project root)
├── config
│ ├── settings
│ │ ├── __init__.py
│ │ ├── local.py
│ │ ├── test.py
│ │ └── production.py
│ ├── __init__.py
│ ├── urls.py
│ └── wsgi.py
├── indicator_celery
│ ├── __init__.py
│ └── celery.py
└── indicators (app)
├── __init__.py
└── tasks.py
为了进一步说明@mislav 的观点,celery 文档中有一个关于 configuration of your celery app instance 的重要警告:
Django users now uses the exact same template as above, but make sure that the module that defines your Celery app instance also sets a default value for DJANGO_SETTINGS_MODULE
as shown in the example Django project in First steps with Django.
(强调)
因此请确保您的 celery.py
中出现类似于以下行的内容
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'my_app.settings')
from django.conf import settings # noqa
app = Celery('my_project_name')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
如果所有其他方法都失败了,请在调用 celery 之前更新 initd 脚本以获取您的虚拟环境。
由于 django celery 任务不是正确的项目模块,我们需要在各自的任务模块中调用 django.setup() 方法。
在我的例子中,解决方案是在任何其他项目相关导入之前在 task.py 文件中设置此行:
import django
django.setup()
我将使用 celery 设置周期性任务,以尝试使用脚本(在官方网站上引用)在测试服务器中守护 django celery 进程:
所以,这是我的 celeryd 文件:
# Names of nodes to start
# most people will only start one node:
CELERYD_NODES="indicators"
# but you can also start multiple and configure settings
# for each in CELERYD_OPTS (see `celery multi --help` for examples):
#CELERYD_NODES="worker1 worker2 worker3"
# alternatively, you can specify the number of nodes to start:
#CELERYD_NODES=10
# Absolute or relative path to the 'celery' command:
#CELERY_BIN="/usr/local/bin/celery"
CELERY_BIN="/opt/pymis/envs/indicators/indicators_test/bin/celery"
# App instance to use
# comment out this line if you don't use an app
#CELERY_APP="sgiprocess"
# or fully qualified:
CELERY_APP="indicator.tasks:app"
# Where to chdir at start.
CELERYD_CHDIR="/opt/pymis/reps/indicator_repository/indicator_ms/"
# Extra command-line arguments to the worker
CELERYD_OPTS="--time-limit=300 --concurrency=8"
# %N will be replaced with the first part of the nodename.
CELERYD_LOG_FILE="/var/log/celery/worker1.log"
CELERYD_PID_FILE="/var/run/celery/worker1.pid"
# Workers should run as an unprivileged user.
# You need to create this user manually (or you can choose
# a user/group combination that already exists, e.g. nobody).
CELERYD_USER="celeryuser"
CELERYD_GROUP="mygroup"
# If enabled pid and log directories will be created if missing,
# and owned by the userid/group configured.
CELERY_CREATE_DIRS=1
这是我的 celerybeat 文件:
# Absolute or relative path to the 'celery' command:
#CELERY_BIN="/usr/local/bin/celery"
CELERY_BIN="/opt/pymis/envs/indicators/indicators_test/bin/celery"
# App instance to use
# comment out this line if you don't use an app
#CELERY_APP="sgiprocess"
# or fully qualified:
CELERY_APP="indicator.tasks:app"
# Where to chdir at start.
CELERYBEAT_CHDIR="/opt/pymis/reps/indicators_repository/indicator_ms/"
# Extra arguments to celerybeat
CELERYBEAT_OPTS="--schedule=/var/run/celery/celerybeat-schedule"
这是我的错误轨迹:
(indicators_test) user@server:/opt/pymis/reps/indicator_repository/indicator_ms$ sudo /etc/init.d/celeryd restart
celery init v10.1.
Using config script: /etc/default/celeryd
celery multi v4.0.2 (latentcall)
Traceback (most recent call last):
File "/usr/lib/python3.5/runpy.py", line 184, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.5/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/__main__.py", line 18, in <module>
main()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/__main__.py", line 14, in main
_main()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/celery.py", line 326, in main
cmd.execute_from_commandline(argv)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/celery.py", line 488, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 279, in execute_from_commandline
argv = self.setup_app_from_commandline(argv)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 481, in setup_app_from_commandline
self.app = self.find_app(app)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 503, in find_app
return find_app(app, symbol_by_name=self.symbol_by_name)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/app/utils.py", line 355, in find_app
sym = symbol_by_name(app, imp=imp)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/bin/base.py", line 506, in symbol_by_name
return imports.symbol_by_name(name, imp=imp)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/kombu/utils/imports.py", line 56, in symbol_by_name
module = imp(module_name, package=package, **kwargs)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/celery/utils/imports.py", line 101, in import_from_cwd
return imp(module, package=package)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/importlib/__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 986, in _gcd_import
File "<frozen importlib._bootstrap>", line 969, in _find_and_load
File "<frozen importlib._bootstrap>", line 958, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 673, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 665, in exec_module
File "<frozen importlib._bootstrap>", line 222, in _call_with_frames_removed
File "/opt/pymis/reps/indicator_repository/indicator_ms/indicator/tasks.py", line 7, in <module>
from .models import Value
File "/opt/pymis/reps/indicator_repository/indicator_ms/indicator/models.py", line 1, in <module>
from django.contrib.auth.models import User
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/contrib/auth/models.py", line 4, in <module>
from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/contrib/auth/base_user.py", line 52, in <module>
class AbstractBaseUser(models.Model):
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/db/models/base.py", line 105, in __new__
app_config = apps.get_containing_app_config(module)
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/apps/registry.py", line 237, in get_containing_app_config
self.check_apps_ready()
File "/opt/pymis/envs/indicators/indicators_test/lib/python3.5/site-packages/django/apps/registry.py", line 124, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
> Restarting node indicators@EMT-LIBREAPP: * Child terminated with exit code 1
FAILED
事情是,当我手动启动 celery 和 celerybeat 时:
celery -A config worker -l info
celery -A config beat
我没有收到任何错误,并且一切正常加载。
更新:这是我的 celery.py 文件
from __future__ import absolute_import
from celery import Celery
from django.conf import settings
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.test')
app = Celery('indicators')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
@app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
更新 2:这是我的目录树
indicator_ms (project root)
├── config
│ ├── settings
│ │ ├── __init__.py
│ │ ├── local.py
│ │ ├── test.py
│ │ └── production.py
│ ├── __init__.py
│ ├── celery.py
│ ├── urls.py
│ └── wsgi.py
│
└── indicators (app)
├── __init__.py
└── tasks.py
这是配置文件夹(不是设置文件夹)中的 init.py 文件
from __future__ import absolute_import
from .celery import app as celery_app
首先...您是否喜欢这个项目结构? :D
因为你刚开始吃芹菜,所以我要冒险说你可以移动celery.py
。
如果您的 celeryd
文件位于自定义路径中,您可以尝试导出您的设置模块。
# regular celeryd config stuff...
# projects settings module.
export DJANGO_SETTINGS_MODULE=my_project.settings.production
export PYTHONPATH=$PYTHONPATH:/path/to/my_project/
并在您的 celery.py
文件中使用 'indicator_celery.settings'
,而不是 config.settings.local
# celery.py file
from __future__ import absolute_import
import os
from celery import Celery
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'indicator_celery.settings')
from django.conf import settings # noqa
app = Celery('indicator_celery')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
也indicator_celery.__init__
文件应该是这样的:
# celery app init file
# -*- coding: utf-8 -*-
from __future__ import absolute_import
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app # noqa
最终目录树:
indicator_ms (project root)
├── config
│ ├── settings
│ │ ├── __init__.py
│ │ ├── local.py
│ │ ├── test.py
│ │ └── production.py
│ ├── __init__.py
│ ├── urls.py
│ └── wsgi.py
├── indicator_celery
│ ├── __init__.py
│ └── celery.py
└── indicators (app)
├── __init__.py
└── tasks.py
为了进一步说明@mislav 的观点,celery 文档中有一个关于 configuration of your celery app instance 的重要警告:
Django users now uses the exact same template as above, but make sure that the module that defines your Celery app instance also sets a default value for
DJANGO_SETTINGS_MODULE
as shown in the example Django project in First steps with Django.
(强调)
因此请确保您的 celery.py
中出现类似于以下行的内容# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'my_app.settings')
from django.conf import settings # noqa
app = Celery('my_project_name')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
如果所有其他方法都失败了,请在调用 celery 之前更新 initd 脚本以获取您的虚拟环境。
由于 django celery 任务不是正确的项目模块,我们需要在各自的任务模块中调用 django.setup() 方法。
在我的例子中,解决方案是在任何其他项目相关导入之前在 task.py 文件中设置此行:
import django
django.setup()