如何对相关请求日志条目进行分组 GAE python 3.7 标准环境
How to group related request log entries GAE python 3.7 standard env
我正在使用 Google App Engine python 3.7 标准，我正在尝试对相关的请求日志条目进行分组。
根据 Writing Application Logs 文档，我应该：
Set the trace identifier in the LogEntry trace field of your app log
entries. The expected format is
projects/[PROJECT_ID]/traces/[TRACE_ID]
Where/How 应该使用 LogEntry 吗？
Stackdriver Logging documentation 没有说明它是如何可能的。我错过了什么吗？
代码示例将不胜感激。
[更新]
按照 Duck Hunt Duo 的建议，我尝试了以下方法，但没有成功：
trace_id = request.headers.get('X-Cloud-Trace-Context', 'no_trace_id').split('/')[0]
client = logging.Client()
logger = client.logger('appengine.googleapis.com%2Fstdout') # Not shown
# logger = client.logger('projects/{}/logs/stdout'.format(GOOGLE_CLOUD_PROJECT)) # error
# logger = client.logger('projects/{}/logs/appengine.googleapis.com%2Fstdout'.format(GOOGLE_CLOUD_PROJECT)) # error
logger.log_text('log_message', trace=trace_id)
日志没有出现在 GAE service log web console
Stackdriver Logging Client Library can be used to achieve this. The logger.log_text 函数将 LogEntry object 发送到 API。示例：
from google.cloud import logging
client = logging.Client()
logger = client.logger('appengine.googleapis.com%2Fstdout')
logger.log_text('log_message', trace=trace_id)
trace_id 应该从请求 headers 中检索，如文档所述。执行此操作的方法将取决于您如何处理请求，但例如在 Flask 中它会很简单 trace_id = request.headers['X-Cloud-Trace-Context'].split('/')[0]
您可能想看看我提供的答案 here。
（此答案解决了如何向写入 Stackdriver 的 Cloud Functions 日志添加日志记录严重性，但基本工作流程是相同的）
引用它：
[...], you can still create logs with certain severity by using the
Stackdriver Logging Client
Libraries.
Check this
documentation
in reference to the Python libraries, and this
one
for some usage-case examples.
Notice that in order to let the logs be under the correct resource,
you will have to manually configure them, see this
list
for the supported resource types. As well, each resource type has
some required
labels
that need to be present in the log structure.
编辑：
使用 App Engine 示例更新之前的答案：
from google.cloud import logging
from google.cloud.logging.resource import Resource
from flask import Flask
app = Flask(__name__)
@app.route('/')
def logger():
log_client = logging.Client()
log_name = 'appengine.googleapis.com%2Fstdout'
res = Resource( type='gae_app',
labels={
"project_id": "MY-PROJECT-ID",
"module_id": "MY-SERVICE-NAME"
})
logger = log_client.logger(log_name)
logger.log_struct({"message": "message string to log"}, resource=res, severity='ERROR') # As an example log message with a ERROR warning level
return 'Wrote logs to {}.'.format(logger.name)
以此代码为例，将日志的资源类型更改为 appengine.googleapis.com%2Fstdout 应该可行，并将 Resource 字段更改为与 gae_app 中的相同here.
中描述的标签
这是我的基本解决方案：
trace_id = request.headers.get('X-Cloud-Trace-Context', 'no_trace_id').split('/')[0]
trace_str = "projects/{}/traces/{}".format(os.getenv('GOOGLE_CLOUD_PROJECT'), trace_id)
log_client = logging.Client()
# This is the resource type of the log
log_name = 'stdout'
# Inside the resource, nest the required labels specific to the resource type
labels = {
'module_id': os.getenv('GAE_SERVICE'),
'project_id': os.getenv('GOOGLE_CLOUD_PROJECT'),
'version_id': os.getenv('GAE_VERSION')
}
res = Resource(type="gae_app",
labels=labels,
)
logger = log_client.logger(log_name)
logger.log_text("MESSAGE_STRING_TO_LOG", resource=res, severity='ERROR', trace=trace_str)
在它开始工作后，我将它包装在一个文件中，这样它就可以像 Google 的 python2.7 记录器一样工作。
这里是my_gae_logging.py:
import logging as python_logging
import os
from flask import request
from google.cloud import logging as gcp_logging
from google.cloud.logging.resource import Resource
# From GCP logging lib for Python2.7
CRITICAL = 50
FATAL = CRITICAL
ERROR = 40
WARNING = 30
WARN = WARNING
INFO = 20
DEBUG = 10
NOTSET = 0
_levelNames = {
CRITICAL: 'CRITICAL',
ERROR: 'ERROR',
WARNING: 'WARNING',
INFO: 'INFO',
DEBUG: 'DEBUG',
NOTSET: 'NOTSET',
'CRITICAL': CRITICAL,
'ERROR': ERROR,
'WARN': WARNING,
'WARNING': WARNING,
'INFO': INFO,
'DEBUG': DEBUG,
'NOTSET': NOTSET,
}
def get_trace_id():
trace_str = None
try:
trace_id = request.headers.get('X-Cloud-Trace-Context', 'no_trace_id').split('/')[0]
trace_str = "projects/{project_id}/traces/{trace_id}".format(
project_id=os.getenv('GOOGLE_CLOUD_PROJECT'),
trace_id=trace_id)
except:
pass
return trace_str
class Logging:
def __init__(self):
self._logger = None
@property
def logger(self):
if self._logger is not None:
return self._logger
log_client = gcp_logging.Client()
# This is the resource type of the log
log_name = 'appengine.googleapis.com%2Fstdout'
# Inside the resource, nest the required labels specific to the resource type
self._logger = log_client.logger(log_name)
return self._logger
@property
def resource(self):
resource = Resource(
type="gae_app",
labels={
'module_id': os.getenv('GAE_SERVICE'),
'project_id': os.getenv('GOOGLE_CLOUD_PROJECT'),
'version_id': os.getenv('GAE_VERSION')
}
)
return resource
def log(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, trace=get_trace_id())
def debug(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(DEBUG), trace=get_trace_id())
def info(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(INFO), trace=get_trace_id())
def warning(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(WARNING), trace=get_trace_id())
def warn(self, text):
return self.warning(text)
def error(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(ERROR), trace=get_trace_id())
def critical(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(CRITICAL), trace=get_trace_id())
if os.getenv('GAE_VERSION'): # check if running under gcp env
logging = Logging()
else:
# when not running under gcp env, use standard python_logging
logging = python_logging
用法：
from my_gae_logging import logging
logging.warn('this is my warning')
使用来自 Google Cloud Logging 的 AppEngineHandler 提供了很多基础设施。这允许附加到 python 日志记录模块，以便标准日志记录导入工作。
设置起来非常简单：
# Setup google cloud logging.
import logging
import google.cloud.logging # Don't conflict with standard logging
from google.cloud.logging.handlers import AppEngineHandler, setup_logging
client = google.cloud.logging.Client()
handler = AppEngineHandler(client, name='stdout')
logging.getLogger().setLevel(logging.INFO)
setup_logging(handler)
https://googleapis.dev/python/logging/latest/usage.html#cloud-logging-handler 处的文档建议非常相似，但不是使用 AppEngineHandler，而是使用 "CloudLoggingHandler"。它还指出 "AppEngineHandler" 适用于灵活的环境，但这适用于标准 python3 环境。
我正在使用 Google App Engine python 3.7 标准，我正在尝试对相关的请求日志条目进行分组。 根据 Writing Application Logs 文档，我应该：
Set the trace identifier in the LogEntry trace field of your app log entries. The expected format is projects/[PROJECT_ID]/traces/[TRACE_ID]
Where/How 应该使用 LogEntry 吗？
Stackdriver Logging documentation 没有说明它是如何可能的。我错过了什么吗？
代码示例将不胜感激。
[更新] 按照 Duck Hunt Duo 的建议，我尝试了以下方法，但没有成功：
trace_id = request.headers.get('X-Cloud-Trace-Context', 'no_trace_id').split('/')[0]
client = logging.Client()
logger = client.logger('appengine.googleapis.com%2Fstdout') # Not shown
# logger = client.logger('projects/{}/logs/stdout'.format(GOOGLE_CLOUD_PROJECT)) # error
# logger = client.logger('projects/{}/logs/appengine.googleapis.com%2Fstdout'.format(GOOGLE_CLOUD_PROJECT)) # error
logger.log_text('log_message', trace=trace_id)
日志没有出现在 GAE service log web console
Stackdriver Logging Client Library can be used to achieve this. The logger.log_text 函数将 LogEntry object 发送到 API。示例：
from google.cloud import logging
client = logging.Client()
logger = client.logger('appengine.googleapis.com%2Fstdout')
logger.log_text('log_message', trace=trace_id)
trace_id 应该从请求 headers 中检索，如文档所述。执行此操作的方法将取决于您如何处理请求，但例如在 Flask 中它会很简单 trace_id = request.headers['X-Cloud-Trace-Context'].split('/')[0]
您可能想看看我提供的答案 here。
（此答案解决了如何向写入 Stackdriver 的 Cloud Functions 日志添加日志记录严重性，但基本工作流程是相同的）
引用它：
[...], you can still create logs with certain severity by using the Stackdriver Logging Client Libraries. Check this documentation in reference to the Python libraries, and this one for some usage-case examples.
Notice that in order to let the logs be under the correct resource, you will have to manually configure them, see this list for the supported resource types. As well, each resource type has some required labels that need to be present in the log structure.
编辑：
使用 App Engine 示例更新之前的答案：
from google.cloud import logging
from google.cloud.logging.resource import Resource
from flask import Flask
app = Flask(__name__)
@app.route('/')
def logger():
log_client = logging.Client()
log_name = 'appengine.googleapis.com%2Fstdout'
res = Resource( type='gae_app',
labels={
"project_id": "MY-PROJECT-ID",
"module_id": "MY-SERVICE-NAME"
})
logger = log_client.logger(log_name)
logger.log_struct({"message": "message string to log"}, resource=res, severity='ERROR') # As an example log message with a ERROR warning level
return 'Wrote logs to {}.'.format(logger.name)
以此代码为例，将日志的资源类型更改为 appengine.googleapis.com%2Fstdout 应该可行，并将 Resource 字段更改为与 gae_app 中的相同here.
这是我的基本解决方案：
trace_id = request.headers.get('X-Cloud-Trace-Context', 'no_trace_id').split('/')[0]
trace_str = "projects/{}/traces/{}".format(os.getenv('GOOGLE_CLOUD_PROJECT'), trace_id)
log_client = logging.Client()
# This is the resource type of the log
log_name = 'stdout'
# Inside the resource, nest the required labels specific to the resource type
labels = {
'module_id': os.getenv('GAE_SERVICE'),
'project_id': os.getenv('GOOGLE_CLOUD_PROJECT'),
'version_id': os.getenv('GAE_VERSION')
}
res = Resource(type="gae_app",
labels=labels,
)
logger = log_client.logger(log_name)
logger.log_text("MESSAGE_STRING_TO_LOG", resource=res, severity='ERROR', trace=trace_str)
在它开始工作后，我将它包装在一个文件中，这样它就可以像 Google 的 python2.7 记录器一样工作。
这里是my_gae_logging.py:
import logging as python_logging
import os
from flask import request
from google.cloud import logging as gcp_logging
from google.cloud.logging.resource import Resource
# From GCP logging lib for Python2.7
CRITICAL = 50
FATAL = CRITICAL
ERROR = 40
WARNING = 30
WARN = WARNING
INFO = 20
DEBUG = 10
NOTSET = 0
_levelNames = {
CRITICAL: 'CRITICAL',
ERROR: 'ERROR',
WARNING: 'WARNING',
INFO: 'INFO',
DEBUG: 'DEBUG',
NOTSET: 'NOTSET',
'CRITICAL': CRITICAL,
'ERROR': ERROR,
'WARN': WARNING,
'WARNING': WARNING,
'INFO': INFO,
'DEBUG': DEBUG,
'NOTSET': NOTSET,
}
def get_trace_id():
trace_str = None
try:
trace_id = request.headers.get('X-Cloud-Trace-Context', 'no_trace_id').split('/')[0]
trace_str = "projects/{project_id}/traces/{trace_id}".format(
project_id=os.getenv('GOOGLE_CLOUD_PROJECT'),
trace_id=trace_id)
except:
pass
return trace_str
class Logging:
def __init__(self):
self._logger = None
@property
def logger(self):
if self._logger is not None:
return self._logger
log_client = gcp_logging.Client()
# This is the resource type of the log
log_name = 'appengine.googleapis.com%2Fstdout'
# Inside the resource, nest the required labels specific to the resource type
self._logger = log_client.logger(log_name)
return self._logger
@property
def resource(self):
resource = Resource(
type="gae_app",
labels={
'module_id': os.getenv('GAE_SERVICE'),
'project_id': os.getenv('GOOGLE_CLOUD_PROJECT'),
'version_id': os.getenv('GAE_VERSION')
}
)
return resource
def log(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, trace=get_trace_id())
def debug(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(DEBUG), trace=get_trace_id())
def info(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(INFO), trace=get_trace_id())
def warning(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(WARNING), trace=get_trace_id())
def warn(self, text):
return self.warning(text)
def error(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(ERROR), trace=get_trace_id())
def critical(self, text):
text = str(text)
self.logger.log_text(text, resource=self.resource, severity=_levelNames.get(CRITICAL), trace=get_trace_id())
if os.getenv('GAE_VERSION'): # check if running under gcp env
logging = Logging()
else:
# when not running under gcp env, use standard python_logging
logging = python_logging
用法：
from my_gae_logging import logging
logging.warn('this is my warning')
使用来自 Google Cloud Logging 的 AppEngineHandler 提供了很多基础设施。这允许附加到 python 日志记录模块，以便标准日志记录导入工作。
设置起来非常简单：
# Setup google cloud logging.
import logging
import google.cloud.logging # Don't conflict with standard logging
from google.cloud.logging.handlers import AppEngineHandler, setup_logging
client = google.cloud.logging.Client()
handler = AppEngineHandler(client, name='stdout')
logging.getLogger().setLevel(logging.INFO)
setup_logging(handler)
https://googleapis.dev/python/logging/latest/usage.html#cloud-logging-handler 处的文档建议非常相似，但不是使用 AppEngineHandler，而是使用 "CloudLoggingHandler"。它还指出 "AppEngineHandler" 适用于灵活的环境，但这适用于标准 python3 环境。