使用 pprof 配置 kubectl
profile kubectl using pprof
在 kubernetes 源代码中有一段代码处理分析部分,但我无法访问端点:
in kubernetes/pkgs/kubelet/server/stats/server.go
func (s *Server) InstallProfilingHandler(enableProfilingLogHandler bool, enableContentionProfiling bool) {
s.addMetricsBucketMatcher("debug")
if !enableProfilingLogHandler {
s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled."))
return
}
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
case "profile":
pprof.Profile(resp, req.Request)
case "symbol":
pprof.Symbol(resp, req.Request)
case "cmdline":
pprof.Cmdline(resp, req.Request)
case "trace":
pprof.Trace(resp, req.Request)
default:
pprof.Index(resp, req.Request)
}
}
// Setup pprof handlers.
ws := new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint")
s.restfulCont.Add(ws)
if enableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}
我不知道 pprof 工具使用的端口,但我发现它使用:
controller-0:/home/sysadmin/go/bin# netstat -atlpn | grep kubelet
tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 184856/kubelet
tcp 0 0 192.168.206.2:49720 192.168.206.1:6443 ESTABLISHED 184856/kubelet
tcp6 0 0 :::10250 :::* LISTEN 184856/kubelet
那我试过了
controller-0:/home/sysadmin/go/bin# ./go tool pprof http://localhost:6443/debug/pprof/mutex
Fetching profile over HTTP from http://localhost:6443/debug/pprof/mutex
http://localhost:6443/debug/pprof/mutex: server response: 400 Bad Request
failed to fetch any source profiles
有没有人知道我应该如何尝试访问 pprof 端点?或者如何尝试不同的方法来分析 kubelet 进程?
尝试:
$ kubectl proxy &
Starting to serve on 127.0.0.1:8001
$ go tool pprof "http://localhost:8001/api/v1/nodes/${NODE}/proxy/debug/pprof/profile"
当您启动 kubectl proxy
时,对 http://localhost:8001/api/v1/nodes/${NODE}/proxy/
的所有请求都会转到 ${NODE}
上的 kubelet 运行。您可以添加任何您想要的路径。让它成为 /debug/pprof/profile
或 debug/pprof/heap
或任何你想要的。
在 kubernetes 源代码中有一段代码处理分析部分,但我无法访问端点:
in kubernetes/pkgs/kubelet/server/stats/server.go
func (s *Server) InstallProfilingHandler(enableProfilingLogHandler bool, enableContentionProfiling bool) {
s.addMetricsBucketMatcher("debug")
if !enableProfilingLogHandler {
s.restfulCont.Handle(pprofBasePath, getHandlerForDisabledEndpoint("profiling endpoint is disabled."))
return
}
handlePprofEndpoint := func(req *restful.Request, resp *restful.Response) {
name := strings.TrimPrefix(req.Request.URL.Path, pprofBasePath)
switch name {
case "profile":
pprof.Profile(resp, req.Request)
case "symbol":
pprof.Symbol(resp, req.Request)
case "cmdline":
pprof.Cmdline(resp, req.Request)
case "trace":
pprof.Trace(resp, req.Request)
default:
pprof.Index(resp, req.Request)
}
}
// Setup pprof handlers.
ws := new(restful.WebService).Path(pprofBasePath)
ws.Route(ws.GET("/{subpath:*}").To(handlePprofEndpoint)).Doc("pprof endpoint")
s.restfulCont.Add(ws)
if enableContentionProfiling {
goruntime.SetBlockProfileRate(1)
}
}
我不知道 pprof 工具使用的端口,但我发现它使用:
controller-0:/home/sysadmin/go/bin# netstat -atlpn | grep kubelet
tcp 0 0 127.0.0.1:10248 0.0.0.0:* LISTEN 184856/kubelet
tcp 0 0 192.168.206.2:49720 192.168.206.1:6443 ESTABLISHED 184856/kubelet
tcp6 0 0 :::10250 :::* LISTEN 184856/kubelet
那我试过了
controller-0:/home/sysadmin/go/bin# ./go tool pprof http://localhost:6443/debug/pprof/mutex
Fetching profile over HTTP from http://localhost:6443/debug/pprof/mutex
http://localhost:6443/debug/pprof/mutex: server response: 400 Bad Request
failed to fetch any source profiles
有没有人知道我应该如何尝试访问 pprof 端点?或者如何尝试不同的方法来分析 kubelet 进程?
尝试:
$ kubectl proxy &
Starting to serve on 127.0.0.1:8001
$ go tool pprof "http://localhost:8001/api/v1/nodes/${NODE}/proxy/debug/pprof/profile"
当您启动 kubectl proxy
时,对 http://localhost:8001/api/v1/nodes/${NODE}/proxy/
的所有请求都会转到 ${NODE}
上的 kubelet 运行。您可以添加任何您想要的路径。让它成为 /debug/pprof/profile
或 debug/pprof/heap
或任何你想要的。