runtime/cgo: pthread_create 失败：资源暂时不可用
runtime/cgo: pthread_create failed: Resource temporarily unavailable
我目前有这个功能，通过 ffmpeg 传输多个 youtubedl 命令，然后将 ffmpeg 的输出传输到 HTTP 客户端。
func pipeThruFfmpegToMp4(vi *VideoInfo, rw web.ResponseWriter) error {
var ffmpeg *exec.Cmd
ffmpeg = exec.Command(
"ffmpeg",
"-i", "-",
"-i", "pipe:3",
"-c:v", "copy", "-c:a", "copy",
"-preset", "veryfast",
"-metadata", fmt.Sprintf(`title=%s`, vi.GetTitle()),
"-movflags", "frag_keyframe+empty_moov",
"-f", "mp4",
"-")
youtubevideo := exec.Command(YoutubeDLPath, "-c", "-f", fmt.Sprintf("%s/bestvideo[ext=mp4]/bestvideo/best", vi.GetFormat()), "--no-cache-dir", "--restrict-filenames", "--hls-prefer-native", "-o", "-", fmt.Sprintf("%s", vi.GetVideoUrl()))
fmt.Println(youtubevideo)
youtube := exec.Command(YoutubeDLPath, "-c", "-f", "bestaudio[ext=m4a]/bestaudio/best", "--no-cache-dir", "--restrict-filenames", "--hls-prefer-native", "-o", "-", fmt.Sprintf("%s", vi.GetVideoUrl()))
fmt.Println(youtube)
var ytvbuf, ytbuf, ffbuf bytes.Buffer
youtubevideo.Stderr = &ytvbuf
youtube.Stderr = &ytbuf
ffmpeg.Stderr = &ffbuf
video, err := youtubevideo.StdoutPipe()
if err != nil {
log.Printf("pipeThruFfmpegToMp4: %v\n", err)
return err
}
pipe3, err := youtube.StdoutPipe()
if err != nil {
log.Printf("pipeThruFfmpegToMp4: %v\n", err)
return err
}
ffmpeg.Stdin = video
ffmpeg.ExtraFiles = []*os.File{pipe3.(*os.File)}
ffmpeg.Stdout = rw
// Headers sent, no turning back now
rw.Header().Set("Content-Type", "video/mp4")
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment;filename=\"%s.mp4\"", vi.GetSlug()))
rw.Flush()
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
youtubevideo.Wait()
youtube.Wait()
// check ytvbuf, ytbuf, ffbuf for stderr errors
if ffbuf.Len() != 0 {
rollbar.Error(rollbar.ERR, err, &rollbar.Field{"stderr", ffbuf.String()})
log.Printf("pipeThruFfmpegToMp4: %v\n", ffbuf.String())
}
if ytvbuf.Len() != 0 {
rollbar.Error(rollbar.ERR, err, &rollbar.Field{"stderr", ytvbuf.String()})
log.Printf("pipeThruYouTubevDLToMp4: %v\n", ytvbuf.String())
}
if ytbuf.Len() != 0 {
rollbar.Error(rollbar.ERR, err, &rollbar.Field{"stderr", ytbuf.String()})
log.Printf("pipeThruYouTubeDLToMp4: %v\n", ytbuf.String())
}
return nil
}
问题是一切运行正常，但过了一会儿，ram 开始在服务器上填满，直到出现错误 runtime/cgo: pthread_create failed: Resource temporarily unavailable
我不确定这是内存泄漏，还是 youtube-dl 的任一实例没有正确关闭，或者 ffmpeg 是否没有正确关闭，只是随着程序运行越来越多而消耗越来越多的内存，直到程序因此错误而崩溃
runtime/cgo: pthread_create failed: Resource temporarily unavailable
SIGABRT: abort
PC=0x7f083501fe97 m=128 sigcode=18446744073709551610
goroutine 0 [idle]:
runtime: unknown pc 0x7f083501fe97
stack: frame={sp:0x7f05ff7fd7d0, fp:0x0} stack=[0x7f05feffe288,0x7f05ff7fde88)
00007f05ff7fd6d0: 00007f05ff7fd700 0000ffff00001fa0
00007f05ff7fd6e0: 00007f05ff7fdbe0 00007f05f8000da0
00007f05ff7fd6f0: 0000000000000000 000000000093032c
00007f05ff7fd700: 0000000000000000 00007f0835600ec3
00007f05ff7fd710: 0000000000000005 0000000000000000
00007f05ff7fd720: 000000c0000ce120 00007f0834ff1ce0
00007f05ff7fd730: 00007f05ff7fdaf0 00007f083560870a
00007f05ff7fd740: 0000000000000000 0000000000000000
00007f05ff7fd750: 0000000000000000 00007f05ff7fdbe0
00007f05ff7fd760: 2525252525252525 2525252525252525
00007f05ff7fd770: 000000ffffffffff 0000000000000000
00007f05ff7fd780: 000000ffffffffff 0000000000000000
00007f05ff7fd790: 000000c00010d1a0 000000c000953740
00007f05ff7fd7a0: 000000c0000ce120 000000c000cf2300
00007f05ff7fd7b0: 000000c00010d260 000000c001f4e180
00007f05ff7fd7c0: 000000c001f4e000 000000c00169f680
00007f05ff7fd7d0: <0000000000000000 000000c001f34180
00007f05ff7fd7e0: 6e75720000000000 6f67632f656d6974
00007f05ff7fd7f0: 0000000000000000 0000000000000000
00007f05ff7fd800: 000000c000cf2300 000000c00010d260
00007f05ff7fd810: 000000c001f4e180 000000c001f4e000
00007f05ff7fd820: 000000c00169f680 000000c00169f500
00007f05ff7fd830: 000000c001f34180 000000c001f34000
00007f05ff7fd840: 000000c000c92780 000000c001ec2600
00007f05ff7fd850: fffffffe7fffffff ffffffffffffffff
00007f05ff7fd860: ffffffffffffffff ffffffffffffffff
00007f05ff7fd870: ffffffffffffffff ffffffffffffffff
00007f05ff7fd880: ffffffffffffffff ffffffffffffffff
00007f05ff7fd890: ffffffffffffffff ffffffffffffffff
00007f05ff7fd8a0: ffffffffffffffff ffffffffffffffff
00007f05ff7fd8b0: ffffffffffffffff ffffffffffffffff
00007f05ff7fd8c0: ffffffffffffffff ffffffffffffffff
runtime: unknown pc 0x7f083501fe97
stack: frame={sp:0x7f05ff7fd7d0, fp:0x0} stack=[0x7f05feffe288,0x7f05ff7fde88)
00007f05ff7fd6d0: 00007f05ff7fd700 0000ffff00001fa0
00007f05ff7fd6e0: 00007f05ff7fdbe0 00007f05f8000da0
00007f05ff7fd6f0: 0000000000000000 000000000093032c
00007f05ff7fd700: 0000000000000000 00007f0835600ec3
00007f05ff7fd710: 0000000000000005 0000000000000000
我也尝试过使用 CGO_ENABLED=0 构建二进制文件，即使我什至没有 import "c"，但最终也会出错
runtime: failed to create new OS thread (have 21 already; errno=11)
runtime: may need to increase max user processes (ulimit -u)
fatal error: newosproc
我的限额已经很慷慨了，这也可能是个问题？还是我可能需要增加管道尺寸？
$ ulimit -a
core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimited
scheduling priority (-e) 0
file size (blocks, -f) unlimited
pending signals (-i) 192907
max locked memory (kbytes, -l) 16384
max memory size (kbytes, -m) unlimited
open files (-n) 100000
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0
stack size (kbytes, -s) 65536
cpu time (seconds, -t) unlimited
max user processes (-u) 63883
virtual memory (kbytes, -v) unlimited
无论哪种方式，我都感谢我能得到的所有帮助。这是关于这个问题的link to the entire program and here is a thread in the golang-nuts forum
我有时会遇到问题，您的代码不会终止 youtube-dl 进程。视频被转换了，但是过程被保留了下来。作为第一个指标，请观察进程计数，如下所示：
ps | grep youtube-dl | wc -l
可以让子进程超时，对于一个子进程，这看起来像这样：
package main
import (
"bytes"
"fmt"
"os/exec"
"time"
)
func main() {
// Sleep is used, so we can control how long it runs.
cmd := exec.Command("sleep", "2")
// Use a bytes.Buffer to get the output
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Start()
// Use a channel to signal completion so we can use a select statement
done := make(chan error)
go func() { done <- cmd.Wait() }()
// Start a timer, should be higher for your video conversion,
// I suggest you use a value that fits both your videos and server capabilities
timeout := time.After(2 * time.Second)
// The select statement allows us to execute based on which channel
// we get a message from first.
select {
case <-timeout:
// Timeout happened first, kill the process and print a message.
cmd.Process.Kill()
fmt.Println("Command timed out")
case err := <-done:
// Command completed before timeout. Print output and error if it exists.
fmt.Println("Output:", buf.String())
if err != nil {
fmt.Println("Non-zero exit code:", err)
}
}
}
在你的情况下你可以替换
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
youtubevideo.Wait()
youtube.Wait()
和
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
commands := 3
done := make(chan error)
go func() { done <- ffmpeg.Wait() }()
go func() { done <- youtubevideo.Wait() }()
go func() { done <- youtube.Wait() }()
timeout := time.After(1 * time.Hour)
Loop:
for {
select {
case <-timeout:
ffmpeg.Process.Kill()
youtubevideo.Process.Kill()
youtube.Process.Kill()
log.Println("Conversion timed out")
break Loop
case err := <-done:
if err != nil {
log.Println("Non-zero exit code:", err)
}
commands = commands - 1
if commands == 0 {
break Loop
}
}
}
所以经过一些调试并在凯文的回答的帮助下。好像成功后youtube-dl并没有关闭。
鉴于如果 ffmpeg 完成，那么拥有 youtube-dl 实例 运行 没有任何意义，并且由于 Wait() 函数正在等待 youtube-dl 完成，但它似乎从来没有发送它的完成信号，我继续并不得不添加一个 kill 来杀死子进程。
所以我不得不添加更改这个
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
youtubevideo.Wait()
youtube.Wait()
对此
youtube.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
youtubevideo.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
syscall.Kill(-youtube.Process.Pid, syscall.SIGKILL)
syscall.Kill(-youtubevideo.Process.Pid, syscall.SIGKILL)
youtubevideo.Wait()
youtube.Wait()
请注意，因为它似乎生成了某种子进程，youtube.Process.Kill() 实际上并没有终止进程，这就是我不得不使用 syscall 以及 [=19= 的原因]
我目前有这个功能，通过 ffmpeg 传输多个 youtubedl 命令，然后将 ffmpeg 的输出传输到 HTTP 客户端。
func pipeThruFfmpegToMp4(vi *VideoInfo, rw web.ResponseWriter) error {
var ffmpeg *exec.Cmd
ffmpeg = exec.Command(
"ffmpeg",
"-i", "-",
"-i", "pipe:3",
"-c:v", "copy", "-c:a", "copy",
"-preset", "veryfast",
"-metadata", fmt.Sprintf(`title=%s`, vi.GetTitle()),
"-movflags", "frag_keyframe+empty_moov",
"-f", "mp4",
"-")
youtubevideo := exec.Command(YoutubeDLPath, "-c", "-f", fmt.Sprintf("%s/bestvideo[ext=mp4]/bestvideo/best", vi.GetFormat()), "--no-cache-dir", "--restrict-filenames", "--hls-prefer-native", "-o", "-", fmt.Sprintf("%s", vi.GetVideoUrl()))
fmt.Println(youtubevideo)
youtube := exec.Command(YoutubeDLPath, "-c", "-f", "bestaudio[ext=m4a]/bestaudio/best", "--no-cache-dir", "--restrict-filenames", "--hls-prefer-native", "-o", "-", fmt.Sprintf("%s", vi.GetVideoUrl()))
fmt.Println(youtube)
var ytvbuf, ytbuf, ffbuf bytes.Buffer
youtubevideo.Stderr = &ytvbuf
youtube.Stderr = &ytbuf
ffmpeg.Stderr = &ffbuf
video, err := youtubevideo.StdoutPipe()
if err != nil {
log.Printf("pipeThruFfmpegToMp4: %v\n", err)
return err
}
pipe3, err := youtube.StdoutPipe()
if err != nil {
log.Printf("pipeThruFfmpegToMp4: %v\n", err)
return err
}
ffmpeg.Stdin = video
ffmpeg.ExtraFiles = []*os.File{pipe3.(*os.File)}
ffmpeg.Stdout = rw
// Headers sent, no turning back now
rw.Header().Set("Content-Type", "video/mp4")
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment;filename=\"%s.mp4\"", vi.GetSlug()))
rw.Flush()
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
youtubevideo.Wait()
youtube.Wait()
// check ytvbuf, ytbuf, ffbuf for stderr errors
if ffbuf.Len() != 0 {
rollbar.Error(rollbar.ERR, err, &rollbar.Field{"stderr", ffbuf.String()})
log.Printf("pipeThruFfmpegToMp4: %v\n", ffbuf.String())
}
if ytvbuf.Len() != 0 {
rollbar.Error(rollbar.ERR, err, &rollbar.Field{"stderr", ytvbuf.String()})
log.Printf("pipeThruYouTubevDLToMp4: %v\n", ytvbuf.String())
}
if ytbuf.Len() != 0 {
rollbar.Error(rollbar.ERR, err, &rollbar.Field{"stderr", ytbuf.String()})
log.Printf("pipeThruYouTubeDLToMp4: %v\n", ytbuf.String())
}
return nil
}
问题是一切运行正常，但过了一会儿，ram 开始在服务器上填满，直到出现错误 runtime/cgo: pthread_create failed: Resource temporarily unavailable
我不确定这是内存泄漏，还是 youtube-dl 的任一实例没有正确关闭，或者 ffmpeg 是否没有正确关闭，只是随着程序运行越来越多而消耗越来越多的内存，直到程序因此错误而崩溃
runtime/cgo: pthread_create failed: Resource temporarily unavailable
SIGABRT: abort
PC=0x7f083501fe97 m=128 sigcode=18446744073709551610
goroutine 0 [idle]:
runtime: unknown pc 0x7f083501fe97
stack: frame={sp:0x7f05ff7fd7d0, fp:0x0} stack=[0x7f05feffe288,0x7f05ff7fde88)
00007f05ff7fd6d0: 00007f05ff7fd700 0000ffff00001fa0
00007f05ff7fd6e0: 00007f05ff7fdbe0 00007f05f8000da0
00007f05ff7fd6f0: 0000000000000000 000000000093032c
00007f05ff7fd700: 0000000000000000 00007f0835600ec3
00007f05ff7fd710: 0000000000000005 0000000000000000
00007f05ff7fd720: 000000c0000ce120 00007f0834ff1ce0
00007f05ff7fd730: 00007f05ff7fdaf0 00007f083560870a
00007f05ff7fd740: 0000000000000000 0000000000000000
00007f05ff7fd750: 0000000000000000 00007f05ff7fdbe0
00007f05ff7fd760: 2525252525252525 2525252525252525
00007f05ff7fd770: 000000ffffffffff 0000000000000000
00007f05ff7fd780: 000000ffffffffff 0000000000000000
00007f05ff7fd790: 000000c00010d1a0 000000c000953740
00007f05ff7fd7a0: 000000c0000ce120 000000c000cf2300
00007f05ff7fd7b0: 000000c00010d260 000000c001f4e180
00007f05ff7fd7c0: 000000c001f4e000 000000c00169f680
00007f05ff7fd7d0: <0000000000000000 000000c001f34180
00007f05ff7fd7e0: 6e75720000000000 6f67632f656d6974
00007f05ff7fd7f0: 0000000000000000 0000000000000000
00007f05ff7fd800: 000000c000cf2300 000000c00010d260
00007f05ff7fd810: 000000c001f4e180 000000c001f4e000
00007f05ff7fd820: 000000c00169f680 000000c00169f500
00007f05ff7fd830: 000000c001f34180 000000c001f34000
00007f05ff7fd840: 000000c000c92780 000000c001ec2600
00007f05ff7fd850: fffffffe7fffffff ffffffffffffffff
00007f05ff7fd860: ffffffffffffffff ffffffffffffffff
00007f05ff7fd870: ffffffffffffffff ffffffffffffffff
00007f05ff7fd880: ffffffffffffffff ffffffffffffffff
00007f05ff7fd890: ffffffffffffffff ffffffffffffffff
00007f05ff7fd8a0: ffffffffffffffff ffffffffffffffff
00007f05ff7fd8b0: ffffffffffffffff ffffffffffffffff
00007f05ff7fd8c0: ffffffffffffffff ffffffffffffffff
runtime: unknown pc 0x7f083501fe97
stack: frame={sp:0x7f05ff7fd7d0, fp:0x0} stack=[0x7f05feffe288,0x7f05ff7fde88)
00007f05ff7fd6d0: 00007f05ff7fd700 0000ffff00001fa0
00007f05ff7fd6e0: 00007f05ff7fdbe0 00007f05f8000da0
00007f05ff7fd6f0: 0000000000000000 000000000093032c
00007f05ff7fd700: 0000000000000000 00007f0835600ec3
00007f05ff7fd710: 0000000000000005 0000000000000000
我也尝试过使用 CGO_ENABLED=0 构建二进制文件，即使我什至没有 import "c"，但最终也会出错
runtime: failed to create new OS thread (have 21 already; errno=11)
runtime: may need to increase max user processes (ulimit -u)
fatal error: newosproc
我的限额已经很慷慨了，这也可能是个问题？还是我可能需要增加管道尺寸？
$ ulimit -a
core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimited
scheduling priority (-e) 0
file size (blocks, -f) unlimited
pending signals (-i) 192907
max locked memory (kbytes, -l) 16384
max memory size (kbytes, -m) unlimited
open files (-n) 100000
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0
stack size (kbytes, -s) 65536
cpu time (seconds, -t) unlimited
max user processes (-u) 63883
virtual memory (kbytes, -v) unlimited
无论哪种方式，我都感谢我能得到的所有帮助。这是关于这个问题的link to the entire program and here is a thread in the golang-nuts forum
我有时会遇到问题，您的代码不会终止 youtube-dl 进程。视频被转换了，但是过程被保留了下来。作为第一个指标，请观察进程计数，如下所示：
ps | grep youtube-dl | wc -l
可以让子进程超时，对于一个子进程，这看起来像这样：
package main
import (
"bytes"
"fmt"
"os/exec"
"time"
)
func main() {
// Sleep is used, so we can control how long it runs.
cmd := exec.Command("sleep", "2")
// Use a bytes.Buffer to get the output
var buf bytes.Buffer
cmd.Stdout = &buf
cmd.Start()
// Use a channel to signal completion so we can use a select statement
done := make(chan error)
go func() { done <- cmd.Wait() }()
// Start a timer, should be higher for your video conversion,
// I suggest you use a value that fits both your videos and server capabilities
timeout := time.After(2 * time.Second)
// The select statement allows us to execute based on which channel
// we get a message from first.
select {
case <-timeout:
// Timeout happened first, kill the process and print a message.
cmd.Process.Kill()
fmt.Println("Command timed out")
case err := <-done:
// Command completed before timeout. Print output and error if it exists.
fmt.Println("Output:", buf.String())
if err != nil {
fmt.Println("Non-zero exit code:", err)
}
}
}
在你的情况下你可以替换
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
youtubevideo.Wait()
youtube.Wait()
和
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
commands := 3
done := make(chan error)
go func() { done <- ffmpeg.Wait() }()
go func() { done <- youtubevideo.Wait() }()
go func() { done <- youtube.Wait() }()
timeout := time.After(1 * time.Hour)
Loop:
for {
select {
case <-timeout:
ffmpeg.Process.Kill()
youtubevideo.Process.Kill()
youtube.Process.Kill()
log.Println("Conversion timed out")
break Loop
case err := <-done:
if err != nil {
log.Println("Non-zero exit code:", err)
}
commands = commands - 1
if commands == 0 {
break Loop
}
}
}
所以经过一些调试并在凯文的回答的帮助下。好像成功后youtube-dl并没有关闭。
鉴于如果 ffmpeg 完成，那么拥有 youtube-dl 实例 运行 没有任何意义，并且由于 Wait() 函数正在等待 youtube-dl 完成，但它似乎从来没有发送它的完成信号，我继续并不得不添加一个 kill 来杀死子进程。
所以我不得不添加更改这个
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
youtubevideo.Wait()
youtube.Wait()
对此
youtube.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
youtubevideo.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
ffmpeg.Start()
youtubevideo.Start()
youtube.Start()
ffmpeg.Wait()
syscall.Kill(-youtube.Process.Pid, syscall.SIGKILL)
syscall.Kill(-youtubevideo.Process.Pid, syscall.SIGKILL)
youtubevideo.Wait()
youtube.Wait()
请注意，因为它似乎生成了某种子进程，youtube.Process.Kill() 实际上并没有终止进程，这就是我不得不使用 syscall 以及 [=19= 的原因]