c++ socket accept() get Resource deadlock avoided errno with epoll 小程序

Question

我是C++初学者学习I/O多路复用

这是我的代码：

test.cpp（不使用 epoll() 并且效果很好）：

#include <iostream>
#include <cstdio>
#include <netinet/in.h>
#include <unistd.h>
#include <string.h>

int main() {
    std::cout << "Hello" << std::endl;

    char buffer[1024];
    buffer[0] = 'f';
    fprintf(stdout, "%s", buffer);
    std::cout << "Hello" << std::endl;

    int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    // bind & listen
    struct sockaddr_in serverAddr;
    memset(&serverAddr, 0, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(80);
    int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
    if (bindResult < 0) {
        fprintf(stderr, "Fail to bind\n");
        return 1;
    }
    int listenResult = listen(serverFd, 1024);
    if (listenResult < 0) {
        fprintf(stderr, "Fail to listen\n");
        return 1;
    }
    struct sockaddr clientAddr;
    unsigned int clientlen = sizeof(clientAddr);
    int acceptFd = accept(serverFd, &clientAddr, &clientlen);
    if (acceptFd < 0) {
        fprintf(stderr, "Fail to create client connection file descriptor\n");
        return 1;
    }
    int fd = acceptFd;
    ssize_t received = recv(fd, &buffer, 1024, 0);
    if (received < 0) {
        fprintf(stderr, "Fail to received bytess from client\n");
        if (errno == EINTR) {
            fprintf(stderr, "Reason: EINTR\n");
        } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
            fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
        } else {
            fprintf(stderr, "Reason: %d\n", errno);
            close(fd);
            return 1;
        }
    } else if (received == 0) {
        close(fd);
    } else {
        buffer[received] = '[=10=]';
        fprintf(stdout, "%s", buffer);
    }
}

test_2.cpp（确实使用了epoll()但效果不佳）：

#include <iostream>
#include <sys/epoll.h>
#include <unistd.h>
#include <netinet/in.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <signal.h>
#include <string.h>

int main() {
    // TODO: too much error message to handle, so it's necessary to deal with it (maybe macros can)
    std::cout << "Hello" << std::endl;

    // process ignore SIGPIPE which is caused by send(), or process will exit, which is hard to find out
    signal(SIGPIPE, SIG_IGN);

    // here needs a socket fd or other fd
    // well, AF_INET is okay;socket(PF_INET, SOCK_SEQPACKET, 0) is sctp, tcp cannot use SOCK_SEQPACKET :(
    // when using tcp, watch out **record boundaries**
    int serverFd = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP);
    if (serverFd < 0) {
        fprintf(stderr, "Fail to create socket file descriptor\n");
        return 1;
    }
    // nonblock
    // nonblock
    int flags = fcntl(serverFd, F_GETFL, 0);
    if (flags < 0) {
        fprintf(stderr, "Fail to get flags\n");
        return 1;
    }
    int setFlagResult = fcntl(serverFd, F_SETFL, flags | O_NONBLOCK);
    if (setFlagResult < 0) {
        fprintf(stderr, "Fail to set flags\n");
        return 1;
    }

    // bind & listen
    struct sockaddr_in serverAddr;
    memset(&serverAddr, 0, sizeof(serverAddr));
    serverAddr.sin_family = AF_INET;
    serverAddr.sin_addr.s_addr = htonl(INADDR_ANY);
    serverAddr.sin_port = htons(80);
    int bindResult = bind(serverFd, (struct sockaddr *) &serverAddr, sizeof(serverAddr));
    if (bindResult < 0) {
        fprintf(stderr, "Fail to bind\n");
        return 1;
    }
    int listenResult = listen(serverFd, 1024);
    if (listenResult < 0) {
        fprintf(stderr, "Fail to listen\n");
        return 1;
    }

    // epoll fd
    int epollFd = epoll_create(1);
    if (epollFd < 0) {
        fprintf(stderr, "Fail to create epoll file descriptor\n");
        return 1;
    }
    // event
    struct epoll_event event, events[1024];
    event.events = EPOLLIN;
    event.data.fd = serverFd;
    // ctl
    int ctlResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, serverFd, &event);
    if (ctlResult < 0) {
        fprintf(stderr, "Fail to run epoll_ctl\n");
        return 1;
    }
    // wait
    while (1) {
        int event_count = epoll_wait(epollFd, events, 1024, -1);

        for (int i = 0; i < event_count; i++) {
            struct epoll_event event = events[i];
            // accept
            if (event.data.fd == serverFd) {
                unsigned int clientlen = sizeof(clientAddr);
                int acceptFd = accept(serverFd, (struct sockaddr *) &clientAddr, &clientlen);
                if (acceptFd < 0) {
                    fprintf(stderr, "Fail to create client connection file descriptor\n");
                    fprintf(stderr, "Fail Reason: %d\n", errno);
                    return 1;
                }

                // nonblock
                int flags = fcntl(acceptFd, F_GETFL, 0);
                if (flags < 0) {
                    fprintf(stderr, "Fail to get flags\n");
                    return 1;
                }
                int setFlagResult = fcntl(acceptFd, F_SETFL, flags | O_NONBLOCK);
                if (setFlagResult < 0) {
                    fprintf(stderr, "Fail to set flags\n");
                    return 1;
                }

                struct epoll_event event;
                event.events = EPOLLIN;
                event.data.fd = serverFd;
                int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);
                if (ctlClientResult < 0) {
                    fprintf(stderr, "Fail to run epoll_ctl\n");
                    return 1;
                }
                // client recv
            } else if (event.events & EPOLLIN) {
                int fd = event.data.fd;
                char buffer[1024+1];
                ssize_t received = recv(fd, &buffer, 1024, 0);
                if (received < 0) {
                    fprintf(stderr, "Fail to received bytess from client\n");
                    if (errno == EINTR) {
                        fprintf(stderr, "Reason: EINTR\n");
                    } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
                    } else {
                        fprintf(stderr, "Reason: %d\n", errno);
                        close(fd);
                        int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                        if (ctlClientResult < 0) {
                            fprintf(stderr, "Fail to run epoll_ctl\n");
                            return 1;
                        }
                        return 1;
                    }
                } else if (received == 0) {
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                    close(fd);
                } else {
                    buffer[received] = '[=11=]';
                    fprintf(stdout, "%s", buffer);
                    // if you want to send something...
                    event.events |= EPOLLOUT;
                    // here is some data that event can hold
                    event.data.u32 = (uint32_t) 1;
                    // you can now send data or just put event in epoll, which is maybe easier
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                }
                // client send
            }  else if (event.events & EPOLLOUT) {
                int fd = event.data.fd;
                char buffer[] = "I see you";
                ssize_t sendResult = send(fd, &buffer, 1024, 0);
                if (sendResult < 0) {
                    fprintf(stderr, "Fail to received bytess from client\n");
                    if (errno == EINTR) {
                        fprintf(stderr, "Reason: EINTR\n");
                    } else if (errno == EAGAIN || errno == EWOULDBLOCK) {
                        fprintf(stderr, "Reason: EAGAIN or EWOULDBLOCK\n");
                    } else {
                        if (errno == EPIPE) {
                            fprintf(stderr, "Reason: EPIPE\n");
                        } else {
                            fprintf(stderr, "Reason: %d\n", errno);
                        }
                        close(fd);
                        int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_DEL, fd, &event);
                        if (ctlClientResult < 0) {
                            fprintf(stderr, "Fail to run epoll_ctl\n");
                            return 1;
                        }
                        return 1;
                    }
                } else if (sendResult == 0) {
                    event.events = EPOLLIN;
                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
                    if (ctlClientResult < 0) {
                        fprintf(stderr, "Fail to run epoll_ctl\n");
                        return 1;
                    }
                } else {

                    // if you want to recv something...
//                    event.events |= EPOLLIN;
//                    // you can now send data or just put event in epoll, which is maybe easier
//                    int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_MOD, fd, &event);
//                    if (ctlClientResult < 0) {
//                        fprintf(stderr, "Fail to run epoll_ctl\n");
//                        return 1;
//                    }
                }
            }


        }
    }

    return 0;
}

当我尝试建立 TCP 套接字连接时（例如 curl -v "http://host:80/"，它可以在 行 81 中建立 test2.cpp 运行） , acceptFd 是 < 0 而 errno 是 11 根据 line 84, 这意味着 "Resource deadlock avoided".

为什么？没有线程相关的代码吧？

Answer 1

错误代码 11 是 EAGAIN，这是处理非阻塞套接字时经常遇到的错误代码 I/O。表示请求的操作没有任何关系，请稍后再试。

在accept()的情况下，表示：

EAGAIN or EWOULDBLOCK

The socket is marked nonblocking and no connections are present to be accepted. POSIX.1-2001 and POSIX.1-2008 allow either error to be returned for this case, and do not require these constants to have the same value, so a portable application should check for both possibilities.

这意味着您在错误的时间调用 accept()，此时侦听套接字没有任何客户端可以接受。仔细检查你的 epoll() 用法，它可能有一个逻辑错误导致你过早地调用 accept()。

例如，在 accept() 成功接受客户端后，您正在分配 listening 套接字而不是 client调用 epoll_ctl(EPOLL_CTL_ADD) 时套接字到 event.data.fd:

struct epoll_event event;
event.events = EPOLLIN;
event.data.fd = serverFd; // <-- HERE, SHOULD BE acceptFd INSTEAD!
int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

因此，当 client 有数据等待读取时，您的循环将最终调用 listening[=53] 上的 accept() =] 套接字，而不是在 客户端 套接字上调用 recv()。

此外，您没有检查报告的 event.events 字段中的套接字错误。如果客户端套接字发生错误，报告的 events 可能包含 EPOLLERR and/or EPOLLHUP 标志。您应该检查这些标志，如果存在则关闭客户端套接字，然后再检查 EPOLLIN 标志以调用 recv().

另外，请注意，在您的示例中，it is not necessary to call epoll_ctl(EPOLL_CTL_DEL) 在套接字上调用 close() 之前。关闭套接字文件描述符会自动将其从监视它的 epoll 实例中删除。

Answer 2

            struct epoll_event event;
            event.events = EPOLLIN;
            event.data.fd = serverFd;
            int ctlClientResult = epoll_ctl(epollFd, EPOLL_CTL_ADD, acceptFd, &event);

当您将新接受的连接添加到 epoll 集时，您告诉它在 serverFd 上将其报告为命中。因此，当客户端向您发送数据时，您尝试接受新连接。

将 event.data.fd = serverFd 更改为 event.data.fd = acceptFd。

然后您可以继续处理下一个错误：

            char buffer[] = "I see you";
            ssize_t sendResult = send(fd, &buffer, 1024, 0);

1024?!

此外，无论何时使用非阻塞套接字，都应添加代码来处理 EAGAIN 或 EWOULDBLOCK 非致命错误。

c++ socket accept() get Resource deadlock avoided errno with epoll 小程序

c++ socket accept() get Resource deadlock avoided errno with epoll

c++

sockets

epoll