使用 OpenSSL 的 DTLS 中随机缺少连接和消息延迟
Random lack of connection and message delay in DTLS with OpenSSL
正在尝试为 DTLS 编写一个服务器，该服务器当前仅输出它接收到的文本。工作客户端取自 https://github.com/stepheny/openssl-dtls-custom-bio，它可以很好地发送和接收到自己的服务器。
但是，当它发送到此服务器时，发生了一些奇怪的事情。首先，连接只是有时发生，似乎无法确定连接是否会启动。其次，更奇怪的是数据“延迟”了。需要发送 6 条消息才能到达 1 条消息。
情况是这样的：
- 启动服务器。
- 启动客户端。
- 希望联系。
- 如果在客户端连接类型 5 消息发送到服务器，它们被发送，但服务器一直在解码它们时出错。
- 发送第 6 条消息后，您会注意到第 1 条消息已到达服务器。
- 发送第 7 个后，您将获得第 2 个。等等
需要注意的是，我们不是在谈论时间延迟，没有办法在服务器启动时简单地读取5条空消息，队列是空的。只有第 6 条消息发送后，队列才会填充第 1 条真实消息。
代码：
//server.cpp
#include "DTLSConnection.hpp"
#include <iostream>
#include <chrono>
#include <thread>
int main(int argc, char *argv[])
{
try
{
DTLSConnection con("192.168.31.177:1235");
std::cout << "Connection created" << std::endl;
ssize_t ret;
for(;;)
{
ret = con.recv([](Client* c) {
try{
std::cout << c->SSL_read_alt() << std::endl;
std::cout << "I am in onmessage" << std::endl;
}
catch(std::string &e)
{
std::cerr << "EXCEPTION: " << e << std::endl;
}
});
std::cout << "Returned value is " << ret << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(1));
}
}
catch(std::string &e)
{
std::cerr << "EXCEPTION: " << e << std::endl;
}
return 0;
}
// CustomBIO.hpp
#include <memory>
#include <deque>
#include <vector>
#include <unordered_map>
#include <cstdio> // temporary
#include <cstring>
#include <cassert>
#include <openssl/ssl.h>
#include <signal.h>
const char *sdump_addr(const struct sockaddr *sa)
{
static char buf[1024];
switch (sa->sa_family)
{
case AF_INET:
memmove(buf, "INET: ", 6);
inet_ntop(AF_INET, &((struct sockaddr_in *)sa)->sin_addr, buf+6, sizeof(buf)-6);
sprintf(buf+strlen(buf), ":%d", ntohs(((struct sockaddr_in *)sa)->sin_port));
break;
case AF_INET6:
memmove(buf, "INET6: [", 8);
inet_ntop(AF_INET6, &((struct sockaddr_in6 *)sa)->sin6_addr, buf+8, sizeof(buf)-8);
sprintf(buf+strlen(buf), "]:%d", ntohs(((struct sockaddr_in6 *)sa)->sin6_port));
break;
default:
memmove(buf, "unknown", 8);
break;
}
return buf;
}
struct Packet
{
size_t capacity = 0;
size_t len = 0;
char * data = nullptr;
Packet() = default;
Packet(size_t cap)
{
init(cap);
}
Packet(char *b, size_t cap)
{
capacity=cap;
len=capacity;
data = new char[capacity];
memcpy(data, b, capacity);
}
Packet(char* b, char* e)
{
capacity=e-b;
len=capacity;
data = new char[capacity];
memcpy(data, b, capacity);
}
char* begin()
{
return data;
}
char* end()
{
return data ? data+len : nullptr;
}
void swap(Packet& that)
{
std::swap(this->capacity, that.capacity);
std::swap(this->len, that.len);
std::swap(this->data, that.data);
}
void init(size_t cap)
{
data = new char[cap];
len = 0;
capacity = cap;
}
void free()
{
if(!data) return;
delete data;
len = 0;
capacity = 0;
data = nullptr;
}
};
// used for both reading and writing
struct CustomBIO
{
using dataBuffer = Packet;
int sockfd;
sockaddr_storage thisAddr{};
socklen_t thisAddr_len{sizeof(sockaddr_storage)};
sockaddr_storage thatAddr{};
socklen_t thatAddr_len{sizeof(sockaddr_storage)};
template<typename T>
T* getThat()
{
return reinterpret_cast<T*>(&thatAddr);
}
std::deque<dataBuffer> receivingQueue{};
bool peekmode{false};
};
inline CustomBIO* BIO_get_CBIO(BIO* b)
{
return reinterpret_cast<CustomBIO *>(BIO_get_data(b));
}
extern "C"
{
int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written);
int BIO_s_custom_write(BIO *b, const char *data, int dlen);
int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes);
int BIO_s_custom_read(BIO *b, char *data, int dlen);
int BIO_s_custom_gets(BIO *b, char *data, int size);
int BIO_s_custom_puts(BIO *b, const char *data);
long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs);
int BIO_s_custom_create(BIO *b);
int BIO_s_custom_destroy(BIO *b);
// long BIO_s_custom_callback_ctrl(BIO *, int, BIO_info_cb *);
BIO_METHOD *BIO_s_custom();
void BIO_s_custom_meth_free();
int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written)
{
fprintf(stderr, "BIO_s_custom_write_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *written[%ld])\n", (long unsigned int)b, (long unsigned int)data, dlen, *written);
fflush(stderr);
return -1;
}
int BIO_s_custom_write(BIO *b, const char *data, int dlen)
{
int ret;
CustomBIO *cbio;
ret = -1;
fprintf(stderr, "BIO_s_custom_write(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (unsigned long)b, (unsigned long)data, (long)dlen);
fflush(stderr);
cbio = BIO_get_CBIO(b);
// dump_addr((struct sockaddr *)&cbio->txaddr, ">> ");
// dump_hex((unsigned const char *)data, dlen, " ");
ret = sendto(cbio->sockfd, data, dlen, 0, cbio->getThat<const sockaddr>(), cbio->thatAddr_len);
if (ret >= 0)
{
fprintf(stderr, " %d bytes sent\n", ret);
}
else
{
fprintf(stderr, " ret: %d errno: [%d] %s\n", ret, errno, strerror(errno));
fprintf(stderr, " socket: %d\n", cbio->sockfd);
fprintf(stderr, " thatAddrLen: %d\n", cbio->thatAddr_len);
fprintf(stderr, " thatAddr: %s\n", sdump_addr(cbio->getThat<sockaddr>()));
}
return ret;
}
int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes)
{
fprintf(stderr, "BIO_s_custom_read_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *readbytes[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen, *readbytes);
fflush(stderr);
return -1;
}
int BIO_s_custom_read(BIO *b, char *data, int dlen)
{
int ret;
CustomBIO *cbio;
ret = -1;
fprintf(stderr, "BIO_s_custom_read(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen);
fprintf(stderr, " probe peekmode %d\n", ((CustomBIO *)BIO_get_data(b))->peekmode);
fflush(stderr);
cbio = BIO_get_CBIO(b);
if(!cbio->receivingQueue.empty())
{
if(cbio->receivingQueue.front().len > (size_t)dlen)
{
fprintf(stderr, "if(cbio->receivingQueue.front().len > (size_t)dlen)");
memmove(data, cbio->receivingQueue.front().data, dlen);
ret = dlen;
if(!cbio->peekmode)
{
CustomBIO::dataBuffer rest{cbio->receivingQueue.front().begin()+ret, cbio->receivingQueue.front().end()};
cbio->receivingQueue.front().swap(rest);
}
}
else
{
Packet &pac = cbio->receivingQueue.front();
ret = pac.len;
memmove(data, pac.data, ret);
if(!cbio->peekmode)
{
pac.free();
cbio->receivingQueue.pop_front();
}
}
fprintf(stderr, " %d bytes read from queue\n", ret);
fflush(stderr);
}
else
{
fprintf(stderr, " The queue is empty\n");
/*ret = recvfrom(cbio->sockfd, data, dlen, 0, cbio->getThat<sockaddr>(), &cbio->thatAddr_len); // not right
if(ret>0 && cbio->peekmode)
{
// todo
}*/
}
return ret;
}
int BIO_s_custom_gets(BIO *b, char *data, int size)
{
fprintf(stderr, "BIO_s_custom_gets(BIO[0x%016lX], data[0x%016lX], size[%d]\n", (long unsigned int)b, (long unsigned int)data, size);
if(size <= 1)
{
return 0;
}
else
{
size = BIO_s_custom_read(b, data, size-1);
data[size] = '[=10=]';
return size;
}
}
int BIO_s_custom_puts(BIO *b, const char *buf)
{
fprintf(stderr, "BIO_s_custom_puts(BIO[0x%016lX], buf[0x%016lX]\n", (long unsigned int)b, (long unsigned int)buf);
size_t size = std::strlen(buf);
return size > 0 ? BIO_s_custom_write(b, buf, size) : 0;
}
long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs)
{
long ret = 0;
fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
if(pargs)
{
for(int i=0; ; ++i)
{
fprintf(stderr, "[%d]=%X ", i, (int)((unsigned char*)pargs)[i]);
if(((unsigned char*)pargs)[i] == 0) break;
}
}
fprintf(stderr, "\n");
fflush(stderr);
switch(cmd)
{
case BIO_CTRL_FLUSH: // 11
case BIO_CTRL_DGRAM_SET_CONNECTED: // 32
case BIO_CTRL_DGRAM_SET_PEER: // 44
case BIO_CTRL_DGRAM_GET_PEER: // 46
ret = 1;
break;
case BIO_CTRL_WPENDING: // 13
ret = 0;
break;
case BIO_CTRL_DGRAM_QUERY_MTU: // 40
case BIO_CTRL_DGRAM_GET_FALLBACK_MTU: // 47
ret = 1500;
// ret = 9000; // jumbo?
break;
case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD: // 49
ret = 96; // random guess
break;
case BIO_CTRL_DGRAM_SET_PEEK_MODE: // 71
BIO_get_CBIO(b)->peekmode = (larg != 0);
ret = 1;
break;
case BIO_CTRL_PUSH: // 6
case BIO_CTRL_POP: // 7
case BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT: // 45
ret = 0;
break;
default:
fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
fprintf(stderr, " unknown cmd: %d\n", cmd);
fflush(stderr);
ret = 0;
break;
}
return ret;
}
int BIO_s_custom_create(BIO *b)
{
fprintf(stderr, "BIO_s_custom_create(BIO[0x%016lX])\n", (long unsigned int)b);
fflush(stderr);
return 1;
}
int BIO_s_custom_destroy(BIO *b)
{
fprintf(stderr, "BIO_s_custom_destroy(BIO[0x%016lX])\n", (long unsigned int)b);
fflush(stderr);
return 1;
}
BIO_METHOD *_BIO_s_custom = nullptr;
BIO_METHOD *BIO_s_custom()
{
if (!_BIO_s_custom)
{
_BIO_s_custom = BIO_meth_new(BIO_get_new_index()|BIO_TYPE_SOURCE_SINK, "BIO_s_custom");
//BIO_meth_set_callback_ctrl(_BIO_s_custom, BIO_s_custom_callback_ctrl);
BIO_meth_set_create(_BIO_s_custom, BIO_s_custom_create);
BIO_meth_set_ctrl(_BIO_s_custom, BIO_s_custom_ctrl);
BIO_meth_set_destroy(_BIO_s_custom, BIO_s_custom_destroy);
BIO_meth_set_gets(_BIO_s_custom, BIO_s_custom_gets);
BIO_meth_set_puts(_BIO_s_custom, BIO_s_custom_puts);
BIO_meth_set_read_ex(_BIO_s_custom, BIO_s_custom_read_ex);
BIO_meth_set_read(_BIO_s_custom, BIO_s_custom_read);
BIO_meth_set_write_ex(_BIO_s_custom, BIO_s_custom_write_ex);
BIO_meth_set_write(_BIO_s_custom, BIO_s_custom_write);
}
return _BIO_s_custom;
}
void BIO_s_custom_meth_free()
{
if (_BIO_s_custom)
BIO_meth_free(_BIO_s_custom);
_BIO_s_custom = NULL;
}
}
// DTLSConnection.hpp
#include <string>
#include <list>
#include <functional>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <sys/epoll.h>
#include <cerrno>
#include <iostream> // temp
#include "CustomBIO.hpp"
constexpr int TIME_OUT = 10000; // ms
char cookie_str[] = "BISCUIT!"; // how to change this
//нужен способ, чтобы клиент был извествен (knownclients) но еще не подключен (не прошел ssl_accept())
// (void) BIO_dgram_get_peer(SSL_get_rbio(ssl), &peer);
// see https://github.com/nplab/DTLS-Examples/blob/master/src/dtls_udp_echo.c
int generate_cookie([[maybe_unused]] SSL *ssl, unsigned char *cookie, unsigned int *cookie_len)
{
memmove(cookie, cookie_str, sizeof(cookie_str)-1);
*cookie_len = sizeof(cookie_str)-1;
return 1;
}
int verify_cookie([[maybe_unused]] SSL *ssl, const unsigned char *cookie, unsigned int cookie_len)
{
return sizeof(cookie_str)-1==cookie_len && memcmp(cookie, cookie_str, sizeof(cookie_str)-1)==0;
}
void throw_SSL_error_if_error(SSL* ssl, int ret, std::string str)
{
if(ret>0) return; // SSL_ERROR_NONE
str += " ret="+std::to_string(ret)+' ';
auto sslError = SSL_get_error(ssl, ret);
if(sslError == SSL_ERROR_SYSCALL){
throw std::string{str+"SSL_ERROR_SYSCALL + error "}+std::to_string(errno);
}
}
namespace std
{
template<> struct hash<sockaddr>
{
size_t operator()(sockaddr const& val) const noexcept
{
size_t res = 0;
for(unsigned long h : val.sa_data)
{
res = (res << 1) ^ h;
}
return res;
}
};
}
bool operator==(const sockaddr& l, const sockaddr& r)
{
if(l.sa_family != r.sa_family) return false;
for(int i=0; i<14; ++i)
{
if(l.sa_data[i] != r.sa_data[i]) return false;
}
return true;
}
class DTLSConnection;
class SSLSetterUpper
{
friend DTLSConnection;
SSL_CTX *ctx;
SSLSetterUpper()
{
int ret; // because it is C;
SSL_load_error_strings();
SSL_library_init();
const SSL_METHOD *mtd = DTLS_server_method();
ctx = SSL_CTX_new(mtd);
SSL_CTX_set_min_proto_version(ctx, DTLS1_2_VERSION);
SSL_CTX_use_certificate_chain_file(ctx, "server-cert.pem");
SSL_CTX_use_PrivateKey_file(ctx, "server-key.pem", SSL_FILETYPE_PEM);
ret = SSL_CTX_load_verify_locations(ctx, "root-ca.pem", nullptr);
if(ret != 1)
{
throw std::string{"SSL_CTX_load_verify_locations failed"};
}
ret = SSL_CTX_set_default_verify_file(ctx);
if(ret != 1)
{
throw std::string{"SSL_CTX_set_default_verify_file failed"};
}
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
SSL_CTX_set_cookie_generate_cb(ctx, generate_cookie);
SSL_CTX_set_cookie_verify_cb(ctx, verify_cookie);
}
SSL* generateSSL() const
{
return SSL_new(ctx);
}
};
struct Client
{
CustomBIO cbio;
SSL *ssl;
explicit Client(SSL *ssl)
:cbio{}, ssl{ssl}
{
cbio.peekmode = false;
BIO *bio = BIO_new(BIO_s_custom());
BIO_set_data(bio, (void *)&cbio);
BIO_set_init(bio, 1);
SSL_set_bio(ssl, bio, bio);
}
bool on_init() const
{
int ret = DTLSv1_listen(this->ssl, nullptr);
//throw_SSL_error_if_error(ssl, ret, "DTLSv1_listen failed");
std::cout << "DTLSv1_listen " << ret << std::endl;
return (ret==1);
}
bool on_connect() const
{
int ret = SSL_accept(ssl);
if(ret!=1) return false;
std::cout << "ssl = " << SSL_state_string_long(ssl) << std::endl;
std::cout << "SSL_accept successful!" << std::endl;
return true;
}
std::string SSL_read_alt() const
{
Packet p(2000);
std::cout << "sizeA = " << cbio.receivingQueue.size() << std::endl;
int ret = ::SSL_read(ssl,p.data,p.capacity);
std::cout << "ret = " << ret << std::endl;
std::cout << "sizeB = " << cbio.receivingQueue.size() << std::endl;
std::cout << "pdata0 " << p.data[0] << std::endl;
//std::cout << SSL_get_error(ssl, ret) << std::endl;
throw_SSL_error_if_error(ssl, ret, "sslread");
p.len = ret;
std::cerr << "plen" << std::endl;
std::cerr << p.len << std::endl;
std::string result(p.data,p.len);
if(result[0] == '[=10=]') std::cout << "res = " << result << std::endl;
p.free();
return result;
}
};
class DTLSConnection
{
static const SSLSetterUpper sslSetup;
int epoll_fd;
std::unordered_map<sockaddr, std::shared_ptr<Client>> knownClients;
std::unordered_map<sockaddr, std::shared_ptr<Client>> connectedClients;
std::shared_ptr<Client> incomingClient;
public:
// 127.0.0.1:1234 or [::1]:1234
explicit DTLSConnection(std::string thisAddress)
{
std::list<sockaddr_storage> addresses;
if (thisAddress[0]=='[')
{
auto pos = thisAddress.find(']', 1);
if (pos == std::string::npos)
{
throw std::string{"invalid target"};
}
int port = std::stoi(thisAddress.substr(pos+2));
if (port<1||port>65535)
{
throw std::string{"invalid port"};
}
addresses.emplace_back();
auto* thisAddr = (sockaddr_in6 *)&(addresses.back());
thisAddr->sin6_family = AF_INET6;
if ( ! inet_pton(AF_INET6, thisAddress.substr(1, pos).c_str(), &thisAddr->sin6_addr) )
{
throw std::string{"invalid ipv6 address"};
}
thisAddr->sin6_port = htons(port);
}
else
{
auto pos = thisAddress.find(':');
if (pos == std::string::npos)
{
throw std::string{"invalid target"};
}
int port = std::stoi(thisAddress.substr(pos+1));
if (port<1||port>65535)
{
throw std::string{"invalid port"};
}
addresses.emplace_back();
auto * thisAddr = (sockaddr_in *)&(addresses.back());
thisAddr->sin_family = AF_INET;
if ( ! inet_pton(AF_INET, thisAddress.substr(0, pos).c_str(), &thisAddr->sin_addr) )
{
throw std::string{"invalid ipv4 address"};
}
thisAddr->sin_port = htons(port);
}
epoll_fd = epoll_create1(EPOLL_CLOEXEC);
for(auto &address : addresses)
{
epoll_event epe {};
epe.data.fd = socket(address.ss_family, SOCK_DGRAM/*|SOCK_NONBLOCK*/|SOCK_CLOEXEC, 0);
if(bind(epe.data.fd, (const sockaddr*)&address, sizeof(address)) != 0)
{
throw std::string{"failed to bind"};
}
epe.events = EPOLLIN|EPOLLET;
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, epe.data.fd, &epe);
}
//signal(SIGINT, signal_handler); // do i need this?
incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
}
ssize_t recv(const std::function<void(Client *)>& onmessage)
{
std::cout << "recv(onmessage)" << std::endl;
epoll_event epe{};
int ret;
ret = epoll_wait(epoll_fd, &epe, 1, TIME_OUT);
if (ret==-1)
{
throw std::string{"epoll_wait failed"};
}
if(ret==0)
{
return ret; // wait longer
}
Packet packet{2000};
ret = recvfrom(epe.data.fd, packet.data, packet.capacity, 0, incomingClient->cbio.getThat<sockaddr>(), &incomingClient->cbio.thatAddr_len);
packet.len = ret;
if(ret==0) return ret;
if(ret<0)
{
switch(errno)
{
case EAGAIN:
//case EWOULDBLOCK:
return ret;
case EBADF:
throw std::string{"EBADF"};
case ECONNREFUSED:
throw std::string{"ECONNREFUSED"};
case EFAULT:
throw std::string{"EFAULT"};
case EINTR:
throw std::string{"EINTR"};
case EINVAL:
throw std::string{"EINVAL"};
case ENOMEM:
throw std::string{"ENOMEM"};
case ENOTCONN:
throw std::string{"ENOTCONN"};
case ENOTSOCK:
throw std::string{"ENOTSOCK"};
default:
throw std::string{"Unknwon errno with negative return from recvfrom: "}+std::to_string(errno);
}
}
auto known = knownClients.find(*incomingClient->cbio.getThat<sockaddr>());
auto connected = connectedClients.find(*incomingClient->cbio.getThat<sockaddr>());
std::cout << "START" << std::endl;
for(auto &pair : knownClients)
{
std::cout << sdump_addr(&pair.first) << std::endl;
}
std::cout << "END" << std::endl;
if(known == knownClients.end())
{
std::cout << "inetaddr = " << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << std::endl;
ret = 0;
incomingClient->cbio.receivingQueue.push_back(std::move(packet));
incomingClient->cbio.sockfd = epe.data.fd;
if( incomingClient->on_init() )
{
std::cout << "inc = " << incomingClient->on_connect() << std::endl; //скорее всего, всегда будет ложь
std::cout << "on_init if" << std::endl;
knownClients[*incomingClient->cbio.getThat<sockaddr>()] = incomingClient;
incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
}
}
else if(connected == connectedClients.end())
{
std::cout << "elseif" << std::endl;
ret = 0;
auto cli = known->second;
cli->cbio.receivingQueue.push_back(std::move(packet));
if( cli->on_connect() )
{
std::cout << "cli->cbio.receivingQueue.size()" << cli->cbio.receivingQueue.size() << std::endl;
connectedClients[*cli->cbio.getThat<sockaddr>()] = cli;
SSL_write(cli->ssl, "hello", 6);
}
}
else
{
std::cout << "else" << std::endl;
std::cout << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << " has been found as connected" << std::endl;
connected->second->cbio.receivingQueue.push_back(std::move(packet));
onmessage(connected->second.get());
}
return ret;
}
};
const SSLSetterUpper DTLSConnection::sslSetup{};
运行 服务器的输出（CustomBIO 输出被截断）是：
Connection created
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
probe peekmode 0
211 bytes read from queue
36 bytes sent
probe peekmode 0
The queue is empty
DTLSv1_listen -1
Returned value is 0
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
probe peekmode 0
219 bytes read from queue
DTLSv1_listen 1
1180 bytes sent
probe peekmode 0
The queue is empty
inc = 0
on_init if
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
86 bytes sent
823 bytes sent
167 bytes sent
79 bytes sent
25 bytes sent
probe peekmode 0
219 bytes read from queue
probe peekmode 0
The queue is empty
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
probe peekmode 0
1088 bytes read from queue
618 bytes sent
ssl = SSL negotiation finished successfully
SSL_accept successful!
cli->cbio.receivingQueue.size()0
43 bytes sent
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
824 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 824
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
58 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 `
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 58
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
131 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 131
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
14 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 14
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
The queue is empty
ret = -1
sizeB = 0
pdata0
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 61
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
55 bytes read from queue
ret = 18
sizeB = 0
pdata0 d
plen
18
dfksaiopfjiaosjfio
I am in onmessage
Returned value is 55
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
46 bytes read from queue
ret = 9
sizeB = 0
pdata0 s
plen
9
sdasdasda
I am in onmessage
Returned value is 46
recv(onmessage)
Returned value is 0
recv(onmessage)
在您看到 EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0 的地方，这意味着 SSL_read 已返回 -1。抱歉有些脏代码。
以防其他人遇到类似问题。问题是调用服务器的 recv 函数之间的等待时间为 1 秒。那时客户端认为服务器没有响应并开始做奇怪的事情。降低延迟解决了问题。
正在尝试为 DTLS 编写一个服务器，该服务器当前仅输出它接收到的文本。工作客户端取自 https://github.com/stepheny/openssl-dtls-custom-bio，它可以很好地发送和接收到自己的服务器。
但是，当它发送到此服务器时，发生了一些奇怪的事情。首先，连接只是有时发生，似乎无法确定连接是否会启动。其次，更奇怪的是数据“延迟”了。需要发送 6 条消息才能到达 1 条消息。
情况是这样的：
- 启动服务器。
- 启动客户端。
- 希望联系。
- 如果在客户端连接类型 5 消息发送到服务器，它们被发送，但服务器一直在解码它们时出错。
- 发送第 6 条消息后，您会注意到第 1 条消息已到达服务器。
- 发送第 7 个后，您将获得第 2 个。等等
需要注意的是，我们不是在谈论时间延迟，没有办法在服务器启动时简单地读取5条空消息，队列是空的。只有第 6 条消息发送后，队列才会填充第 1 条真实消息。
代码：
//server.cpp
#include "DTLSConnection.hpp"
#include <iostream>
#include <chrono>
#include <thread>
int main(int argc, char *argv[])
{
try
{
DTLSConnection con("192.168.31.177:1235");
std::cout << "Connection created" << std::endl;
ssize_t ret;
for(;;)
{
ret = con.recv([](Client* c) {
try{
std::cout << c->SSL_read_alt() << std::endl;
std::cout << "I am in onmessage" << std::endl;
}
catch(std::string &e)
{
std::cerr << "EXCEPTION: " << e << std::endl;
}
});
std::cout << "Returned value is " << ret << std::endl;
std::this_thread::sleep_for(std::chrono::seconds(1));
}
}
catch(std::string &e)
{
std::cerr << "EXCEPTION: " << e << std::endl;
}
return 0;
}
// CustomBIO.hpp
#include <memory>
#include <deque>
#include <vector>
#include <unordered_map>
#include <cstdio> // temporary
#include <cstring>
#include <cassert>
#include <openssl/ssl.h>
#include <signal.h>
const char *sdump_addr(const struct sockaddr *sa)
{
static char buf[1024];
switch (sa->sa_family)
{
case AF_INET:
memmove(buf, "INET: ", 6);
inet_ntop(AF_INET, &((struct sockaddr_in *)sa)->sin_addr, buf+6, sizeof(buf)-6);
sprintf(buf+strlen(buf), ":%d", ntohs(((struct sockaddr_in *)sa)->sin_port));
break;
case AF_INET6:
memmove(buf, "INET6: [", 8);
inet_ntop(AF_INET6, &((struct sockaddr_in6 *)sa)->sin6_addr, buf+8, sizeof(buf)-8);
sprintf(buf+strlen(buf), "]:%d", ntohs(((struct sockaddr_in6 *)sa)->sin6_port));
break;
default:
memmove(buf, "unknown", 8);
break;
}
return buf;
}
struct Packet
{
size_t capacity = 0;
size_t len = 0;
char * data = nullptr;
Packet() = default;
Packet(size_t cap)
{
init(cap);
}
Packet(char *b, size_t cap)
{
capacity=cap;
len=capacity;
data = new char[capacity];
memcpy(data, b, capacity);
}
Packet(char* b, char* e)
{
capacity=e-b;
len=capacity;
data = new char[capacity];
memcpy(data, b, capacity);
}
char* begin()
{
return data;
}
char* end()
{
return data ? data+len : nullptr;
}
void swap(Packet& that)
{
std::swap(this->capacity, that.capacity);
std::swap(this->len, that.len);
std::swap(this->data, that.data);
}
void init(size_t cap)
{
data = new char[cap];
len = 0;
capacity = cap;
}
void free()
{
if(!data) return;
delete data;
len = 0;
capacity = 0;
data = nullptr;
}
};
// used for both reading and writing
struct CustomBIO
{
using dataBuffer = Packet;
int sockfd;
sockaddr_storage thisAddr{};
socklen_t thisAddr_len{sizeof(sockaddr_storage)};
sockaddr_storage thatAddr{};
socklen_t thatAddr_len{sizeof(sockaddr_storage)};
template<typename T>
T* getThat()
{
return reinterpret_cast<T*>(&thatAddr);
}
std::deque<dataBuffer> receivingQueue{};
bool peekmode{false};
};
inline CustomBIO* BIO_get_CBIO(BIO* b)
{
return reinterpret_cast<CustomBIO *>(BIO_get_data(b));
}
extern "C"
{
int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written);
int BIO_s_custom_write(BIO *b, const char *data, int dlen);
int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes);
int BIO_s_custom_read(BIO *b, char *data, int dlen);
int BIO_s_custom_gets(BIO *b, char *data, int size);
int BIO_s_custom_puts(BIO *b, const char *data);
long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs);
int BIO_s_custom_create(BIO *b);
int BIO_s_custom_destroy(BIO *b);
// long BIO_s_custom_callback_ctrl(BIO *, int, BIO_info_cb *);
BIO_METHOD *BIO_s_custom();
void BIO_s_custom_meth_free();
int BIO_s_custom_write_ex(BIO *b, const char *data, size_t dlen, size_t *written)
{
fprintf(stderr, "BIO_s_custom_write_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *written[%ld])\n", (long unsigned int)b, (long unsigned int)data, dlen, *written);
fflush(stderr);
return -1;
}
int BIO_s_custom_write(BIO *b, const char *data, int dlen)
{
int ret;
CustomBIO *cbio;
ret = -1;
fprintf(stderr, "BIO_s_custom_write(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (unsigned long)b, (unsigned long)data, (long)dlen);
fflush(stderr);
cbio = BIO_get_CBIO(b);
// dump_addr((struct sockaddr *)&cbio->txaddr, ">> ");
// dump_hex((unsigned const char *)data, dlen, " ");
ret = sendto(cbio->sockfd, data, dlen, 0, cbio->getThat<const sockaddr>(), cbio->thatAddr_len);
if (ret >= 0)
{
fprintf(stderr, " %d bytes sent\n", ret);
}
else
{
fprintf(stderr, " ret: %d errno: [%d] %s\n", ret, errno, strerror(errno));
fprintf(stderr, " socket: %d\n", cbio->sockfd);
fprintf(stderr, " thatAddrLen: %d\n", cbio->thatAddr_len);
fprintf(stderr, " thatAddr: %s\n", sdump_addr(cbio->getThat<sockaddr>()));
}
return ret;
}
int BIO_s_custom_read_ex(BIO *b, char *data, size_t dlen, size_t *readbytes)
{
fprintf(stderr, "BIO_s_custom_read_ex(BIO[0x%016lX], data[0x%016lX], dlen[%ld], *readbytes[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen, *readbytes);
fflush(stderr);
return -1;
}
int BIO_s_custom_read(BIO *b, char *data, int dlen)
{
int ret;
CustomBIO *cbio;
ret = -1;
fprintf(stderr, "BIO_s_custom_read(BIO[0x%016lX], data[0x%016lX], dlen[%ld])\n", (long unsigned int)b, (long unsigned int)data, (long int)dlen);
fprintf(stderr, " probe peekmode %d\n", ((CustomBIO *)BIO_get_data(b))->peekmode);
fflush(stderr);
cbio = BIO_get_CBIO(b);
if(!cbio->receivingQueue.empty())
{
if(cbio->receivingQueue.front().len > (size_t)dlen)
{
fprintf(stderr, "if(cbio->receivingQueue.front().len > (size_t)dlen)");
memmove(data, cbio->receivingQueue.front().data, dlen);
ret = dlen;
if(!cbio->peekmode)
{
CustomBIO::dataBuffer rest{cbio->receivingQueue.front().begin()+ret, cbio->receivingQueue.front().end()};
cbio->receivingQueue.front().swap(rest);
}
}
else
{
Packet &pac = cbio->receivingQueue.front();
ret = pac.len;
memmove(data, pac.data, ret);
if(!cbio->peekmode)
{
pac.free();
cbio->receivingQueue.pop_front();
}
}
fprintf(stderr, " %d bytes read from queue\n", ret);
fflush(stderr);
}
else
{
fprintf(stderr, " The queue is empty\n");
/*ret = recvfrom(cbio->sockfd, data, dlen, 0, cbio->getThat<sockaddr>(), &cbio->thatAddr_len); // not right
if(ret>0 && cbio->peekmode)
{
// todo
}*/
}
return ret;
}
int BIO_s_custom_gets(BIO *b, char *data, int size)
{
fprintf(stderr, "BIO_s_custom_gets(BIO[0x%016lX], data[0x%016lX], size[%d]\n", (long unsigned int)b, (long unsigned int)data, size);
if(size <= 1)
{
return 0;
}
else
{
size = BIO_s_custom_read(b, data, size-1);
data[size] = '[=10=]';
return size;
}
}
int BIO_s_custom_puts(BIO *b, const char *buf)
{
fprintf(stderr, "BIO_s_custom_puts(BIO[0x%016lX], buf[0x%016lX]\n", (long unsigned int)b, (long unsigned int)buf);
size_t size = std::strlen(buf);
return size > 0 ? BIO_s_custom_write(b, buf, size) : 0;
}
long BIO_s_custom_ctrl(BIO *b, int cmd, long larg, void *pargs)
{
long ret = 0;
fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
if(pargs)
{
for(int i=0; ; ++i)
{
fprintf(stderr, "[%d]=%X ", i, (int)((unsigned char*)pargs)[i]);
if(((unsigned char*)pargs)[i] == 0) break;
}
}
fprintf(stderr, "\n");
fflush(stderr);
switch(cmd)
{
case BIO_CTRL_FLUSH: // 11
case BIO_CTRL_DGRAM_SET_CONNECTED: // 32
case BIO_CTRL_DGRAM_SET_PEER: // 44
case BIO_CTRL_DGRAM_GET_PEER: // 46
ret = 1;
break;
case BIO_CTRL_WPENDING: // 13
ret = 0;
break;
case BIO_CTRL_DGRAM_QUERY_MTU: // 40
case BIO_CTRL_DGRAM_GET_FALLBACK_MTU: // 47
ret = 1500;
// ret = 9000; // jumbo?
break;
case BIO_CTRL_DGRAM_GET_MTU_OVERHEAD: // 49
ret = 96; // random guess
break;
case BIO_CTRL_DGRAM_SET_PEEK_MODE: // 71
BIO_get_CBIO(b)->peekmode = (larg != 0);
ret = 1;
break;
case BIO_CTRL_PUSH: // 6
case BIO_CTRL_POP: // 7
case BIO_CTRL_DGRAM_SET_NEXT_TIMEOUT: // 45
ret = 0;
break;
default:
fprintf(stderr, "BIO_s_custom_ctrl(BIO[0x%016lX], cmd[%d], larg[%ld], pargs[0x%016lX])\n", (long unsigned int)b, cmd, larg, (long unsigned int)pargs);
fprintf(stderr, " unknown cmd: %d\n", cmd);
fflush(stderr);
ret = 0;
break;
}
return ret;
}
int BIO_s_custom_create(BIO *b)
{
fprintf(stderr, "BIO_s_custom_create(BIO[0x%016lX])\n", (long unsigned int)b);
fflush(stderr);
return 1;
}
int BIO_s_custom_destroy(BIO *b)
{
fprintf(stderr, "BIO_s_custom_destroy(BIO[0x%016lX])\n", (long unsigned int)b);
fflush(stderr);
return 1;
}
BIO_METHOD *_BIO_s_custom = nullptr;
BIO_METHOD *BIO_s_custom()
{
if (!_BIO_s_custom)
{
_BIO_s_custom = BIO_meth_new(BIO_get_new_index()|BIO_TYPE_SOURCE_SINK, "BIO_s_custom");
//BIO_meth_set_callback_ctrl(_BIO_s_custom, BIO_s_custom_callback_ctrl);
BIO_meth_set_create(_BIO_s_custom, BIO_s_custom_create);
BIO_meth_set_ctrl(_BIO_s_custom, BIO_s_custom_ctrl);
BIO_meth_set_destroy(_BIO_s_custom, BIO_s_custom_destroy);
BIO_meth_set_gets(_BIO_s_custom, BIO_s_custom_gets);
BIO_meth_set_puts(_BIO_s_custom, BIO_s_custom_puts);
BIO_meth_set_read_ex(_BIO_s_custom, BIO_s_custom_read_ex);
BIO_meth_set_read(_BIO_s_custom, BIO_s_custom_read);
BIO_meth_set_write_ex(_BIO_s_custom, BIO_s_custom_write_ex);
BIO_meth_set_write(_BIO_s_custom, BIO_s_custom_write);
}
return _BIO_s_custom;
}
void BIO_s_custom_meth_free()
{
if (_BIO_s_custom)
BIO_meth_free(_BIO_s_custom);
_BIO_s_custom = NULL;
}
}
// DTLSConnection.hpp
#include <string>
#include <list>
#include <functional>
#include <sys/types.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <sys/epoll.h>
#include <cerrno>
#include <iostream> // temp
#include "CustomBIO.hpp"
constexpr int TIME_OUT = 10000; // ms
char cookie_str[] = "BISCUIT!"; // how to change this
//нужен способ, чтобы клиент был извествен (knownclients) но еще не подключен (не прошел ssl_accept())
// (void) BIO_dgram_get_peer(SSL_get_rbio(ssl), &peer);
// see https://github.com/nplab/DTLS-Examples/blob/master/src/dtls_udp_echo.c
int generate_cookie([[maybe_unused]] SSL *ssl, unsigned char *cookie, unsigned int *cookie_len)
{
memmove(cookie, cookie_str, sizeof(cookie_str)-1);
*cookie_len = sizeof(cookie_str)-1;
return 1;
}
int verify_cookie([[maybe_unused]] SSL *ssl, const unsigned char *cookie, unsigned int cookie_len)
{
return sizeof(cookie_str)-1==cookie_len && memcmp(cookie, cookie_str, sizeof(cookie_str)-1)==0;
}
void throw_SSL_error_if_error(SSL* ssl, int ret, std::string str)
{
if(ret>0) return; // SSL_ERROR_NONE
str += " ret="+std::to_string(ret)+' ';
auto sslError = SSL_get_error(ssl, ret);
if(sslError == SSL_ERROR_SYSCALL){
throw std::string{str+"SSL_ERROR_SYSCALL + error "}+std::to_string(errno);
}
}
namespace std
{
template<> struct hash<sockaddr>
{
size_t operator()(sockaddr const& val) const noexcept
{
size_t res = 0;
for(unsigned long h : val.sa_data)
{
res = (res << 1) ^ h;
}
return res;
}
};
}
bool operator==(const sockaddr& l, const sockaddr& r)
{
if(l.sa_family != r.sa_family) return false;
for(int i=0; i<14; ++i)
{
if(l.sa_data[i] != r.sa_data[i]) return false;
}
return true;
}
class DTLSConnection;
class SSLSetterUpper
{
friend DTLSConnection;
SSL_CTX *ctx;
SSLSetterUpper()
{
int ret; // because it is C;
SSL_load_error_strings();
SSL_library_init();
const SSL_METHOD *mtd = DTLS_server_method();
ctx = SSL_CTX_new(mtd);
SSL_CTX_set_min_proto_version(ctx, DTLS1_2_VERSION);
SSL_CTX_use_certificate_chain_file(ctx, "server-cert.pem");
SSL_CTX_use_PrivateKey_file(ctx, "server-key.pem", SSL_FILETYPE_PEM);
ret = SSL_CTX_load_verify_locations(ctx, "root-ca.pem", nullptr);
if(ret != 1)
{
throw std::string{"SSL_CTX_load_verify_locations failed"};
}
ret = SSL_CTX_set_default_verify_file(ctx);
if(ret != 1)
{
throw std::string{"SSL_CTX_set_default_verify_file failed"};
}
SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_FAIL_IF_NO_PEER_CERT, nullptr);
SSL_CTX_set_cookie_generate_cb(ctx, generate_cookie);
SSL_CTX_set_cookie_verify_cb(ctx, verify_cookie);
}
SSL* generateSSL() const
{
return SSL_new(ctx);
}
};
struct Client
{
CustomBIO cbio;
SSL *ssl;
explicit Client(SSL *ssl)
:cbio{}, ssl{ssl}
{
cbio.peekmode = false;
BIO *bio = BIO_new(BIO_s_custom());
BIO_set_data(bio, (void *)&cbio);
BIO_set_init(bio, 1);
SSL_set_bio(ssl, bio, bio);
}
bool on_init() const
{
int ret = DTLSv1_listen(this->ssl, nullptr);
//throw_SSL_error_if_error(ssl, ret, "DTLSv1_listen failed");
std::cout << "DTLSv1_listen " << ret << std::endl;
return (ret==1);
}
bool on_connect() const
{
int ret = SSL_accept(ssl);
if(ret!=1) return false;
std::cout << "ssl = " << SSL_state_string_long(ssl) << std::endl;
std::cout << "SSL_accept successful!" << std::endl;
return true;
}
std::string SSL_read_alt() const
{
Packet p(2000);
std::cout << "sizeA = " << cbio.receivingQueue.size() << std::endl;
int ret = ::SSL_read(ssl,p.data,p.capacity);
std::cout << "ret = " << ret << std::endl;
std::cout << "sizeB = " << cbio.receivingQueue.size() << std::endl;
std::cout << "pdata0 " << p.data[0] << std::endl;
//std::cout << SSL_get_error(ssl, ret) << std::endl;
throw_SSL_error_if_error(ssl, ret, "sslread");
p.len = ret;
std::cerr << "plen" << std::endl;
std::cerr << p.len << std::endl;
std::string result(p.data,p.len);
if(result[0] == '[=10=]') std::cout << "res = " << result << std::endl;
p.free();
return result;
}
};
class DTLSConnection
{
static const SSLSetterUpper sslSetup;
int epoll_fd;
std::unordered_map<sockaddr, std::shared_ptr<Client>> knownClients;
std::unordered_map<sockaddr, std::shared_ptr<Client>> connectedClients;
std::shared_ptr<Client> incomingClient;
public:
// 127.0.0.1:1234 or [::1]:1234
explicit DTLSConnection(std::string thisAddress)
{
std::list<sockaddr_storage> addresses;
if (thisAddress[0]=='[')
{
auto pos = thisAddress.find(']', 1);
if (pos == std::string::npos)
{
throw std::string{"invalid target"};
}
int port = std::stoi(thisAddress.substr(pos+2));
if (port<1||port>65535)
{
throw std::string{"invalid port"};
}
addresses.emplace_back();
auto* thisAddr = (sockaddr_in6 *)&(addresses.back());
thisAddr->sin6_family = AF_INET6;
if ( ! inet_pton(AF_INET6, thisAddress.substr(1, pos).c_str(), &thisAddr->sin6_addr) )
{
throw std::string{"invalid ipv6 address"};
}
thisAddr->sin6_port = htons(port);
}
else
{
auto pos = thisAddress.find(':');
if (pos == std::string::npos)
{
throw std::string{"invalid target"};
}
int port = std::stoi(thisAddress.substr(pos+1));
if (port<1||port>65535)
{
throw std::string{"invalid port"};
}
addresses.emplace_back();
auto * thisAddr = (sockaddr_in *)&(addresses.back());
thisAddr->sin_family = AF_INET;
if ( ! inet_pton(AF_INET, thisAddress.substr(0, pos).c_str(), &thisAddr->sin_addr) )
{
throw std::string{"invalid ipv4 address"};
}
thisAddr->sin_port = htons(port);
}
epoll_fd = epoll_create1(EPOLL_CLOEXEC);
for(auto &address : addresses)
{
epoll_event epe {};
epe.data.fd = socket(address.ss_family, SOCK_DGRAM/*|SOCK_NONBLOCK*/|SOCK_CLOEXEC, 0);
if(bind(epe.data.fd, (const sockaddr*)&address, sizeof(address)) != 0)
{
throw std::string{"failed to bind"};
}
epe.events = EPOLLIN|EPOLLET;
epoll_ctl(epoll_fd, EPOLL_CTL_ADD, epe.data.fd, &epe);
}
//signal(SIGINT, signal_handler); // do i need this?
incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
}
ssize_t recv(const std::function<void(Client *)>& onmessage)
{
std::cout << "recv(onmessage)" << std::endl;
epoll_event epe{};
int ret;
ret = epoll_wait(epoll_fd, &epe, 1, TIME_OUT);
if (ret==-1)
{
throw std::string{"epoll_wait failed"};
}
if(ret==0)
{
return ret; // wait longer
}
Packet packet{2000};
ret = recvfrom(epe.data.fd, packet.data, packet.capacity, 0, incomingClient->cbio.getThat<sockaddr>(), &incomingClient->cbio.thatAddr_len);
packet.len = ret;
if(ret==0) return ret;
if(ret<0)
{
switch(errno)
{
case EAGAIN:
//case EWOULDBLOCK:
return ret;
case EBADF:
throw std::string{"EBADF"};
case ECONNREFUSED:
throw std::string{"ECONNREFUSED"};
case EFAULT:
throw std::string{"EFAULT"};
case EINTR:
throw std::string{"EINTR"};
case EINVAL:
throw std::string{"EINVAL"};
case ENOMEM:
throw std::string{"ENOMEM"};
case ENOTCONN:
throw std::string{"ENOTCONN"};
case ENOTSOCK:
throw std::string{"ENOTSOCK"};
default:
throw std::string{"Unknwon errno with negative return from recvfrom: "}+std::to_string(errno);
}
}
auto known = knownClients.find(*incomingClient->cbio.getThat<sockaddr>());
auto connected = connectedClients.find(*incomingClient->cbio.getThat<sockaddr>());
std::cout << "START" << std::endl;
for(auto &pair : knownClients)
{
std::cout << sdump_addr(&pair.first) << std::endl;
}
std::cout << "END" << std::endl;
if(known == knownClients.end())
{
std::cout << "inetaddr = " << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << std::endl;
ret = 0;
incomingClient->cbio.receivingQueue.push_back(std::move(packet));
incomingClient->cbio.sockfd = epe.data.fd;
if( incomingClient->on_init() )
{
std::cout << "inc = " << incomingClient->on_connect() << std::endl; //скорее всего, всегда будет ложь
std::cout << "on_init if" << std::endl;
knownClients[*incomingClient->cbio.getThat<sockaddr>()] = incomingClient;
incomingClient = std::make_shared<Client>(sslSetup.generateSSL());
}
}
else if(connected == connectedClients.end())
{
std::cout << "elseif" << std::endl;
ret = 0;
auto cli = known->second;
cli->cbio.receivingQueue.push_back(std::move(packet));
if( cli->on_connect() )
{
std::cout << "cli->cbio.receivingQueue.size()" << cli->cbio.receivingQueue.size() << std::endl;
connectedClients[*cli->cbio.getThat<sockaddr>()] = cli;
SSL_write(cli->ssl, "hello", 6);
}
}
else
{
std::cout << "else" << std::endl;
std::cout << sdump_addr(incomingClient->cbio.getThat<sockaddr>()) << " has been found as connected" << std::endl;
connected->second->cbio.receivingQueue.push_back(std::move(packet));
onmessage(connected->second.get());
}
return ret;
}
};
const SSLSetterUpper DTLSConnection::sslSetup{};
运行 服务器的输出（CustomBIO 输出被截断）是：
Connection created
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
probe peekmode 0
211 bytes read from queue
36 bytes sent
probe peekmode 0
The queue is empty
DTLSv1_listen -1
Returned value is 0
recv(onmessage)
START
END
inetaddr = INET: 192.168.31.177:58897
probe peekmode 0
219 bytes read from queue
DTLSv1_listen 1
1180 bytes sent
probe peekmode 0
The queue is empty
inc = 0
on_init if
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
86 bytes sent
823 bytes sent
167 bytes sent
79 bytes sent
25 bytes sent
probe peekmode 0
219 bytes read from queue
probe peekmode 0
The queue is empty
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
elseif
probe peekmode 0
1088 bytes read from queue
618 bytes sent
ssl = SSL negotiation finished successfully
SSL_accept successful!
cli->cbio.receivingQueue.size()0
43 bytes sent
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
824 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 824
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
58 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 `
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 58
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
131 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 131
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
14 bytes read from queue
probe peekmode 0
The queue is empty
ret = -1
sizeB = 0
pdata0 �
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 14
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
The queue is empty
ret = -1
sizeB = 0
pdata0
EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0
Returned value is 61
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
55 bytes read from queue
ret = 18
sizeB = 0
pdata0 d
plen
18
dfksaiopfjiaosjfio
I am in onmessage
Returned value is 55
recv(onmessage)
Returned value is 0
recv(onmessage)
START
INET: 192.168.31.177:58897
END
else
INET: 192.168.31.177:58897 has been found as connected
sizeA = 1
probe peekmode 0
46 bytes read from queue
ret = 9
sizeB = 0
pdata0 s
plen
9
sdasdasda
I am in onmessage
Returned value is 46
recv(onmessage)
Returned value is 0
recv(onmessage)
在您看到 EXCEPTION: sslread ret=-1 SSL_ERROR_SYSCALL + error 0 的地方，这意味着 SSL_read 已返回 -1。抱歉有些脏代码。
以防其他人遇到类似问题。问题是调用服务器的 recv 函数之间的等待时间为 1 秒。那时客户端认为服务器没有响应并开始做奇怪的事情。降低延迟解决了问题。