From f4c612f83296c03c5bd2be6d6e7dc6db18a060b5 Mon Sep 17 00:00:00 2001 From: Linus Yang Date: Fri, 9 Mar 2018 04:29:45 +0800 Subject: [PATCH 1/6] Support TCP fast open on Windows --- configure.ac | 2 +- src/local.c | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/local.h | 8 ++++++ src/resolv.c | 3 +- src/server.c | 78 +++++++++++++++++++++++++++++++++++++++++++++++++- src/server.h | 8 ++++++ src/winsock.c | 57 +++++++++++++++++++++++++++++++++++++ src/winsock.h | 35 ++++++++++++++++++++++- 8 files changed, 265 insertions(+), 5 deletions(-) diff --git a/configure.ac b/configure.ac index a7a17943..243a1a3f 100755 --- a/configure.ac +++ b/configure.ac @@ -167,7 +167,7 @@ AC_CHECK_HEADERS([net/if.h], [], [], case $host in *-mingw*) AC_DEFINE([CONNECT_IN_PROGRESS], [WSAEWOULDBLOCK], [errno for incomplete non-blocking connect(2)]) - AC_CHECK_HEADERS([winsock2.h ws2tcpip.h], [], [AC_MSG_ERROR([Missing MinGW headers])], []) + AC_CHECK_HEADERS([winsock2.h ws2tcpip.h mswsock.h], [], [AC_MSG_ERROR([Missing MinGW headers])], []) ;; *-linux*) AC_DEFINE([CONNECT_IN_PROGRESS], [EINPROGRESS], [errno for incomplete non-blocking connect(2)]) diff --git a/src/local.c b/src/local.c index 47ff491c..3da1302a 100644 --- a/src/local.c +++ b/src/local.c @@ -371,11 +371,57 @@ server_recv_cb(EV_P_ ev_io *w, int revents) ev_io_start(EV_A_ & remote->send_ctx->io); ev_timer_start(EV_A_ & remote->send_ctx->watcher); } else { - int s = -1; #if defined(MSG_FASTOPEN) && !defined(TCP_FASTOPEN_CONNECT) + int s = -1; s = sendto(remote->fd, remote->buf->data, remote->buf->len, MSG_FASTOPEN, (struct sockaddr *)&(remote->addr), remote->addr_len); +#elif defined(TCP_FASTOPEN_WINSOCK) + DWORD s = -1; + DWORD err = 0; + do { + int optval = 1; + // Set fast open option + if(setsockopt(remote->fd, IPPROTO_TCP, TCP_FASTOPEN, + &optval, sizeof(optval)) != 0) { + ERROR("setsockopt"); + err = WSAEOPNOTSUPP; + break; + } + // Load ConnectEx function + LPFN_CONNECTEX ConnectEx = winsock_getconnectex(); + if (ConnectEx == NULL) { + LOGE("Cannot load ConnectEx() function"); + err = WSAEOPNOTSUPP; + break; + } + // ConnectEx requires a bound socket + if (winsock_dummybind(remote->fd, + (struct sockaddr *)&(remote->addr)) != 0) { + ERROR("bind"); + break; + } + // Call ConnectEx to send data + memset(&remote->olap, 0, sizeof(remote->olap)); + remote->connect_ex_done = 0; + if (ConnectEx(remote->fd, (const struct sockaddr *)&(remote->addr), + remote->addr_len, remote->buf->data, remote->buf->len, + &s, &remote->olap)) { + remote->connect_ex_done = 1; + break; + }; + // XXX: ConnectEx pending, check later in remote_send + if (WSAGetLastError() == ERROR_IO_PENDING) { + err = CONNECT_IN_PROGRESS; + break; + } + ERROR("ConnectEx"); + } while(0); + // Set error number + if (err) { + SetLastError(err); + } #else + int s = -1; #if defined(CONNECT_DATA_IDEMPOTENT) ((struct sockaddr_in *)&(remote->addr))->sin_len = sizeof(struct sockaddr_in); sa_endpoints_t endpoints; @@ -958,6 +1004,37 @@ remote_send_cb(EV_P_ ev_io *w, int revents) server_t *server = remote->server; if (!remote_send_ctx->connected) { +#ifdef TCP_FASTOPEN_WINSOCK + if (fast_open) { + // Check if ConnectEx is done + if (!remote->connect_ex_done) { + DWORD numBytes; + DWORD flags; + // Non-blocking way to fetch ConnectEx result + if (WSAGetOverlappedResult(remote->fd, &remote->olap, + &numBytes, FALSE, &flags)) { + remote->buf->len -= numBytes; + remote->buf->idx = numBytes; + remote->connect_ex_done = 1; + } else if (WSAGetLastError() == WSA_IO_INCOMPLETE) { + // XXX: ConnectEx still not connected, wait for next time + return; + } else { + ERROR("WSAGetOverlappedResult"); + // not connected + close_and_free_remote(EV_A_ remote); + close_and_free_server(EV_A_ server); + return; + }; + } + + // Make getpeername work + if (setsockopt(remote->fd, SOL_SOCKET, + SO_UPDATE_CONNECT_CONTEXT, NULL, 0) != 0) { + ERROR("setsockopt"); + } + } +#endif struct sockaddr_storage addr; socklen_t len = sizeof addr; int r = getpeername(remote->fd, (struct sockaddr *)&addr, &len); diff --git a/src/local.h b/src/local.h index b948afa9..411273bb 100644 --- a/src/local.h +++ b/src/local.h @@ -31,6 +31,10 @@ #include #endif +#ifdef TCP_FASTOPEN_WINSOCK +#include "winsock.h" +#endif + #include "crypto.h" #include "jconf.h" #include "protocol.h" @@ -85,6 +89,10 @@ typedef struct remote { int direct; int addr_len; uint32_t counter; +#ifdef TCP_FASTOPEN_WINSOCK + OVERLAPPED olap; + int connect_ex_done; +#endif buffer_t *buf; diff --git a/src/resolv.c b/src/resolv.c index 7e7e42b6..ed8ea76f 100644 --- a/src/resolv.c +++ b/src/resolv.c @@ -38,6 +38,8 @@ #include #include #include +#else +#include "winsock.h" // Should be before #endif #include @@ -52,7 +54,6 @@ #include "resolv.h" #include "utils.h" #include "netutils.h" -#include "winsock.h" #ifdef __MINGW32__ #define CONV_STATE_CB (ares_sock_state_cb) diff --git a/src/server.c b/src/server.c index ab3ce1a0..222e6cab 100644 --- a/src/server.c +++ b/src/server.c @@ -517,11 +517,56 @@ connect_to_remote(EV_P_ struct addrinfo *res, remote_t *remote = new_remote(sockfd); if (fast_open) { - int s = -1; #if defined(MSG_FASTOPEN) && !defined(TCP_FASTOPEN_CONNECT) + int s = -1; s = sendto(sockfd, server->buf->data, server->buf->len, MSG_FASTOPEN, res->ai_addr, res->ai_addrlen); +#elif defined(TCP_FASTOPEN_WINSOCK) + DWORD s = -1; + DWORD err = 0; + do { + int optval = 1; + // Set fast open option + if(setsockopt(sockfd, IPPROTO_TCP, TCP_FASTOPEN, + &optval, sizeof(optval)) != 0) { + ERROR("setsockopt"); + err = WSAEOPNOTSUPP; + break; + } + // Load ConnectEx function + LPFN_CONNECTEX ConnectEx = winsock_getconnectex(); + if (ConnectEx == NULL) { + LOGE("Cannot load ConnectEx() function"); + err = WSAEOPNOTSUPP; + break; + } + // ConnectEx requires a bound socket + if (winsock_dummybind(sockfd, res->ai_addr) != 0) { + ERROR("bind"); + break; + } + // Call ConnectEx to send data + memset(&remote->olap, 0, sizeof(remote->olap)); + remote->connect_ex_done = 0; + if (ConnectEx(sockfd, res->ai_addr, res->ai_addrlen, + server->buf->data, server->buf->len, + &s, &remote->olap)) { + remote->connect_ex_done = 1; + break; + }; + // XXX: ConnectEx pending, check later in remote_send + if (WSAGetLastError() == ERROR_IO_PENDING) { + err = CONNECT_IN_PROGRESS; + break; + } + ERROR("ConnectEx"); + } while(0); + // Set error number + if (err) { + SetLastError(err); + } #else + int s = -1; #if defined(TCP_FASTOPEN_CONNECT) int optval = 1; if(setsockopt(sockfd, IPPROTO_TCP, TCP_FASTOPEN_CONNECT, @@ -1175,6 +1220,37 @@ remote_send_cb(EV_P_ ev_io *w, int revents) } if (!remote_send_ctx->connected) { +#ifdef TCP_FASTOPEN_WINSOCK + if (fast_open) { + // Check if ConnectEx is done + if (!remote->connect_ex_done) { + DWORD numBytes; + DWORD flags; + // Non-blocking way to fetch ConnectEx result + if (WSAGetOverlappedResult(remote->fd, &remote->olap, + &numBytes, FALSE, &flags)) { + remote->buf->len -= numBytes; + remote->buf->idx = numBytes; + remote->connect_ex_done = 1; + } else if (WSAGetLastError() == WSA_IO_INCOMPLETE) { + // XXX: ConnectEx still not connected, wait for next time + return; + } else { + ERROR("WSAGetOverlappedResult"); + // not connected + close_and_free_remote(EV_A_ remote); + close_and_free_server(EV_A_ server); + return; + }; + } + + // Make getpeername work + if (setsockopt(remote->fd, SOL_SOCKET, + SO_UPDATE_CONNECT_CONTEXT, NULL, 0) != 0) { + ERROR("setsockopt"); + } + } +#endif struct sockaddr_storage addr; socklen_t len = sizeof(struct sockaddr_storage); memset(&addr, 0, len); diff --git a/src/server.h b/src/server.h index d7a2d1b0..2daed0f3 100644 --- a/src/server.h +++ b/src/server.h @@ -32,6 +32,10 @@ #include #endif +#ifdef TCP_FASTOPEN_WINSOCK +#include "winsock.h" +#endif + #include "crypto.h" #include "jconf.h" #include "resolv.h" @@ -104,6 +108,10 @@ typedef struct remote_ctx { typedef struct remote { int fd; +#ifdef TCP_FASTOPEN_WINSOCK + OVERLAPPED olap; + int connect_ex_done; +#endif buffer_t *buf; struct remote_ctx *recv_ctx; struct remote_ctx *send_ctx; diff --git a/src/winsock.c b/src/winsock.c index 68a3af16..2f065102 100644 --- a/src/winsock.c +++ b/src/winsock.c @@ -72,4 +72,61 @@ ss_error(const char *s) } } +#ifdef TCP_FASTOPEN_WINSOCK +LPFN_CONNECTEX +winsock_getconnectex(void) +{ + static LPFN_CONNECTEX pConnectEx = NULL; + if (pConnectEx != NULL) { + return pConnectEx; + } + + // Dummy socket for WSAIoctl + SOCKET s = socket(AF_INET, SOCK_STREAM, 0); + if (s == INVALID_SOCKET) { + ERROR("socket"); + return NULL; + } + + // Load ConnectEx function + GUID guid = WSAID_CONNECTEX; + DWORD numBytes; + int ret = -1; + ret = WSAIoctl(s, SIO_GET_EXTENSION_FUNCTION_POINTER, + (void *)&guid, sizeof(guid), + (void *)&pConnectEx, sizeof(pConnectEx), + &numBytes, NULL, NULL); + if (ret != 0) { + ERROR("WSAIoctl"); + closesocket(s); + return NULL; + } + closesocket(s); + return pConnectEx; +} + +int +winsock_dummybind(SOCKET fd, struct sockaddr *sa) +{ + struct sockaddr_storage ss; + memset(&ss, 0, sizeof(ss)); + if (sa->sa_family == AF_INET) { + struct sockaddr_in *sin = (struct sockaddr_in *)&ss; + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = INADDR_ANY; + } else if (sa->sa_family == AF_INET6) { + struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)&ss; + sin6->sin6_family = AF_INET6; + sin6->sin6_addr = in6addr_any; + } else { + return -1; + } + if (bind(fd, (struct sockaddr *)&ss, sizeof(ss)) < 0 && + WSAGetLastError() != WSAEINVAL) { + return -1; + } + return 0; +} +#endif + #endif // __MINGW32__ diff --git a/src/winsock.h b/src/winsock.h index a20259bb..0b2a7c21 100644 --- a/src/winsock.h +++ b/src/winsock.h @@ -25,6 +25,7 @@ #ifdef __MINGW32__ +// Target NT6 #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif @@ -37,11 +38,13 @@ #define _WIN32_WINNT 0x0600 #endif +// Winsock headers #include #include #include +#include -// Override error number +// Override POSIX error number #ifdef errno #undef errno #endif @@ -57,6 +60,32 @@ #endif #define CONNECT_IN_PROGRESS WSAEWOULDBLOCK +#ifdef EOPNOTSUPP +#undef EOPNOTSUPP +#endif +#define EOPNOTSUPP WSAEOPNOTSUPP + +#ifdef EPROTONOSUPPORT +#undef EPROTONOSUPPORT +#endif +#define EPROTONOSUPPORT WSAEPROTONOSUPPORT + +#ifdef ENOPROTOOPT +#undef ENOPROTOOPT +#endif +#define ENOPROTOOPT WSAENOPROTOOPT + +// Check if ConnectEx supported in header +#ifdef WSAID_CONNECTEX +// Hardcode TCP fast open option +#ifdef TCP_FASTOPEN +#undef TCP_FASTOPEN +#endif +#define TCP_FASTOPEN 15 +// Enable TFO support +#define TCP_FASTOPEN_WINSOCK 1 +#endif + // Override close function #define close(fd) closesocket(fd) @@ -80,6 +109,10 @@ void ss_error(const char *s); int setnonblocking(SOCKET socket); void winsock_init(void); void winsock_cleanup(void); +#ifdef TCP_FASTOPEN_WINSOCK +LPFN_CONNECTEX winsock_getconnectex(void); +int winsock_dummybind(SOCKET fd, struct sockaddr *sa); +#endif #endif // __MINGW32__ From 87b7dd85a9d4be029cbcf24e60d760b92496c3af Mon Sep 17 00:00:00 2001 From: Linus Yang Date: Sat, 10 Mar 2018 16:34:28 +0800 Subject: [PATCH 2/6] Correct header inclusion --- src/local.h | 2 +- src/server.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/local.h b/src/local.h index 411273bb..67161781 100644 --- a/src/local.h +++ b/src/local.h @@ -31,7 +31,7 @@ #include #endif -#ifdef TCP_FASTOPEN_WINSOCK +#ifdef __MINGW32__ #include "winsock.h" #endif diff --git a/src/server.h b/src/server.h index 2daed0f3..5233a4e8 100644 --- a/src/server.h +++ b/src/server.h @@ -32,7 +32,7 @@ #include #endif -#ifdef TCP_FASTOPEN_WINSOCK +#ifdef __MINGW32__ #include "winsock.h" #endif From b4c5021c654515365cc871cfa54269a4ac351ff6 Mon Sep 17 00:00:00 2001 From: Linus Yang Date: Sat, 10 Mar 2018 23:44:27 +0800 Subject: [PATCH 3/6] Missing macro for MinGW --- src/server.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server.c b/src/server.c index 222e6cab..a968a2bc 100644 --- a/src/server.c +++ b/src/server.c @@ -307,7 +307,7 @@ setfastopen(int fd) int s = 0; #ifdef TCP_FASTOPEN if (fast_open) { -#ifdef __APPLE__ +#if defined(__APPLE__) || defined(__MINGW32__) int opt = 1; #else int opt = 5; From 0a11c87aa5903b83a0cebb5cc1b0efaa63b61c05 Mon Sep 17 00:00:00 2001 From: Linus Yang Date: Mon, 12 Mar 2018 12:41:51 +0800 Subject: [PATCH 4/6] Do not override error code of setsockopt --- src/local.c | 3 +-- src/server.c | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/local.c b/src/local.c index 3da1302a..01a1a26a 100644 --- a/src/local.c +++ b/src/local.c @@ -384,14 +384,13 @@ server_recv_cb(EV_P_ ev_io *w, int revents) if(setsockopt(remote->fd, IPPROTO_TCP, TCP_FASTOPEN, &optval, sizeof(optval)) != 0) { ERROR("setsockopt"); - err = WSAEOPNOTSUPP; break; } // Load ConnectEx function LPFN_CONNECTEX ConnectEx = winsock_getconnectex(); if (ConnectEx == NULL) { LOGE("Cannot load ConnectEx() function"); - err = WSAEOPNOTSUPP; + err = WSAENOPROTOOPT; break; } // ConnectEx requires a bound socket diff --git a/src/server.c b/src/server.c index a968a2bc..cc361e6b 100644 --- a/src/server.c +++ b/src/server.c @@ -530,14 +530,13 @@ connect_to_remote(EV_P_ struct addrinfo *res, if(setsockopt(sockfd, IPPROTO_TCP, TCP_FASTOPEN, &optval, sizeof(optval)) != 0) { ERROR("setsockopt"); - err = WSAEOPNOTSUPP; break; } // Load ConnectEx function LPFN_CONNECTEX ConnectEx = winsock_getconnectex(); if (ConnectEx == NULL) { LOGE("Cannot load ConnectEx() function"); - err = WSAEOPNOTSUPP; + err = WSAENOPROTOOPT; break; } // ConnectEx requires a bound socket From ca51c60ee7f004ccb3d51c9e71abd0f50b5ebfb0 Mon Sep 17 00:00:00 2001 From: Linus Yang Date: Mon, 12 Mar 2018 12:57:52 +0800 Subject: [PATCH 5/6] Only define TCP_FASTOPEN if not exists --- src/winsock.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/winsock.h b/src/winsock.h index 0b2a7c21..50c8c168 100644 --- a/src/winsock.h +++ b/src/winsock.h @@ -78,10 +78,9 @@ // Check if ConnectEx supported in header #ifdef WSAID_CONNECTEX // Hardcode TCP fast open option -#ifdef TCP_FASTOPEN -#undef TCP_FASTOPEN -#endif +#ifndef TCP_FASTOPEN #define TCP_FASTOPEN 15 +#endif // Enable TFO support #define TCP_FASTOPEN_WINSOCK 1 #endif From 3f00e533bf24cb3584d1c3131a5ef5260d92376a Mon Sep 17 00:00:00 2001 From: Linus Yang Date: Mon, 12 Mar 2018 13:16:34 +0800 Subject: [PATCH 6/6] Minor format fix --- src/local.c | 4 ++-- src/server.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/local.c b/src/local.c index 01a1a26a..6b04f098 100644 --- a/src/local.c +++ b/src/local.c @@ -381,8 +381,8 @@ server_recv_cb(EV_P_ ev_io *w, int revents) do { int optval = 1; // Set fast open option - if(setsockopt(remote->fd, IPPROTO_TCP, TCP_FASTOPEN, - &optval, sizeof(optval)) != 0) { + if (setsockopt(remote->fd, IPPROTO_TCP, TCP_FASTOPEN, + &optval, sizeof(optval)) != 0) { ERROR("setsockopt"); break; } diff --git a/src/server.c b/src/server.c index cc361e6b..fd9061a3 100644 --- a/src/server.c +++ b/src/server.c @@ -527,8 +527,8 @@ connect_to_remote(EV_P_ struct addrinfo *res, do { int optval = 1; // Set fast open option - if(setsockopt(sockfd, IPPROTO_TCP, TCP_FASTOPEN, - &optval, sizeof(optval)) != 0) { + if (setsockopt(sockfd, IPPROTO_TCP, TCP_FASTOPEN, + &optval, sizeof(optval)) != 0) { ERROR("setsockopt"); break; }