diff --git a/Knot.files b/Knot.files
index 5f5370a1ff29474eac56d5aee38463f3c8a17c78..5d2df54469c3dd44d34f945d38de216ec78567fd 100644
--- a/Knot.files
+++ b/Knot.files
@@ -6,6 +6,8 @@ src/contrib/base64.h
src/contrib/base64url.c
src/contrib/base64url.h
src/contrib/color.h
+src/contrib/conn_pool.c
+src/contrib/conn_pool.h
src/contrib/ctype.h
src/contrib/dnstap/convert.c
src/contrib/dnstap/convert.h
diff --git a/doc/man/knot.conf.5in b/doc/man/knot.conf.5in
index 1b0c746e39ea809ba32330a667116bf6cc832cd8..cac59ff779ed64b34278ab2e697c62d288b621b6 100644
--- a/doc/man/knot.conf.5in
+++ b/doc/man/knot.conf.5in
@@ -191,6 +191,8 @@ server:
tcp\-max\-clients: INT
tcp\-reuseport: BOOL
tcp\-fastopen: BOOL
+ remote\-pool\-limit: INT
+ remote\-pool\-timeout: TIME
socket\-affinity: BOOL
udp\-max\-payload: SIZE
udp\-max\-payload\-ipv4: SIZE
@@ -358,6 +360,21 @@ is \fB1\fP for server side, and \fBnet.inet.tcp.fastopen.client_enable\fP is
.UNINDENT
.sp
\fIDefault:\fP off
+.SS remote\-pool\-limit
+.sp
+If nonzero, the server will keep up to this number of outgoing TCP connections
+open for later use. This is an optimization to avoid frequent opening of
+TCP connections to the same remote.
+.sp
+Change of this parameter requires restart of the Knot server to take effect.
+.sp
+\fIDefault:\fP 0
+.SS remote\-pool\-timeout
+.sp
+The timeout in seconds after which the unused kept\-open outgoing TCP connections
+to remote servers are closed.
+.sp
+\fIDefault:\fP 5
.SS socket\-affinity
.sp
If enabled and if SO_REUSEPORT is available on Linux, all configured network
diff --git a/doc/reference.rst b/doc/reference.rst
index 63542210e9f622fd6c5d7802fffc4f02f0d2f5ca..2cea76fffc9969c7a356623bfcc0179b45188da1 100644
--- a/doc/reference.rst
+++ b/doc/reference.rst
@@ -142,6 +142,8 @@ General options related to the server.
tcp-max-clients: INT
tcp-reuseport: BOOL
tcp-fastopen: BOOL
+ remote-pool-limit: INT
+ remote-pool-timeout: TIME
socket-affinity: BOOL
udp-max-payload: SIZE
udp-max-payload-ipv4: SIZE
@@ -351,6 +353,29 @@ configuration as it's enabled automatically if supported by OS.
*Default:* off
+.. _server_remote-pool-limit:
+
+remote-pool-limit
+-----------------
+
+If nonzero, the server will keep up to this number of outgoing TCP connections
+open for later use. This is an optimization to avoid frequent opening of
+TCP connections to the same remote.
+
+Change of this parameter requires restart of the Knot server to take effect.
+
+*Default:* 0
+
+.. _server_remote-pool-timeout:
+
+remote-pool-timeout
+-------------------
+
+The timeout in seconds after which the unused kept-open outgoing TCP connections
+to remote servers are closed.
+
+*Default:* 5
+
.. _server_socket-affinity:
socket-affinity
diff --git a/src/contrib/Makefile.inc b/src/contrib/Makefile.inc
index ee987071d893ccda78f9d7fe2869159269a1e2e4..32fb30750c5c602ea088c141907b768f1f1641ef 100644
--- a/src/contrib/Makefile.inc
+++ b/src/contrib/Makefile.inc
@@ -29,6 +29,8 @@ libcontrib_la_SOURCES = \
contrib/base64.h \
contrib/base64url.c \
contrib/base64url.h \
+ contrib/conn_pool.c \
+ contrib/conn_pool.h \
contrib/color.h \
contrib/ctype.h \
contrib/files.c \
diff --git a/src/contrib/conn_pool.c b/src/contrib/conn_pool.c
new file mode 100644
index 0000000000000000000000000000000000000000..cb8af8255d79896a2a1e86ac2baa3e8bd56947ec
--- /dev/null
+++ b/src/contrib/conn_pool.c
@@ -0,0 +1,253 @@
+/* Copyright (C) 2021 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "contrib/conn_pool.h"
+
+#include "contrib/sockaddr.h"
+
+conn_pool_t *global_conn_pool = NULL;
+
+static void *closing_thread(void *_arg)
+{
+ conn_pool_t *pool = _arg;
+
+ while (true) {
+ knot_time_t now = knot_time(), next = 0;
+ knot_timediff_t timeout = conn_pool_timeout(pool, 0);
+ assert(timeout != 0);
+
+ while (true) {
+ int old_fd = conn_pool_get_old(pool, now - timeout + 1, &next);
+ if (old_fd >= 0) {
+ close(old_fd);
+ } else {
+ break;
+ }
+ }
+
+ if (next == 0) {
+ sleep(timeout);
+ } else {
+ sleep(next + timeout - now);
+ }
+ }
+
+ return NULL; // we never get here since the thread will be cancelled instead
+}
+
+conn_pool_t *conn_pool_init(size_t capacity, knot_timediff_t timeout)
+{
+ if (capacity == 0 || timeout == 0) {
+ return NULL;
+ }
+
+ conn_pool_t *pool = calloc(1, sizeof(*pool) + capacity * sizeof(pool->conns[0]));
+ if (pool != NULL) {
+ pool->capacity = capacity;
+ pool->timeout = timeout;
+ if (pthread_mutex_init(&pool->mutex, 0) != 0) {
+ free(pool);
+ return NULL;
+ }
+ if (pthread_create(&pool->closing_thread, NULL, closing_thread, pool) != 0) {
+ pthread_mutex_destroy(&pool->mutex);
+ free(pool);
+ return NULL;
+ }
+ }
+ return pool;
+}
+
+void conn_pool_deinit(conn_pool_t *pool)
+{
+ if (pool != NULL) {
+ pthread_cancel(pool->closing_thread);
+
+ int fd;
+ knot_time_t unused;
+ while ((fd = conn_pool_get_old(pool, 0, &unused)) >= 0) {
+ close(fd);
+ }
+
+ pthread_mutex_destroy(&pool->mutex);
+ free(pool);
+ }
+}
+
+knot_timediff_t conn_pool_timeout(conn_pool_t *pool,
+ knot_timediff_t new_timeout)
+{
+ if (pool == NULL) {
+ return 0;
+ }
+
+ pthread_mutex_lock(&pool->mutex);
+
+ knot_timediff_t prev = pool->timeout;
+ if (new_timeout != 0) {
+ pool->timeout = new_timeout;
+ }
+
+ pthread_mutex_unlock(&pool->mutex);
+ return prev;
+}
+
+static int pool_pop(conn_pool_t *pool, size_t i)
+{
+ conn_pool_memb_t *conn = &pool->conns[i];
+ assert(conn->last_active != 0);
+ assert(pool->usage > 0);
+ int fd = conn->fd;
+ memset(conn, 0, sizeof(*conn));
+ pool->usage--;
+ return fd;
+}
+
+int conn_pool_get(conn_pool_t *pool,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst)
+{
+ if (pool == NULL) {
+ return -1;
+ }
+
+ int fd = -1;
+ pthread_mutex_lock(&pool->mutex);
+
+ for (size_t i = 0; i < pool->capacity; i++) {
+ if (pool->conns[i].last_active != 0 &&
+ sockaddr_cmp(&pool->conns[i].dst, dst, false) == 0 &&
+ sockaddr_cmp(&pool->conns[i].src, src, false) == 0) {
+ fd = pool_pop(pool, i);
+ break;
+ }
+ }
+
+ pthread_mutex_unlock(&pool->mutex);
+
+ if (fd >= 0) {
+ uint8_t unused;
+ int peek = recv(fd, &unused, 1, MSG_PEEK | MSG_DONTWAIT); // returns 0 if fd closed by other side; -1 if still open
+ if (peek == 0) {
+ close(fd);
+ fd = -1;
+ }
+ }
+
+ return fd;
+}
+
+int conn_pool_get_old(conn_pool_t *pool,
+ knot_time_t older_than,
+ knot_time_t *next_oldest)
+{
+ *next_oldest = 0;
+ if (pool == NULL) {
+ return -1;
+ }
+
+ int fd = -1;
+ pthread_mutex_lock(&pool->mutex);
+
+ for (size_t i = 0; i < pool->capacity; i++) {
+ knot_time_t la = pool->conns[i].last_active;
+ if (fd == -1 && knot_time_cmp(la, older_than) < 0) {
+ fd = pool_pop(pool, i);
+ } else if (knot_time_cmp(la, *next_oldest) < 0) {
+ *next_oldest = la;
+ }
+ }
+
+ pthread_mutex_unlock(&pool->mutex);
+ return fd;
+}
+
+static void pool_push(conn_pool_t *pool, size_t i,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst,
+ int fd)
+{
+ conn_pool_memb_t *conn = &pool->conns[i];
+ assert(conn->last_active == 0);
+ assert(pool->usage < pool->capacity);
+ conn->last_active = knot_time();
+ conn->fd = fd;
+ memcpy(&conn->src, src, sizeof(conn->src));
+ memcpy(&conn->dst, dst, sizeof(conn->dst));
+ pool->usage++;
+}
+
+bool conn_pool_put(conn_pool_t *pool,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst,
+ int fd)
+{
+ if (pool == NULL) {
+ return false;
+ }
+
+ pthread_mutex_lock(&pool->mutex);
+
+ for (size_t i = 0; i < pool->capacity; i++) {
+ if (pool->conns[i].last_active == 0) {
+ pool_push(pool, i, src, dst, fd);
+ pthread_mutex_unlock(&pool->mutex);
+ return true;
+ }
+ }
+
+ pthread_mutex_unlock(&pool->mutex);
+ return false;
+}
+
+int conn_pool_put_force(conn_pool_t *pool,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst,
+ int fd)
+{
+ if (pool == NULL || pool->capacity == 0) {
+ return fd;
+ }
+
+ knot_time_t oldest_time = 0;
+ size_t oldest_i = pool->capacity;
+
+ pthread_mutex_lock(&pool->mutex);
+
+ for (size_t i = 0; i < pool->capacity; i++) {
+ knot_time_t la = pool->conns[i].last_active;
+ if (la == 0) {
+ pool_push(pool, i, src, dst, fd);
+ pthread_mutex_unlock(&pool->mutex);
+ return -1;
+ } else if (knot_time_cmp(la, oldest_time) < 0) {
+ oldest_time = la;
+ oldest_i = i;
+ }
+ }
+
+ assert(oldest_i < pool->capacity);
+ int oldest_fd = pool_pop(pool, oldest_i);
+ pool_push(pool, oldest_i, src, dst, fd);
+ pthread_mutex_unlock(&pool->mutex);
+ return oldest_fd;
+}
diff --git a/src/contrib/conn_pool.h b/src/contrib/conn_pool.h
new file mode 100644
index 0000000000000000000000000000000000000000..4b2316fed4f16dd67e4484354b33090d77194b01
--- /dev/null
+++ b/src/contrib/conn_pool.h
@@ -0,0 +1,130 @@
+/* Copyright (C) 2021 CZ.NIC, z.s.p.o. <knot-dns@labs.nic.cz>
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <sys/socket.h>
+
+#include "contrib/time.h"
+
+typedef struct {
+ struct sockaddr_storage src;
+ struct sockaddr_storage dst;
+ int fd;
+ knot_time_t last_active;
+} conn_pool_memb_t;
+
+typedef struct {
+ size_t capacity;
+ size_t usage;
+ knot_timediff_t timeout;
+ pthread_mutex_t mutex;
+ pthread_t closing_thread;
+ conn_pool_memb_t conns[];
+} conn_pool_t;
+
+extern conn_pool_t *global_conn_pool;
+
+/*!
+ * \brief Allocate connection pool.
+ *
+ * \param capacity Connection pool capacity (must be positive number).
+ * \param timeout Connection timeout (must be positive number).
+ *
+ * \return Connection pool or NULL if error.
+ */
+conn_pool_t *conn_pool_init(size_t capacity, knot_timediff_t timeout);
+
+/*!
+ * \brief Deallocate the pool, close all connections, terminate closing thread.
+ *
+ * \param pool Connection pool.
+ */
+void conn_pool_deinit(conn_pool_t *pool);
+
+/*!
+ * \brief Get and/or set connection timeout.
+ *
+ * \param pool Connection pool.
+ * \param new_timeout Optional: set new timeout (if positive number).
+ *
+ * \return Previous value of timeout.
+ */
+knot_timediff_t conn_pool_timeout(conn_pool_t *pool,
+ knot_timediff_t new_timeout);
+
+/*!
+ * \brief Try to get an open connection if present, check if alive.
+ *
+ * \param pool Pool to search in.
+ * \param src Connection source address.
+ * \param dst Connection destination address.
+ *
+ * \retval -1 If error (no such connection).
+ * \return >= 0 File descriptor of the connection.
+ */
+int conn_pool_get(conn_pool_t *pool,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst);
+
+/*!
+ * \brief Try to get an open connection older than specified timestamp.
+ *
+ * \param pool Pool to search in.
+ * \param older_than Timestamp that the connection must be older than.
+ * \param next_oldest Out: the timestamp of the oldest connection (other than the returned).
+ *
+ * \return -1 if error (no such connection), >= 0 connection file descriptor.
+ *
+ * \warning The returned connection is not necessarily the oldest one.
+ */
+int conn_pool_get_old(conn_pool_t *pool,
+ knot_time_t older_than,
+ knot_time_t *next_oldest);
+
+/*!
+ * \brief Put an open connection to the pool.
+ *
+ * \param pool Pool to insert into.
+ * \param src Connestion source address.
+ * \param dst Connection destination adress.
+ * \param fd Connection file descriptor.
+ *
+ * \return True if connection stored, false if not.
+ */
+bool conn_pool_put(conn_pool_t *pool,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst,
+ int fd);
+
+/*!
+ * \brief Put an open connection to the pool, possibly displacing the oldest one there.
+ *
+ * \param pool Pool to insert into.
+ * \param src Connestion source address.
+ * \param dst Connection destination adress.
+ * \param fd Connection file descriptor.
+ *
+ * \retval -1 If connection stored to free slot.
+ * \retval fd If not able to store connection.
+ * \return >= 0 File descriptor of the displaced old connection.
+ */
+int conn_pool_put_force(conn_pool_t *pool,
+ struct sockaddr_storage *src,
+ struct sockaddr_storage *dst,
+ int fd);
diff --git a/src/knot/conf/schema.c b/src/knot/conf/schema.c
index 1b42449aa97577bda7b0996d9c3bd360e305995f..4cb321a9cf7256729dab8f7f37442c28f96eb023 100644
--- a/src/knot/conf/schema.c
+++ b/src/knot/conf/schema.c
@@ -195,6 +195,8 @@ static const yp_item_t desc_server[] = {
{ C_TCP_MAX_CLIENTS, YP_TINT, YP_VINT = { 0, INT32_MAX, YP_NIL } },
{ C_TCP_REUSEPORT, YP_TBOOL, YP_VNONE },
{ C_TCP_FASTOPEN, YP_TBOOL, YP_VNONE },
+ { C_RMT_POOL_LIMIT, YP_TINT, YP_VINT = { 0, INT32_MAX, 0 } },
+ { C_RMT_POOL_TIMEOUT, YP_TINT, YP_VINT = { 1, INT32_MAX, 5, YP_STIME } },
{ C_SOCKET_AFFINITY, YP_TBOOL, YP_VNONE },
{ C_UDP_MAX_PAYLOAD, YP_TINT, YP_VINT = { KNOT_EDNS_MIN_DNSSEC_PAYLOAD,
KNOT_EDNS_MAX_UDP_PAYLOAD,
diff --git a/src/knot/conf/schema.h b/src/knot/conf/schema.h
index d3806d72375c4d41346d926602d4f85bc4deb18e..7e12b8d44d0a62aa349fceb505c1a1144a98e2c8 100644
--- a/src/knot/conf/schema.h
+++ b/src/knot/conf/schema.h
@@ -95,6 +95,8 @@
#define C_REFRESH_MIN_INTERVAL "\x14""refresh-min-interval"
#define C_REPRO_SIGNING "\x14""reproducible-signing"
#define C_RMT "\x06""remote"
+#define C_RMT_POOL_LIMIT "\x11""remote-pool-limit"
+#define C_RMT_POOL_TIMEOUT "\x13""remote-pool-timeout"
#define C_ROUTE_CHECK "\x0B""route-check"
#define C_RRSIG_LIFETIME "\x0E""rrsig-lifetime"
#define C_RRSIG_PREREFRESH "\x11""rrsig-pre-refresh"
diff --git a/src/knot/query/requestor.c b/src/knot/query/requestor.c
index 59d7dcac91e68525a838ae76af81257e7279b158..77ab1be948b66e18327d69afbd627dc878a43190 100644
--- a/src/knot/query/requestor.c
+++ b/src/knot/query/requestor.c
@@ -19,6 +19,7 @@
#include "libknot/attribute.h"
#include "knot/query/requestor.h"
#include "libknot/errcode.h"
+#include "contrib/conn_pool.h"
#include "contrib/mempattern.h"
#include "contrib/net.h"
#include "contrib/sockaddr.h"
@@ -41,6 +42,16 @@ static int request_ensure_connected(knot_request_t *request)
}
int sock_type = use_tcp(request) ? SOCK_STREAM : SOCK_DGRAM;
+
+ if (sock_type == SOCK_STREAM) {
+ request->fd = conn_pool_get(global_conn_pool,
+ &request->source,
+ &request->remote);
+ if (request->fd >= 0) {
+ return KNOT_EOK;
+ }
+ }
+
request->fd = net_connected_socket(sock_type,
&request->remote,
&request->source,
@@ -156,6 +167,12 @@ void knot_request_free(knot_request_t *request, knot_mm_t *mm)
return;
}
+ if (request->fd >= 0 && use_tcp(request)) {
+ request->fd = conn_pool_put_force(global_conn_pool,
+ &request->source,
+ &request->remote,
+ request->fd);
+ }
if (request->fd >= 0) {
close(request->fd);
}
diff --git a/src/knot/server/server.c b/src/knot/server/server.c
index d88d241a1f7a188318fd64605e2dbabc256085f6..e9b42d88b6fe57351a6f570653b579a5233b6cb8 100644
--- a/src/knot/server/server.c
+++ b/src/knot/server/server.c
@@ -38,6 +38,7 @@
#include "knot/zone/timers.h"
#include "knot/zone/zonedb-load.h"
#include "knot/worker/pool.h"
+#include "contrib/conn_pool.h"
#include "contrib/net.h"
#include "contrib/openbsd/strlcat.h"
#include "contrib/os.h"
@@ -696,6 +697,10 @@ void server_deinit(server_t *server)
/* Close journal database if open. */
knot_lmdb_deinit(&server->journaldb);
+
+ /* Close and deinit connection pool. */
+ conn_pool_deinit(global_conn_pool);
+ global_conn_pool = NULL;
}
static int server_init_handler(server_t *server, int index, int thread_count,
@@ -1190,6 +1195,18 @@ int server_reconfigure(conf_t *conf, server_t *server)
knot_strerror(ret));
}
+ /* Reconfigure connection pool. */
+ conf_val_t val = conf_get(conf, C_SRV, C_RMT_POOL_LIMIT);
+ size_t cp_size = conf_int(&val);
+ val = conf_get(conf, C_SRV, C_RMT_POOL_TIMEOUT);
+ knot_timediff_t cp_timeout = conf_int(&val);
+ if (global_conn_pool == NULL && cp_size > 0) {
+ conn_pool_t *new_pool = conn_pool_init(cp_size, cp_timeout);
+ global_conn_pool = new_pool;
+ } else {
+ conn_pool_timeout(global_conn_pool, cp_timeout);
+ }
+
return KNOT_EOK;
}
diff --git a/tests-extra/tools/dnstest/server.py b/tests-extra/tools/dnstest/server.py
index 90c0dda2edea6fa4311dfa934fb160436622ff56..68d967511404a849ef58febc45dbb49b9072ed69 100644
--- a/tests-extra/tools/dnstest/server.py
+++ b/tests-extra/tools/dnstest/server.py
@@ -1241,6 +1241,7 @@ class Knot(Server):
self._str(s, "udp-max-payload", self.udp_max_payload)
self._str(s, "udp-max-payload-ipv4", self.udp_max_payload_ipv4)
self._str(s, "udp-max-payload-ipv6", self.udp_max_payload_ipv6)
+ self._str(s, "remote-pool-limit", str(random.randint(0,6)))
s.end()
s.begin("control")