diff --git a/doc/man/knot.conf.5in b/doc/man/knot.conf.5in
index 5b160b5263a1e784638d3c7ac84a4f1566022830..55c7cc600783ea6ce97a6eaebbb174e13ca672b9 100644
--- a/doc/man/knot.conf.5in
+++ b/doc/man/knot.conf.5in
@@ -379,7 +379,7 @@ to remote servers are closed.
 .SS remote\-retry\-delay
 .sp
 When a connection attempt times out to some remote address, this information will be
-kept for this specified time in seconds and other connections to the same address won\(aqt
+kept for this specified time (in milliseconds) and other connections to the same address won\(aqt
 be attempted. This prevents repetitive waiting for timeout on an unreachable remote.
 .sp
 \fIDefault:\fP 0
diff --git a/doc/reference.rst b/doc/reference.rst
index 988a66036e2c7edf723322f83d55b770ba1557a9..17a538e2bd6acf232639bf487f0875917493cdd6 100644
--- a/doc/reference.rst
+++ b/doc/reference.rst
@@ -383,7 +383,7 @@ remote-retry-delay
 ------------------
 
 When a connection attempt times out to some remote address, this information will be
-kept for this specified time in seconds and other connections to the same address won't
+kept for this specified time (in milliseconds) and other connections to the same address won't
 be attempted. This prevents repetitive waiting for timeout on an unreachable remote.
 
 *Default:* 0
diff --git a/src/knot/common/unreachable.c b/src/knot/common/unreachable.c
index c36500198fa16a7c4e09132379d63951254cd253..e137f3dc74790a41bfbd00f47cea08279e82c271 100644
--- a/src/knot/common/unreachable.c
+++ b/src/knot/common/unreachable.c
@@ -26,22 +26,38 @@ static uint32_t get_timestamp(void)
 {
 	struct timespec t;
 	clock_gettime(CLOCK_MONOTONIC, &t);
-	uint64_t res = (uint64_t)t.tv_sec * 1000000;
-	res += (uint64_t)t.tv_nsec / 1000;
+	uint64_t res = (uint64_t)t.tv_sec * 1000;
+	res += (uint64_t)t.tv_nsec / 1000000;
 	return res & 0xffffffff; // overflow does not matter since we are working with differences
 }
 
-knot_unreachables_t *knot_unreachables_init(uint32_t ttl)
+knot_unreachables_t *knot_unreachables_init(uint32_t ttl_ms)
 {
 	knot_unreachables_t *res = calloc(1, sizeof(*res));
 	if (res != NULL) {
 		pthread_mutex_init(&res->mutex, NULL);
-		res->ttl = ttl;
+		res->ttl_ms = ttl_ms;
 		init_list(&res->urs);
 	}
 	return res;
 }
 
+uint32_t knot_unreachables_ttl(knot_unreachables_t *urs, uint32_t new_ttl_ms)
+{
+	if (urs == NULL) {
+		return 0;
+	}
+
+	pthread_mutex_lock(&urs->mutex);
+
+	uint32_t prev = urs->ttl_ms;
+	urs->ttl_ms = new_ttl_ms;
+
+	pthread_mutex_unlock(&urs->mutex);
+
+	return prev;
+}
+
 void knot_unreachables_deinit(knot_unreachables_t **urs)
 {
 	if (urs != NULL && *urs != NULL) {
@@ -56,9 +72,9 @@ void knot_unreachables_deinit(knot_unreachables_t **urs)
 	}
 }
 
-static bool clear_old(knot_unreachable_t *ur, uint32_t now, uint32_t ttl)
+static bool clear_old(knot_unreachable_t *ur, uint32_t now, uint32_t ttl_ms)
 {
-	if (ur->time != 0 && now - ur->time > ttl) {
+	if (ur->time_ms != 0 && now - ur->time_ms > ttl_ms) {
 		rem_node((node_t *)ur);
 		free(ur);
 		return true;
@@ -76,7 +92,7 @@ static knot_unreachable_t *get_ur(knot_unreachables_t *urs,
 	uint32_t now = get_timestamp();
 	knot_unreachable_t *ur, *nxt;
 	WALK_LIST_DELSAFE(ur, nxt, urs->urs) {
-		if (clear_old(ur, now, urs->ttl)) {
+		if (clear_old(ur, now, urs->ttl_ms)) {
 			continue;
 		}
 
@@ -124,7 +140,7 @@ void knot_unreachable_add(knot_unreachables_t *urs,
 	if (ur != NULL) {
 		memcpy(&ur->addr, addr, sizeof(ur->addr));
 		memcpy(&ur->via, via, sizeof(ur->via));
-		ur->time = get_timestamp();
+		ur->time_ms = get_timestamp();
 		add_head(&urs->urs, (node_t *)ur);
 	}
 
diff --git a/src/knot/common/unreachable.h b/src/knot/common/unreachable.h
index f64f85a2b475360c58f2815407eaaf8aea96eebf..40094f9344c95b2408e0f628b4541be0c6c11dac 100644
--- a/src/knot/common/unreachable.h
+++ b/src/knot/common/unreachable.h
@@ -27,12 +27,12 @@ typedef struct {
 	node_t n;
 	struct sockaddr_storage addr;
 	struct sockaddr_storage via;
-	uint32_t time;
+	uint32_t time_ms;
 } knot_unreachable_t;
 
 typedef struct {
 	pthread_mutex_t mutex;
-	uint32_t ttl;
+	uint32_t ttl_ms;
 	list_t urs;
 } knot_unreachables_t;
 
@@ -41,17 +41,27 @@ extern knot_unreachables_t *global_unreachables;
 /*!
  * \brief Allocate Unreachables structure.
  *
- * \param ttl   TTL for unreachable in usecs.
+ * \param ttl   TTL for unreachable in milliseconds.
  *
  * \return Allocated structure, or NULL.
  */
-knot_unreachables_t *knot_unreachables_init(uint32_t ttl);
+knot_unreachables_t *knot_unreachables_init(uint32_t ttl_ms);
 
 /*!
  * \brief Free Unreachables structure.
  */
 void knot_unreachables_deinit(knot_unreachables_t **urs);
 
+/*!
+ * \brief Get and/or set the TTL.
+ *
+ * \param urs          Unreachables structure.
+ * \param new_ttl_ms   New TTL value in milliseconds.
+ *
+ * \return Previous value of TTL.
+ */
+uint32_t knot_unreachables_ttl(knot_unreachables_t *urs, uint32_t new_ttl_ms);
+
 /*!
  * \brief Determine if given address is unreachable.
  *
diff --git a/src/knot/conf/schema.c b/src/knot/conf/schema.c
index 52cbe44affce5316febf5dd78187ab59c5e73d75..16bdc1f7687286bf1be30e2bb789a4f4161a9297 100644
--- a/src/knot/conf/schema.c
+++ b/src/knot/conf/schema.c
@@ -197,7 +197,7 @@ static const yp_item_t desc_server[] = {
 	{ C_TCP_FASTOPEN,         YP_TBOOL, YP_VNONE },
 	{ C_RMT_POOL_LIMIT,       YP_TINT,  YP_VINT = { 0, INT32_MAX, 0 } },
 	{ C_RMT_POOL_TIMEOUT,     YP_TINT,  YP_VINT = { 1, INT32_MAX, 5, YP_STIME } },
-	{ C_RMT_RETRY_DELAY,      YP_TINT,  YP_VINT = { 0, INT32_MAX, 0, YP_STIME } },
+	{ C_RMT_RETRY_DELAY,      YP_TINT,  YP_VINT = { 0, INT32_MAX, 0 } },
 	{ C_SOCKET_AFFINITY,      YP_TBOOL, YP_VNONE },
 	{ C_UDP_MAX_PAYLOAD,      YP_TINT,  YP_VINT = { KNOT_EDNS_MIN_DNSSEC_PAYLOAD,
 	                                                KNOT_EDNS_MAX_UDP_PAYLOAD,
diff --git a/src/knot/server/server.c b/src/knot/server/server.c
index 5d2ac57024df8747a3f6b616e4610b67e99bb276..26e2e6d4dd3a9bd34298c8c4edada366ce7f19c6 100644
--- a/src/knot/server/server.c
+++ b/src/knot/server/server.c
@@ -1165,9 +1165,11 @@ static int reconfigure_remote_pool(conf_t *conf)
 	}
 
 	val = conf_get(conf, C_SRV, C_RMT_RETRY_DELAY);
-	int delay = conf_int(&val);
-	if (global_unreachables == NULL && delay > 0) {
-		global_unreachables = knot_unreachables_init(delay * 1000000); // secs -> usecs
+	int delay_ms = conf_int(&val);
+	if (global_unreachables == NULL && delay_ms > 0) {
+		global_unreachables = knot_unreachables_init(delay_ms);
+	} else {
+		(void)knot_unreachables_ttl(global_unreachables, delay_ms);
 	}
 
 	return KNOT_EOK;
diff --git a/tests-extra/tools/dnstest/server.py b/tests-extra/tools/dnstest/server.py
index 7e692bbcace34eb1690707ad5af03040f0534f58..e45fa5bcb3cbe1652fd76aad619bf4a86247eb10 100644
--- a/tests-extra/tools/dnstest/server.py
+++ b/tests-extra/tools/dnstest/server.py
@@ -1252,6 +1252,7 @@ class Knot(Server):
         self._str(s, "udp-max-payload-ipv4", self.udp_max_payload_ipv4)
         self._str(s, "udp-max-payload-ipv6", self.udp_max_payload_ipv6)
         self._str(s, "remote-pool-limit", str(random.randint(0,6)))
+        self._str(s, "remote-retry-delay", str(random.choice([0, 1, 5])))
         s.end()
 
         s.begin("control")
diff --git a/tests/knot/test_unreachable.c b/tests/knot/test_unreachable.c
index d22a34050202794dad2f7a0dbb5fa4d988fabab7..b0a23eee838a7e9689fb166ccafe43120a1eade7 100644
--- a/tests/knot/test_unreachable.c
+++ b/tests/knot/test_unreachable.c
@@ -28,7 +28,7 @@ int main(int argc, char *argv[])
 {
 	plan_lazy();
 
-	global_unreachables = knot_unreachables_init(1000);
+	global_unreachables = knot_unreachables_init(10);
 	ok(global_unreachables != NULL, "unreachables: init");
 
 	// ur_test_via[0] left empty - AF_UNSPEC
@@ -45,7 +45,7 @@ int main(int argc, char *argv[])
 		ok(knot_unreachable_is(global_unreachables, s, via), "unreachables: post[%d]", i);
 		ok(!knot_unreachable_is(global_unreachables, s, not_via), "unreachables: via[%d]", i);
 
-		usleep(100);
+		usleep(1000);
 		if (i >= 10) {
 			ok(!knot_unreachable_is(global_unreachables, &ur_test_addrs[i - 10], via),
 			   "unreachables: expired[%d]", i - 10);