redis: add multi-db and/or sentinel support

This commit is contained in:
Daniel Salzman 2025-09-25 15:26:57 +02:00
parent 853d8ad2ef
commit 357706157a
11 changed files with 309 additions and 79 deletions

View file

@ -1214,7 +1214,7 @@ Configuration of databases for zone contents, DNSSEC metadata, or event timers.
timer-db-max-size: SIZE
catalog-db: str
catalog-db-max-size: SIZE
zone-db-listen: ADDR[@INT] | STR[@INT]
zone-db-listen: ADDR[@INT] | STR[@INT] ...
zone-db-tls: BOOL
zone-db-cert-key: BASE64 ...
zone-db-cert-hostname: STR ...
@ -1348,10 +1348,15 @@ The hard limit for the catalog database maximum size.
zone-db-listen
--------------
An IP address or a hostname and optionally a port (default is 6379) or an
absolute UNIX socket path (starting with ``/``) of a running instance of
a Redis (or compatible) database to be used for reading and/or writing zone
contents. See :ref:`zone_zone-db-input` and :ref:`zone_zone-db-output`.
An ordered list of IP addresses or hostnames, and optionally ports (default is 6379),
or absolute UNIX socket paths (starting with ``/``) of running Redis (or compatible)
instances to be used for reading and/or writing zone contents.
See :ref:`zone_zone-db-input` and :ref:`zone_zone-db-output`.
The listen parameters are tried sequentially until a usable connection
is established. The connected database can be a master, a replica, or a sentinel.
If it is a sentinel, it is used to acquire connection parameters of a master
database.
*Default:* not set

View file

@ -9,6 +9,7 @@
#include "contrib/sockaddr.h"
#include "contrib/strtonum.h"
#include "knot/common/log.h"
#include "knot/zone/redis.h"
#include "libknot/errcode.h"
#ifdef ENABLE_REDIS_TLS
@ -131,57 +132,20 @@ static int hiredis_attach_gnutls(redisContext *ctx, struct knot_creds *local_cre
}
#endif // ENABLE_REDIS_TLS
redisContext *rdb_connect(conf_t *conf)
static redisContext *connect_addr(conf_t *conf, const char *addr_str, int port)
{
conf_val_t db_listen = conf_db_param(conf, C_ZONE_DB_LISTEN);
struct sockaddr_storage addr = conf_addr(&db_listen, NULL);
redisContext *rdb = (void *)conn_pool_get(global_redis_pool, &addr, &addr);
if (rdb != NULL && (intptr_t)rdb != CONN_POOL_FD_INVALID) {
return rdb;
}
int port = 0;
char addr_str[SOCKADDR_STRLEN];
if (addr.ss_family == AF_UNIX) {
const char *path = ((struct sockaddr_un *)&addr)->sun_path;
if (path[0] != '/') { // hostname
strlcpy(addr_str, path, sizeof(addr_str));
char *port_sep = strchr(addr_str, '@');
if (port_sep != NULL) {
*port_sep = '\0';
uint16_t num;
int ret = str_to_u16(port_sep + 1, &num);
if (ret != KNOT_EOK || num == 0) {
return NULL;
}
port = num;
} else {
port = CONF_REDIS_PORT;
}
}
} else {
port = sockaddr_port(&addr);
sockaddr_port_set(&addr, 0);
if (sockaddr_tostr(addr_str, sizeof(addr_str), &addr) <= 0) {
return NULL;
}
}
const struct timeval timeout = { 10, 0 };
redisContext *rdb;
if (port == 0) {
rdb = redisConnectUnixWithTimeout(addr_str, timeout);
} else {
rdb = redisConnectWithTimeout(addr_str, port, timeout);
}
if (rdb == NULL) {
log_error("rdb, failed to connect");
} else if (rdb->err) {
log_error("rdb, failed to connect (%s)", rdb->errstr);
if (rdb == NULL || rdb->err != REDIS_OK) {
log_debug("rdb, failed to connect, remote %s%s%.0u (%s)",
addr_str, (port != 0 ? "@" : ""), port,
(rdb != NULL ? rdb->errstr : "no reply"));
return NULL;
}
@ -211,6 +175,8 @@ redisContext *rdb_connect(conf_t *conf)
free(key_file);
free(cert_file);
if (ret != KNOT_EOK) {
log_error("rdb, failed to initialize credentials or to load certificates (%s)",
knot_strerror(ret));
redisFree(rdb);
return NULL;
}
@ -233,6 +199,7 @@ redisContext *rdb_connect(conf_t *conf)
struct knot_creds *creds = knot_creds_init_peer(local_creds, hostnames, pins);
if (creds == NULL) {
log_debug("rdb, failed to use TLS (%s)", knot_strerror(KNOT_ENOMEM));
knot_creds_free(local_creds);
redisFree(rdb);
return NULL;
@ -240,6 +207,7 @@ redisContext *rdb_connect(conf_t *conf)
int ret = hiredis_attach_gnutls(rdb, local_creds, creds);
if (ret != KNOT_EOK) {
log_debug("rdb, failed to use TLS (%s)", knot_strerror(ret));
knot_creds_free(local_creds);
knot_creds_free(creds);
redisFree(rdb);
@ -251,6 +219,170 @@ redisContext *rdb_connect(conf_t *conf)
return rdb;
}
int rdb_addr_to_str(struct sockaddr_storage *addr, char *out, size_t out_len, int *port)
{
*port = 0;
if (addr->ss_family == AF_UNIX) {
const char *path = ((struct sockaddr_un *)addr)->sun_path;
if (path[0] != '/') { // hostname
size_t len = strlcpy(out, path, out_len);
if (len == 0 || len >= out_len) {
return KNOT_EINVAL;
}
char *port_sep = strchr(out, '@');
if (port_sep != NULL) {
*port_sep = '\0';
uint16_t num;
int ret = str_to_u16(port_sep + 1, &num);
if (ret != KNOT_EOK || num == 0) {
return KNOT_EINVAL;
}
*port = num;
} else {
*port = CONF_REDIS_PORT;
}
}
} else {
*port = sockaddr_port(addr);
sockaddr_port_set(addr, 0);
if (sockaddr_tostr(out, out_len, addr) <= 0 || *port == 0) {
return KNOT_EINVAL;
}
}
return KNOT_EOK;
}
static int get_master(redisContext *rdb, char *out, size_t out_len, int *port)
{
redisReply *masters_reply = redisCommand(rdb, "SENTINEL masters");
if (masters_reply == NULL || masters_reply->type != REDIS_REPLY_ARRAY ||
masters_reply->elements == 0) {
if (masters_reply != NULL) {
freeReplyObject(masters_reply);
}
return KNOT_ENOENT;
}
redisReply *first_master = masters_reply->element[0];
const char *master_name = NULL;
for (size_t j = 0; j < first_master->elements; j += 2) {
const char *field = first_master->element[j]->str;
const char *value = first_master->element[j + 1]->str;
if (strcmp(field, "name") == 0) {
master_name = value;
break;
}
}
if (master_name == NULL) {
freeReplyObject(masters_reply);
return KNOT_ENOENT;
}
redisReply *addr_reply = redisCommand(rdb, "SENTINEL get-master-addr-by-name %s",
master_name);
freeReplyObject(masters_reply);
if (addr_reply == NULL || addr_reply->type != REDIS_REPLY_ARRAY ||
addr_reply->elements != 2) {
if (addr_reply != NULL) {
freeReplyObject(addr_reply);
}
return KNOT_ENOENT;
}
const char *ip_str = addr_reply->element[0]->str;
const char *port_str = addr_reply->element[1]->str;
size_t len = strlcpy(out, ip_str, out_len);
if (len == 0 || len >= out_len) {
freeReplyObject(addr_reply);
return KNOT_ERANGE;
}
uint16_t num;
int ret = str_to_u16(port_str, &num);
if (ret != KNOT_EOK || num == 0) {
freeReplyObject(addr_reply);
return KNOT_EINVAL;
}
*port = num;
freeReplyObject(addr_reply);
return KNOT_EOK;
}
redisContext *rdb_connect(conf_t *conf, bool require_master)
{
int port = 0;
int role = -1;
char addr_str[SOCKADDR_STRLEN - SOCKADDR_STRLEN_EXT] = "\0";
redisContext *rdb = NULL;
conf_val_t db_listen = conf_db_param(conf, C_ZONE_DB_LISTEN);
while (db_listen.code == KNOT_EOK) {
struct sockaddr_storage addr = conf_addr(&db_listen, NULL);
rdb = (void *)conn_pool_get(global_redis_pool, &addr, &addr);
if (rdb != NULL && (intptr_t)rdb != CONN_POOL_FD_INVALID) {
role = zone_redis_role(rdb);
if (!require_master || role == 0) {
goto connected;
}
redisFree(rdb);
}
conf_val_next(&db_listen);
}
conf_val_reset(&db_listen);
while (db_listen.code == KNOT_EOK) {
struct sockaddr_storage addr = conf_addr(&db_listen, NULL);
if (rdb_addr_to_str(&addr, addr_str, sizeof(addr_str), &port) != KNOT_EOK ||
(rdb = connect_addr(conf, addr_str, port)) == NULL) {
conf_val_next(&db_listen);
continue;
}
role = zone_redis_role(rdb);
if (role == 0) { // Master
goto connected;
} else if (role == 1 && !require_master) { // Replica
goto connected;
} else if (role == 2) { // Sentinel
if (get_master(rdb, addr_str, sizeof(addr_str), &port) == KNOT_EOK &&
(rdb = connect_addr(conf, addr_str, port)) == KNOT_EOK) {
goto connected;
}
}
conf_val_next(&db_listen);
}
return NULL;
connected:
if (log_enabled_debug()) {
bool tcp = rdb->connection_type == REDIS_CONN_TCP;
bool tls = rdb->privctx != NULL;
bool pool = addr_str[0] == '\0';
log_debug("rdb, connected, remote %s%s%.0u%s%s%s",
(tcp ? rdb->tcp.host : rdb->unix_sock.path),
(tcp ? "@" : ""),
(tcp ? rdb->tcp.port : 0),
(tls ? " TLS" : ""),
(role == 1 ? " replica" : ""),
(pool ? " pool" : ""));
}
return rdb;
}
void rdb_disconnect(redisContext *rdb, bool pool_save)
{
if (rdb != NULL && pool_save) {

View file

@ -13,7 +13,9 @@
#include "knot/conf/conf.h"
redisContext *rdb_connect(conf_t *conf);
int rdb_addr_to_str(struct sockaddr_storage *addr, char *out, size_t out_len, int *port);
redisContext *rdb_connect(conf_t *conf, bool require_master);
void rdb_disconnect(redisContext *rdb, bool pool_save);

View file

@ -313,7 +313,7 @@ static const yp_item_t desc_database[] = {
{ C_CATALOG_DB, YP_TSTR, YP_VSTR = { "catalog" } },
{ C_CATALOG_DB_MAX_SIZE, YP_TINT, YP_VINT = { MEGA(5), VIRT_MEM_LIMIT(GIGA(100)),
VIRT_MEM_LIMIT(GIGA(20)), YP_SSIZE } },
{ C_ZONE_DB_LISTEN, YP_TADDR, YP_VADDR = { CONF_REDIS_PORT }, YP_FNONE, { check_rdb, check_listen } },
{ C_ZONE_DB_LISTEN, YP_TADDR, YP_VADDR = { CONF_REDIS_PORT }, YP_FMULTI, { check_db_listen } },
{ C_ZONE_DB_TLS, YP_TBOOL, YP_VNONE },
{ C_ZONE_DB_CERT_KEY, YP_TB64, YP_VNONE, YP_FMULTI, { check_cert_pin } },
{ C_ZONE_DB_CERT_HOSTNAME, YP_TSTR, YP_VNONE, YP_FMULTI },

View file

@ -41,6 +41,9 @@
#include "contrib/sockaddr.h"
#include "contrib/string.h"
#include "contrib/wire_ctx.h"
#ifdef ENABLE_REDIS
#include "knot/common/hiredis.h"
#endif
#define MAX_INCLUDE_DEPTH 5
@ -291,12 +294,33 @@ int check_listen(
return KNOT_EOK;
}
int check_db_listen(
knotd_conf_check_args_t *args)
{
#ifndef ENABLE_REDIS
args->err_str = "zone database backend is not available";
return KNOT_ENOTSUP;
#else
bool no_port;
struct sockaddr_storage ss = yp_addr(args->data, &no_port);
int port;
char addr_str[SOCKADDR_STRLEN - SOCKADDR_STRLEN_EXT] = "\0";
if (rdb_addr_to_str(&ss, addr_str, sizeof(addr_str), &port) != KNOT_EOK) {
args->err_str = "invalid value";
return KNOT_EINVAL;
}
return KNOT_EOK;
#endif
}
int check_xdp_listen(
knotd_conf_check_args_t *args)
{
#ifndef ENABLE_XDP
args->err_str = "XDP is not available";
return KNOT_ENOTSUP;
args->err_str = "XDP is not available";
return KNOT_ENOTSUP;
#else
bool no_port;
struct sockaddr_storage ss = yp_addr(args->data, &no_port);
@ -1213,17 +1237,6 @@ int check_catalog_tpl(
return check_zone_or_tpl(args);
}
int check_rdb(
knotd_conf_check_args_t *args)
{
#ifndef ENABLE_REDIS
args->err_str = "Zone database support not available";
return KNOT_ENOTSUP;
#else
return KNOT_EOK;
#endif
}
int check_db_instance(
knotd_conf_check_args_t *args)
{

View file

@ -67,6 +67,10 @@ int check_listen(
knotd_conf_check_args_t *args
);
int check_db_listen(
knotd_conf_check_args_t *args
);
int check_xdp_listen(
knotd_conf_check_args_t *args
);
@ -163,10 +167,6 @@ int check_catalog_tpl(
knotd_conf_check_args_t *args
);
int check_rdb(
knotd_conf_check_args_t *args
);
int check_db_instance(
knotd_conf_check_args_t *args
);

View file

@ -109,7 +109,7 @@ int event_load(conf_t *conf, zone_t *zone)
bool db_enabled = conf_zone_rdb_enabled(conf, zone->name, true, &db_instance);
if (db_enabled) {
zone_src = "database";
db_ctx = zone_redis_connect(conf);
db_ctx = zone_redis_connect(conf, false);
}
// Attempt to load changes from database. If fails, load full zone from there later.

View file

@ -54,7 +54,7 @@
#endif
#define SESSION_TICKET_POOL_TIMEOUT 1200
#define REDIS_CONN_POOL_TIMEOUT (4 * 60)
#define REDIS_CONN_POOL_TIMEOUT 30
#define QUIC_LOG "QUIC/TLS, "
@ -940,7 +940,7 @@ static int rdb_listener_run(struct dthread *thread)
while (thread->state & ThreadActive) {
if (s->rdb_ctx == NULL) {
s->rdb_ctx = rdb_connect(conf());
s->rdb_ctx = rdb_connect(conf(), false);
if (s->rdb_ctx == NULL) {
log_error("rdb, failed to connect");
sleep(2);

View file

@ -735,7 +735,7 @@ static int commit_redis(conf_t *conf, zone_update_t *update)
return KNOT_EOK;
}
struct redisContext *db_ctx = zone_redis_connect(conf);
struct redisContext *db_ctx = zone_redis_connect(conf, true);
if (db_ctx == NULL) {
return KNOT_ECONN;
}

View file

@ -3,6 +3,7 @@
* For more information, see <https://www.knot-dns.cz/>
*/
#include <poll.h>
#include <string.h>
#include "knot/zone/redis.h"
@ -14,9 +15,9 @@
#define UNREAD_MAX 20 // Redis write batch length.
struct redisContext *zone_redis_connect(conf_t *conf)
struct redisContext *zone_redis_connect(conf_t *conf, bool require_master)
{
return rdb_connect(conf);
return rdb_connect(conf, require_master);
}
void zone_redis_disconnect(struct redisContext *ctx, bool pool_save)
@ -30,10 +31,72 @@ bool zone_redis_ping(struct redisContext *ctx)
return false;
}
redisReply *reply = redisCommand(ctx, "PING");
bool res = (reply != NULL &&
reply->type == REDIS_REPLY_STATUS &&
strcmp(reply->str, "PONG") == 0);
if (redisAppendCommand(ctx, "PING") != REDIS_OK) {
return false;
}
int done = 0;
while (!done) {
if (redisBufferWrite(ctx, &done) != REDIS_OK) {
return false;
}
}
struct pollfd pfd = { .fd = ctx->fd, .events = POLLIN };
if (poll(&pfd, 1, 500) == 0) {
return false;
}
redisReply *reply;
if (redisGetReply(ctx, (void **)&reply) != REDIS_OK) {
return false;
}
bool res = reply->type == REDIS_REPLY_STATUS &&
strcmp(reply->str, "PONG") == 0;
freeReplyObject(reply);
return res;
}
int zone_redis_role(struct redisContext *ctx)
{
if (ctx == NULL) {
return -1;
}
if (redisAppendCommand(ctx, "ROLE") != REDIS_OK) {
return -1;
}
int done = 0;
while (!done) {
if (redisBufferWrite(ctx, &done) != REDIS_OK) {
return -1;
}
}
struct pollfd pfd = { .fd = ctx->fd, .events = POLLIN };
if (poll(&pfd, 1, 1000) == 0) {
return -1;
}
redisReply *reply;
if (redisGetReply(ctx, (void **)&reply) != REDIS_OK) {
return -1;
}
int res = -1;
if (reply->type == REDIS_REPLY_ARRAY) {
if (strcmp(reply->element[0]->str, "master") == 0) {
res = 0;
} else if (strcmp(reply->element[0]->str, "sentinel") == 0) {
res = 2;
} else {
res = 1;
}
}
freeReplyObject(reply);
@ -409,7 +472,7 @@ int zone_redis_load_upd(struct redisContext *rdb, uint8_t instance,
#else // ENABLE_REDIS
struct redisContext *zone_redis_connect(conf_t *conf)
struct redisContext *zone_redis_connect(conf_t *conf, bool require_master)
{
return NULL;
}
@ -424,6 +487,11 @@ bool zone_redis_ping(struct redisContext *ctx)
return false;
}
int zone_redis_role(struct redisContext *ctx)
{
return -1;
}
int zone_redis_txn_begin(zone_redis_txn_t *txn, struct redisContext *rdb,
uint8_t instance, const knot_dname_t *zone_name,
bool incremental)

View file

@ -37,7 +37,7 @@ typedef struct {
/*!
* \brief Wrappers to rdb_connect and rdb_disconnect not needing #ifdef ENABLE_REDIS around.
*/
struct redisContext *zone_redis_connect(conf_t *conf);
struct redisContext *zone_redis_connect(conf_t *conf, bool require_master);
void zone_redis_disconnect(struct redisContext *ctx, bool pool_save);
/*!
@ -45,6 +45,16 @@ void zone_redis_disconnect(struct redisContext *ctx, bool pool_save);
*/
bool zone_redis_ping(struct redisContext *ctx);
/*!
* \brief Check the connected DB role.
*
* \retval -1 Error
* \retval 0 Master
* \retval 1 Replica
* \retval 2 Sentinel
*/
int zone_redis_role(struct redisContext *ctx);
/*!
* \brief Start a writing stransaction into Redis zone database.
*