Add rdatavec

Add an implementation of rdataset specialized for authoritative
workloads. For now, it is a copy of rdataslab, with redundant fields
from the header removed.

lib/dns/include/dns/rdataset.h

@@ -51,6 +51,7 @@
 #include <isc/stdtime.h>
 
 #include <dns/rdataslab.h>
+#include <dns/rdatavec.h>
 #include <dns/rdatastruct.h>
 #include <dns/types.h>
 
@@ -200,6 +201,20 @@ struct dns_rdataset {
 			dns_slabheader_proof_t *noqname, *closest;
 		} slab;
 
+		/*
+		 * A vec rdataset provides access to an rdatavec. In
+		 * a QP database, 'header' points to the vecheader
+		 * structure. (There is an exception in the case of
+		 * rdatasets returned by the `getnoqname` and `getclosest`
+		 * methods; see comments in rdatavec.c for details.)
+		 */
+		struct {
+			struct dns_db           *db;
+			dns_dbnode_t           *node;
+			dns_vecheader_t        *header;
+			rdatavec_iter_t        iter;
+		} vec;
+
 		/*
 		 * A simple rdatalist, plus an optional dbnode used by
 		 * builtin and sdlz.

lib/dns/include/dns/rdatavec.h

@@ -0,0 +1,272 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+/*! \file dns/rdatavec.h
+ * \brief
+ * Implements storage of rdatasets into vectors of memory.
+ *
+ * MP:
+ *\li	Clients of this module must impose any required synchronization.
+ *
+ * Reliability:
+ *\li	This module deals with low-level byte streams.  Errors in any of
+ *	the functions are likely to crash the server or corrupt memory.
+ *
+ *\li	If the caller passes invalid memory references, these functions are
+ *	likely to crash the server or corrupt memory.
+ *
+ * Resources:
+ *\li	None.
+ *
+ * Security:
+ *\li	None.
+ *
+ * Standards:
+ *\li	None.
+ */
+
+/***
+ *** Imports
+ ***/
+
+#include <stdbool.h>
+
+#include <isc/atomic.h>
+#include <isc/heap.h>
+#include <isc/slist.h>
+#include <isc/stdtime.h>
+#include <isc/urcu.h>
+
+#include <dns/name.h>
+#include <dns/rdataset.h>
+#include <dns/types.h>
+
+#define DNS_RDATAVEC_FORCE 0x1
+#define DNS_RDATAVEC_EXACT 0x2
+
+#define DNS_RDATAVEC_OFFLINE 0x01 /* RRSIG is for offline DNSKEY */
+
+
+typedef struct dns_vectop dns_vectop_t;
+typedef struct dns_vecheader dns_vecheader_t;
+
+struct rdatavec_iter {
+	unsigned char *iter_pos;
+	unsigned int   iter_count;
+	dns_rdataclass_t iter_rdclass;
+	dns_rdatatype_t  iter_type;
+};
+
+typedef struct rdatavec_iter rdatavec_iter_t;
+
+struct dns_vectop {
+	ISC_SLINK(dns_vectop_t) next_type;
+	ISC_SLIST(dns_vecheader_t) headers;
+
+	dns_typepair_t typepair;
+};
+
+struct dns_vecheader {
+	_Atomic(uint16_t)    attributes;
+	_Atomic(dns_trust_t) trust;
+
+	/*%
+	 * Locked by the owning node's lock.
+	 */
+	uint16_t       resign_lsb : 1;
+	unsigned int   heap_index : 31;
+	uint32_t       serial;
+	dns_ttl_t      ttl;
+	dns_typepair_t typepair;
+
+	/* resigning (zone). The lsb is not adjacent for struct packing reasons */
+	isc_stdtime_t resign;
+
+	/*%
+	 * Link to the other versions of this rdataset.
+	 */
+	ISC_SLINK(dns_vecheader_t) next_header;
+
+	/*%
+	 * The database node objects containing this rdataset, if any.
+	 */
+	dns_dbnode_t   *node;
+
+	/*%
+	 * Cached glue records for an rdataset of type NS (zone only).
+	 */
+	dns_gluelist_t *gluelist;
+
+	/*%
+	 * Case vector.  If the bit is set then the corresponding
+	 * character in the owner name needs to be AND'd with 0x20,
+	 * rendering that character upper case.
+	 */
+	unsigned char	upper[32];
+
+	/*%
+	 * Flexible member indicates the address of the raw data
+	 * following this header.
+	 */
+	unsigned char	raw[];
+};
+
+
+enum {
+	DNS_VECHEADERATTR_NONEXISTENT = 1 << 0,
+	DNS_VECHEADERATTR_IGNORE = 1 << 1,
+	DNS_VECHEADERATTR_RESIGN = 1 << 2,
+	DNS_VECHEADERATTR_OPTOUT = 1 << 3,
+	DNS_VECHEADERATTR_CASESET = 1 << 4,
+	DNS_VECHEADERATTR_ZEROTTL = 1 << 5,
+	DNS_VECHEADERATTR_CASEFULLYLOWER = 1 << 6,
+};
+
+/* clang-format off : RemoveParentheses */
+#define DNS_VECHEADER_GETATTR(header, attribute) \
+	(atomic_load_acquire(&(header)->attributes) & (attribute))
+/* clang-format on */
+#define DNS_VECHEADER_SETATTR(header, attribute) \
+	atomic_fetch_or_release(&(header)->attributes, attribute)
+#define DNS_VECHEADER_CLRATTR(header, attribute) \
+	atomic_fetch_and_release(&(header)->attributes, ~(attribute))
+
+extern dns_rdatasetmethods_t dns_rdatavec_rdatasetmethods;
+
+/***
+ *** Functions
+ ***/
+
+isc_result_t
+dns_rdatavec_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
+			   isc_region_t *region, uint32_t limit);
+/*%<
+ * Allocate space for a vec to hold the data in rdataset, and copy the
+ * data into it.  The resulting vec will be returned in 'region'.
+ *
+ * dns_rdatavec_fromrdataset() allocates space for a dns_vecheader object
+ * and the memory needed for a raw vec, and partially initializes
+ * it, setting the type, trust, and TTL fields to match rdataset->type,
+ * rdataset->covers, rdataset->trust, and rdataset->ttl.  (Note that the
+ * last field needs to be overridden when used in the cache database,
+ * since cache headers use an expire time instead of a TTL.)
+ *
+ * Requires:
+ *\li	'rdataset' is valid.
+ *
+ * Ensures:
+ *\li	'region' will have base pointing to the start of allocated memory,
+ *	with the vecified region beginning at region->base + reservelen.
+ *	region->length contains the total length allocated.
+ *
+ * Returns:
+ *\li	ISC_R_SUCCESS		- successful completion
+ *\li	ISC_R_NOSPACE		- more than 64k RRs
+ *\li	DNS_R_TOOMANYRECORDS	- more than max-records-per-rrset RRs
+ *\li	DNS_R_SINGLETON		- singleton type has more than one RR
+ */
+
+unsigned int
+dns_rdatavec_size(dns_vecheader_t *header);
+/*%<
+ * Return the total size of the rdatavec following 'header'.
+ *
+ * Requires:
+ *\li	'header' points to a vecheader with an rdatavec following it.
+ *
+ * Returns:
+ *\li	The number of bytes in the vec, plus the header.
+ */
+
+unsigned int
+dns_rdatavec_count(dns_vecheader_t *header);
+/*%<
+ * Return the number of records in the rdatavec following 'header'.
+ *
+ * Requires:
+ *\li	'header' points to a vecheader with an rdatavec following it.
+ *
+ * Returns:
+ *\li	The number of records in the vec.
+ */
+
+isc_result_t
+dns_rdatavec_merge(dns_vecheader_t *oheader, dns_vecheader_t *nheader,
+		    isc_mem_t *mctx, dns_rdataclass_t rdclass,
+		    dns_rdatatype_t type, unsigned int flags,
+		    uint32_t maxrrperset, dns_vecheader_t **theaderp);
+/*%<
+ * Merge the vecs following 'oheader' and 'nheader'.
+ */
+
+isc_result_t
+dns_rdatavec_subtract(dns_vecheader_t *mheader, dns_vecheader_t *sheader,
+		       isc_mem_t *mctx, dns_rdataclass_t rdclass,
+		       dns_rdatatype_t type, unsigned int flags,
+		       dns_vecheader_t **theaderp);
+/*%<
+ * Subtract the vec following 'sheader' from the one following 'mheader'.
+ * If 'exact' is true then all elements from the 'sheader' vec must exist
+ * in the 'mheader' vec.
+ *
+ * XXX
+ * valid flags are DNS_RDATAVEC_EXACT
+ */
+
+
+void
+dns_vecheader_setownercase(dns_vecheader_t *header, const dns_name_t *name);
+/*%<
+ * Store the casing of 'name', into a bitfield in 'header'.
+ *
+ * Requires:
+ * \li	'header' is a valid vecheader.
+ * \li	'name' is a valid name.
+ */
+
+
+void
+dns_vecheader_reset(dns_vecheader_t *h, dns_dbnode_t *node);
+/*%<
+ * Reset an rdatavec header 'h' so it can be used to store data in
+ * database node 'node'.
+ */
+
+dns_vecheader_t *
+dns_vecheader_new(isc_mem_t *mctx, dns_dbnode_t *node);
+/*%<
+ * Allocate memory for an rdatavec header and initialize it for use
+ * in database node 'node'.
+ */
+
+void
+dns_vecheader_destroy(dns_vecheader_t **headerp);
+/*%<
+ * Free all memory associated with '*headerp'.
+ */
+
+
+dns_vectop_t *
+dns_vectop_new(isc_mem_t *mctx, dns_typepair_t typepair);
+/*%<
+ * Allocate memory for an rdatavec top and initialize it for use
+ * with 'typepair' type and covers pair.
+ */
+
+void
+dns_vectop_destroy(isc_mem_t *mctx, dns_vectop_t **topp);
+/*%<
+ * Free all memory associated with '*vectopp'.
+ */

lib/dns/meson.build

@@ -140,6 +140,7 @@ dns_srcset.add(
         'rdataset.c',
         'rdatasetiter.c',
         'rdataslab.c',
+        'rdatavec.c',
         'remote.c',
         'request.c',
         'resconf.c',

lib/dns/rdatavec.c

@@ -0,0 +1,995 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+/*! \file */
+
+#include <ctype.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdlib.h>
+
+#include <isc/ascii.h>
+#include <isc/atomic.h>
+#include <isc/mem.h>
+#include <isc/region.h>
+#include <isc/result.h>
+#include <isc/string.h>
+#include <isc/util.h>
+
+#include <dns/db.h>
+#include <dns/rdata.h>
+#include <dns/rdataset.h>
+#include <dns/rdatavec.h>
+#include <dns/stats.h>
+
+#include "rdatavec_p.h"
+
+/*
+ * The memory structure of an rdatavec is as follows:
+ *
+ *	header		(dns_vecheader_t)
+ *	record count	(2 bytes)
+ *	data records
+ *		data length	(2 bytes)
+ *		order		(2 bytes)
+ *		meta data	(1 byte for RRSIG, 0 for all other types)
+ *		data		(data length bytes)
+ *
+ * A "bare" rdatavec is everything after "header".
+ *
+ * When a vec is created, data records are sorted into DNSSEC order.
+ */
+
+static void
+rdataset_disassociate(dns_rdataset_t *rdataset DNS__DB_FLARG);
+static isc_result_t
+rdataset_first(dns_rdataset_t *rdataset);
+static isc_result_t
+rdataset_next(dns_rdataset_t *rdataset);
+static void
+rdataset_current(dns_rdataset_t *rdataset, dns_rdata_t *rdata);
+static void
+rdataset_clone(dns_rdataset_t *source, dns_rdataset_t *target DNS__DB_FLARG);
+static unsigned int
+rdataset_count(dns_rdataset_t *rdataset);
+static void
+rdataset_settrust(dns_rdataset_t *rdataset, dns_trust_t trust);
+static void
+rdataset_getownercase(const dns_rdataset_t *rdataset, dns_name_t *name);
+
+dns_rdatasetmethods_t dns_rdatavec_rdatasetmethods = {
+	.disassociate = rdataset_disassociate,
+	.first = rdataset_first,
+	.next = rdataset_next,
+	.current = rdataset_current,
+	.clone = rdataset_clone,
+	.count = rdataset_count,
+	.settrust = rdataset_settrust,
+	.expire = NULL,
+	.clearprefetch = NULL,
+	.getownercase = rdataset_getownercase,
+};
+
+/*% Note: the "const void *" are just to make qsort happy.  */
+static int
+compare_rdata(const void *p1, const void *p2) {
+	return dns_rdata_compare(p1, p2);
+}
+
+static size_t
+header_size(const dns_vecheader_t* header) {
+	UNUSED(header);
+	return sizeof(dns_vecheader_t);
+}
+
+static unsigned char*
+rdatavec_raw(dns_vecheader_t* header) {
+	unsigned char *as_char_star = (unsigned char*) header;
+	unsigned char* raw = as_char_star + header_size(header);
+
+	return raw;
+}
+
+static unsigned char*
+rdatavec_data(dns_vecheader_t* header) {
+	return rdatavec_raw(header) + 2;
+}
+
+static unsigned int 
+rdatavec_count(dns_vecheader_t* header) {
+	unsigned char* raw = rdatavec_raw(header);
+	unsigned int count = get_uint16(raw);
+
+	return count;
+}
+
+static isc_result_t
+makevec(dns_rdataset_t *rdataset, isc_mem_t *mctx, isc_region_t *region,
+	 uint32_t maxrrperset) {
+	/*
+	 * Use &removed as a sentinel pointer for duplicate
+	 * rdata as rdata.data == NULL is valid.
+	 */
+	static unsigned char removed;
+	dns_rdata_t *rdata = NULL;
+	unsigned char *rawbuf = NULL;
+	unsigned int headerlen = sizeof(dns_vecheader_t);
+	unsigned int buflen = headerlen + 2;
+	isc_result_t result;
+	unsigned int nitems;
+	unsigned int nalloc;
+	unsigned int length;
+	size_t i;
+	size_t rdatasize;
+
+	/*
+	 * If the source rdataset is also a vec, we don't need
+	 * to do anything special, just copy the whole vec to a
+	 * new buffer.
+	 */
+	if (rdataset->methods == &dns_rdatavec_rdatasetmethods) {
+		dns_vecheader_t *header = dns_vecheader_getheader(rdataset);
+		buflen = dns_rdatavec_size(header);
+
+		rawbuf = isc_mem_get(mctx, buflen);
+		region->base = rawbuf;
+		region->length = buflen;
+
+		memmove(rawbuf, header, buflen);
+		return ISC_R_SUCCESS;
+	}
+
+	/*
+	 * If there are no rdata then we just need to allocate a header
+	 * with a zero record count.
+	 */
+	nitems = dns_rdataset_count(rdataset);
+	if (nitems == 0) {
+		if (rdataset->type != 0) {
+			return ISC_R_FAILURE;
+		}
+		rawbuf = isc_mem_get(mctx, buflen);
+		region->base = rawbuf;
+		region->length = buflen;
+		rawbuf += headerlen;
+		put_uint16(rawbuf, 0);
+		return ISC_R_SUCCESS;
+	}
+
+	if (maxrrperset > 0 && nitems > maxrrperset) {
+		return DNS_R_TOOMANYRECORDS;
+	}
+
+	if (nitems > 0xffff) {
+		return ISC_R_NOSPACE;
+	}
+
+	/*
+	 * Remember the original number of items.
+	 */
+	nalloc = nitems;
+
+	RUNTIME_CHECK(!ckd_mul(&rdatasize, nalloc, sizeof(rdata[0])));
+	rdata = isc_mem_get(mctx, rdatasize);
+
+	/*
+	 * Save all of the rdata members into an array.
+	 */
+	result = dns_rdataset_first(rdataset);
+	if (result != ISC_R_SUCCESS && result != ISC_R_NOMORE) {
+		goto free_rdatas;
+	}
+	for (i = 0; i < nalloc && result == ISC_R_SUCCESS; i++) {
+		INSIST(result == ISC_R_SUCCESS);
+		dns_rdata_init(&rdata[i]);
+		dns_rdataset_current(rdataset, &rdata[i]);
+		INSIST(rdata[i].data != &removed);
+		result = dns_rdataset_next(rdataset);
+	}
+	if (i != nalloc || result != ISC_R_NOMORE) {
+		/*
+		 * Somehow we iterated over fewer rdatas than
+		 * dns_rdataset_count() said there were or there
+		 * were more items than dns_rdataset_count said
+		 * there were.
+		 */
+		result = ISC_R_FAILURE;
+		goto free_rdatas;
+	}
+
+	/*
+	 * Put into DNSSEC order.
+	 */
+	if (nalloc > 1U) {
+		qsort(rdata, nalloc, sizeof(rdata[0]), compare_rdata);
+	}
+
+	/*
+	 * Remove duplicates and compute the total storage required.
+	 *
+	 * If an rdata is not a duplicate, accumulate the storage size
+	 * required for the rdata.  We do not store the class, type, etc,
+	 * just the rdata, so our overhead is 2 bytes for the number of
+	 * records, and 2 bytes for the length of each rdata, plus the
+	 * rdata itself.
+	 */
+	for (i = 1; i < nalloc; i++) {
+		if (compare_rdata(&rdata[i - 1], &rdata[i]) == 0) {
+			rdata[i - 1].data = &removed;
+			nitems--;
+		} else {
+			buflen += (2 + rdata[i - 1].length);
+			/*
+			 * Provide space to store the per RR meta data.
+			 */
+			if (rdataset->type == dns_rdatatype_rrsig) {
+				buflen++;
+			}
+		}
+	}
+
+	/*
+	 * Don't forget the last item!
+	 */
+	buflen += (2 + rdata[i - 1].length);
+
+	/*
+	 * Provide space to store the per RR meta data.
+	 */
+	if (rdataset->type == dns_rdatatype_rrsig) {
+		buflen++;
+	}
+
+	/*
+	 * Ensure that singleton types are actually singletons.
+	 */
+	if (nitems > 1 && dns_rdatatype_issingleton(rdataset->type)) {
+		/*
+		 * We have a singleton type, but there's more than one
+		 * RR in the rdataset.
+		 */
+		result = DNS_R_SINGLETON;
+		goto free_rdatas;
+	}
+
+	/*
+	 * Allocate the memory, set up a buffer, start copying in
+	 * data.
+	 */
+	rawbuf = isc_mem_get(mctx, buflen);
+
+	region->base = rawbuf;
+	region->length = buflen;
+	rawbuf += headerlen;
+	put_uint16(rawbuf, nitems);
+
+	for (i = 0; i < nalloc; i++) {
+		if (rdata[i].data == &removed) {
+			continue;
+		}
+		length = rdata[i].length;
+		if (rdataset->type == dns_rdatatype_rrsig) {
+			length++;
+		}
+		INSIST(length <= 0xffff);
+
+		put_uint16(rawbuf, length);
+
+		/*
+		 * Store the per RR meta data.
+		 */
+		if (rdataset->type == dns_rdatatype_rrsig) {
+			*rawbuf++ = (rdata[i].flags & DNS_RDATA_OFFLINE)
+					    ? DNS_RDATAVEC_OFFLINE
+					    : 0;
+		}
+		if (rdata[i].length != 0) {
+			memmove(rawbuf, rdata[i].data, rdata[i].length);
+		}
+		rawbuf += rdata[i].length;
+	}
+
+	result = ISC_R_SUCCESS;
+
+free_rdatas:
+	isc_mem_put(mctx, rdata, rdatasize);
+	return result;
+}
+
+isc_result_t
+dns_rdatavec_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
+			   isc_region_t *region, uint32_t maxrrperset) {
+	isc_result_t result;
+
+	if (rdataset->type == dns_rdatatype_none &&
+	    rdataset->covers == dns_rdatatype_none)
+	{
+		return DNS_R_DISALLOWED;
+	}
+
+	result = makevec(rdataset, mctx, region, maxrrperset);
+	if (result == ISC_R_SUCCESS) {
+		dns_vecheader_t *new = (dns_vecheader_t *)region->base;
+		dns_typepair_t typepair;
+
+		if (rdataset->attributes.negative) {
+			INSIST(rdataset->type == dns_rdatatype_none);
+			INSIST(rdataset->covers != dns_rdatatype_none);
+			typepair = DNS_TYPEPAIR_VALUE(rdataset->covers,
+						      dns_rdatatype_none);
+		} else {
+			INSIST(rdataset->type != dns_rdatatype_none);
+			INSIST(dns_rdatatype_issig(rdataset->type) ||
+			       rdataset->covers == dns_rdatatype_none);
+			typepair = DNS_TYPEPAIR_VALUE(rdataset->type,
+						      rdataset->covers);
+		}
+
+		*new = (dns_vecheader_t){
+			.next_header = ISC_SLINK_INITIALIZER,
+			.typepair = typepair,
+			.trust = rdataset->trust,
+			.ttl = rdataset->ttl,
+		};
+	}
+
+	return result;
+}
+
+unsigned int
+dns_rdatavec_size(dns_vecheader_t *header) {
+	REQUIRE(header != NULL);
+
+	unsigned char *vec = rdatavec_raw(header);
+	INSIST(vec != NULL);
+
+	unsigned char *current = rdatavec_data(header);
+	uint16_t count = rdatavec_count(header);
+
+	while (count-- > 0) {
+		uint16_t length = get_uint16(current);
+		current += length;
+	}
+
+	return (unsigned int)(current - vec) + header_size(header);
+}
+
+unsigned int
+dns_rdatavec_count(dns_vecheader_t *header) {
+	REQUIRE(header != NULL);
+
+	return rdatavec_count(header);
+}
+
+/*
+ * Make the dns_rdata_t 'rdata' refer to the vec item
+ * beginning at '*current' (which is part of a vec of type
+ * 'type' and class 'rdclass') and advance '*current' to
+ * point to the next item in the vec.
+ */
+static void
+rdata_from_vecitem(unsigned char **current, dns_rdataclass_t rdclass,
+		    dns_rdatatype_t type, dns_rdata_t *rdata) {
+	unsigned char *tcurrent = *current;
+	isc_region_t region;
+	bool offline = false;
+	uint16_t length = get_uint16(tcurrent);
+
+	if (type == dns_rdatatype_rrsig) {
+		if ((*tcurrent & DNS_RDATAVEC_OFFLINE) != 0) {
+			offline = true;
+		}
+		length--;
+		tcurrent++;
+	}
+	region.length = length;
+	region.base = tcurrent;
+	tcurrent += region.length;
+	dns_rdata_fromregion(rdata, rdclass, type, &region);
+	if (offline) {
+		rdata->flags |= DNS_RDATA_OFFLINE;
+	}
+	*current = tcurrent;
+}
+
+static void
+rdata_to_vecitem(unsigned char **current, dns_rdatatype_t type,
+		  dns_rdata_t *rdata) {
+	unsigned int length = rdata->length;
+	unsigned char *data = rdata->data;
+	unsigned char *p = *current;
+
+	if (type == dns_rdatatype_rrsig) {
+		length++;
+		data--;
+	}
+
+	put_uint16(p, length);
+	memmove(p, data, length);
+	p += length;
+
+	*current = p;
+}
+
+typedef struct vecinfo {
+	unsigned char *pos;
+	dns_rdata_t rdata;
+	bool dup;
+} vecinfo_t;
+
+isc_result_t
+dns_rdatavec_merge(dns_vecheader_t *oheader, dns_vecheader_t *nheader,
+		    isc_mem_t *mctx, dns_rdataclass_t rdclass,
+		    dns_rdatatype_t type, unsigned int flags,
+		    uint32_t maxrrperset, dns_vecheader_t **theaderp) {
+	isc_result_t result = ISC_R_SUCCESS;
+	unsigned char *ocurrent = NULL, *ncurrent = NULL, *tcurrent = NULL;
+	unsigned int ocount, ncount, tlength, tcount = 0;
+	vecinfo_t *oinfo = NULL, *ninfo = NULL;
+	size_t o = 0, n = 0;
+
+	REQUIRE(theaderp != NULL && *theaderp == NULL);
+	REQUIRE(oheader != NULL && nheader != NULL);
+
+	ocurrent = rdatavec_data(oheader);
+	ocount = rdatavec_count(oheader);
+
+	ncurrent = rdatavec_data(nheader);
+	ncount = rdatavec_count(nheader);
+
+	INSIST(ocount > 0 && ncount > 0);
+
+	if (maxrrperset > 0 && ocount + ncount > maxrrperset) {
+		return DNS_R_TOOMANYRECORDS;
+	}
+
+	/*
+	 * Figure out the target length. Start with the header,
+	 * plus 2 octets for the count.
+	 */
+	tlength = header_size(oheader) + 2;
+
+	/*
+	 * Gather the rdatas in the old vec and add their lengths to
+	 * the larget length.
+	 */
+	oinfo = isc_mem_cget(mctx, ocount, sizeof(struct vecinfo));
+	for (size_t i = 0; i < ocount; i++) {
+		oinfo[i].pos = ocurrent;
+		dns_rdata_init(&oinfo[i].rdata);
+		rdata_from_vecitem(&ocurrent, rdclass, type, &oinfo[i].rdata);
+		tlength += ocurrent - oinfo[i].pos;
+	}
+
+	/*
+	 * Then add the length of rdatas in the new vec that aren't
+	 * duplicated in the old vec.
+	 */
+	ninfo = isc_mem_cget(mctx, ncount, sizeof(struct vecinfo));
+	for (size_t i = 0; i < ncount; i++) {
+		ninfo[i].pos = ncurrent;
+		dns_rdata_init(&ninfo[i].rdata);
+		rdata_from_vecitem(&ncurrent, rdclass, type, &ninfo[i].rdata);
+
+		for (size_t j = 0; j < ocount; j++) {
+			if (oinfo[j].dup) {
+				/*
+				 * This was already found to be
+				 * duplicated; no need to compare
+				 * it again.
+				 */
+				continue;
+			}
+
+			if (dns_rdata_compare(&oinfo[j].rdata,
+					      &ninfo[i].rdata) == 0)
+			{
+				/*
+				 * Found a dup. Mark the old copy as a
+				 * duplicate so we don't check it again;
+				 * mark the new copy as a duplicate so we
+				 * don't copy it to the target.
+				 */
+				oinfo[j].dup = ninfo[i].dup = true;
+				break;
+			}
+		}
+
+		if (ninfo[i].dup) {
+			continue;
+		}
+
+		/*
+		 * We will be copying this item to the target, so
+		 * add its length to tlength and increment tcount.
+		 */
+		tlength += ncurrent - ninfo[i].pos;
+		tcount++;
+	}
+
+	/*
+	 * If the EXACT flag is set, there can't be any rdata in
+	 * the new vec that was also in the old. If tcount is less
+	 * than ncount, then we found such a duplicate.
+	 */
+	if (((flags & DNS_RDATAVEC_EXACT) != 0) && (tcount < ncount)) {
+		CLEANUP(DNS_R_NOTEXACT);
+	}
+
+	/*
+	 * If nothing's being copied in from the new vec, and the
+	 * FORCE flag isn't set, we're done.
+	 */
+	if (tcount == 0 && (flags & DNS_RDATAVEC_FORCE) == 0) {
+		CLEANUP(DNS_R_UNCHANGED);
+	}
+
+	/* Add to tcount the total number of items from the old vec. */
+	tcount += ocount;
+
+	/* Resposition ncurrent at the first item. */
+	ncurrent = rdatavec_data(nheader);
+
+	/* Single types can't have more than one RR. */
+	if (tcount > 1 && dns_rdatatype_issingleton(type)) {
+		CLEANUP(DNS_R_SINGLETON);
+	}
+
+	if (tcount > 0xffff) {
+		CLEANUP(ISC_R_NOSPACE);
+	}
+
+	/* Allocate the target buffer and copy the new vec's header */
+	unsigned char *tstart = isc_mem_get(mctx, tlength);
+	dns_vecheader_t *as_header = (dns_vecheader_t*) tstart;
+
+	/* 
+	 * Preserve the case of the old header, but the rest from the new
+	 * header
+	 */
+	memmove(tstart, nheader, header_size(nheader));
+	memmove(as_header->upper, oheader->upper, sizeof(oheader->upper));
+	uint16_t case_attrs = DNS_VECHEADER_GETATTR(
+		oheader,
+		DNS_VECHEADERATTR_CASESET | DNS_VECHEADERATTR_CASEFULLYLOWER);
+	DNS_VECHEADER_CLRATTR(as_header,
+			      DNS_VECHEADERATTR_CASESET |
+				      DNS_VECHEADERATTR_CASEFULLYLOWER);
+	DNS_VECHEADER_SETATTR(as_header, case_attrs);
+
+	tcurrent = tstart + header_size(nheader);
+
+	/* Write the new count, then start merging the vecs. */
+	put_uint16(tcurrent, tcount);
+
+	/*
+	 * Now walk the sets together, adding each item in DNSSEC order,
+	 * and skipping over any more dups in the new vec.
+	 */
+	while (o < ocount || n < ncount) {
+		bool fromold;
+
+		/* Skip to the next non-duplicate in the new vec. */
+		for (; n < ncount && ninfo[n].dup; n++)
+			;
+
+		if (o == ocount) {
+			fromold = false;
+		} else if (n == ncount) {
+			fromold = true;
+		} else {
+			fromold = dns_rdata_compare(&oinfo[o].rdata,
+						    &ninfo[n].rdata) < 0;
+		}
+
+		if (fromold) {
+			rdata_to_vecitem(&tcurrent, type, &oinfo[o].rdata);
+			if (++o < ocount) {
+				/* Skip to the next rdata in the old vec */
+				continue;
+			}
+		} else {
+			rdata_to_vecitem(&tcurrent, type, &ninfo[n++].rdata);
+		}
+	}
+
+	INSIST(tcurrent == tstart + tlength);
+
+	*theaderp = (dns_vecheader_t *)tstart;
+
+cleanup:
+	isc_mem_cput(mctx, oinfo, ocount, sizeof(struct vecinfo));
+	isc_mem_cput(mctx, ninfo, ncount, sizeof(struct vecinfo));
+
+	return result;
+}
+
+isc_result_t
+dns_rdatavec_subtract(dns_vecheader_t *oheader, dns_vecheader_t *sheader,
+		       isc_mem_t *mctx, dns_rdataclass_t rdclass,
+		       dns_rdatatype_t type, unsigned int flags,
+		       dns_vecheader_t **theaderp) {
+	isc_result_t result = ISC_R_SUCCESS;
+	unsigned char *ocurrent = NULL, *scurrent = NULL;
+	unsigned char *tstart = NULL, *tcurrent = NULL;
+	unsigned int ocount, scount, tlength;
+	unsigned int tcount = 0, rcount = 0;
+	vecinfo_t *oinfo = NULL, *sinfo = NULL;
+
+	REQUIRE(theaderp != NULL && *theaderp == NULL);
+	REQUIRE(oheader != NULL && sheader != NULL);
+
+	ocurrent = rdatavec_data(oheader);
+	ocount = rdatavec_count(oheader);
+
+	scurrent = rdatavec_data(sheader);
+	scount = rdatavec_count(sheader);
+
+	INSIST(ocount > 0 && scount > 0);
+
+	/* Get info about the rdatas being subtracted */
+	sinfo = isc_mem_cget(mctx, scount, sizeof(struct vecinfo));
+	for (size_t i = 0; i < scount; i++) {
+		sinfo[i].pos = scurrent;
+		dns_rdata_init(&sinfo[i].rdata);
+		rdata_from_vecitem(&scurrent, rdclass, type, &sinfo[i].rdata);
+	}
+
+	/*
+	 * Figure out the target length. Start with the header,
+	 * plus 2 octets for the count.
+	 */
+	tlength = header_size(oheader) + 2;
+
+	/*
+	 * Add the length of the rdatas in the old vec that
+	 * aren't being subtracted.
+	 */
+	oinfo = isc_mem_cget(mctx, ocount, sizeof(struct vecinfo));
+	for (size_t i = 0; i < ocount; i++) {
+		bool matched = false;
+
+		oinfo[i].pos = ocurrent;
+		dns_rdata_init(&oinfo[i].rdata);
+		rdata_from_vecitem(&ocurrent, rdclass, type, &oinfo[i].rdata);
+
+		for (size_t j = 0; j < scount; j++) {
+			if (sinfo[j].dup) {
+				continue;
+			} else if (dns_rdata_compare(&oinfo[i].rdata,
+						     &sinfo[j].rdata) == 0)
+			{
+				matched = true;
+				oinfo[i].dup = sinfo[j].dup = true;
+				break;
+			}
+		}
+
+		if (matched) {
+			/* This item will be subtracted. */
+			rcount++;
+		} else {
+			/*
+			 * This rdata wasn't in the vec to be subtracted,
+			 * so copy it to the target.  Add its length to
+			 * tlength and increment tcount.
+			 */
+			tlength += ocurrent - oinfo[i].pos;
+			tcount++;
+		}
+	}
+
+	/*
+	 * If the EXACT flag wasn't set, check that all the records that
+	 * were to be subtracted actually did exist in the original vec.
+	 * (The numeric check works here because rdatavecs do not contain
+	 * duplicates.)
+	 */
+	if ((flags & DNS_RDATAVEC_EXACT) != 0 && rcount != scount) {
+		CLEANUP(DNS_R_NOTEXACT);
+	}
+
+	/*
+	 * If the resulting rdatavec would be empty, don't bother to
+	 * create a new buffer, just return.
+	 */
+	if (tcount == 0) {
+		CLEANUP(DNS_R_NXRRSET);
+	}
+
+	/*
+	 * If nothing is going to change, stop.
+	 */
+	if (rcount == 0) {
+		CLEANUP(DNS_R_UNCHANGED);
+	}
+
+	/*
+	 * Allocate the target buffer and copy the old vec's header.
+	 */
+	tstart = isc_mem_get(mctx, tlength);
+	memmove(tstart, oheader, header_size(oheader));
+	tcurrent = tstart + header_size(oheader);
+
+	/*
+	 * Write the new count.
+	 */
+	put_uint16(tcurrent, tcount);
+
+	/*
+	 * Copy the parts of the old vec that didn't have duplicates.
+	 */
+	for (size_t i = 0; i < ocount; i++) {
+		if (!oinfo[i].dup) {
+			rdata_to_vecitem(&tcurrent, type, &oinfo[i].rdata);
+		}
+	}
+
+	INSIST(tcurrent == tstart + tlength);
+
+	*theaderp = (dns_vecheader_t *)tstart;
+
+cleanup:
+	isc_mem_cput(mctx, oinfo, ocount, sizeof(struct vecinfo));
+	isc_mem_cput(mctx, sinfo, scount, sizeof(struct vecinfo));
+
+	return result;
+}
+
+
+void
+dns_vecheader_setownercase(dns_vecheader_t *header, const dns_name_t *name) {
+	REQUIRE(!CASESET(header));
+
+	bool casefullylower = true;
+
+	/*
+	 * We do not need to worry about label lengths as they are all
+	 * less than or equal to 63.
+	 */
+	memset(header->upper, 0, sizeof(header->upper));
+	for (size_t i = 0; i < name->length; i++) {
+		if (isupper(name->ndata[i])) {
+			header->upper[i / 8] |= 1 << (i % 8);
+			casefullylower = false;
+		}
+	}
+	if (casefullylower) {
+		DNS_VECHEADER_SETATTR(header,
+				       DNS_VECHEADERATTR_CASEFULLYLOWER);
+	}
+	DNS_VECHEADER_SETATTR(header, DNS_VECHEADERATTR_CASESET);
+}
+
+void
+dns_vecheader_reset(dns_vecheader_t *h, dns_dbnode_t *node) {
+	h->heap_index = 0;
+	h->node = node;
+
+	atomic_init(&h->attributes, 0);
+
+	STATIC_ASSERT(sizeof(h->attributes) == 2,
+		      "The .attributes field of dns_vecheader_t needs to be "
+		      "16-bit int type exactly.");
+}
+
+dns_vecheader_t *
+dns_vecheader_new(isc_mem_t *mctx, dns_dbnode_t *node) {
+	dns_vecheader_t *h = NULL;
+
+	h = isc_mem_get(mctx, sizeof(*h));
+	*h = (dns_vecheader_t){
+		.node = node,
+	};
+	return h;
+}
+
+void
+dns_vecheader_destroy(dns_vecheader_t **headerp) {
+	unsigned int size;
+	dns_vecheader_t *header = *headerp;
+
+	*headerp = NULL;
+
+	isc_mem_t *mctx = header->node->mctx;
+	dns_db_deletedata(header->node, header);
+
+	if (EXISTS(header)) {
+		size = dns_rdatavec_size(header);
+	} else {
+		size = sizeof(*header);
+	}
+
+	isc_mem_put(mctx, header, size);
+}
+
+/* Iterators for already bound rdatavec */
+
+isc_result_t
+vecheader_first(rdatavec_iter_t *iter, dns_vecheader_t *header, dns_rdataclass_t rdclass) {
+	unsigned char *raw = rdatavec_data(header);
+	uint16_t count = rdatavec_count(header);
+	if (count == 0) {
+		iter->iter_pos = NULL;
+		iter->iter_count = 0;
+		return ISC_R_NOMORE;
+	}
+
+	/*
+	 * iter.iter_count is the number of rdata beyond the cursor
+	 * position, so we decrement the total count by one before
+	 * storing it.
+	 *
+	 * 'raw' points to the first record.
+	 */
+	iter->iter_pos = raw;
+	iter->iter_count = count - 1;
+	iter->iter_rdclass = rdclass;
+	iter->iter_type = DNS_TYPEPAIR_TYPE(header->typepair);
+
+	return ISC_R_SUCCESS;
+}
+
+isc_result_t
+vecheader_next(rdatavec_iter_t *iter) {
+	uint16_t count = iter->iter_count;
+	if (count == 0) {
+		iter->iter_pos = NULL;
+		return ISC_R_NOMORE;
+	}
+	iter->iter_count = count - 1;
+
+	/*
+	 * Skip forward one record (length + 4) or one offset (4).
+	 */
+	unsigned char *raw = iter->iter_pos;
+	uint16_t length = peek_uint16(raw);
+	raw += length;
+	iter->iter_pos = raw + sizeof(uint16_t);
+
+	return ISC_R_SUCCESS;
+}
+
+void
+vecheader_current(rdatavec_iter_t *iter, dns_rdata_t *rdata) {
+	unsigned char *raw = NULL;
+	unsigned int length;
+	isc_region_t r;
+	unsigned int flags = 0;
+
+	raw = iter->iter_pos;
+	REQUIRE(raw != NULL);
+
+	/*
+	 * Find the start of the record if not already in iter_pos
+	 * then skip the length and order fields.
+	 */
+	length = get_uint16(raw);
+
+	if (iter->iter_type == dns_rdatatype_rrsig) {
+		if (*raw & DNS_RDATAVEC_OFFLINE) {
+			flags |= DNS_RDATA_OFFLINE;
+		}
+		length--;
+		raw++;
+	}
+	r.length = length;
+	r.base = raw;
+	dns_rdata_fromregion(rdata, iter->iter_rdclass, iter->iter_type, &r);
+	rdata->flags |= flags;
+}
+
+
+/* Fixed RRSet helper macros */
+
+static void
+rdataset_disassociate(dns_rdataset_t *rdataset DNS__DB_FLARG) {
+	dns_dbnode_t *node = rdataset->vec.node;
+
+	dns__db_detachnode(&node DNS__DB_FLARG_PASS);
+}
+
+static isc_result_t
+rdataset_first(dns_rdataset_t *rdataset) {
+	return vecheader_first(&rdataset->vec.iter, rdataset->vec.header, rdataset->rdclass);
+}
+
+static isc_result_t
+rdataset_next(dns_rdataset_t *rdataset) {
+	return vecheader_next(&rdataset->vec.iter);
+}
+
+static void
+rdataset_current(dns_rdataset_t *rdataset, dns_rdata_t *rdata) {
+	vecheader_current(&rdataset->vec.iter, rdata);
+}
+
+static void
+rdataset_clone(dns_rdataset_t *source, dns_rdataset_t *target DNS__DB_FLARG) {
+	dns_dbnode_t *node = source->vec.node;
+	dns_dbnode_t *cloned_node = NULL;
+
+	dns__db_attachnode(node, &cloned_node DNS__DB_FLARG_PASS);
+	INSIST(!ISC_LINK_LINKED(target, link));
+	*target = *source;
+	ISC_LINK_INIT(target, link);
+
+	target->vec.iter.iter_pos = NULL;
+	target->vec.iter.iter_count = 0;
+}
+
+static unsigned int
+rdataset_count(dns_rdataset_t *rdataset) {
+	return rdatavec_count(rdataset->vec.header);
+}
+
+static void
+rdataset_settrust(dns_rdataset_t *rdataset, dns_trust_t trust) {
+	dns_vecheader_t *header = dns_vecheader_getheader(rdataset);
+
+	rdataset->trust = trust;
+	atomic_store(&header->trust, trust);
+}
+
+static void
+rdataset_getownercase(const dns_rdataset_t *rdataset, dns_name_t *name) {
+	dns_vecheader_t *header = dns_vecheader_getheader(rdataset);
+	uint8_t mask = (1 << 7);
+	uint8_t bits = 0;
+
+	if (!CASESET(header)) {
+		return;
+	}
+
+	if (CASEFULLYLOWER(header)) {
+		isc_ascii_lowercopy(name->ndata, name->ndata, name->length);
+		return;
+	}
+
+	uint8_t *nd = name->ndata;
+	for (size_t i = 0; i < name->length; i++) {
+		if (mask == (1 << 7)) {
+			bits = header->upper[i / 8];
+			mask = 1;
+		} else {
+			mask <<= 1;
+		}
+		nd[i] = (bits & mask) ? isc_ascii_toupper(nd[i])
+				      : isc_ascii_tolower(nd[i]);
+	}
+}
+
+dns_vecheader_t *
+dns_vecheader_getheader(const dns_rdataset_t *rdataset) {
+	return rdataset->vec.header;
+}
+
+dns_vectop_t *
+dns_vectop_new(isc_mem_t *mctx, dns_typepair_t typepair) {
+	dns_vectop_t *top = isc_mem_get(mctx, sizeof(*top));
+	*top = (dns_vectop_t){
+		.next_type = ISC_SLINK_INITIALIZER,
+		.headers = ISC_SLIST_INITIALIZER,
+		.typepair = typepair,
+	};
+
+	return top;
+}
+
+void
+dns_vectop_destroy(isc_mem_t *mctx, dns_vectop_t **topp) {
+	REQUIRE(topp != NULL && *topp != NULL);
+	dns_vectop_t *top = *topp;
+	*topp = NULL;
+	isc_mem_put(mctx, top, sizeof(*top));
+}

lib/dns/rdatavec_p.h

@@ -0,0 +1,69 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * SPDX-License-Identifier: MPL-2.0
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+#pragma once
+
+#include <isc/endian.h>
+
+#include <dns/rdatavec.h>
+
+#define CASEFULLYLOWER(header)                         \
+	((atomic_load_acquire(&(header)->attributes) & \
+	  DNS_VECHEADERATTR_CASEFULLYLOWER) != 0)
+#define CASESET(header)                                \
+	((atomic_load_acquire(&(header)->attributes) & \
+	  DNS_VECHEADERATTR_CASESET) != 0)
+#define EXISTS(header)                                 \
+	((atomic_load_acquire(&(header)->attributes) & \
+	  DNS_VECHEADERATTR_NONEXISTENT) == 0)
+#define IGNORE(header)                                 \
+	((atomic_load_acquire(&(header)->attributes) & \
+	  DNS_VECHEADERATTR_IGNORE) != 0)
+#define OPTOUT(header)                                 \
+	((atomic_load_acquire(&(header)->attributes) & \
+	  DNS_VECHEADERATTR_OPTOUT) != 0)
+#define RESIGN(header)                                 \
+	((atomic_load_acquire(&(header)->attributes) & \
+	  DNS_VECHEADERATTR_RESIGN) != 0)
+
+#define peek_uint16(buffer) ISC_U8TO16_BE(buffer)
+#define get_uint16(buffer)                            \
+	({                                            \
+		uint16_t __ret = peek_uint16(buffer); \
+		buffer += sizeof(uint16_t);           \
+		__ret;                                \
+	})
+#define put_uint16(buffer, val)               \
+	{                                     \
+		ISC_U16TO8_BE(buffer, val);   \
+		(buffer) += sizeof(uint16_t); \
+	}
+
+dns_vecheader_t *
+dns_vecheader_getheader(const dns_rdataset_t *rdataset);
+/*%<
+ * Return a pointer to the vecheader for a vec rdataset.
+ *
+ * Requires:
+ * \li	'rdataset' is a valid rdataset using rdatavec methods.
+ */
+
+
+isc_result_t
+vecheader_first(rdatavec_iter_t *iter, dns_vecheader_t *header, dns_rdataclass_t rdclass);
+
+isc_result_t
+vecheader_next(rdatavec_iter_t *iter);
+
+void
+vecheader_current(rdatavec_iter_t *iter, dns_rdata_t *rdata);