create 'journal' system test

tests that version 1 journal files containing version 1 transaction
headers are rolled forward correctly on server startup, then updated
into version 2 journals. also checks journal file consistency and
'max-journal-size' behavior.

(cherry picked from commit a0aefa1de6)

bin/tests/system/conf.sh.common

@@ -88,6 +88,7 @@ idna \
 inline \
 integrity \
 ixfr \
+journal \
 kasp \
 keepalive \
 legacy \

bin/tests/system/journal/clean.sh

@@ -0,0 +1,14 @@
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+rm -f */named.memstats
+rm -f */named.run
+rm -f */named.conf
+rm -f */*.db */*.jnl
+rm -f dig.out*

bin/tests/system/journal/ns1/generic.db.in

@@ -0,0 +1,16 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@		SOA     ns hostmaster 2012010901 3600 1200 604800 1200
+                NS      ns
+ns		A       192.0.2.1
+
+addr1	A      10.53.0.1

bin/tests/system/journal/ns1/ixfr.db.in

@@ -0,0 +1,17 @@
+; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+;
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+;
+; See the COPYRIGHT file distributed with this work for additional
+; information regarding copyright ownership.
+
+
+$TTL 600
+@		SOA     ns hostmaster 2012010902 3600 1200 604800 1200
+                NS      ns
+ns		A       192.0.2.1
+
+addr1	A      10.53.0.1
+addr2	A      10.53.0.2

bin/tests/system/journal/ns1/named.conf.in

@@ -0,0 +1,90 @@
+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+options {
+	query-source address 10.53.0.1;
+	notify-source 10.53.0.1;
+	transfer-source 10.53.0.1;
+	port @PORT@;
+	session-keyfile "session.key";
+	pid-file "named.pid";
+	listen-on { 10.53.0.1; };
+	listen-on-v6 { none; };
+	dnssec-validation yes;
+	minimal-responses no;
+	recursion no;
+	notify yes;
+};
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.1 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+zone changed {
+	type primary;
+	update-policy local;
+	file "changed.db";
+};
+
+zone unchanged {
+	type primary;
+	update-policy local;
+	file "unchanged.db";
+};
+
+zone changed2 {
+	type primary;
+	update-policy local;
+	file "changed2.db";
+};
+
+zone unchanged2 {
+	type primary;
+	update-policy local;
+	file "unchanged2.db";
+};
+
+zone hdr1d1d2d1d2 {
+	type primary;
+	update-policy local;
+	file "d1212.db";
+};
+
+zone hdr1d2d1d2d1 {
+	type primary;
+	update-policy local;
+	file "d2121.db";
+};
+
+zone ixfr {
+	type primary;
+	ixfr-from-differences yes;
+	file "ixfr.db";
+};
+
+zone maxjournal {
+	type primary;
+	max-journal-size 1k;
+	update-policy local;
+	file "maxjournal.db";
+};
+
+zone maxjournal2 {
+	type primary;
+	max-journal-size 1k;
+	update-policy local;
+	file "maxjournal2.db";
+};

bin/tests/system/journal/setup.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+$SHELL clean.sh
+
+copy_setports ns1/named.conf.in ns1/named.conf
+cp ns1/generic.db.in ns1/changed.db
+cp ns1/changed.ver1.jnl.saved ns1/changed.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged.db
+cp ns1/unchanged.ver1.jnl.saved ns1/unchanged.db.jnl
+
+cp ns1/generic.db.in ns1/changed2.db
+cp ns1/changed.ver2.jnl.saved ns1/changed2.db.jnl
+
+cp ns1/generic.db.in ns1/unchanged2.db
+cp ns1/unchanged.ver2.jnl.saved ns1/unchanged2.db.jnl
+
+cp ns1/ixfr.db.in ns1/ixfr.db
+cp ns1/ixfr.ver1.jnl.saved ns1/ixfr.db.jnl
+
+cp ns1/generic.db.in ns1/d1212.db
+cp ns1/d1212.jnl.saved ns1/d1212.db.jnl
+
+cp ns1/generic.db.in ns1/d2121.db
+cp ns1/d2121.jnl.saved ns1/d2121.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal.db
+cp ns1/maxjournal.jnl.saved ns1/maxjournal.db.jnl
+
+cp ns1/generic.db.in ns1/maxjournal2.db
+cp ns1/maxjournal2.jnl.saved ns1/maxjournal2.db.jnl

bin/tests/system/journal/tests.sh

@@ -0,0 +1,201 @@
+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, you can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+. ../conf.sh
+
+dig_with_opts() {
+    "$DIG" @10.53.0.1 -p "$PORT" +tcp "$@"
+}
+
+rndc_with_opts() {
+    "$RNDC" -c ../common/rndc.conf -p "$CONTROLPORT" -s "$@"
+}
+
+status=0
+n=0
+
+n=`expr $n + 1`
+echo_i "check outdated journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed/IN: retried using old journal format' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated empty journal did not cause an error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check updated journal has correct RR count (dynamic) ($n)"
+ret=0
+$JOURNALPRINT -x ns1/changed.db.jnl | grep "rrcount 3 " > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journal rolled forward (dynamic) ($n)"
+ret=0
+dig_with_opts changed2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone changed2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format empty journal did not cause error (dynamic) ($n)"
+ret=0
+dig_with_opts unchanged2 soa > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010901' dig.out.test$n > /dev/null || ret=1
+grep 'zone unchanged2/IN: retried using old journal format' ns1/named.run > /dev/null && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check new-format journals were updated or removed (dynamic) ($n)"
+ret=0
+cat -v ns1/changed2.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ -f ns1/unchanged2.db.jnl ] && ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated up-to-date journal succeeded (ixfr-from-differences) ($n)"
+ret=0
+dig_with_opts -t soa ixfr > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010902' dig.out.test$n > /dev/null || ret=1
+grep 'zone ixfr/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check outdated journal was updated (ixfr-from-differences) ($n)"
+ret=0
+cat -v ns1/ixfr.db.jnl | grep "BIND LOG V9.2" > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 1,2,1,2) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d1d2d1d2 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d1d2d1d2/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 1,2,1,2) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d1212.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers succeeded (version 2,1,2,1) ($n)"
+ret=0
+dig_with_opts -t soa hdr1d2d1d2d1 > dig.out.test$n
+grep 'status: NOERROR' dig.out.test$n > /dev/null || ret=1
+grep '2012010905' dig.out.test$n > /dev/null || ret=1
+grep 'zone hdr1d2d1d2d1/IN: journal rollforward completed successfully: recoverable' ns1/named.run > /dev/null || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal with mixed headers was updated (version 2,1,2,1) ($n)"
+ret=0
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 1") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.jnl.saved | grep -c "version 2") -eq 2 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 1") -eq 0 ] || ret=1
+[ $($JOURNALPRINT -x ns1/d2121.db.jnl | grep -c "version 2") -eq 4 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check there are no journals left un-updated ($n)"
+ret=0
+c1=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9")
+c2=$(cat -v ns1/*.jnl | grep -c "BIND LOG V9.2")
+[ ${c1} -eq ${c2} ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal downgrade/upgrade ($n)"
+ret=0
+cp ns1/changed.db.jnl ns1/temp.jnl
+$JOURNALPRINT -d ns1/temp.jnl
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 1") -eq 1 ] || ret=1
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 1" || ret=1
+$JOURNALPRINT -u ns1/temp.jnl
+$JOURNALPRINT -x ns1/temp.jnl | grep -q "Header version = 2" || ret=1
+[ $($JOURNALPRINT -x ns1/temp.jnl | grep -c "version 2") -eq 1 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works after journal update ($n)"
+ret=0
+# a dump should have been triggered by repairing the journal,
+# which would have resulted in the journal already being
+# compacted.
+[ $(wc -c < ns1/maxjournal.db.jnl) -lt 4000 ] || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check max-journal-size works with non-updated journals ($n)"
+ret=0
+# journal was not repaired, so it should still be big
+[ $(wc -c < ns1/maxjournal2.db.jnl) -gt 12000 ] || ret=1
+# the zone hasn't been dumped yet, so 'rndc sync' should work without
+# needing a zone update first.
+rndc_with_opts 10.53.0.1 sync maxjournal2
+check_size() (
+    [ $(wc -c < ns1/maxjournal2.db.jnl) -lt 4000 ]
+)
+retry_quiet 10 check_size || ret=1
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+n=`expr $n + 1`
+echo_i "check journal index consistency ($n)"
+ret=0
+for jnl in ns1/*.jnl; do
+    $JOURNALPRINT -x $jnl 2>&1 | grep -q "Offset mismatch" && ret=1
+done
+[ $ret -eq 0 ] || echo_i "failed"
+status=`expr $status + $ret`
+
+echo_i "exit status: $status"
+[ $status -eq 0 ] || exit 1

util/copyrights

@@ -638,6 +638,18 @@
 ./bin/tests/system/ixfr/prereq.sh		SH	2001,2004,2007,2012,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/ixfr/setup.sh		SH	2001,2004,2007,2011,2012,2013,2014,2016,2018,2019,2020,2021
 ./bin/tests/system/ixfr/tests.sh		SH	2001,2004,2007,2011,2012,2014,2016,2018,2019,2020,2021
+./bin/tests/system/journal/clean.sh		SH	2021
+./bin/tests/system/journal/ns1/changed.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/changed.ver2.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/d1212.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/d2121.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/ixfr.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/maxjournal.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/maxjournal2.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/unchanged.ver1.jnl.saved	X	2021
+./bin/tests/system/journal/ns1/unchanged.ver2.jnl.saved	X	2021
+./bin/tests/system/journal/setup.sh		SH	2021
+./bin/tests/system/journal/tests.sh		SH	2021
 ./bin/tests/system/kasp/README			TXT.BRIEF	2019,2020,2021
 ./bin/tests/system/kasp/clean.sh		SH	2019,2020,2021
 ./bin/tests/system/kasp/ns2/setup.sh		SH	2019,2020,2021