Merge branch '4-remove-embedded-atf' into 'master'

Remove embedded ATF and required ATF >= 0.20 + Kyua to run unittests

See merge request isc-projects/bind9!875
This commit is contained in:
Ondřej Surý 2018-11-15 10:40:28 -05:00
commit 436b6fb354
16 changed files with 165 additions and 162 deletions

29
.gitignore vendored
View file

@ -27,35 +27,6 @@ gen.dSYM/
.deps/
.dirstamp
.libs/
unit/atf-src/atf-c++/atf-c++.pc
unit/atf-src/atf-c/atf-c.pc
unit/atf-src/atf-c/defs.h
unit/atf-src/atf-c/detail/process_helpers
unit/atf-src/atf-config/atf-config
unit/atf-src/atf-report/atf-report
unit/atf-src/atf-report/fail_helper
unit/atf-src/atf-report/misc_helpers
unit/atf-src/atf-report/pass_helper
unit/atf-src/atf-run/atf-run
unit/atf-src/atf-run/bad_metadata_helper
unit/atf-src/atf-run/expect_helpers
unit/atf-src/atf-run/misc_helpers
unit/atf-src/atf-run/pass_helper
unit/atf-src/atf-run/several_tcs_helper
unit/atf-src/atf-run/zero_tcs_helper
unit/atf-src/atf-sh/atf-check
unit/atf-src/atf-sh/atf-sh
unit/atf-src/atf-sh/misc_helpers
unit/atf-src/atf-version/atf-version
unit/atf-src/atf-version/revision.h
unit/atf-src/atf-version/revision.h.stamp
unit/atf-src/bconfig.h
unit/atf-src/bootstrap/atconfig
unit/atf-src/doc/atf.7
unit/atf-src/stamp-h1
unit/atf-src/test-programs/c_helpers
unit/atf-src/test-programs/cpp_helpers
unit/atf-src/test-programs/sh_helpers
# ccc-analyzer store its results in .plist directories
*.plist/
*~

View file

@ -1,3 +1,6 @@
5097. [cleanup] Remove embedded ATF unit testing framework
from BIND source distribution. [GL !875]
5096. [func] Use multiple event loops in socket code, and
make network threads CPU-affinitive. This
significantly improves performance on large

5
README
View file

@ -289,8 +289,9 @@ and will be skipped if these are not available. Some tests require Python
and the 'dnspython' module and will be skipped if these are not available.
See bin/tests/system/README for further details.
Unit tests are implemented using Automated Testing Framework (ATF). To run
them, use configure --with-atf, then run make test or make unit.
Unit tests are implemented using cmocka unit testing framework.
To run them, use configure --with-cmocka, then run make test or
make unit.
Documentation

View file

@ -307,8 +307,8 @@ and will be skipped if these are not available. Some tests require Python
and the 'dnspython' module and will be skipped if these are not available.
See bin/tests/system/README for further details.
Unit tests are implemented using Automated Testing Framework (ATF).
To run them, use `configure --with-atf`, then run `make test` or
Unit tests are implemented using cmocka unit testing framework.
To run them, use `configure --with-cmocka`, then run `make test` or
`make unit`.
### <a name="doc"/> Documentation

View file

@ -6,9 +6,6 @@
/* Use AES for Client Cookie generation */
#undef AES_CC
/* define if ATF unit tests are to be built. */
#undef ATF_TEST
/* Define if you cannot bind() before connect() for TCP sockets. */
#undef BROKEN_TCP_BIND_BEFORE_CONNECT

81
configure vendored
View file

@ -670,6 +670,7 @@ BIND9_ISCCC_BUILDINCLUDE
BIND9_ISC_BUILDINCLUDE
BIND9_TOP_BUILDDIR
LD_WRAP_TESTS
KYUA
UNITTESTS
CMOCKA_LIBS
CMOCKA_CFLAGS
@ -841,7 +842,6 @@ infodir
docdir
oldincludedir
includedir
runstatedir
localstatedir
sharedstatedir
sysconfdir
@ -961,7 +961,8 @@ OPENSSL_LIBS
LIBIDN2_CFLAGS
LIBIDN2_LIBS
CMOCKA_CFLAGS
CMOCKA_LIBS'
CMOCKA_LIBS
KYUA'
# Initialize some variables set by options.
@ -1000,7 +1001,6 @@ datadir='${datarootdir}'
sysconfdir='${prefix}/etc'
sharedstatedir='${prefix}/com'
localstatedir='${prefix}/var'
runstatedir='${localstatedir}/run'
includedir='${prefix}/include'
oldincludedir='/usr/include'
docdir='${datarootdir}/doc/${PACKAGE_TARNAME}'
@ -1253,15 +1253,6 @@ do
| -silent | --silent | --silen | --sile | --sil)
silent=yes ;;
-runstatedir | --runstatedir | --runstatedi | --runstated \
| --runstate | --runstat | --runsta | --runst | --runs \
| --run | --ru | --r)
ac_prev=runstatedir ;;
-runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \
| --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \
| --run=* | --ru=* | --r=*)
runstatedir=$ac_optarg ;;
-sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb)
ac_prev=sbindir ;;
-sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \
@ -1399,7 +1390,7 @@ fi
for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \
datadir sysconfdir sharedstatedir localstatedir includedir \
oldincludedir docdir infodir htmldir dvidir pdfdir psdir \
libdir localedir mandir runstatedir
libdir localedir mandir
do
eval ac_val=\$$ac_var
# Remove trailing slashes.
@ -1552,7 +1543,6 @@ Fine tuning of the installation directories:
--sysconfdir=DIR read-only single-machine data [PREFIX/etc]
--sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com]
--localstatedir=DIR modifiable single-machine data [PREFIX/var]
--runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run]
--libdir=DIR object code libraries [EPREFIX/lib]
--includedir=DIR C header files [PREFIX/include]
--oldincludedir=DIR C header files for non-gcc [/usr/include]
@ -1715,6 +1705,7 @@ Some influential environment variables:
CMOCKA_CFLAGS
C compiler flags for CMOCKA, overriding pkg-config
CMOCKA_LIBS linker flags for CMOCKA, overriding pkg-config
KYUA path to kyua execution engine
Use these variables to override the choices made by `configure' or to help
it to find libraries and programs with nonstandard names/locations.
@ -11916,7 +11907,6 @@ yes)
STD_CDEFINES="$STD_CDEFINES -DISC_MEM_DEFAULTFILL=1 -DISC_LIST_CHECKINIT=1 -DNS_HOOKS_ENABLE=1"
test "${enable_fixed_rrset+set}" = set || enable_fixed_rrset=yes
test "${enable_querytrace+set}" = set || enable_querytrace=yes
test "${with_atf+set}" = set || with_atf=yes
test "${with_cmocka+set}" = set || with_cmocka=yes
test "${with_dlz_filesystem+set}" = set || with_dlz_filesystem=yes
test "${enable_symtable+set}" = set || enable_symtable=all
@ -19884,6 +19874,64 @@ esac
#
# Check for kyua execution engine if ATF was requested
# and bail out if execution engine was not found
#
if test "$with_cmocka" != "no"; then :
for ac_prog in kyua
do
# Extract the first word of "$ac_prog", so it can be a program name with args.
set dummy $ac_prog; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_path_KYUA+:} false; then :
$as_echo_n "(cached) " >&6
else
case $KYUA in
[\\/]* | ?:[\\/]*)
ac_cv_path_KYUA="$KYUA" # Let the user override the test with a path.
;;
*)
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_path_KYUA="$as_dir/$ac_word$ac_exec_ext"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS
;;
esac
fi
KYUA=$ac_cv_path_KYUA
if test -n "$KYUA"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $KYUA" >&5
$as_echo "$KYUA" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
test -n "$KYUA" && break
done
if test -z "$KYUA"; then :
as_fn_error $? "kyua test execution engine not found" "$LINENO" 5
fi
fi
#
# Check for -Wl,--wrap= support
#
@ -19925,6 +19973,9 @@ rm -f core conftest.err conftest.$ac_objext \
LDFLAGS=$save_LDFLAGS
#
# Check for i18n
#
for ac_header in locale.h
do :
ac_fn_c_check_header_mongrel "$LINENO" "locale.h" "ac_cv_header_locale_h" "$ac_includes_default"

View file

@ -79,7 +79,6 @@ yes)
STD_CDEFINES="$STD_CDEFINES -DISC_MEM_DEFAULTFILL=1 -DISC_LIST_CHECKINIT=1 -DNS_HOOKS_ENABLE=1"
test "${enable_fixed_rrset+set}" = set || enable_fixed_rrset=yes
test "${enable_querytrace+set}" = set || enable_querytrace=yes
test "${with_atf+set}" = set || with_atf=yes
test "${with_cmocka+set}" = set || with_cmocka=yes
test "${with_dlz_filesystem+set}" = set || with_dlz_filesystem=yes
test "${enable_symtable+set}" = set || enable_symtable=all
@ -2436,6 +2435,18 @@ AC_SUBST([CMOCKA_CFLAGS])
AC_SUBST([CMOCKA_LIBS])
AC_SUBST(UNITTESTS)
#
# Check for kyua execution engine if ATF was requested
# and bail out if execution engine was not found
#
AC_ARG_VAR([KYUA], [path to kyua execution engine])
AS_IF([test "$with_cmocka" != "no"],
[AC_PATH_PROGS([KYUA], [kyua], [])
AS_IF([test -z "$KYUA"],
[AC_MSG_ERROR([kyua test execution engine not found])])])
AC_SUBST([KYUA])
#
# Check for -Wl,--wrap= support
#
@ -2458,6 +2469,9 @@ AC_SUBST([LD_WRAP_TESTS])
LDFLAGS=$save_LDFLAGS
#
# Check for i18n
#
AC_CHECK_HEADERS(locale.h)
AC_CHECK_FUNCS(setlocale)

View file

@ -197,27 +197,10 @@ points to `rndc`, `SIGNZONE` to `dnssec-signzone`, etc.
#### <a name="unittest"></a> Building unit tests
BIND uses the Automated Testing Framework (ATF), originally from the NetBSD
project, as its unit testing framework. (Note: ATF has been supplanted by
a newer version called Kyua, but BIND is still using the older system.)
BIND uses the cmocka, unit testing framework.
To build BIND with unit tests, run `configure` with the `--with-atf`
option. This causes the ATF source code in the `unit/atf-src`
subdirectory to be built.
To save time on repeated builds, you can build and install ATF
in another directory, and configure BIND to use the pre-built
version. (Be sure to disable shared libraries in the ATF build,
and to build the ATF tools; libraries alone are not sufficient).
$ cd atf-src
$ configure --prefix=<prefix> --enable-tools --disable-shared
$ make
$ make install
After this has been done, specify the ATF prefix when building BIND:
$ configure --with-atf=<prefix>
To build BIND with unit tests, run `configure` with the `--with-cmocka`
option. This requires cmocka >= 1.0.0 to be installed in the system.
#### Running unit tests
@ -249,53 +232,11 @@ redirected:
#### Writing unit tests
Information on writing ATF tests can be found at the
[NetBSD site](http://wiki.netbsd.org/tutorials/atf/).
Information on writing cmocka tests can be found at the
[cmocka website](https://cmocka.org).
New unit tests should be added whenever significant new API
functionality is added to libdns or libisc.
Each unit test file contains at least one unit test case, and
a list of test cases to execute when the test is run. These
will look like the following:
ATF_TC(test_case);
ATF_TC_HEAD(test_case, tc) {
atf_tc_set_md_var(tc, "descr",
"Describe the test case here.");
}
ATF_TC_BODY(serialize_align, tc) {
UNUSED(tc);
result = isc_test_begin(NULL, true);
ATF_REQUIRE_EQ(result, ISC_R_SUCCESS);
ATF_CHECK_EQ(value1, value2);
ATF_CHECK(value1 + value2 < 100);
isc_test_end();
}
/*
* Main
*/
ATF_TP_ADD_TCS(tp) {
ATF_TP_ADD_TC(tp, test_case);
return (atf_no_error());
}
If the conditions specified in `ATF_CHECK` and `ATF_CHECK_EQ`
directives are found to be false, then the test case will fail, but it
will continue running to see if there are any more failures.
If the conditions specified in `ATF_REQUIRE` and `ATF_REQUIRE_EQ` are
found to be false, the test case cannot continue running and will stop
`isc_test_begin()` and `isc_test_end()` set up necessary preconditions
for checking libisc functions, such as starting a task manger and
creating a memory context. Similar functions `dns_test_begin()` and
`dns_test_end()` are available for testing libdns functions.
New unit tests should be added whenever new API functionality is added to the
libraries.
### <a name="arch"></a> BIND system architecture

View file

@ -15,9 +15,13 @@
#include <stdarg.h>
#include <stddef.h>
#include <stdlib.h>
#include <setjmp.h>
#include <stdlib.h>
#include <isc/util.h>
#ifdef NS_HOOKS_ENABLE
#include <stdio.h>
#include <string.h>
#include <unistd.h>
@ -122,6 +126,17 @@ main(void) {
return (cmocka_run_group_tests(tests, NULL, NULL));
}
#else
#include <stdio.h>
int
main(void) {
printf("1..0 # Skipped: libns hooks not enabled\n");
return (0);
}
#endif /* NS_HOOKS_ENABLE */
#else /* HAVE_CMOCKA */
@ -133,4 +148,4 @@ main(void) {
return (0);
}
#endif
#endif /* HAVE_CMOCKA */

View file

@ -15,9 +15,13 @@
#include <stdarg.h>
#include <stddef.h>
#include <stdlib.h>
#include <setjmp.h>
#include <stdlib.h>
#include <isc/util.h>
#ifdef NS_HOOKS_ENABLE
#include <stdio.h>
#include <string.h>
#include <unistd.h>
@ -150,6 +154,17 @@ main(void) {
return (cmocka_run_group_tests(tests, NULL, NULL));
}
#else
#include <stdio.h>
int
main(void) {
printf("1..0 # Skipped: libns hooks not enabled\n");
return (0);
}
#endif /* NS_HOOKS_ENABLE */
#else /* HAVE_CMOCKA */

View file

@ -13,6 +13,8 @@
#include <config.h>
#ifdef NS_HOOKS_ENABLE
#include <inttypes.h>
#include <stdbool.h>
#include <stdlib.h>
@ -925,3 +927,4 @@ ns_test_getdata(const char *file, unsigned char *buf,
isc_stdio_close(f);
return (result);
}
#endif

View file

@ -17,6 +17,10 @@
#include <stddef.h>
#include <setjmp.h>
#include <isc/util.h>
#ifdef NS_HOOKS_ENABLE
#include <inttypes.h>
#include <stdbool.h>
#include <stdlib.h>
@ -27,7 +31,6 @@
#include <dns/badcache.h>
#include <dns/view.h>
#include <isc/util.h>
#include <ns/client.h>
#include <ns/query.h>
#include <isc/util.h>
@ -601,6 +604,17 @@ main(void) {
return (cmocka_run_group_tests(tests, NULL, NULL));
}
#else
#include <stdio.h>
int
main(void) {
printf("1..0 # Skipped: libns hooks not enabled\n");
return (0);
}
#endif /* NS_HOOKS_ENABLE */
#else /* HAVE_CMOCKA */
@ -612,4 +626,4 @@ main(void) {
return (0);
}
#endif
#endif /* HAVE_CMOCKA */

View file

@ -1,21 +1,7 @@
These unit tests for BIND 9 are based on the NetBSD Automated Test Framework
release 0.17.
Unit tests for BIND 9 are based on cmocka and test execution engine Kyua.
To build an external copy of ATF for use by BIND 9:
If your distribution of choice doesn't include packages for kyua or cmocka, the
sources can be found here:
$ cd atf-src
$ configure --prefix=<prefix> --enable-tools --disable-shared
$ make
$ make install
Subsequently, specify the ATF prefix when building BIND 9:
$ configure --with-atf=<prefix>
ATF can also be built automatically during the BIND 9 build,
by specifying --with-atf without an argument:
$ configure --with-atf
This causes BIND 9 to build ATF in the atf-src directory and
link to it directly.
* Kyua 0.13 - https://github.com/jmmv/kyua/releases
* CMocka 1.0 - https://cmocka.org/files/

View file

@ -1,29 +1,27 @@
#!/bin/sh
PATH="@ATFBIN@:${PATH}"
export PATH
KYUA=`command -v kyua 2>/dev/null`
KYUA=@KYUA@
UNITTESTS=@UNITTESTS@
CMOCKA_MESSAGE_OUTPUT=TAP
export CMOCKA_MESSAGE_OUTPUT
status=0
if [ -n "@UNITTESTS@" -a -x "$KYUA" -a -f Kyuafile ]
if [ -n "${UNITTESTS}" ] && [ -f Kyuafile ]
then
echo "S:unit:`date`"
echo "S:unit:$(date)"
echo "T:unit:1:A"
echo "I: unit tests (using kyua)"
$KYUA -v parallelism=${TEST_PARALLEL_JOBS:-1} --logfile kyua.log --loglevel debug test --results-file ${KYUA_RESULT:-NEW}
${KYUA} -v parallelism="${TEST_PARALLEL_JOBS:-1}" --logfile kyua.log --loglevel debug test --results-file "${KYUA_RESULT:-NEW}"
status=$?
$KYUA report --results-file ${KYUA_RESULT:-LATEST}
${KYUA} report --results-file "${KYUA_RESULT:-LATEST}"
if [ $status -eq 0 ]
if [ "${status}" -eq "0" ]
then
rm -f kyua.log
echo R:PASS
echo "R:PASS"
else
echo R:FAIL
echo "R:FAIL"
fi
echo "E:unit:`date`"
echo "E:unit:$(date)"
fi
exit $status
exit ${status}

View file

@ -68,8 +68,6 @@ while (<CHANGES>) {
# Contributed code should maintain its own copyright.
if ($base =~ /\.\/contrib\//) {
$file_types{$_} = "X";
} elsif ($base =~ /\.\/unit\/atf-src\//) {
$file_types{$_} = "X";
} elsif ($base =~ /\/openssl-[a-z0-9.]*-patch$/) {
$file_types{$_} = "X";
} elsif ($base =~ /\.(c|h|css)$/) {
@ -109,7 +107,7 @@ while (<CHANGES>) {
$file_types{$_} = "CONF-SH";
} elsif ($base =~ /\.md$/) {
$file_types{$_} = "MKD";
} elsif ($base =~ /(\/\.(gitignore|gitattributes)|Atffile|Kyuafile|\.(gif|jpg))$/i) {
} elsif ($base =~ /(\/\.(gitignore|gitattributes)|Kyuafile|\.(gif|jpg))$/i) {
$file_types{$_} = "X";
} elsif ($base =~ /\.(def|dep|dsp|dsw|mak|sln)$/i) {
$file_types{$_} = "X";

View file

@ -94,7 +94,3 @@ void isc__mempool_put(void *mem, void *ptr FLARG) {
if (!mem) __coverity_panic__();
__coverity_free__(ptr);
}
void atf_tc_fail_requirement(const char *, const size_t, const char *, ...) {
__coverity_panic__();
}