upstream/opnsense-src
1
0
Fork 0
mirror of https://github.com/opnsense/src.git synced 2026-06-05 06:42:56 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

ipsec_offload: kernel infrastructure

Browse source 
Inline IPSEC offload moves almost whole IPSEC processing from the
CPU/MCU and possibly crypto accelerator, to the network card.

The transmitted packet content is not touched by CPU during TX
operations, kernel only does the required policy and security
association lookups to find out that given flow is offloaded, and then
packet is transmitted as plain text to the card. For driver convenience,
a metadata is attached to the packet identifying SA which must process
the packet. Card does encryption of the payload, padding, calculates
authentication, and does the reformat according to the policy.

Similarly, on receive, card does the decapsulation, decryption, and
authentification.  Kernel receives the identifier of SA that was
used to process the packet, together with the plain-text packet.

Overall, payload octets are only read or written by card DMA engine,
removing a lot of memory subsystem overhead, and saving CPU time because
IPSEC algos calculations are avoided.

If driver declares support for inline IPSEC offload (with the
IFCAP2_IPSEC_OFFLOAD capability set and registering method table struct
if_ipsec_accel_methods), kernel offers the SPD and SAD to driver.
Driver decides which policies and SAs can be offloaded based on
hardware capacity, and acks/nacks each SA for given interface to
kernel.  Kernel needs to keep this information to make a decision to
skip software processing on TX, and to assume processing already done
on RX.  This shadow SPD/SAD database of offloads is rooted from
policies (struct secpolicy accel_ifps, struct ifp_handle_sp) and SAs
(struct secasvar accel_ipfs, struct ifp_handle_sav).

Some extensions to the PF_KEY socket allow to limit interfaces for
which given SP/SA could be offloaded (proposed for offload).  Also,
additional statistics extensions allow to observe allocation/octet/use
counters for specific SA.

Since SPs and SAs are typically instantiated in non-sleepable context,
while offloading them into card is expected to require costly async
manipulations of the card state, calls to the driver for offload and
termination are executed in the threaded taskqueue.  It also solves
the issue of allocating resources needed for the offload database.
Neither ipf_handle_sp nor ipf_handle_sav do not add reference to the
owning SP/SA, the offload must be terminated before last reference is
dropped.  ipsec_accel only adds transient references to ensure safe
pointer ownership by taskqueue.

Maintaining the SA counters for hardware-accelerated packets is the
duty of the driver.  The helper ipsec_accel_drv_sa_lifetime_update()
is provided to hide accel infrastructure from drivers which would use
expected callout to query hardware periodically for updates.

Reviewed by:	rscheff	(transport, stack integration), np
Sponsored by:	NVIDIA networking
Differential revision:	https://reviews.freebsd.org/D44219
This commit is contained in:
Konstantin Belousov 2021-08-22 22:38:04 +03:00
parent b6919741b7
commit ef2a572bf6
14 changed files with 1630 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
sys/conf/files
View file

@ -4464,6 +4464,8 @@ netipsec/ipsec.c optional ipsec inet | ipsec inet6
netipsec/ipsec_input.c optional ipsec inet | ipsec inet6
netipsec/ipsec_mbuf.c optional ipsec inet | ipsec inet6
netipsec/ipsec_mod.c optional ipsec inet | ipsec inet6
netipsec/ipsec_offload.c optional ipsec ipsec_offload inet | \
ipsec ipsec_offload inet6
netipsec/ipsec_output.c optional ipsec inet | ipsec inet6
netipsec/ipsec_pcb.c optional ipsec inet | ipsec inet6 | \
ipsec_support inet | ipsec_support inet6

1
sys/conf/options
View file

@ -466,6 +466,7 @@ IPFIREWALL_PMOD opt_ipfw.h
IPSEC opt_ipsec.h
IPSEC_DEBUG opt_ipsec.h
IPSEC_SUPPORT opt_ipsec.h
IPSEC_OFFLOAD opt_ipsec.h
IPSTEALTH
KERN_TLS
KRPC

5
sys/modules/ipsec/Makefile
View file

@ -2,8 +2,9 @@
.PATH: ${SRCTOP}/sys/net ${SRCTOP}/sys/netipsec
KMOD= ipsec
SRCS= if_ipsec.c ipsec.c ipsec_input.c ipsec_mbuf.c ipsec_mod.c \
ipsec_output.c xform_ah.c xform_esp.c xform_ipcomp.c \
SRCS= if_ipsec.c ipsec.c ipsec_input.c ipsec_mbuf.c \
ipsec_mod.c ipsec_offload.c ipsec_output.c \
xform_ah.c xform_esp.c xform_ipcomp.c \
opt_inet.h opt_inet6.h opt_ipsec.h opt_kern_tls.h opt_sctp.h
.if "${MK_INET}" != "no" || "${MK_INET6}" != "no"
SRCS+= udpencap.c

17
sys/netipsec/ipsec.c
View file

@ -85,6 +85,7 @@
#ifdef INET6
#include <netipsec/ipsec6.h>
#endif
#include <netipsec/ipsec_offload.h>
#include <netipsec/ah_var.h>
#include <netipsec/esp_var.h>
#include <netipsec/ipcomp.h> /*XXX*/
@ -636,8 +637,16 @@ int
ipsec4_in_reject(const struct mbuf *m, struct inpcb *inp)
{
struct secpolicy *sp;
#ifdef IPSEC_OFFLOAD
struct ipsec_accel_in_tag *tag;
#endif
int result;
#ifdef IPSEC_OFFLOAD
tag = ipsec_accel_input_tag_lookup(m);
if (tag != NULL)
return (0);
#endif
sp = ipsec4_getpolicy(m, inp, IPSEC_DIR_INBOUND, 0);
result = ipsec_in_reject(sp, inp, m);
key_freesp(&sp);
@ -802,8 +811,16 @@ int
ipsec6_in_reject(const struct mbuf *m, struct inpcb *inp)
{
struct secpolicy *sp;
#ifdef IPSEC_OFFLOAD
struct ipsec_accel_in_tag *tag;
#endif
int result;
#ifdef IPSEC_OFFLOAD
tag = ipsec_accel_input_tag_lookup(m);
if (tag != NULL)
return (0);
#endif
sp = ipsec6_getpolicy(m, inp, IPSEC_DIR_INBOUND, 0);
result = ipsec_in_reject(sp, inp, m);
key_freesp(&sp);

11
sys/netipsec/ipsec.h
View file

@ -71,6 +71,12 @@ struct ipsecrequest {
u_int level; /* IPsec level defined below. */
};
struct ipsec_accel_adddel_sp_tq {
struct vnet *adddel_vnet;
struct task adddel_task;
int adddel_scheduled;
};
/* Security Policy Data Base */
struct secpolicy {
TAILQ_ENTRY(secpolicy) chain;
@ -102,6 +108,11 @@ struct secpolicy {
time_t lastused; /* updated every when kernel sends a packet */
long lifetime; /* duration of the lifetime of this policy */
long validtime; /* duration this policy is valid without use */
CK_LIST_HEAD(, ifp_handle_sp) accel_ifps;
struct ipsec_accel_adddel_sp_tq accel_add_tq;
struct ipsec_accel_adddel_sp_tq accel_del_tq;
struct inpcb *ipsec_accel_add_sp_inp;
const char *accel_ifname;
};
/*

11
sys/netipsec/ipsec_input.c
View file

@ -90,6 +90,7 @@
#include <netipsec/esp.h>
#include <netipsec/esp_var.h>
#include <netipsec/ipcomp_var.h>
#include <netipsec/ipsec_offload.h>
#include <netipsec/key.h>
#include <netipsec/keydb.h>
@ -237,6 +238,11 @@ ipsec_common_input(struct mbuf *m, int skip, int protoff, int af, int sproto)
int
ipsec4_input(struct mbuf *m, int offset, int proto)
{
int error;
error = ipsec_accel_input(m, offset, proto);
if (error != ENXIO)
return (error);
switch (proto) {
case IPPROTO_AH:
@ -536,7 +542,12 @@ ipsec6_lasthdr(int proto)
int
ipsec6_input(struct mbuf *m, int offset, int proto)
{
int error;
error = ipsec_accel_input(m, offset, proto);
if (error != ENXIO)
return (error);
switch (proto) {
case IPPROTO_AH:
case IPPROTO_ESP:

1061
sys/netipsec/ipsec_offload.c Normal file
View file

File diff suppressed because it is too large Load diff

191
sys/netipsec/ipsec_offload.h Normal file
View file

@ -0,0 +1,191 @@
/*-
* Copyright (c) 2021,2022 NVIDIA CORPORATION & AFFILIATES. ALL RIGHTS RESERVED.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#ifndef _NETIPSEC_IPSEC_OFFLOAD_H_
#define _NETIPSEC_IPSEC_OFFLOAD_H_
#ifdef _KERNEL
#include <sys/errno.h>
#include <net/if.h>
#include <net/if_var.h>
struct secpolicy;
struct secasvar;
struct inpcb;
struct ipsec_accel_out_tag {
struct m_tag tag;
uint16_t drv_spi;
};
struct ipsec_accel_in_tag {
struct m_tag tag;
uint16_t drv_spi;
};
#define IPSEC_ACCEL_DRV_SPI_BYPASS 2
#define IPSEC_ACCEL_DRV_SPI_MIN 3
#define IPSEC_ACCEL_DRV_SPI_MAX 0xffff
extern void (*ipsec_accel_sa_newkey_p)(struct secasvar *sav);
extern void (*ipsec_accel_sa_install_input_p)(struct secasvar *sav,
const union sockaddr_union *dst_address, int sproto, uint32_t spi);
extern void (*ipsec_accel_forget_sav_p)(struct secasvar *sav);
extern void (*ipsec_accel_spdadd_p)(struct secpolicy *sp, struct inpcb *inp);
extern void (*ipsec_accel_spddel_p)(struct secpolicy *sp);
extern int (*ipsec_accel_sa_lifetime_op_p)(struct secasvar *sav,
struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
struct rm_priotracker *sahtree_trackerp);
extern void (*ipsec_accel_sync_p)(void);
extern bool (*ipsec_accel_is_accel_sav_p)(struct secasvar *sav);
extern struct mbuf *(*ipsec_accel_key_setaccelif_p)(struct secasvar *sav);
#ifdef IPSEC_OFFLOAD
/*
* Have to use ipsec_accel_sa_install_input_p indirection because
* key.c is unconditionally included into the static kernel.
*/
static inline void
ipsec_accel_sa_newkey(struct secasvar *sav)
{
void (*p)(struct secasvar *sav);
p = atomic_load_ptr(&ipsec_accel_sa_newkey_p);
if (p != NULL)
p(sav);
}
static inline void
ipsec_accel_forget_sav(struct secasvar *sav)
{
void (*p)(struct secasvar *sav);
p = atomic_load_ptr(&ipsec_accel_forget_sav_p);
if (p != NULL)
p(sav);
}
static inline void
ipsec_accel_spdadd(struct secpolicy *sp, struct inpcb *inp)
{
void (*p)(struct secpolicy *sp, struct inpcb *inp);
p = atomic_load_ptr(&ipsec_accel_spdadd_p);
if (p != NULL)
p(sp, inp);
}
static inline void
ipsec_accel_spddel(struct secpolicy *sp)
{
void (*p)(struct secpolicy *sp);
p = atomic_load_ptr(&ipsec_accel_spddel_p);
if (p != NULL)
p(sp);
}
static inline int
ipsec_accel_sa_lifetime_op(struct secasvar *sav,
struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
struct rm_priotracker *sahtree_trackerp)
{
int (*p)(struct secasvar *sav, struct seclifetime *lft_c, if_t ifp,
enum IF_SA_CNT_WHICH op, struct rm_priotracker *sahtree_trackerp);
p = atomic_load_ptr(&ipsec_accel_sa_lifetime_op_p);
if (p != NULL)
return (p(sav, lft_c, ifp, op, sahtree_trackerp));
return (ENOTSUP);
}
static inline void
ipsec_accel_sync(void)
{
void (*p)(void);
p = atomic_load_ptr(&ipsec_accel_sync_p);
if (p != NULL)
p();
}
static inline bool
ipsec_accel_is_accel_sav(struct secasvar *sav)
{
bool (*p)(struct secasvar *sav);
p = atomic_load_ptr(&ipsec_accel_is_accel_sav_p);
if (p != NULL)
return (p(sav));
return (false);
}
static inline struct mbuf *
ipsec_accel_key_setaccelif(struct secasvar *sav)
{
struct mbuf *(*p)(struct secasvar *sav);
p = atomic_load_ptr(&ipsec_accel_key_setaccelif_p);
if (p != NULL)
return (p(sav));
return (NULL);
}
#else
#define ipsec_accel_sa_newkey(a)
#define ipsec_accel_forget_sav(a)
#define ipsec_accel_spdadd(a, b)
#define ipsec_accel_spddel(a)
#define ipsec_accel_sa_lifetime_op(a, b, c, d, e)
#define ipsec_accel_sync()
#define ipsec_accel_is_accel_sav(a)
#define ipsec_accel_key_setaccelif(a)
#endif
void ipsec_accel_forget_sav_impl(struct secasvar *sav);
void ipsec_accel_spdadd_impl(struct secpolicy *sp, struct inpcb *inp);
void ipsec_accel_spddel_impl(struct secpolicy *sp);
#ifdef IPSEC_OFFLOAD
int ipsec_accel_input(struct mbuf *m, int offset, int proto);
bool ipsec_accel_output(struct ifnet *ifp, struct mbuf *m,
struct inpcb *inp, struct secpolicy *sp, struct secasvar *sav, int af,
int mtu);
void ipsec_accel_forget_sav(struct secasvar *sav);
#else
#define ipsec_accel_input(a, b, c) (ENXIO)
#define ipsec_accel_output(a, b, c, d, e, f, g) (false)
#define ipsec_accel_forget_sav(a)
#endif
struct ipsec_accel_in_tag *ipsec_accel_input_tag_lookup(const struct mbuf *);
void ipsec_accel_on_ifdown(struct ifnet *ifp);
void ipsec_accel_drv_sa_lifetime_update(struct secasvar *sav, if_t ifp,
u_int drv_spi, uint64_t octets, uint64_t allocs);
#endif /* _KERNEL */
#endif /* _NETIPSEC_IPSEC_OFFLOAD_H_ */

15
sys/netipsec/ipsec_output.c
View file

@ -84,6 +84,7 @@
#include <netipsec/ipsec6.h>
#endif
#include <netipsec/ipsec_support.h>
#include <netipsec/ipsec_offload.h>
#include <netipsec/ah_var.h>
#include <netipsec/esp_var.h>
#include <netipsec/ipcomp_var.h>
@ -210,6 +211,8 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
sav = ipsec4_allocsa(ifp, m, sp, &idx, &error);
if (sav == NULL) {
if (error == EJUSTRETURN) { /* No IPsec required */
(void)ipsec_accel_output(ifp, m, inp, sp, NULL,
AF_INET, mtu);
key_freesp(&sp);
return (error);
}
@ -222,6 +225,9 @@ ipsec4_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
goto bad;
if (ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET, mtu))
return (EJUSTRETURN);
ip = mtod(m, struct ip *);
dst = &sav->sah->saidx.dst;
/* Do the appropriate encapsulation, if necessary */
@ -597,6 +603,8 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
sav = ipsec6_allocsa(ifp, m, sp, &idx, &error);
if (sav == NULL) {
if (error == EJUSTRETURN) { /* No IPsec required */
(void)ipsec_accel_output(ifp, m, inp, sp, NULL,
AF_INET6, mtu);
key_freesp(&sp);
return (error);
}
@ -611,6 +619,9 @@ ipsec6_perform_request(struct ifnet *ifp, struct mbuf *m, struct secpolicy *sp,
if ((error = ipsec_run_hhooks(&ctx, HHOOK_TYPE_IPSEC_OUT)) != 0)
goto bad;
if (ipsec_accel_output(ifp, m, inp, sp, sav, AF_INET6, mtu))
return (EJUSTRETURN);
ip6 = mtod(m, struct ip6_hdr *); /* pfil can change mbuf */
dst = &sav->sah->saidx.dst;
@ -859,6 +870,10 @@ ipsec_process_done(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
struct m_tag *mtag;
int error;
if (sav->state >= SADB_SASTATE_DEAD) {
error = ESRCH;
goto bad;
}
saidx = &sav->sah->saidx;
switch (saidx->dst.sa.sa_family) {
#ifdef INET

42
sys/netipsec/ipsec_pcb.c
View file

@ -49,6 +49,7 @@
#include <netipsec/ipsec_support.h>
#include <netipsec/key.h>
#include <netipsec/key_debug.h>
#include <netipsec/ipsec_offload.h>
MALLOC_DEFINE(M_IPSEC_INPCB, "inpcbpolicy", "inpcb-resident ipsec policy");
@ -166,18 +167,26 @@ ipsec_init_pcbpolicy(struct inpcb *inp)
int
ipsec_delete_pcbpolicy(struct inpcb *inp)
{
struct inpcbpolicy *inp_sp;
if (inp->inp_sp == NULL)
inp_sp = inp->inp_sp;
if (inp_sp == NULL)
return (0);
if (inp->inp_sp->sp_in != NULL)
key_freesp(&inp->inp_sp->sp_in);
if (inp->inp_sp->sp_out != NULL)
key_freesp(&inp->inp_sp->sp_out);
free(inp->inp_sp, M_IPSEC_INPCB);
inp->inp_sp = NULL;
if (inp_sp->sp_in != NULL) {
if ((inp_sp->flags & INP_INBOUND_POLICY) != 0)
ipsec_accel_spddel(inp_sp->sp_in);
key_freesp(&inp_sp->sp_in);
}
if (inp_sp->sp_out != NULL) {
if ((inp_sp->flags & INP_OUTBOUND_POLICY) != 0)
ipsec_accel_spddel(inp_sp->sp_out);
key_freesp(&inp_sp->sp_out);
}
free(inp_sp, M_IPSEC_INPCB);
return (0);
}
@ -248,20 +257,26 @@ ipsec_copy_pcbpolicy(struct inpcb *old, struct inpcb *new)
if (sp == NULL)
return (ENOBUFS);
ipsec_setspidx_inpcb(new, &sp->spidx, IPSEC_DIR_INBOUND);
if (new->inp_sp->sp_in != NULL)
if (new->inp_sp->sp_in != NULL) {
ipsec_accel_spddel(new->inp_sp->sp_in);
key_freesp(&new->inp_sp->sp_in);
}
new->inp_sp->sp_in = sp;
new->inp_sp->flags |= INP_INBOUND_POLICY;
ipsec_accel_spdadd(sp, new);
}
if (old->inp_sp->flags & INP_OUTBOUND_POLICY) {
sp = ipsec_deepcopy_pcbpolicy(old->inp_sp->sp_out);
if (sp == NULL)
return (ENOBUFS);
ipsec_setspidx_inpcb(new, &sp->spidx, IPSEC_DIR_OUTBOUND);
if (new->inp_sp->sp_out != NULL)
if (new->inp_sp->sp_out != NULL) {
ipsec_accel_spddel(new->inp_sp->sp_out);
key_freesp(&new->inp_sp->sp_out);
}
new->inp_sp->sp_out = sp;
new->inp_sp->flags |= INP_OUTBOUND_POLICY;
ipsec_accel_spdadd(sp, new);
}
return (0);
}
@ -339,8 +354,10 @@ ipsec_set_pcbpolicy(struct inpcb *inp, struct ucred *cred,
flags = INP_OUTBOUND_POLICY;
}
/* Clear old SP and set new SP. */
if (*spp != NULL)
if (*spp != NULL) {
ipsec_accel_spddel(*spp);
key_freesp(spp);
}
*spp = newsp;
KEYDBG(IPSEC_DUMP,
printf("%s: new SP(%p)\n", __func__, newsp));
@ -348,6 +365,7 @@ ipsec_set_pcbpolicy(struct inpcb *inp, struct ucred *cred,
inp->inp_sp->flags &= ~flags;
else {
inp->inp_sp->flags |= flags;
ipsec_accel_spdadd(newsp, inp);
KEYDBG(IPSEC_DUMP, kdebug_secpolicy(newsp));
}
INP_WUNLOCK(inp);

270
sys/netipsec/key.c
View file

@ -83,6 +83,7 @@
#include <netipsec/key.h>
#include <netipsec/keysock.h>
#include <netipsec/key_debug.h>
#include <netipsec/ipsec_offload.h>
#include <netipsec/ipsec.h>
#ifdef INET6
@ -90,12 +91,26 @@
#endif
#include <netipsec/xform.h>
#include <netipsec/ipsec_offload.h>
#include <machine/in_cksum.h>
#include <machine/stdarg.h>
/* randomness */
#include <sys/random.h>
#ifdef IPSEC_OFFLOAD
void (*ipsec_accel_sa_newkey_p)(struct secasvar *sav);
void (*ipsec_accel_forget_sav_p)(struct secasvar *sav);
void (*ipsec_accel_spdadd_p)(struct secpolicy *sp, struct inpcb *inp);
void (*ipsec_accel_spddel_p)(struct secpolicy *sp);
int (*ipsec_accel_sa_lifetime_op_p)(struct secasvar *sav,
struct seclifetime *lft_c, if_t ifp, enum IF_SA_CNT_WHICH op,
struct rm_priotracker *sahtree_trackerp);
void (*ipsec_accel_sync_p)(void);
bool (*ipsec_accel_is_accel_sav_p)(struct secasvar *sav);
struct mbuf *(*ipsec_accel_key_setaccelif_p)(struct secasvar *sav);
#endif
#define FULLMASK 0xff
#define _BITS(bytes) ((bytes) << 3)
@ -391,6 +406,9 @@ static const int minsize[] = {
[SADB_X_EXT_SA_REPLAY] = sizeof(struct sadb_x_sa_replay),
[SADB_X_EXT_NEW_ADDRESS_SRC] = sizeof(struct sadb_address),
[SADB_X_EXT_NEW_ADDRESS_DST] = sizeof(struct sadb_address),
[SADB_X_EXT_LFT_CUR_SW_OFFL] = sizeof(struct sadb_lifetime),
[SADB_X_EXT_LFT_CUR_HW_OFFL] = sizeof(struct sadb_lifetime),
[SADB_X_EXT_IF_HW_OFFL] = sizeof(struct sadb_x_if_hw_offl),
};
_Static_assert(nitems(minsize) == SADB_EXT_MAX + 1, "minsize size mismatch");
@ -424,6 +442,9 @@ static const int maxsize[] = {
[SADB_X_EXT_SA_REPLAY] = sizeof(struct sadb_x_sa_replay),
[SADB_X_EXT_NEW_ADDRESS_SRC] = 0,
[SADB_X_EXT_NEW_ADDRESS_DST] = 0,
[SADB_X_EXT_LFT_CUR_SW_OFFL] = sizeof(struct sadb_lifetime),
[SADB_X_EXT_LFT_CUR_HW_OFFL] = sizeof(struct sadb_lifetime),
[SADB_X_EXT_IF_HW_OFFL] = sizeof(struct sadb_x_if_hw_offl),
};
_Static_assert(nitems(maxsize) == SADB_EXT_MAX + 1, "maxsize size mismatch");
@ -661,7 +682,7 @@ static int key_updateaddresses(struct socket *, struct mbuf *,
const struct sadb_msghdr *, struct secasvar *, struct secasindex *);
static struct mbuf *key_setdumpsa(struct secasvar *, u_int8_t,
u_int8_t, u_int32_t, u_int32_t);
u_int8_t, u_int32_t, u_int32_t, struct rm_priotracker *);
static struct mbuf *key_setsadbmsg(u_int8_t, u_int16_t, u_int8_t,
u_int32_t, pid_t, u_int16_t);
static struct mbuf *key_setsadbsa(struct secasvar *);
@ -1227,6 +1248,11 @@ key_freesp(struct secpolicy **spp)
KEYDBG(IPSEC_DATA, kdebug_secpolicy(sp));
*spp = NULL;
#ifdef IPSEC_OFFLOAD
KASSERT(CK_LIST_EMPTY(&sp->accel_ifps),
("key_freesp: sp %p still offloaded", sp));
free(__DECONST(char *, sp->accel_ifname), M_IPSEC_MISC);
#endif
while (sp->tcount > 0)
ipsec_delisr(sp->req[--sp->tcount]);
free(sp, M_IPSEC_SP);
@ -1240,6 +1266,7 @@ key_unlink(struct secpolicy *sp)
SPTREE_WUNLOCK();
if (SPDCACHE_ENABLED())
spdcache_clear();
ipsec_accel_sync();
key_freesp(&sp);
}
@ -1258,6 +1285,7 @@ key_detach(struct secpolicy *sp)
return;
}
sp->state = IPSEC_SPSTATE_DEAD;
ipsec_accel_spddel(sp);
TAILQ_REMOVE(&V_sptree[sp->spidx.dir], sp, chain);
V_spd_size--;
LIST_REMOVE(sp, idhash);
@ -1285,6 +1313,7 @@ done:
newsp->state = IPSEC_SPSTATE_ALIVE;
V_spd_size++;
V_sp_genid++;
ipsec_accel_spdadd(newsp, NULL);
}
/*
@ -1329,6 +1358,7 @@ key_register_ifnet(struct secpolicy **spp, u_int count)
*/
LIST_INSERT_HEAD(SPHASH_HASH(spp[i]->id), spp[i], idhash);
spp[i]->state = IPSEC_SPSTATE_IFNET;
ipsec_accel_spdadd(spp[i], NULL);
}
SPTREE_WUNLOCK();
/*
@ -1357,6 +1387,7 @@ key_unregister_ifnet(struct secpolicy **spp, u_int count)
if (spp[i]->state != IPSEC_SPSTATE_IFNET)
continue;
spp[i]->state = IPSEC_SPSTATE_DEAD;
ipsec_accel_spddel(spp[i]);
TAILQ_REMOVE(&V_sptree_ifnet[spp[i]->spidx.dir],
spp[i], chain);
V_spd_size--;
@ -1365,6 +1396,7 @@ key_unregister_ifnet(struct secpolicy **spp, u_int count)
SPTREE_WUNLOCK();
if (SPDCACHE_ENABLED())
spdcache_clear();
ipsec_accel_sync();
for (i = 0; i < count; i++) {
m = key_setdumpsp(spp[i], SADB_X_SPDDELETE, 0, 0);
@ -1424,6 +1456,7 @@ key_unlinksav(struct secasvar *sav)
/* Unlink from SPI hash */
LIST_REMOVE(sav, spihash);
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
sah = sav->sah;
SAHTREE_WUNLOCK();
key_freesav(&sav);
@ -1821,6 +1854,9 @@ key_sp2msg(struct secpolicy *sp, void *request, size_t *len)
size_t xlen, ilen;
caddr_t p;
int error, i;
#ifdef IPSEC_OFFLOAD
struct sadb_x_if_hw_offl *xif;
#endif
IPSEC_ASSERT(sp != NULL, ("null policy"));
@ -1876,6 +1912,18 @@ key_sp2msg(struct secpolicy *sp, void *request, size_t *len)
}
}
xpl->sadb_x_policy_len = PFKEY_UNIT64(xlen);
#ifdef IPSEC_OFFLOAD
if (error == 0 && sp->accel_ifname != NULL) {
xif = (struct sadb_x_if_hw_offl *)(xpl + 1);
bzero(xif, sizeof(*xif));
xif->sadb_x_if_hw_offl_len = PFKEY_UNIT64(sizeof(*xif));
xif->sadb_x_if_hw_offl_exttype = SADB_X_EXT_IF_HW_OFFL;
xif->sadb_x_if_hw_offl_flags = 0;
strncpy(xif->sadb_x_if_hw_offl_if, sp->accel_ifname,
sizeof(xif->sadb_x_if_hw_offl_if));
xlen += sizeof(*xif);
}
#endif
if (error == 0)
*len = xlen;
else
@ -2088,6 +2136,27 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
newsp->lifetime = lft ? lft->sadb_lifetime_addtime : 0;
newsp->validtime = lft ? lft->sadb_lifetime_usetime : 0;
bcopy(&spidx, &newsp->spidx, sizeof(spidx));
#ifdef IPSEC_OFFLOAD
if (!SADB_CHECKHDR(mhp, SADB_X_EXT_IF_HW_OFFL) &&
!SADB_CHECKLEN(mhp, SADB_X_EXT_IF_HW_OFFL)) {
struct sadb_x_if_hw_offl *xof;
xof = (struct sadb_x_if_hw_offl *)mhp->ext[
SADB_X_EXT_IF_HW_OFFL];
newsp->accel_ifname = malloc(sizeof(xof->sadb_x_if_hw_offl_if),
M_IPSEC_MISC, M_NOWAIT);
if (newsp->accel_ifname == NULL) {
ipseclog((LOG_DEBUG, "%s: cannot alloc accel_ifname.\n",
__func__));
key_freesp(&newsp);
return (key_senderror(so, m, error));
}
strncpy(__DECONST(char *, newsp->accel_ifname),
xof->sadb_x_if_hw_offl_if,
sizeof(xof->sadb_x_if_hw_offl_if));
}
#endif
SPTREE_WLOCK();
if ((newsp->id = key_getnewspid()) == 0) {
@ -2095,6 +2164,7 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
key_detach(oldsp);
SPTREE_WUNLOCK();
if (oldsp != NULL) {
ipsec_accel_sync();
key_freesp(&oldsp); /* first for key_detach */
IPSEC_ASSERT(oldsp != NULL, ("null oldsp: refcount bug"));
key_freesp(&oldsp); /* second for our reference */
@ -2109,6 +2179,7 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
key_insertsp(newsp);
SPTREE_WUNLOCK();
if (oldsp != NULL) {
ipsec_accel_sync();
key_freesp(&oldsp); /* first for key_detach */
IPSEC_ASSERT(oldsp != NULL, ("null oldsp: refcount bug"));
key_freesp(&oldsp); /* second for our reference */
@ -2290,6 +2361,7 @@ key_spddelete(struct socket *so, struct mbuf *m,
KEYDBG(KEY_STAMP,
printf("%s: SP(%p)\n", __func__, sp));
KEYDBG(KEY_DATA, kdebug_secpolicy(sp));
ipsec_accel_spddel(sp);
key_unlink(sp);
key_freesp(&sp);
@ -2561,6 +2633,7 @@ key_spdflush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
*/
TAILQ_FOREACH(sp, &drainq, chain) {
sp->state = IPSEC_SPSTATE_DEAD;
ipsec_accel_spddel(sp);
LIST_REMOVE(sp, idhash);
}
V_sp_genid++;
@ -2764,6 +2837,10 @@ key_getspreqmsglen(struct secpolicy *sp)
tlen += PFKEY_ALIGN8(len);
}
#ifdef IPSEC_OFFLOAD
if (sp->accel_ifname != NULL)
tlen += sizeof(struct sadb_x_if_hw_offl);
#endif
return (tlen);
}
@ -3005,6 +3082,32 @@ key_newsav(const struct sadb_msghdr *mhp, struct secasindex *saidx,
sav->state = SADB_SASTATE_LARVAL;
sav->pid = (pid_t)mhp->msg->sadb_msg_pid;
SAV_INITREF(sav);
#ifdef IPSEC_OFFLOAD
CK_LIST_INIT(&sav->accel_ifps);
sav->accel_forget_tq = 0;
sav->accel_lft_sw = uma_zalloc_pcpu(ipsec_key_lft_zone,
M_NOWAIT | M_ZERO);
if (sav->accel_lft_sw == NULL) {
*errp = ENOBUFS;
goto done;
}
if (!SADB_CHECKHDR(mhp, SADB_X_EXT_IF_HW_OFFL) &&
!SADB_CHECKLEN(mhp, SADB_X_EXT_IF_HW_OFFL)) {
struct sadb_x_if_hw_offl *xof;
xof = (struct sadb_x_if_hw_offl *)mhp->ext[
SADB_X_EXT_IF_HW_OFFL];
sav->accel_ifname = malloc(sizeof(xof->sadb_x_if_hw_offl_if),
M_IPSEC_MISC, M_NOWAIT);
if (sav->accel_ifname == NULL) {
*errp = ENOBUFS;
goto done;
}
strncpy(__DECONST(char *, sav->accel_ifname),
xof->sadb_x_if_hw_offl_if,
sizeof(xof->sadb_x_if_hw_offl_if));
}
#endif
again:
sah = key_getsah(saidx);
if (sah == NULL) {
@ -3068,9 +3171,10 @@ again:
SAH_ADDREF(sah);
}
/* Link SAV with SAH */
if (sav->state == SADB_SASTATE_MATURE)
if (sav->state == SADB_SASTATE_MATURE) {
TAILQ_INSERT_HEAD(&sah->savtree_alive, sav, chain);
else
ipsec_accel_sa_newkey(sav);
} else
TAILQ_INSERT_HEAD(&sah->savtree_larval, sav, chain);
/* Add SAV into SPI hash */
LIST_INSERT_HEAD(SAVHASH_HASH(sav->spi), sav, spihash);
@ -3085,6 +3189,13 @@ done:
}
if (sav->lft_c != NULL)
uma_zfree_pcpu(ipsec_key_lft_zone, sav->lft_c);
#ifdef IPSEC_OFFLOAD
if (sav->accel_lft_sw != NULL)
uma_zfree_pcpu(ipsec_key_lft_zone,
sav->accel_lft_sw);
free(__DECONST(char *, sav->accel_ifname),
M_IPSEC_MISC);
#endif
free(sav, M_IPSEC_SA), sav = NULL;
}
if (sah != NULL)
@ -3153,6 +3264,10 @@ key_delsav(struct secasvar *sav)
("attempt to free non DEAD SA %p", sav));
IPSEC_ASSERT(sav->refcnt == 0, ("reference count %u > 0",
sav->refcnt));
#ifdef IPSEC_OFFLOAD
KASSERT(CK_LIST_EMPTY(&sav->accel_ifps),
("key_unlinksav: sav %p still offloaded", sav));
#endif
/*
* SA must be unlinked from the chain and hashtbl.
@ -3165,6 +3280,11 @@ key_delsav(struct secasvar *sav)
free(sav->lock, M_IPSEC_MISC);
uma_zfree_pcpu(ipsec_key_lft_zone, sav->lft_c);
}
#ifdef IPSEC_OFFLOAD
/* XXXKIB should this be moved to key_cleansav()? */
uma_zfree_pcpu(ipsec_key_lft_zone, sav->accel_lft_sw);
free(__DECONST(char *, sav->accel_ifname), M_IPSEC_MISC);
#endif
free(sav, M_IPSEC_SA);
}
@ -3588,7 +3708,7 @@ fail:
*/
static struct mbuf *
key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
uint32_t seq, uint32_t pid)
uint32_t seq, uint32_t pid, struct rm_priotracker *sahtree_trackerp)
{
struct seclifetime lft_c;
struct mbuf *result = NULL, *tres = NULL, *m;
@ -3604,8 +3724,15 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
SADB_X_EXT_NAT_T_SPORT, SADB_X_EXT_NAT_T_DPORT,
SADB_X_EXT_NAT_T_OAI, SADB_X_EXT_NAT_T_OAR,
SADB_X_EXT_NAT_T_FRAG,
#ifdef IPSEC_OFFLOAD
SADB_X_EXT_LFT_CUR_SW_OFFL, SADB_X_EXT_LFT_CUR_HW_OFFL,
SADB_X_EXT_IF_HW_OFFL,
#endif
};
uint32_t replay_count;
#ifdef IPSEC_OFFLOAD
int error;
#endif
SECASVAR_RLOCK_TRACKER;
@ -3752,6 +3879,44 @@ key_setdumpsa(struct secasvar *sav, uint8_t type, uint8_t satype,
case SADB_X_EXT_NAT_T_FRAG:
/* We do not (yet) support those. */
continue;
#ifdef IPSEC_OFFLOAD
case SADB_X_EXT_LFT_CUR_SW_OFFL:
if (!ipsec_accel_is_accel_sav(sav))
continue;
SAV_ADDREF(sav);
error = ipsec_accel_sa_lifetime_op(sav, &lft_c,
NULL, IF_SA_CNT_TOTAL_SW_VAL, sahtree_trackerp);
if (error != 0) {
m = NULL;
goto fail;
}
m = key_setlifetime(&lft_c, dumporder[i]);
if (m == NULL)
goto fail;
key_freesav(&sav);
if (sav == NULL) {
m_freem(m);
goto fail;
}
break;
case SADB_X_EXT_LFT_CUR_HW_OFFL:
if (!ipsec_accel_is_accel_sav(sav))
continue;
memset(&lft_c, 0, sizeof(lft_c));
lft_c.bytes = sav->accel_hw_octets;
lft_c.allocations = sav->accel_hw_allocs;
m = key_setlifetime(&lft_c, dumporder[i]);
if (m == NULL)
goto fail;
break;
case SADB_X_EXT_IF_HW_OFFL:
if (!ipsec_accel_is_accel_sav(sav))
continue;
m = ipsec_accel_key_setaccelif(sav);
if (m == NULL)
continue; /* benigh */
break;
#endif
case SADB_EXT_ADDRESS_PROXY:
case SADB_EXT_IDENTITY_SRC:
@ -4502,6 +4667,7 @@ key_flush_spd(time_t now)
V_spd_size--;
LIST_REMOVE(sp, idhash);
sp->state = IPSEC_SPSTATE_DEAD;
ipsec_accel_spddel(sp);
sp = nextsp;
}
V_sp_genid++;
@ -4625,6 +4791,7 @@ key_flush_sad(time_t now)
TAILQ_REMOVE(&sav->sah->savtree_larval, sav, chain);
LIST_REMOVE(sav, spihash);
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
sav = nextsav;
}
/* Unlink all SAs with expired HARD lifetime */
@ -4641,6 +4808,7 @@ key_flush_sad(time_t now)
TAILQ_REMOVE(&sav->sah->savtree_alive, sav, chain);
LIST_REMOVE(sav, spihash);
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
sav = nextsav;
}
/* Mark all SAs with expired SOFT lifetime as DYING */
@ -5239,6 +5407,30 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
/* Clone SA's content into newsav */
SAV_INITREF(newsav);
bcopy(sav, newsav, offsetof(struct secasvar, chain));
#ifdef IPSEC_OFFLOAD
CK_LIST_INIT(&newsav->accel_ifps);
newsav->accel_forget_tq = 0;
newsav->accel_lft_sw = uma_zalloc_pcpu(ipsec_key_lft_zone,
M_NOWAIT | M_ZERO);
if (newsav->accel_lft_sw == NULL) {
error = ENOBUFS;
goto fail;
}
if (sav->accel_ifname != NULL) {
struct sadb_x_if_hw_offl xof;
newsav->accel_ifname = malloc(sizeof(xof.sadb_x_if_hw_offl_if),
M_IPSEC_MISC, M_NOWAIT);
if (newsav->accel_ifname == NULL) {
error = ENOBUFS;
goto fail;
}
strncpy(__DECONST(char *, sav->accel_ifname),
newsav->accel_ifname,
sizeof(xof.sadb_x_if_hw_offl_if));
}
#endif
/*
* We create new NAT-T config if it is needed.
* Old NAT-T config will be freed by key_cleansav() when
@ -5269,6 +5461,7 @@ key_updateaddresses(struct socket *so, struct mbuf *m,
TAILQ_REMOVE(&sav->sah->savtree_alive, sav, chain);
LIST_REMOVE(sav, spihash);
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
/*
* Link new SA with SAH. Keep SAs ordered by
@ -5326,6 +5519,10 @@ fail:
if (isnew != 0)
key_freesah(&sah);
if (newsav != NULL) {
#ifdef IPSEC_OFFLOAD
uma_zfree_pcpu(ipsec_key_lft_zone, newsav->accel_lft_sw);
free(__DECONST(char *, newsav->accel_ifname), M_IPSEC_MISC);
#endif
if (newsav->natt != NULL)
free(newsav->natt, M_IPSEC_MISC);
free(newsav, M_IPSEC_SA);
@ -5540,6 +5737,7 @@ key_update(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
KEYDBG(KEY_STAMP,
printf("%s: SA(%p)\n", __func__, sav));
KEYDBG(KEY_DATA, kdebug_secasv(sav));
ipsec_accel_sa_newkey(sav);
key_freesav(&sav);
{
@ -5692,6 +5890,7 @@ key_add(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
KEYDBG(KEY_STAMP,
printf("%s: return SA(%p)\n", __func__, sav));
KEYDBG(KEY_DATA, kdebug_secasv(sav));
ipsec_accel_sa_newkey(sav);
/*
* If SADB_ADD was in response to SADB_ACQUIRE, we need to schedule
* ACQ for deletion.
@ -6196,6 +6395,7 @@ key_delete_all(struct socket *so, struct mbuf *m,
/* Unlink all queued SAs from SPI hash */
TAILQ_FOREACH(sav, &drainq, chain) {
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
LIST_REMOVE(sav, spihash);
}
SAHTREE_WUNLOCK();
@ -6264,6 +6464,7 @@ key_delete_xform(const struct xformsw *xsp)
/* Unlink all queued SAs from SPI hash */
TAILQ_FOREACH(sav, &drainq, chain) {
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
LIST_REMOVE(sav, spihash);
}
SAHTREE_WUNLOCK();
@ -6372,7 +6573,7 @@ key_get(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* create new sadb_msg to reply. */
n = key_setdumpsa(sav, SADB_GET, satype, mhp->msg->sadb_msg_seq,
mhp->msg->sadb_msg_pid);
mhp->msg->sadb_msg_pid, NULL);
key_freesav(&sav);
if (!n)
@ -7614,9 +7815,11 @@ key_flush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
*/
TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
}
TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
}
}
SAHTREE_WUNLOCK();
@ -7638,10 +7841,12 @@ key_flush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
LIST_REMOVE(sav, spihash);
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
}
TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
LIST_REMOVE(sav, spihash);
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
}
/* Add SAH into flushq */
TAILQ_INSERT_HEAD(&flushq, sah, chain);
@ -7705,6 +7910,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* count sav entries to be sent to the userland. */
cnt = 0;
IFNET_RLOCK();
SAHTREE_RLOCK();
TAILQ_FOREACH(sah, &V_sahtree, chain) {
if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC &&
@ -7719,6 +7925,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
if (cnt == 0) {
SAHTREE_RUNLOCK();
IFNET_RUNLOCK();
return key_senderror(so, m, ENOENT);
}
@ -7731,30 +7938,34 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* map proto to satype */
if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
SAHTREE_RUNLOCK();
IFNET_RUNLOCK();
ipseclog((LOG_DEBUG, "%s: there was invalid proto in "
"SAD.\n", __func__));
return key_senderror(so, m, EINVAL);
}
TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
n = key_setdumpsa(sav, SADB_DUMP, satype,
--cnt, mhp->msg->sadb_msg_pid);
--cnt, mhp->msg->sadb_msg_pid, &sahtree_tracker);
if (n == NULL) {
SAHTREE_RUNLOCK();
IFNET_RUNLOCK();
return key_senderror(so, m, ENOBUFS);
}
key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
n = key_setdumpsa(sav, SADB_DUMP, satype,
--cnt, mhp->msg->sadb_msg_pid);
--cnt, mhp->msg->sadb_msg_pid, &sahtree_tracker);
if (n == NULL) {
SAHTREE_RUNLOCK();
IFNET_RUNLOCK();
return key_senderror(so, m, ENOBUFS);
}
key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
}
SAHTREE_RUNLOCK();
IFNET_RUNLOCK();
m_freem(m);
return (0);
}
@ -8175,6 +8386,11 @@ key_align(struct mbuf *m, struct sadb_msghdr *mhp)
case SADB_X_EXT_SA_REPLAY:
case SADB_X_EXT_NEW_ADDRESS_SRC:
case SADB_X_EXT_NEW_ADDRESS_DST:
#ifdef IPSEC_OFFLOAD
case SADB_X_EXT_LFT_CUR_SW_OFFL:
case SADB_X_EXT_LFT_CUR_HW_OFFL:
case SADB_X_EXT_IF_HW_OFFL:
#endif
/* duplicate check */
/*
* XXX Are there duplication payloads of either
@ -8483,9 +8699,11 @@ key_vnet_destroy(void *arg __unused)
sah->state = SADB_SASTATE_DEAD;
TAILQ_FOREACH(sav, &sah->savtree_larval, chain) {
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
}
TAILQ_FOREACH(sav, &sah->savtree_alive, chain) {
sav->state = SADB_SASTATE_DEAD;
ipsec_accel_forget_sav(sav);
}
}
SAHTREE_WUNLOCK();
@ -8633,6 +8851,32 @@ key_setkey(struct seckey *src, uint16_t exttype)
return m;
}
#ifdef IPSEC_OFFLOAD
struct mbuf *
key_setaccelif(const char *ifname)
{
struct mbuf *m = NULL;
struct sadb_x_if_hw_offl *p;
int len = PFKEY_ALIGN8(sizeof(*p));
m = m_get2(len, M_NOWAIT, MT_DATA, 0);
if (m == NULL)
return (m);
m_align(m, len);
m->m_len = len;
p = mtod(m, struct sadb_x_if_hw_offl *);
bzero(p, len);
p->sadb_x_if_hw_offl_len = PFKEY_UNIT64(len);
p->sadb_x_if_hw_offl_exttype = SADB_X_EXT_IF_HW_OFFL;
p->sadb_x_if_hw_offl_flags = 0;
strncpy(p->sadb_x_if_hw_offl_if, ifname,
sizeof(p->sadb_x_if_hw_offl_if));
return (m);
}
#endif
/*
* Take one of the kernel's lifetime data structures and convert it
* into a PF_KEY structure within an mbuf, suitable for sending up to
@ -8708,3 +8952,15 @@ comp_algorithm_lookup(int alg)
return (supported_calgs[i].xform);
return (NULL);
}
void
ipsec_sahtree_runlock(struct rm_priotracker *sahtree_trackerp)
{
rm_runlock(&sahtree_lock, sahtree_trackerp);
}
void
ipsec_sahtree_rlock(struct rm_priotracker *sahtree_trackerp)
{
rm_rlock(&sahtree_lock, sahtree_trackerp);
}

6
sys/netipsec/key.h
View file

@ -36,6 +36,7 @@
#ifdef _KERNEL
struct mbuf;
struct secpolicy;
struct secpolicyindex;
struct secasvar;
@ -60,6 +61,7 @@ int key_havesp_any(void);
void key_bumpspgen(void);
uint32_t key_getspgen(void);
uint32_t key_newreqid(void);
struct mbuf *key_setaccelif(const char *ifname);
struct secasvar *key_allocsa(union sockaddr_union *, uint8_t, uint32_t);
struct secasvar *key_allocsa_tunnel(union sockaddr_union *,
@ -85,6 +87,10 @@ extern void key_sa_recordxfer(struct secasvar *, struct mbuf *);
uint16_t key_portfromsaddr(struct sockaddr *);
void key_porttosaddr(struct sockaddr *, uint16_t port);
struct rm_priotracker;
void ipsec_sahtree_runlock(struct rm_priotracker *);
void ipsec_sahtree_rlock(struct rm_priotracker *);
#ifdef MALLOC_DECLARE
MALLOC_DECLARE(M_IPSEC_SA);
MALLOC_DECLARE(M_IPSEC_SAH);

5
sys/netipsec/key_debug.c
View file

@ -155,6 +155,8 @@ kdebug_sadb_exttype(uint16_t type)
X_NAME(SA_REPLAY);
X_NAME(NEW_ADDRESS_SRC);
X_NAME(NEW_ADDRESS_DST);
X_NAME(LFT_CUR_SW_OFFL);
X_NAME(LFT_CUR_HW_OFFL);
default:
return ("UNKNOWN");
};
@ -251,6 +253,9 @@ kdebug_sadb(struct sadb_msg *base)
case SADB_X_EXT_NAT_T_DPORT:
kdebug_sadb_x_natt(ext);
break;
case SADB_X_EXT_LFT_CUR_SW_OFFL:
case SADB_X_EXT_LFT_CUR_HW_OFFL:
kdebug_sadb_lifetime(ext);
default:
printf("%s: invalid ext_type %u\n", __func__,
ext->sadb_ext_type);

14
sys/netipsec/keydb.h
View file

@ -36,9 +36,11 @@
#ifdef _KERNEL
#include <sys/counter.h>
#include <sys/ck.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/rmlock.h>
#include <sys/_task.h>
#include <netipsec/key_var.h>
#include <opencrypto/_cryptodev.h>
@ -125,6 +127,7 @@ struct xformsw;
struct enc_xform;
struct auth_hash;
struct comp_algo;
struct ifp_handle_sav;
/*
* Security Association
@ -185,8 +188,19 @@ struct secasvar {
uint64_t cntr; /* counter for GCM and CTR */
volatile u_int refcnt; /* reference count */
CK_LIST_HEAD(, ifp_handle_sav) accel_ifps;
uintptr_t accel_forget_tq;
const char *accel_ifname;
uint32_t accel_flags;
counter_u64_t accel_lft_sw;
uint64_t accel_hw_allocs;
uint64_t accel_hw_octets;
uint64_t accel_firstused;
};
#define SADB_KEY_ACCEL_INST 0x00000001
#define SADB_KEY_ACCEL_DEINST 0x00000002
#define SECASVAR_RLOCK_TRACKER struct rm_priotracker _secas_tracker
#define SECASVAR_RLOCK(_sav) rm_rlock((_sav)->lock, &_secas_tracker)
#define SECASVAR_RUNLOCK(_sav) rm_runlock((_sav)->lock, &_secas_tracker)