diff options
| author | Steffen Klassert <steffen.klassert@secunet.com> | 2021-07-02 13:30:31 +0200 |
|---|---|---|
| committer | Steffen Klassert <steffen.klassert@secunet.com> | 2022-03-04 14:05:32 +0100 |
| commit | 21371896c05d7b8093b64aa754fe633a5d0ac3b5 (patch) | |
| tree | 884f744c244181ec7f70cde33dcb5b0551a54460 | |
| parent | 65c7b193c14e4e53d197f29d7671bb46c9403313 (diff) | |
| download | linux-ft-xfrm-pcpu-v2.tar.gz | |
xfrm: Add an inbound percpu state cache.xfrm-pcpu-v2
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
| -rw-r--r-- | include/net/netns/xfrm.h | 1 | ||||
| -rw-r--r-- | include/net/xfrm.h | 1 | ||||
| -rw-r--r-- | net/xfrm/xfrm_state.c | 51 |
3 files changed, 50 insertions, 3 deletions
diff --git a/include/net/netns/xfrm.h b/include/net/netns/xfrm.h index 947733a639a6f1..f3e8f8049dfc0c 100644 --- a/include/net/netns/xfrm.h +++ b/include/net/netns/xfrm.h @@ -43,6 +43,7 @@ struct netns_xfrm { struct hlist_head __rcu *state_bysrc; struct hlist_head __rcu *state_byspi; struct hlist_head __rcu *state_byseq; + struct hlist_head __rcu __percpu *state_cache_input; unsigned int state_hmask; unsigned int state_num; struct work_struct state_hash_work; diff --git a/include/net/xfrm.h b/include/net/xfrm.h index 1a8a859be0ca86..7e091de0021b7b 100644 --- a/include/net/xfrm.h +++ b/include/net/xfrm.h @@ -163,6 +163,7 @@ struct xfrm_state { struct hlist_node byspi; struct hlist_node byseq; struct hlist_node state_cache; + struct hlist_node state_cache_input; refcount_t refcnt; spinlock_t lock; diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c index 13d98dd48dad49..d7c23ffff96c15 100644 --- a/net/xfrm/xfrm_state.c +++ b/net/xfrm/xfrm_state.c @@ -694,6 +694,9 @@ int __xfrm_state_delete(struct xfrm_state *x) hlist_del_rcu(&x->byseq); if (!hlist_unhashed(&x->state_cache)) hlist_del_rcu(&x->state_cache); + if (!hlist_unhashed(&x->state_cache_input)) + hlist_del_rcu(&x->state_cache_input); + if (x->id.spi) hlist_del_rcu(&x->byspi); net->xfrm.state_num--; @@ -963,7 +966,26 @@ static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, unsigned short family) { unsigned int h = xfrm_spi_hash(net, daddr, spi, proto, family); - struct xfrm_state *x; + struct hlist_head *state_cache_input; + struct xfrm_state *x = NULL; + int cpu = get_cpu(); + + state_cache_input = per_cpu_ptr(net->xfrm.state_cache_input, cpu); + + hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) { + if (x->props.family != family || + x->id.spi != spi || + x->id.proto != proto || + !xfrm_addr_equal(&x->id.daddr, daddr, family)) + continue; + + if ((mark & x->mark.m) != x->mark.v) + continue; + if (!xfrm_state_hold_rcu(x)) + continue; + goto out; + } + hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) { if (x->props.family != family || @@ -976,10 +998,25 @@ static struct xfrm_state *__xfrm_state_lookup(struct net *net, u32 mark, continue; if (!xfrm_state_hold_rcu(x)) continue; - return x; + + if (x->km.state == XFRM_STATE_VALID) { + spin_lock_bh(&net->xfrm.xfrm_state_lock); + if (hlist_unhashed(&x->state_cache_input)) { + hlist_add_head_rcu(&x->state_cache_input, state_cache_input); + } else { + hlist_del_rcu(&x->state_cache_input); + hlist_add_head_rcu(&x->state_cache_input, state_cache_input); + } + spin_unlock_bh(&net->xfrm.xfrm_state_lock); + + } + + goto out; } - return NULL; +out: + put_cpu(); + return x; } static struct xfrm_state *__xfrm_state_lookup_byaddr(struct net *net, u32 mark, @@ -2807,6 +2844,11 @@ int __net_init xfrm_state_init(struct net *net) net->xfrm.state_byseq = xfrm_hash_alloc(sz); if (!net->xfrm.state_byseq) goto out_byseq; + + net->xfrm.state_cache_input = alloc_percpu(struct hlist_head); + if (!net->xfrm.state_cache_input) + goto out_state_cache_input; + net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1); net->xfrm.state_num = 0; @@ -2816,6 +2858,8 @@ int __net_init xfrm_state_init(struct net *net) &net->xfrm.xfrm_state_lock); return 0; +out_state_cache_input: + xfrm_hash_free(net->xfrm.state_byseq, sz); out_byseq: xfrm_hash_free(net->xfrm.state_byspi, sz); out_byspi: @@ -2845,6 +2889,7 @@ void xfrm_state_fini(struct net *net) xfrm_hash_free(net->xfrm.state_bysrc, sz); WARN_ON(!hlist_empty(net->xfrm.state_bydst)); xfrm_hash_free(net->xfrm.state_bydst, sz); + free_percpu(net->xfrm.state_cache_input); } #ifdef CONFIG_AUDITSYSCALL |