Skip to content

Commit 09eed11

Browse files
Eric Dumazetkuba-moo
Eric Dumazet
authored and
kuba-moo
committed
neighbour: switch to standard rcu, instead of rcu_bh
rcu_bh is no longer a win, especially for objects freed with standard call_rcu(). Switch neighbour code to no longer disable BH when not necessary. Signed-off-by: Eric Dumazet <[email protected]> Signed-off-by: Jakub Kicinski <[email protected]>
1 parent 4c5c496 commit 09eed11

File tree

13 files changed

+87
-83
lines changed

13 files changed

+87
-83
lines changed

‎include/net/arp.h

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,11 @@ static inline struct neighbour *__ipv4_neigh_lookup(struct net_device *dev, u32
3838
{
3939
struct neighbour *n;
4040

41-
rcu_read_lock_bh();
41+
rcu_read_lock();
4242
n = __ipv4_neigh_lookup_noref(dev, key);
4343
if (n && !refcount_inc_not_zero(&n->refcnt))
4444
n = NULL;
45-
rcu_read_unlock_bh();
45+
rcu_read_unlock();
4646

4747
return n;
4848
}
@@ -51,10 +51,10 @@ static inline void __ipv4_confirm_neigh(struct net_device *dev, u32 key)
5151
{
5252
struct neighbour *n;
5353

54-
rcu_read_lock_bh();
54+
rcu_read_lock();
5555
n = __ipv4_neigh_lookup_noref(dev, key);
5656
neigh_confirm(n);
57-
rcu_read_unlock_bh();
57+
rcu_read_unlock();
5858
}
5959

6060
void arp_init(void);

‎include/net/ndisc.h

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -395,11 +395,11 @@ static inline struct neighbour *__ipv6_neigh_lookup(struct net_device *dev, cons
395395
{
396396
struct neighbour *n;
397397

398-
rcu_read_lock_bh();
398+
rcu_read_lock();
399399
n = __ipv6_neigh_lookup_noref(dev, pkey);
400400
if (n && !refcount_inc_not_zero(&n->refcnt))
401401
n = NULL;
402-
rcu_read_unlock_bh();
402+
rcu_read_unlock();
403403

404404
return n;
405405
}
@@ -409,21 +409,21 @@ static inline void __ipv6_confirm_neigh(struct net_device *dev,
409409
{
410410
struct neighbour *n;
411411

412-
rcu_read_lock_bh();
412+
rcu_read_lock();
413413
n = __ipv6_neigh_lookup_noref(dev, pkey);
414414
neigh_confirm(n);
415-
rcu_read_unlock_bh();
415+
rcu_read_unlock();
416416
}
417417

418418
static inline void __ipv6_confirm_neigh_stub(struct net_device *dev,
419419
const void *pkey)
420420
{
421421
struct neighbour *n;
422422

423-
rcu_read_lock_bh();
423+
rcu_read_lock();
424424
n = __ipv6_neigh_lookup_noref_stub(dev, pkey);
425425
neigh_confirm(n);
426-
rcu_read_unlock_bh();
426+
rcu_read_unlock();
427427
}
428428

429429
/* uses ipv6_stub and is meant for use outside of IPv6 core */

‎include/net/neighbour.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -299,14 +299,14 @@ static inline struct neighbour *___neigh_lookup_noref(
299299
const void *pkey,
300300
struct net_device *dev)
301301
{
302-
struct neigh_hash_table *nht = rcu_dereference_bh(tbl->nht);
302+
struct neigh_hash_table *nht = rcu_dereference(tbl->nht);
303303
struct neighbour *n;
304304
u32 hash_val;
305305

306306
hash_val = hash(pkey, dev, nht->hash_rnd) >> (32 - nht->hash_shift);
307-
for (n = rcu_dereference_bh(nht->hash_buckets[hash_val]);
307+
for (n = rcu_dereference(nht->hash_buckets[hash_val]);
308308
n != NULL;
309-
n = rcu_dereference_bh(n->next)) {
309+
n = rcu_dereference(n->next)) {
310310
if (n->dev == dev && key_eq(n, pkey))
311311
return n;
312312
}

‎include/net/nexthop.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -498,7 +498,7 @@ static inline struct fib6_nh *nexthop_fib6_nh(struct nexthop *nh)
498498
}
499499

500500
/* Variant of nexthop_fib6_nh().
501-
* Caller should either hold rcu_read_lock_bh(), or RTNL.
501+
* Caller should either hold rcu_read_lock(), or RTNL.
502502
*/
503503
static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh)
504504
{
@@ -507,13 +507,13 @@ static inline struct fib6_nh *nexthop_fib6_nh_bh(struct nexthop *nh)
507507
if (nh->is_group) {
508508
struct nh_group *nh_grp;
509509

510-
nh_grp = rcu_dereference_bh_rtnl(nh->nh_grp);
510+
nh_grp = rcu_dereference_rtnl(nh->nh_grp);
511511
nh = nexthop_mpath_select(nh_grp, 0);
512512
if (!nh)
513513
return NULL;
514514
}
515515

516-
nhi = rcu_dereference_bh_rtnl(nh->nh_info);
516+
nhi = rcu_dereference_rtnl(nh->nh_info);
517517
if (nhi->family == AF_INET6)
518518
return &nhi->fib6_nh;
519519

‎net/core/filter.c

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2204,7 +2204,7 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
22042204
return -ENOMEM;
22052205
}
22062206

2207-
rcu_read_lock_bh();
2207+
rcu_read_lock();
22082208
if (!nh) {
22092209
dst = skb_dst(skb);
22102210
nexthop = rt6_nexthop(container_of(dst, struct rt6_info, dst),
@@ -2217,10 +2217,12 @@ static int bpf_out_neigh_v6(struct net *net, struct sk_buff *skb,
22172217
int ret;
22182218

22192219
sock_confirm_neigh(skb, neigh);
2220+
local_bh_disable();
22202221
dev_xmit_recursion_inc();
22212222
ret = neigh_output(neigh, skb, false);
22222223
dev_xmit_recursion_dec();
2223-
rcu_read_unlock_bh();
2224+
local_bh_enable();
2225+
rcu_read_unlock();
22242226
return ret;
22252227
}
22262228
rcu_read_unlock_bh();
@@ -2302,7 +2304,7 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
23022304
return -ENOMEM;
23032305
}
23042306

2305-
rcu_read_lock_bh();
2307+
rcu_read_lock();
23062308
if (!nh) {
23072309
struct dst_entry *dst = skb_dst(skb);
23082310
struct rtable *rt = container_of(dst, struct rtable, dst);
@@ -2314,21 +2316,23 @@ static int bpf_out_neigh_v4(struct net *net, struct sk_buff *skb,
23142316
} else if (nh->nh_family == AF_INET) {
23152317
neigh = ip_neigh_gw4(dev, nh->ipv4_nh);
23162318
} else {
2317-
rcu_read_unlock_bh();
2319+
rcu_read_unlock();
23182320
goto out_drop;
23192321
}
23202322

23212323
if (likely(!IS_ERR(neigh))) {
23222324
int ret;
23232325

23242326
sock_confirm_neigh(skb, neigh);
2327+
local_bh_disable();
23252328
dev_xmit_recursion_inc();
23262329
ret = neigh_output(neigh, skb, is_v6gw);
23272330
dev_xmit_recursion_dec();
2328-
rcu_read_unlock_bh();
2331+
local_bh_enable();
2332+
rcu_read_unlock();
23292333
return ret;
23302334
}
2331-
rcu_read_unlock_bh();
2335+
rcu_read_unlock();
23322336
out_drop:
23332337
kfree_skb(skb);
23342338
return -ENETDOWN;

‎net/core/neighbour.c

Lines changed: 32 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -614,15 +614,15 @@ struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey,
614614

615615
NEIGH_CACHE_STAT_INC(tbl, lookups);
616616

617-
rcu_read_lock_bh();
617+
rcu_read_lock();
618618
n = __neigh_lookup_noref(tbl, pkey, dev);
619619
if (n) {
620620
if (!refcount_inc_not_zero(&n->refcnt))
621621
n = NULL;
622622
NEIGH_CACHE_STAT_INC(tbl, hits);
623623
}
624624

625-
rcu_read_unlock_bh();
625+
rcu_read_unlock();
626626
return n;
627627
}
628628
EXPORT_SYMBOL(neigh_lookup);
@@ -2184,11 +2184,11 @@ static int neightbl_fill_info(struct sk_buff *skb, struct neigh_table *tbl,
21842184
.ndtc_proxy_qlen = tbl->proxy_queue.qlen,
21852185
};
21862186

2187-
rcu_read_lock_bh();
2188-
nht = rcu_dereference_bh(tbl->nht);
2187+
rcu_read_lock();
2188+
nht = rcu_dereference(tbl->nht);
21892189
ndc.ndtc_hash_rnd = nht->hash_rnd[0];
21902190
ndc.ndtc_hash_mask = ((1 << nht->hash_shift) - 1);
2191-
rcu_read_unlock_bh();
2191+
rcu_read_unlock();
21922192

21932193
if (nla_put(skb, NDTA_CONFIG, sizeof(ndc), &ndc))
21942194
goto nla_put_failure;
@@ -2703,15 +2703,15 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
27032703
if (filter->dev_idx || filter->master_idx)
27042704
flags |= NLM_F_DUMP_FILTERED;
27052705

2706-
rcu_read_lock_bh();
2707-
nht = rcu_dereference_bh(tbl->nht);
2706+
rcu_read_lock();
2707+
nht = rcu_dereference(tbl->nht);
27082708

27092709
for (h = s_h; h < (1 << nht->hash_shift); h++) {
27102710
if (h > s_h)
27112711
s_idx = 0;
2712-
for (n = rcu_dereference_bh(nht->hash_buckets[h]), idx = 0;
2712+
for (n = rcu_dereference(nht->hash_buckets[h]), idx = 0;
27132713
n != NULL;
2714-
n = rcu_dereference_bh(n->next)) {
2714+
n = rcu_dereference(n->next)) {
27152715
if (idx < s_idx || !net_eq(dev_net(n->dev), net))
27162716
goto next;
27172717
if (neigh_ifindex_filtered(n->dev, filter->dev_idx) ||
@@ -2730,7 +2730,7 @@ static int neigh_dump_table(struct neigh_table *tbl, struct sk_buff *skb,
27302730
}
27312731
rc = skb->len;
27322732
out:
2733-
rcu_read_unlock_bh();
2733+
rcu_read_unlock();
27342734
cb->args[1] = h;
27352735
cb->args[2] = idx;
27362736
return rc;
@@ -3075,20 +3075,20 @@ void neigh_for_each(struct neigh_table *tbl, void (*cb)(struct neighbour *, void
30753075
int chain;
30763076
struct neigh_hash_table *nht;
30773077

3078-
rcu_read_lock_bh();
3079-
nht = rcu_dereference_bh(tbl->nht);
3078+
rcu_read_lock();
3079+
nht = rcu_dereference(tbl->nht);
30803080

3081-
read_lock(&tbl->lock); /* avoid resizes */
3081+
read_lock_bh(&tbl->lock); /* avoid resizes */
30823082
for (chain = 0; chain < (1 << nht->hash_shift); chain++) {
30833083
struct neighbour *n;
30843084

3085-
for (n = rcu_dereference_bh(nht->hash_buckets[chain]);
3085+
for (n = rcu_dereference(nht->hash_buckets[chain]);
30863086
n != NULL;
3087-
n = rcu_dereference_bh(n->next))
3087+
n = rcu_dereference(n->next))
30883088
cb(n, cookie);
30893089
}
3090-
read_unlock(&tbl->lock);
3091-
rcu_read_unlock_bh();
3090+
read_unlock_bh(&tbl->lock);
3091+
rcu_read_unlock();
30923092
}
30933093
EXPORT_SYMBOL(neigh_for_each);
30943094

@@ -3138,7 +3138,7 @@ int neigh_xmit(int index, struct net_device *dev,
31383138
tbl = neigh_tables[index];
31393139
if (!tbl)
31403140
goto out;
3141-
rcu_read_lock_bh();
3141+
rcu_read_lock();
31423142
if (index == NEIGH_ARP_TABLE) {
31433143
u32 key = *((u32 *)addr);
31443144

@@ -3150,11 +3150,11 @@ int neigh_xmit(int index, struct net_device *dev,
31503150
neigh = __neigh_create(tbl, addr, dev, false);
31513151
err = PTR_ERR(neigh);
31523152
if (IS_ERR(neigh)) {
3153-
rcu_read_unlock_bh();
3153+
rcu_read_unlock();
31543154
goto out_kfree_skb;
31553155
}
31563156
err = neigh->output(neigh, skb);
3157-
rcu_read_unlock_bh();
3157+
rcu_read_unlock();
31583158
}
31593159
else if (index == NEIGH_LINK_TABLE) {
31603160
err = dev_hard_header(skb, dev, ntohs(skb->protocol),
@@ -3183,7 +3183,7 @@ static struct neighbour *neigh_get_first(struct seq_file *seq)
31833183

31843184
state->flags &= ~NEIGH_SEQ_IS_PNEIGH;
31853185
for (bucket = 0; bucket < (1 << nht->hash_shift); bucket++) {
3186-
n = rcu_dereference_bh(nht->hash_buckets[bucket]);
3186+
n = rcu_dereference(nht->hash_buckets[bucket]);
31873187

31883188
while (n) {
31893189
if (!net_eq(dev_net(n->dev), net))
@@ -3201,7 +3201,7 @@ static struct neighbour *neigh_get_first(struct seq_file *seq)
32013201
if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
32023202
break;
32033203
next:
3204-
n = rcu_dereference_bh(n->next);
3204+
n = rcu_dereference(n->next);
32053205
}
32063206

32073207
if (n)
@@ -3225,7 +3225,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
32253225
if (v)
32263226
return n;
32273227
}
3228-
n = rcu_dereference_bh(n->next);
3228+
n = rcu_dereference(n->next);
32293229

32303230
while (1) {
32313231
while (n) {
@@ -3243,7 +3243,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
32433243
if (READ_ONCE(n->nud_state) & ~NUD_NOARP)
32443244
break;
32453245
next:
3246-
n = rcu_dereference_bh(n->next);
3246+
n = rcu_dereference(n->next);
32473247
}
32483248

32493249
if (n)
@@ -3252,7 +3252,7 @@ static struct neighbour *neigh_get_next(struct seq_file *seq,
32523252
if (++state->bucket >= (1 << nht->hash_shift))
32533253
break;
32543254

3255-
n = rcu_dereference_bh(nht->hash_buckets[state->bucket]);
3255+
n = rcu_dereference(nht->hash_buckets[state->bucket]);
32563256
}
32573257

32583258
if (n && pos)
@@ -3354,17 +3354,17 @@ static void *neigh_get_idx_any(struct seq_file *seq, loff_t *pos)
33543354

33553355
void *neigh_seq_start(struct seq_file *seq, loff_t *pos, struct neigh_table *tbl, unsigned int neigh_seq_flags)
33563356
__acquires(tbl->lock)
3357-
__acquires(rcu_bh)
3357+
__acquires(rcu)
33583358
{
33593359
struct neigh_seq_state *state = seq->private;
33603360

33613361
state->tbl = tbl;
33623362
state->bucket = 0;
33633363
state->flags = (neigh_seq_flags & ~NEIGH_SEQ_IS_PNEIGH);
33643364

3365-
rcu_read_lock_bh();
3366-
state->nht = rcu_dereference_bh(tbl->nht);
3367-
read_lock(&tbl->lock);
3365+
rcu_read_lock();
3366+
state->nht = rcu_dereference(tbl->nht);
3367+
read_lock_bh(&tbl->lock);
33683368

33693369
return *pos ? neigh_get_idx_any(seq, pos) : SEQ_START_TOKEN;
33703370
}
@@ -3399,13 +3399,13 @@ EXPORT_SYMBOL(neigh_seq_next);
33993399

34003400
void neigh_seq_stop(struct seq_file *seq, void *v)
34013401
__releases(tbl->lock)
3402-
__releases(rcu_bh)
3402+
__releases(rcu)
34033403
{
34043404
struct neigh_seq_state *state = seq->private;
34053405
struct neigh_table *tbl = state->tbl;
34063406

3407-
read_unlock(&tbl->lock);
3408-
rcu_read_unlock_bh();
3407+
read_unlock_bh(&tbl->lock);
3408+
rcu_read_unlock();
34093409
}
34103410
EXPORT_SYMBOL(neigh_seq_stop);
34113411

‎net/ipv4/fib_semantics.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2191,7 +2191,7 @@ static bool fib_good_nh(const struct fib_nh *nh)
21912191
if (nh->fib_nh_scope == RT_SCOPE_LINK) {
21922192
struct neighbour *n;
21932193

2194-
rcu_read_lock_bh();
2194+
rcu_read_lock();
21952195

21962196
if (likely(nh->fib_nh_gw_family == AF_INET))
21972197
n = __ipv4_neigh_lookup_noref(nh->fib_nh_dev,
@@ -2204,7 +2204,7 @@ static bool fib_good_nh(const struct fib_nh *nh)
22042204
if (n)
22052205
state = READ_ONCE(n->nud_state);
22062206

2207-
rcu_read_unlock_bh();
2207+
rcu_read_unlock();
22082208
}
22092209

22102210
return !!(state & NUD_VALID);

0 commit comments

Comments
 (0)