From b26bbdae460ba90fb8cda37123a66be15a43a2a9 Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Mon, 26 Jun 2017 19:01:50 +0200 Subject: [PATCH 1/2] udp: move scratch area helpers into the include file So that they can be later used by the IPv6 code, too. Also lift the comments a bit. Signed-off-by: Paolo Abeni Signed-off-by: David S. Miller --- include/net/udp.h | 61 +++++++++++++++++++++++++++++++++++++++++++++++ net/ipv4/udp.c | 60 ---------------------------------------------- 2 files changed, 61 insertions(+), 60 deletions(-) diff --git a/include/net/udp.h b/include/net/udp.h index 1468dbd0f09a..972ce4baab6b 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -302,6 +302,67 @@ struct sock *__udp6_lib_lookup(struct net *net, struct sock *udp6_lib_lookup_skb(struct sk_buff *skb, __be16 sport, __be16 dport); +/* UDP uses skb->dev_scratch to cache as much information as possible and avoid + * possibly multiple cache miss on dequeue() + */ +#if BITS_PER_LONG == 64 + +/* truesize, len and the bit needed to compute skb_csum_unnecessary will be on + * cold cache lines at recvmsg time. + * skb->len can be stored on 16 bits since the udp header has been already + * validated and pulled. + */ +struct udp_dev_scratch { + u32 truesize; + u16 len; + bool is_linear; + bool csum_unnecessary; +}; + +static inline unsigned int udp_skb_len(struct sk_buff *skb) +{ + return ((struct udp_dev_scratch *)&skb->dev_scratch)->len; +} + +static inline bool udp_skb_csum_unnecessary(struct sk_buff *skb) +{ + return ((struct udp_dev_scratch *)&skb->dev_scratch)->csum_unnecessary; +} + +static inline bool udp_skb_is_linear(struct sk_buff *skb) +{ + return ((struct udp_dev_scratch *)&skb->dev_scratch)->is_linear; +} + +#else +static inline unsigned int udp_skb_len(struct sk_buff *skb) +{ + return skb->len; +} + +static inline bool udp_skb_csum_unnecessary(struct sk_buff *skb) +{ + return skb_csum_unnecessary(skb); +} + +static inline bool udp_skb_is_linear(struct sk_buff *skb) +{ + return !skb_is_nonlinear(skb); +} +#endif + +static inline int copy_linear_skb(struct sk_buff *skb, int len, int off, + struct iov_iter *to) +{ + int n, copy = len - off; + + n = copy_to_iter(skb->data + off, copy, to); + if (n == copy) + return 0; + + return -EFAULT; +} + /* * SNMP statistics for UDP and UDP-Lite */ diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 47c7aa0501af..86fad2a14ac4 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -1163,24 +1163,7 @@ out: return ret; } -/* Copy as much information as possible into skb->dev_scratch to avoid - * possibly multiple cache miss on dequeue(); - */ #if BITS_PER_LONG == 64 - -/* we can store multiple info here: truesize, len and the bit needed to - * compute skb_csum_unnecessary will be on cold cache lines at recvmsg - * time. - * skb->len can be stored on 16 bits since the udp header has been already - * validated and pulled. - */ -struct udp_dev_scratch { - u32 truesize; - u16 len; - bool is_linear; - bool csum_unnecessary; -}; - static void udp_set_dev_scratch(struct sk_buff *skb) { struct udp_dev_scratch *scratch; @@ -1197,22 +1180,6 @@ static int udp_skb_truesize(struct sk_buff *skb) { return ((struct udp_dev_scratch *)&skb->dev_scratch)->truesize; } - -static unsigned int udp_skb_len(struct sk_buff *skb) -{ - return ((struct udp_dev_scratch *)&skb->dev_scratch)->len; -} - -static bool udp_skb_csum_unnecessary(struct sk_buff *skb) -{ - return ((struct udp_dev_scratch *)&skb->dev_scratch)->csum_unnecessary; -} - -static bool udp_skb_is_linear(struct sk_buff *skb) -{ - return ((struct udp_dev_scratch *)&skb->dev_scratch)->is_linear; -} - #else static void udp_set_dev_scratch(struct sk_buff *skb) { @@ -1223,21 +1190,6 @@ static int udp_skb_truesize(struct sk_buff *skb) { return skb->dev_scratch; } - -static unsigned int udp_skb_len(struct sk_buff *skb) -{ - return skb->len; -} - -static bool udp_skb_csum_unnecessary(struct sk_buff *skb) -{ - return skb_csum_unnecessary(skb); -} - -static bool udp_skb_is_linear(struct sk_buff *skb) -{ - return !skb_is_nonlinear(skb); -} #endif /* fully reclaim rmem/fwd memory allocated for skb */ @@ -1598,18 +1550,6 @@ busy_check: } EXPORT_SYMBOL_GPL(__skb_recv_udp); -static int copy_linear_skb(struct sk_buff *skb, int len, int off, - struct iov_iter *to) -{ - int n, copy = len - off; - - n = copy_to_iter(skb->data + off, copy, to); - if (n == copy) - return 0; - - return -EFAULT; -} - /* * This should be easy, if there is something there we * return it, otherwise we block. From 67a51780aebb683c3b217b2867e25102cdb0afbc Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Mon, 26 Jun 2017 19:01:51 +0200 Subject: [PATCH 2/2] ipv6: udp: leverage scratch area helpers The commit b65ac44674dd ("udp: try to avoid 2 cache miss on dequeue") leveraged the scratched area helpers for UDP v4 but I forgot to update accordingly the IPv6 code path. This change extends the scratch area usage to the IPv6 code, synching the two implementations and giving some performance benefit. IPv6 is again almost on the same level of IPv4, performance-wide. Signed-off-by: Paolo Abeni Signed-off-by: David S. Miller --- net/ipv6/udp.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index d1d728805729..450829dd6384 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -362,7 +362,7 @@ try_again: if (!skb) return err; - ulen = skb->len; + ulen = udp_skb_len(skb); copied = len; if (copied > ulen - off) copied = ulen - off; @@ -379,14 +379,18 @@ try_again: if (copied < ulen || peeking || (is_udplite && UDP_SKB_CB(skb)->partial_cov)) { - checksum_valid = !udp_lib_checksum_complete(skb); + checksum_valid = udp_skb_csum_unnecessary(skb) || + !__udp_lib_checksum_complete(skb); if (!checksum_valid) goto csum_copy_err; } - if (checksum_valid || skb_csum_unnecessary(skb)) - err = skb_copy_datagram_msg(skb, off, msg, copied); - else { + if (checksum_valid || udp_skb_csum_unnecessary(skb)) { + if (udp_skb_is_linear(skb)) + err = copy_linear_skb(skb, copied, off, &msg->msg_iter); + else + err = skb_copy_datagram_msg(skb, off, msg, copied); + } else { err = skb_copy_and_csum_datagram_msg(skb, off, msg); if (err == -EINVAL) goto csum_copy_err;