[openwrt/openwrt] kernel: backport improved checksum function for ARM64
LEDE Commits
lede-commits at lists.infradead.org
Sun Sep 6 09:36:26 EDT 2020
nbd pushed a commit to openwrt/openwrt.git, branch master:
https://git.openwrt.org/63b6b106702e58cdc333a4e59ff35e46dd919ee7
commit 63b6b106702e58cdc333a4e59ff35e46dd919ee7
Author: Felix Fietkau <nbd at nbd.name>
AuthorDate: Sun Sep 6 13:28:59 2020 +0200
kernel: backport improved checksum function for ARM64
Improves network performance in some cases when checksum offload is not
available
Signed-off-by: Felix Fietkau <nbd at nbd.name>
---
...rm64-Implement-optimised-checksum-routine.patch | 176 +++++++++++++++++++++
...4-csum-Fix-pathological-zero-length-calls.patch | 28 ++++
2 files changed, 204 insertions(+)
diff --git a/target/linux/generic/backport-5.4/041-v5.5-arm64-Implement-optimised-checksum-routine.patch b/target/linux/generic/backport-5.4/041-v5.5-arm64-Implement-optimised-checksum-routine.patch
new file mode 100644
index 0000000000..00ec7d0207
--- /dev/null
+++ b/target/linux/generic/backport-5.4/041-v5.5-arm64-Implement-optimised-checksum-routine.patch
@@ -0,0 +1,176 @@
+From: Robin Murphy <robin.murphy at arm.com>
+Date: Wed, 15 Jan 2020 16:42:39 +0000
+Subject: [PATCH] arm64: Implement optimised checksum routine
+
+Apparently there exist certain workloads which rely heavily on software
+checksumming, for which the generic do_csum() implementation becomes a
+significant bottleneck. Therefore let's give arm64 its own optimised
+version - for ease of maintenance this foregoes assembly or intrisics,
+and is thus not actually arm64-specific, but does rely heavily on C
+idioms that translate well to the A64 ISA and the typical load/store
+capabilities of most ARMv8 CPU cores.
+
+The resulting increase in checksum throughput scales nicely with buffer
+size, tending towards 4x for a small in-order core (Cortex-A53), and up
+to 6x or more for an aggressive big core (Ampere eMAG).
+
+Reported-by: Lingyan Huang <huanglingyan2 at huawei.com>
+Tested-by: Lingyan Huang <huanglingyan2 at huawei.com>
+Signed-off-by: Robin Murphy <robin.murphy at arm.com>
+Signed-off-by: Will Deacon <will at kernel.org>
+---
+ create mode 100644 arch/arm64/lib/csum.c
+
+--- a/arch/arm64/include/asm/checksum.h
++++ b/arch/arm64/include/asm/checksum.h
+@@ -36,6 +36,9 @@ static inline __sum16 ip_fast_csum(const
+ }
+ #define ip_fast_csum ip_fast_csum
+
++extern unsigned int do_csum(const unsigned char *buff, int len);
++#define do_csum do_csum
++
+ #include <asm-generic/checksum.h>
+
+ #endif /* __ASM_CHECKSUM_H */
+--- a/arch/arm64/lib/Makefile
++++ b/arch/arm64/lib/Makefile
+@@ -1,9 +1,9 @@
+ # SPDX-License-Identifier: GPL-2.0
+ lib-y := clear_user.o delay.o copy_from_user.o \
+ copy_to_user.o copy_in_user.o copy_page.o \
+- clear_page.o memchr.o memcpy.o memmove.o memset.o \
+- memcmp.o strcmp.o strncmp.o strlen.o strnlen.o \
+- strchr.o strrchr.o tishift.o
++ clear_page.o csum.o memchr.o memcpy.o memmove.o \
++ memset.o memcmp.o strcmp.o strncmp.o strlen.o \
++ strnlen.o strchr.o strrchr.o tishift.o
+
+ ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
+ obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
+--- /dev/null
++++ b/arch/arm64/lib/csum.c
+@@ -0,0 +1,123 @@
++// SPDX-License-Identifier: GPL-2.0-only
++// Copyright (C) 2019-2020 Arm Ltd.
++
++#include <linux/compiler.h>
++#include <linux/kasan-checks.h>
++#include <linux/kernel.h>
++
++#include <net/checksum.h>
++
++/* Looks dumb, but generates nice-ish code */
++static u64 accumulate(u64 sum, u64 data)
++{
++ __uint128_t tmp = (__uint128_t)sum + data;
++ return tmp + (tmp >> 64);
++}
++
++unsigned int do_csum(const unsigned char *buff, int len)
++{
++ unsigned int offset, shift, sum;
++ const u64 *ptr;
++ u64 data, sum64 = 0;
++
++ offset = (unsigned long)buff & 7;
++ /*
++ * This is to all intents and purposes safe, since rounding down cannot
++ * result in a different page or cache line being accessed, and @buff
++ * should absolutely not be pointing to anything read-sensitive. We do,
++ * however, have to be careful not to piss off KASAN, which means using
++ * unchecked reads to accommodate the head and tail, for which we'll
++ * compensate with an explicit check up-front.
++ */
++ kasan_check_read(buff, len);
++ ptr = (u64 *)(buff - offset);
++ len = len + offset - 8;
++
++ /*
++ * Head: zero out any excess leading bytes. Shifting back by the same
++ * amount should be at least as fast as any other way of handling the
++ * odd/even alignment, and means we can ignore it until the very end.
++ */
++ shift = offset * 8;
++ data = READ_ONCE_NOCHECK(*ptr++);
++#ifdef __LITTLE_ENDIAN
++ data = (data >> shift) << shift;
++#else
++ data = (data << shift) >> shift;
++#endif
++
++ /*
++ * Body: straightforward aligned loads from here on (the paired loads
++ * underlying the quadword type still only need dword alignment). The
++ * main loop strictly excludes the tail, so the second loop will always
++ * run at least once.
++ */
++ while (unlikely(len > 64)) {
++ __uint128_t tmp1, tmp2, tmp3, tmp4;
++
++ tmp1 = READ_ONCE_NOCHECK(*(__uint128_t *)ptr);
++ tmp2 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 2));
++ tmp3 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 4));
++ tmp4 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 6));
++
++ len -= 64;
++ ptr += 8;
++
++ /* This is the "don't dump the carry flag into a GPR" idiom */
++ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++ tmp2 += (tmp2 >> 64) | (tmp2 << 64);
++ tmp3 += (tmp3 >> 64) | (tmp3 << 64);
++ tmp4 += (tmp4 >> 64) | (tmp4 << 64);
++ tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64);
++ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++ tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64);
++ tmp3 += (tmp3 >> 64) | (tmp3 << 64);
++ tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64);
++ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++ tmp1 = ((tmp1 >> 64) << 64) | sum64;
++ tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++ sum64 = tmp1 >> 64;
++ }
++ while (len > 8) {
++ __uint128_t tmp;
++
++ sum64 = accumulate(sum64, data);
++ tmp = READ_ONCE_NOCHECK(*(__uint128_t *)ptr);
++
++ len -= 16;
++ ptr += 2;
++
++#ifdef __LITTLE_ENDIAN
++ data = tmp >> 64;
++ sum64 = accumulate(sum64, tmp);
++#else
++ data = tmp;
++ sum64 = accumulate(sum64, tmp >> 64);
++#endif
++ }
++ if (len > 0) {
++ sum64 = accumulate(sum64, data);
++ data = READ_ONCE_NOCHECK(*ptr);
++ len -= 8;
++ }
++ /*
++ * Tail: zero any over-read bytes similarly to the head, again
++ * preserving odd/even alignment.
++ */
++ shift = len * -8;
++#ifdef __LITTLE_ENDIAN
++ data = (data << shift) >> shift;
++#else
++ data = (data >> shift) << shift;
++#endif
++ sum64 = accumulate(sum64, data);
++
++ /* Finally, folding */
++ sum64 += (sum64 >> 32) | (sum64 << 32);
++ sum = sum64 >> 32;
++ sum += (sum >> 16) | (sum << 16);
++ if (offset & 1)
++ return (u16)swab32(sum);
++
++ return sum >> 16;
++}
diff --git a/target/linux/generic/backport-5.4/042-v5.5-arm64-csum-Fix-pathological-zero-length-calls.patch b/target/linux/generic/backport-5.4/042-v5.5-arm64-csum-Fix-pathological-zero-length-calls.patch
new file mode 100644
index 0000000000..50b210e14f
--- /dev/null
+++ b/target/linux/generic/backport-5.4/042-v5.5-arm64-csum-Fix-pathological-zero-length-calls.patch
@@ -0,0 +1,28 @@
+From: Robin Murphy <robin.murphy at arm.com>
+Date: Fri, 17 Jan 2020 15:48:39 +0000
+Subject: [PATCH] arm64: csum: Fix pathological zero-length calls
+
+In validating the checksumming results of the new routine, I sadly
+neglected to test its not-checksumming results. Thus it slipped through
+that the one case where @buff is already dword-aligned and @len = 0
+manages to defeat the tail-masking logic and behave as if @len = 8.
+For a zero length it doesn't make much sense to deference @buff anyway,
+so just add an early return (which has essentially zero impact on
+performance).
+
+Signed-off-by: Robin Murphy <robin.murphy at arm.com>
+Signed-off-by: Will Deacon <will at kernel.org>
+---
+
+--- a/arch/arm64/lib/csum.c
++++ b/arch/arm64/lib/csum.c
+@@ -20,6 +20,9 @@ unsigned int do_csum(const unsigned char
+ const u64 *ptr;
+ u64 data, sum64 = 0;
+
++ if (unlikely(len == 0))
++ return 0;
++
+ offset = (unsigned long)buff & 7;
+ /*
+ * This is to all intents and purposes safe, since rounding down cannot
More information about the lede-commits
mailing list