diff options
| author | Andrew Morton <akpm@linux-foundation.org> | 2026-05-28 21:32:17 -0700 |
|---|---|---|
| committer | Andrew Morton <akpm@linux-foundation.org> | 2026-05-28 21:32:17 -0700 |
| commit | 8b6270dcbcd3443d4608b81329abb55078ae3ded (patch) | |
| tree | 34858faa2bc48fd62acb343ea6f8394eedfc6225 | |
| parent | d90fdc074685684dfc210e86688cb009c1a327a7 (diff) | |
| parent | c60ffec33ddf24577f6f4da18fe825b2058c5f78 (diff) | |
| download | linux-next-history-8b6270dcbcd3443d4608b81329abb55078ae3ded.tar.gz | |
foo
150 files changed, 3033 insertions, 2122 deletions
diff --git a/.clang-format b/.clang-format index 1cc151e2adcc5..6a3de86ab27a4 100644 --- a/.clang-format +++ b/.clang-format @@ -481,6 +481,7 @@ ForEachMacros: - 'genradix_for_each' - 'genradix_for_each_from' - 'genradix_for_each_reverse' + - 'guard' - 'hash_for_each' - 'hash_for_each_possible' - 'hash_for_each_possible_rcu' @@ -674,6 +675,7 @@ ForEachMacros: - 'rq_list_for_each' - 'rq_list_for_each_safe' - 'sample_read_group__for_each' + - 'scoped_guard' - 'scsi_for_each_prot_sg' - 'scsi_for_each_sg' - 'sctp_for_each_hentry' diff --git a/.mailmap b/.mailmap index 0b9298a55d2de..a6fb141692870 100644 --- a/.mailmap +++ b/.mailmap @@ -436,6 +436,7 @@ John Stultz <johnstul@us.ibm.com> Jonas Gorski <jonas.gorski@gmail.com> <jogo@openwrt.org> Jonathan Cameron <jic23@kernel.org> <jonathan.cameron@huawei.com> Jordan Crouse <jordan@cosmicpenguin.net> <jcrouse@codeaurora.org> +Jorge Ramirez-Ortiz <jorge.ramirez@oss.qualcomm.com> <jorge.ramirez-ortiz@linaro.org> <josh@joshtriplett.org> <josh@freedesktop.org> <josh@joshtriplett.org> <josh@kernel.org> <josh@joshtriplett.org> <josht@linux.vnet.ibm.com> @@ -3368,6 +3368,10 @@ N: Anil Ravindranath E: anil_ravindranath@pmc-sierra.com D: PMC-Sierra MaxRAID driver +N: Dwaipayan Ray +E: dwaipayanray1@gmail.com +D: checkpatch improvements + N: Eric S. Raymond E: esr@thyrsus.com W: http://www.tuxedo.org/~esr/ diff --git a/Documentation/crypto/async-tx-api.rst b/Documentation/crypto/async-tx-api.rst index f88a7809385e2..49fcfc66314ac 100644 --- a/Documentation/crypto/async-tx-api.rst +++ b/Documentation/crypto/async-tx-api.rst @@ -82,9 +82,9 @@ xor_val xor a series of source buffers and set a flag if the pq generate the p+q (raid6 syndrome) from a series of source buffers pq_val validate that a p and or q buffer are in sync with a given series of sources -datap (raid6_datap_recov) recover a raid6 data block and the p block +datap (raid6_recov_datap) recover a raid6 data block and the p block from the given sources -2data (raid6_2data_recov) recover 2 raid6 data blocks from the given +2data (raid6_recov_2data) recover 2 raid6 data blocks from the given sources ======== ==================================================================== diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst index dccede68698ca..6139a08c34cd8 100644 --- a/Documentation/dev-tools/checkpatch.rst +++ b/Documentation/dev-tools/checkpatch.rst @@ -184,6 +184,13 @@ Available options: Override checking of perl version. Runtime errors may be encountered after enabling this flag if the perl version does not meet the minimum specified. + - --spdx-cxx-comments + + Don't force C comments ``/* */`` for SPDX license (required by old + toolchains), allow also C++ comments ``//``. + + NOTE: it should *not* be used for Linux mainline. + - --codespell Use the codespell dictionary for checking spelling errors. @@ -210,6 +217,13 @@ Available options: Display the help text. +Configuration file +================== + +Default configuration options can be stored in ``.checkpatch.conf``, search +path: ``.:$HOME:.scripts`` or in a directory specified by ``$CHECKPATCH_CONFIG_DIR`` +environment variable (falling back to the default search path). + Message Levels ============== diff --git a/Documentation/dev-tools/kcov.rst b/Documentation/dev-tools/kcov.rst index 8127849d40f59..1a739290c8ecc 100644 --- a/Documentation/dev-tools/kcov.rst +++ b/Documentation/dev-tools/kcov.rst @@ -237,6 +237,9 @@ Both ``kcov_remote_start`` and ``kcov_remote_stop`` annotations and the collection sections. The way a handle is used depends on the context where the matching code section executes. +A thread can use two separate KCOV instances to collect remote coverage and +normal coverage at the same time. + KCOV supports collecting remote coverage from the following contexts: 1. Global kernel background tasks. These are the tasks that are spawned during @@ -262,6 +265,9 @@ gets saved to the ``kcov_handle`` field in the current ``task_struct`` and needs to be passed to the newly spawned local tasks via custom kernel code modifications. Those tasks should in turn use the passed handle in their ``kcov_remote_start`` and ``kcov_remote_stop`` annotations. +In the kernel, common handles are wrapped in a ``kcov_common_handle_id``, which +consumes no space in builds without ``CONFIG_KCOV``; subsystems that integrate +with this mechanism should not need to use any ``#ifdef CONFIG_KCOV`` or such. KCOV follows a predefined format for both global and common handles. Each handle is a ``u64`` integer. Currently, only the one top and the lower 4 bytes diff --git a/MAINTAINERS b/MAINTAINERS index fcc46165038cc..c338f957f5fd3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -5969,8 +5969,6 @@ F: drivers/input/keyboard/charlieplex_keypad.c CHECKPATCH M: Andy Whitcroft <apw@canonical.com> M: Joe Perches <joe@perches.com> -R: Dwaipayan Ray <dwaipayanray1@gmail.com> -R: Lukas Bulwahn <lukas.bulwahn@gmail.com> S: Maintained F: scripts/checkpatch.pl @@ -24825,7 +24823,7 @@ F: drivers/md/md* F: drivers/md/raid* F: include/linux/raid/ F: include/uapi/linux/raid/ -F: lib/raid6/ +F: lib/raid/raid6/ SOLIDRUN CLEARFOG SUPPORT M: Russell King <linux@armlinux.org.uk> diff --git a/Makefile b/Makefile index f056c921ea9cd..8813f24b1ff4c 100644 --- a/Makefile +++ b/Makefile @@ -826,12 +826,6 @@ endif # KBUILD_EXTMOD # Defaults to vmlinux, but the arch makefile usually adds further targets all: vmlinux -CFLAGS_GCOV := -fprofile-arcs -ftest-coverage -ifdef CONFIG_CC_IS_GCC -CFLAGS_GCOV += -fno-tree-loop-im -endif -export CFLAGS_GCOV - # The arch Makefiles can override CC_FLAGS_FTRACE. We may also append it later. ifdef CONFIG_FUNCTION_TRACER CC_FLAGS_FTRACE := -pg @@ -1149,6 +1143,27 @@ endif # Ensure compilers do not transform certain loops into calls to wcslen() KBUILD_CFLAGS += -fno-builtin-wcslen +CFLAGS_GCOV := -fprofile-arcs -ftest-coverage +ifdef CONFIG_CC_IS_GCC +CFLAGS_GCOV += -fno-tree-loop-im +# Use atomic counter updates to avoid concurrent-access crashes in GCOV. +# Only enable if -fprofile-update=prefer-atomic does not introduce new +# undefined symbols (e.g. libatomic calls that the kernel cannot link). +CFLAGS_GCOV += $(call try-run,\ + echo 'long long x; void f(void){x++;}' | \ + $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -w -fprofile-arcs \ + -ftest-coverage -x c - -c -o "$$TMP.base" && \ + echo 'long long x; void f(void){x++;}' | \ + $(CC) $(KBUILD_CPPFLAGS) $(KBUILD_CFLAGS) -w -fprofile-arcs \ + -ftest-coverage -fprofile-update=prefer-atomic \ + -x c - -c -o "$$TMP" && \ + $(NM) "$$TMP.base" | grep ' U ' > "$$TMP.ubase" || true ; \ + $(NM) "$$TMP" | grep ' U ' > "$$TMP.utest" || true ; \ + cmp -s "$$TMP.ubase" "$$TMP.utest",\ + -fprofile-update=prefer-atomic) +endif +export CFLAGS_GCOV + # change __FILE__ to the relative path to the source directory ifdef building_out_of_srctree KBUILD_CPPFLAGS += -fmacro-prefix-map=$(srcroot)/= diff --git a/arch/arc/include/asm/uaccess.h b/arch/arc/include/asm/uaccess.h index 1e8809ea000a3..6df2209541ac0 100644 --- a/arch/arc/include/asm/uaccess.h +++ b/arch/arc/include/asm/uaccess.h @@ -628,8 +628,7 @@ static inline unsigned long __clear_user(void __user *to, unsigned long n) return res; } -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER #define __clear_user __clear_user diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index d6ae80b5df36f..1593cf3b98008 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -616,8 +616,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) } #define __clear_user(addr, n) (memset((void __force *)addr, 0, n), 0) #endif -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) { diff --git a/arch/arm64/include/asm/uaccess.h b/arch/arm64/include/asm/uaccess.h index b0c83a08dda97..9f5bd9c69c249 100644 --- a/arch/arm64/include/asm/uaccess.h +++ b/arch/arm64/include/asm/uaccess.h @@ -456,8 +456,7 @@ do { \ unsafe_copy_loop(__ucu_dst, __ucu_src, __ucu_len, u8, label); \ } while (0) -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER extern unsigned long __must_check __arch_clear_user(void __user *to, unsigned long n); static inline unsigned long __must_check __clear_user(void __user *to, unsigned long n) diff --git a/arch/hexagon/include/asm/uaccess.h b/arch/hexagon/include/asm/uaccess.h index bff77efc0d9a9..1aecf60ec4f5a 100644 --- a/arch/hexagon/include/asm/uaccess.h +++ b/arch/hexagon/include/asm/uaccess.h @@ -26,8 +26,7 @@ unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER __kernel_size_t __clear_user_hexagon(void __user *dest, unsigned long count); #define __clear_user(a, s) __clear_user_hexagon((a), (s)) diff --git a/arch/loongarch/include/asm/uaccess.h b/arch/loongarch/include/asm/uaccess.h index 438269313e78c..428f373feabf1 100644 --- a/arch/loongarch/include/asm/uaccess.h +++ b/arch/loongarch/include/asm/uaccess.h @@ -292,8 +292,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) return __copy_user((__force void *)to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * __clear_user: - Zero a block of memory in user space, with less checking. diff --git a/arch/m68k/include/asm/uaccess.h b/arch/m68k/include/asm/uaccess.h index 64914872a5c98..31d133faa45ef 100644 --- a/arch/m68k/include/asm/uaccess.h +++ b/arch/m68k/include/asm/uaccess.h @@ -377,8 +377,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) return __constant_copy_to_user(to, from, n); return __generic_copy_to_user(to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #define __get_kernel_nofault(dst, src, type, err_label) \ do { \ diff --git a/arch/microblaze/include/asm/uaccess.h b/arch/microblaze/include/asm/uaccess.h index 3aab2f17e0462..afa0dd8d013fb 100644 --- a/arch/microblaze/include/asm/uaccess.h +++ b/arch/microblaze/include/asm/uaccess.h @@ -250,8 +250,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_tofrom_user(to, (__force const void __user *)from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * Copy a null terminated string from userspace. diff --git a/arch/mips/include/asm/uaccess.h b/arch/mips/include/asm/uaccess.h index c0cede273c7c0..f00c36676b737 100644 --- a/arch/mips/include/asm/uaccess.h +++ b/arch/mips/include/asm/uaccess.h @@ -433,8 +433,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) return __cu_len_r; } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER extern __kernel_size_t __bzero(void __user *addr, __kernel_size_t size); diff --git a/arch/nios2/include/asm/uaccess.h b/arch/nios2/include/asm/uaccess.h index 6ccc9a232c239..5e6e05cc6efc7 100644 --- a/arch/nios2/include/asm/uaccess.h +++ b/arch/nios2/include/asm/uaccess.h @@ -57,8 +57,7 @@ extern unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n); extern unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n); -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER extern long strncpy_from_user(char *__to, const char __user *__from, long __len); diff --git a/arch/openrisc/include/asm/uaccess.h b/arch/openrisc/include/asm/uaccess.h index d6500a374e183..db934ebc0069f 100644 --- a/arch/openrisc/include/asm/uaccess.h +++ b/arch/openrisc/include/asm/uaccess.h @@ -218,8 +218,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long size) { return __copy_tofrom_user((__force void *)to, from, size); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER extern unsigned long __clear_user(void __user *addr, unsigned long size); diff --git a/arch/parisc/include/asm/uaccess.h b/arch/parisc/include/asm/uaccess.h index 6c531d2c847eb..0d17f81c8b270 100644 --- a/arch/parisc/include/asm/uaccess.h +++ b/arch/parisc/include/asm/uaccess.h @@ -197,7 +197,6 @@ unsigned long __must_check raw_copy_to_user(void __user *dst, const void *src, unsigned long len); unsigned long __must_check raw_copy_from_user(void *dst, const void __user *src, unsigned long len); -#define INLINE_COPY_TO_USER -#define INLINE_COPY_FROM_USER +#define INLINE_COPY_USER #endif /* __PARISC_UACCESS_H */ diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c5754942cf85a..0d10b299bad83 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -404,9 +404,6 @@ config ARCH_RV32I bool "RV32I" depends on NONPORTABLE select 32BIT - select GENERIC_LIB_ASHLDI3 - select GENERIC_LIB_ASHRDI3 - select GENERIC_LIB_LSHRDI3 select GENERIC_LIB_UCMPDI2 config ARCH_RV64I diff --git a/arch/riscv/include/asm/asm-prototypes.h b/arch/riscv/include/asm/asm-prototypes.h index 5b90ba5314ee9..a0ca9efff267e 100644 --- a/arch/riscv/include/asm/asm-prototypes.h +++ b/arch/riscv/include/asm/asm-prototypes.h @@ -5,6 +5,10 @@ #include <linux/ftrace.h> #include <asm-generic/asm-prototypes.h> +long long __lshrdi3(long long a, int b); +long long __ashrdi3(long long a, int b); +long long __ashldi3(long long a, int b); + long long __lshrti3(long long a, int b); long long __ashrti3(long long a, int b); long long __ashlti3(long long a, int b); diff --git a/arch/riscv/kernel/image-vars.h b/arch/riscv/kernel/image-vars.h index 3bd9d06a8b8ff..7b44b94f1283b 100644 --- a/arch/riscv/kernel/image-vars.h +++ b/arch/riscv/kernel/image-vars.h @@ -32,6 +32,15 @@ __efistub___init_text_end = __init_text_end; __efistub_sysfb_primary_display = sysfb_primary_display; #endif +/* + * These double-word integer shifts are used by the library code, and + * the first two of them are required to link EFI stub. Note __ashrdi3() + * is not actually used by the stub but this may change in the future. + */ +PROVIDE(__efistub___lshrdi3 = __lshrdi3); +PROVIDE(__efistub___ashldi3 = __ashldi3); +PROVIDE(__efistub___ashrdi3 = __ashrdi3); + #endif #endif /* __RISCV_KERNEL_IMAGE_VARS_H */ diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile index 6f767b2a349d7..f668b98970bda 100644 --- a/arch/riscv/lib/Makefile +++ b/arch/riscv/lib/Makefile @@ -16,6 +16,7 @@ ifeq ($(CONFIG_MMU), y) lib-$(CONFIG_RISCV_ISA_V) += uaccess_vector.o endif lib-$(CONFIG_MMU) += uaccess.o +lib-$(CONFIG_32BIT) += ashldi3.o ashrdi3.o lshrdi3.o lib-$(CONFIG_64BIT) += tishift.o lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o diff --git a/arch/riscv/lib/ashldi3.S b/arch/riscv/lib/ashldi3.S new file mode 100644 index 0000000000000..c3408862e2f6a --- /dev/null +++ b/arch/riscv/lib/ashldi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/** + * Adopted for the Linux kernel from IPXE project, see + * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/** + * Shift left + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + +SYM_FUNC_START(__ashldi3) + + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a1, a0 + mv a0, zero +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + srl t2, a0, t1 + sll a0, a0, a2 + sll a1, a1, a2 + or a1, a1, t2 +2: ret + +SYM_FUNC_END(__ashldi3) +EXPORT_SYMBOL(__ashldi3) diff --git a/arch/riscv/lib/ashrdi3.S b/arch/riscv/lib/ashrdi3.S new file mode 100644 index 0000000000000..426de09466064 --- /dev/null +++ b/arch/riscv/lib/ashrdi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/** + * Adopted for the Linux kernel from IPXE project, see + * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/** + * Arithmetic shift right + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + +SYM_FUNC_START(__ashrdi3) + + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a0, a1 + srai a1, a1, 31 +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + sll t2, a1, t1 + sra a1, a1, a2 + srl a0, a0, a2 + or a0, a0, t2 +2: ret + +SYM_FUNC_END(__ashrdi3) +EXPORT_SYMBOL(__ashrdi3) diff --git a/arch/riscv/lib/lshrdi3.S b/arch/riscv/lib/lshrdi3.S new file mode 100644 index 0000000000000..1af03985ccb72 --- /dev/null +++ b/arch/riscv/lib/lshrdi3.S @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/** + * Adopted for the Linux kernel from IPXE project, see + * https://github.com/ipxe/ipxe/blob/master/src/arch/riscv32/libgcc/llshift.S + */ + +#include <linux/linkage.h> +#include <asm/asm.h> + +/** + * Logical shift right + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + +SYM_FUNC_START(__lshrdi3) + + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a0, a1 + mv a1, zero +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + sll t2, a1, t1 + srl a1, a1, a2 + srl a0, a0, a2 + or a0, a0, t2 +2: ret + +SYM_FUNC_END(__lshrdi3) +EXPORT_SYMBOL(__lshrdi3) diff --git a/arch/s390/include/asm/uaccess.h b/arch/s390/include/asm/uaccess.h index dff035372601e..a9f32c53f699b 100644 --- a/arch/s390/include/asm/uaccess.h +++ b/arch/s390/include/asm/uaccess.h @@ -30,8 +30,7 @@ void debug_user_asce(int exit); #define uaccess_kmsan_or_inline __always_inline #endif -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER static uaccess_kmsan_or_inline __must_check unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long size) diff --git a/arch/sh/include/asm/uaccess.h b/arch/sh/include/asm/uaccess.h index a79609eb14be4..02e7a066538ec 100644 --- a/arch/sh/include/asm/uaccess.h +++ b/arch/sh/include/asm/uaccess.h @@ -95,8 +95,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) { return __copy_user((__force void *)to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * Clear the area and return remaining number of bytes diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 43284b6ec46a6..5542d5b32994f 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -190,8 +190,7 @@ static inline unsigned long raw_copy_from_user(void *to, const void __user *from return __copy_user((__force void __user *) to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER static inline unsigned long __clear_user(void __user *addr, unsigned long size) { diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index b825a5dd0210e..e2989cfba626d 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -231,8 +231,7 @@ unsigned long __must_check raw_copy_from_user(void *to, unsigned long __must_check raw_copy_to_user(void __user *to, const void *from, unsigned long size); -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER unsigned long __must_check raw_copy_in_user(void __user *to, const void __user *from, diff --git a/arch/um/include/asm/uaccess.h b/arch/um/include/asm/uaccess.h index 0df9ea4abda83..4417c8b1d37a6 100644 --- a/arch/um/include/asm/uaccess.h +++ b/arch/um/include/asm/uaccess.h @@ -27,8 +27,7 @@ static inline int __access_ok(const void __user *ptr, unsigned long size); #define __access_ok __access_ok #define __clear_user __clear_user -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #include <asm-generic/uaccess.h> diff --git a/arch/xtensa/include/asm/uaccess.h b/arch/xtensa/include/asm/uaccess.h index 56aec6d504fee..6538a29a2bbd4 100644 --- a/arch/xtensa/include/asm/uaccess.h +++ b/arch/xtensa/include/asm/uaccess.h @@ -237,8 +237,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) prefetch(from); return __xtensa_copy_user((__force void *)to, from, n); } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER /* * We need to return the number of bytes not cleared. Our memset() diff --git a/certs/Kconfig b/certs/Kconfig index 8e39a80c7abe5..9d2bf7fb5b9e4 100644 --- a/certs/Kconfig +++ b/certs/Kconfig @@ -6,14 +6,14 @@ config MODULE_SIG_KEY default "certs/signing_key.pem" depends on MODULE_SIG || (IMA_APPRAISE_MODSIG && MODULES) help - Provide the file name of a private key/certificate in PEM format, - or a PKCS#11 URI according to RFC7512. The file should contain, or - the URI should identify, both the certificate and its corresponding - private key. + Provide the file name of a private key/certificate in PEM format, + or a PKCS#11 URI according to RFC7512. The file should contain, or + the URI should identify, both the certificate and its corresponding + private key. - If this option is unchanged from its default "certs/signing_key.pem", - then the kernel will automatically generate the private key and - certificate as described in Documentation/admin-guide/module-signing.rst + If this option is unchanged from its default "certs/signing_key.pem", + then the kernel will automatically generate the private key and + certificate as described in Documentation/admin-guide/module-signing.rst choice prompt "Type of module signing key to be generated" diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c index 9e4bb7fbde25e..27f99349e310b 100644 --- a/crypto/async_tx/async_pq.c +++ b/crypto/async_tx/async_pq.c @@ -8,6 +8,7 @@ #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/raid/pq.h> +#include <linux/raid/pq_tables.h> #include <linux/async_tx.h> #include <linux/gfp.h> @@ -119,7 +120,7 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, for (i = 0; i < disks; i++) { if (blocks[i] == NULL) { BUG_ON(i > disks - 3); /* P or Q can't be zero */ - srcs[i] = raid6_get_zero_page(); + srcs[i] = page_address(ZERO_PAGE(0)); } else { srcs[i] = page_address(blocks[i]) + offsets[i]; @@ -131,11 +132,11 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks, } } if (submit->flags & ASYNC_TX_PQ_XOR_DST) { - BUG_ON(!raid6_call.xor_syndrome); + BUG_ON(!raid6_can_xor_syndrome()); if (start >= 0) - raid6_call.xor_syndrome(disks, start, stop, len, srcs); + raid6_xor_syndrome(disks, start, stop, len, srcs); } else - raid6_call.gen_syndrome(disks, len, srcs); + raid6_gen_syndrome(disks, len, srcs); async_tx_sync_epilog(submit); } diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c index 539ea5b378dcd..e53870d84bc55 100644 --- a/crypto/async_tx/async_raid6_recov.c +++ b/crypto/async_tx/async_raid6_recov.c @@ -11,6 +11,7 @@ #include <linux/module.h> #include <linux/dma-mapping.h> #include <linux/raid/pq.h> +#include <linux/raid/pq_tables.h> #include <linux/async_tx.h> #include <linux/dmaengine.h> @@ -414,11 +415,11 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, async_tx_quiesce(&submit->depend_tx); for (i = 0; i < disks; i++) if (blocks[i] == NULL) - ptrs[i] = raid6_get_zero_page(); + ptrs[i] = page_address(ZERO_PAGE(0)); else ptrs[i] = page_address(blocks[i]) + offs[i]; - raid6_2data_recov(disks, bytes, faila, failb, ptrs); + raid6_recov_2data(disks, bytes, faila, failb, ptrs); async_tx_sync_epilog(submit); @@ -497,11 +498,11 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila, async_tx_quiesce(&submit->depend_tx); for (i = 0; i < disks; i++) if (blocks[i] == NULL) - ptrs[i] = raid6_get_zero_page(); + ptrs[i] = page_address(ZERO_PAGE(0)); else ptrs[i] = page_address(blocks[i]) + offs[i]; - raid6_datap_recov(disks, bytes, faila, ptrs); + raid6_recov_datap(disks, bytes, faila, ptrs); async_tx_sync_epilog(submit); diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c index ed037fa883f6f..0de03611252ed 100644 --- a/drivers/dma/bcm-sba-raid.c +++ b/drivers/dma/bcm-sba-raid.c @@ -40,6 +40,7 @@ #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/raid/pq.h> +#include <linux/raid/pq_tables.h> #include "dmaengine.h" diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index cfedb3025c263..77a2b2d74f3f6 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -95,8 +95,10 @@ CFLAGS_zboot-decompress-gzip.o += -I$(srctree)/lib/zlib_inflate zboot-obj-$(CONFIG_KERNEL_ZSTD) := zboot-decompress-zstd.o lib-xxhash.o CFLAGS_zboot-decompress-zstd.o += -I$(srctree)/lib/zstd -zboot-obj-$(CONFIG_RISCV) += lib-clz_ctz.o lib-ashldi3.o -zboot-obj-$(CONFIG_LOONGARCH) += lib-clz_ctz.o lib-ashldi3.o +zboot-riscv-obj-$(CONFIG_32BIT) := lib-ashldi3.o lib-lshrdi3.o +zboot-obj-$(CONFIG_RISCV) += lib-clz_ctz.o $(zboot-riscv-obj-y) +zboot-loongarch-obj-$(CONFIG_32BIT) := lib-ashldi3.o lib-lshrdi3.o +zboot-obj-$(CONFIG_LOONGARCH) += lib-clz_ctz.o $(zboot-loongarch-obj-y) lib-$(CONFIG_EFI_ZBOOT) += zboot.o $(zboot-obj-y) lib-$(CONFIG_UNACCEPTED_MEMORY) += unaccepted_memory.o bitmap.o find.o diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 0d76e82f4506e..ebcb193176702 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -6955,7 +6955,7 @@ raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) if (kstrtoul(page, 10, &new)) return -EINVAL; - if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) + if (new != PARITY_DISABLE_RMW && !raid6_can_xor_syndrome()) return -EINVAL; if (new != PARITY_DISABLE_RMW && @@ -7646,7 +7646,7 @@ static struct r5conf *setup_conf(struct mddev *mddev) conf->level = mddev->new_level; if (conf->level == 6) { conf->max_degraded = 2; - if (raid6_call.xor_syndrome) + if (raid6_can_xor_syndrome()) conf->rmw_level = PARITY_ENABLE_RMW; else conf->rmw_level = PARITY_DISABLE_RMW; diff --git a/drivers/rapidio/devices/tsi721.c b/drivers/rapidio/devices/tsi721.c index 66331e67cf4ef..71b87bf8c31d8 100644 --- a/drivers/rapidio/devices/tsi721.c +++ b/drivers/rapidio/devices/tsi721.c @@ -394,7 +394,6 @@ static void tsi721_db_dpc(struct work_struct *work) idb_work); struct rio_mport *mport; struct rio_dbell *dbell; - int found = 0; u32 wr_ptr, rd_ptr; u64 *idb_entry; u32 regval; @@ -412,6 +411,8 @@ static void tsi721_db_dpc(struct work_struct *work) rd_ptr = ioread32(priv->regs + TSI721_IDQ_RP(IDB_QUEUE)) % IDB_QSIZE; while (wr_ptr != rd_ptr) { + int found = 0; + idb_entry = (u64 *)(priv->idb_base + (TSI721_IDB_ENTRY_SIZE * rd_ptr)); rd_ptr++; diff --git a/drivers/usb/usbip/usbip_common.h b/drivers/usb/usbip/usbip_common.h index 282efca64a012..be4c5e65a7f8c 100644 --- a/drivers/usb/usbip/usbip_common.h +++ b/drivers/usb/usbip/usbip_common.h @@ -282,9 +282,7 @@ struct usbip_device { void (*unusable)(struct usbip_device *); } eh_ops; -#ifdef CONFIG_KCOV - u64 kcov_handle; -#endif + struct kcov_common_handle_id kcov_handle; }; #define kthread_get_run(threadfn, data, namefmt, ...) \ @@ -339,29 +337,4 @@ static inline int interface_to_devnum(struct usb_interface *interface) return udev->devnum; } -#ifdef CONFIG_KCOV - -static inline void usbip_kcov_handle_init(struct usbip_device *ud) -{ - ud->kcov_handle = kcov_common_handle(); -} - -static inline void usbip_kcov_remote_start(struct usbip_device *ud) -{ - kcov_remote_start_common(ud->kcov_handle); -} - -static inline void usbip_kcov_remote_stop(void) -{ - kcov_remote_stop(); -} - -#else /* CONFIG_KCOV */ - -static inline void usbip_kcov_handle_init(struct usbip_device *ud) { } -static inline void usbip_kcov_remote_start(struct usbip_device *ud) { } -static inline void usbip_kcov_remote_stop(void) { } - -#endif /* CONFIG_KCOV */ - #endif /* __USBIP_COMMON_H */ diff --git a/drivers/usb/usbip/vhci_rx.c b/drivers/usb/usbip/vhci_rx.c index a75f4a898a412..a678e7c898375 100644 --- a/drivers/usb/usbip/vhci_rx.c +++ b/drivers/usb/usbip/vhci_rx.c @@ -261,9 +261,9 @@ int vhci_rx_loop(void *data) if (usbip_event_happened(ud)) break; - usbip_kcov_remote_start(ud); + kcov_remote_start_common(ud->kcov_handle); vhci_rx_pdu(ud); - usbip_kcov_remote_stop(); + kcov_remote_stop(); } return 0; diff --git a/drivers/usb/usbip/vhci_sysfs.c b/drivers/usb/usbip/vhci_sysfs.c index 5bc8c47788d45..b98d14c43d13d 100644 --- a/drivers/usb/usbip/vhci_sysfs.c +++ b/drivers/usb/usbip/vhci_sysfs.c @@ -425,7 +425,7 @@ static ssize_t attach_store(struct device *dev, struct device_attribute *attr, vdev->ud.tcp_rx = tcp_rx; vdev->ud.tcp_tx = tcp_tx; vdev->ud.status = VDEV_ST_NOTASSIGNED; - usbip_kcov_handle_init(&vdev->ud); + vdev->ud.kcov_handle = kcov_common_handle(); spin_unlock(&vdev->ud.lock); spin_unlock_irqrestore(&vhci->lock, flags); diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index 4fe99765c5c73..0192ade6e7491 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -44,7 +44,7 @@ struct vhost_worker { /* Used to serialize device wide flushing with worker swapping. */ struct mutex mutex; struct llist_head work_list; - u64 kcov_handle; + struct kcov_common_handle_id kcov_handle; u32 id; int attachment_cnt; bool killed; diff --git a/fs/Kconfig b/fs/Kconfig index 43cb06de297ff..cf6ae64776e62 100644 --- a/fs/Kconfig +++ b/fs/Kconfig @@ -78,7 +78,7 @@ config FS_DAX --map=mem: https://docs.pmem.io/ndctl-user-guide/ndctl-man-pages/ndctl-create-namespace - For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most + For ndctl to work CONFIG_DEV_DAX needs to be enabled as well. For most file systems DAX support needs to be manually enabled globally or per-inode using a mount option as well. See the file documentation in Documentation/filesystems/dax.rst for details. @@ -116,8 +116,8 @@ config FILE_LOCKING default y help This option enables standard file locking support, required - for filesystems like NFS and for the flock() system - call. Disabling this option saves about 11k. + for filesystems like NFS and for the flock() system + call. Disabling this option saves about 11k. source "fs/crypto/Kconfig" diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c index 08ee8f316d96d..dabc9522e8814 100644 --- a/fs/btrfs/raid56.c +++ b/fs/btrfs/raid56.c @@ -1410,7 +1410,7 @@ static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int rbio_qstripe_paddr(rbio, sector_nr, step_nr)); assert_rbio(rbio); - raid6_call.gen_syndrome(rbio->real_stripes, step, pointers); + raid6_gen_syndrome(rbio->real_stripes, step, pointers); } else { /* raid5 */ memcpy(pointers[rbio->nr_data], pointers[0], step); @@ -1987,10 +1987,10 @@ static void recover_vertical_step(struct btrfs_raid_bio *rbio, } if (failb == rbio->real_stripes - 2) { - raid6_datap_recov(rbio->real_stripes, step, + raid6_recov_datap(rbio->real_stripes, step, faila, pointers); } else { - raid6_2data_recov(rbio->real_stripes, step, + raid6_recov_2data(rbio->real_stripes, step, faila, failb, pointers); } } else { @@ -2644,7 +2644,7 @@ static bool verify_one_parity_step(struct btrfs_raid_bio *rbio, if (has_qstripe) { assert_rbio(rbio); /* RAID6, call the library function to fill in our P/Q. */ - raid6_call.gen_syndrome(rbio->real_stripes, step, pointers); + raid6_gen_syndrome(rbio->real_stripes, step, pointers); } else { /* RAID5. */ memcpy(pointers[nr_data], pointers[0], step); diff --git a/fs/fat/fat_test.c b/fs/fat/fat_test.c index 886bf044a9f1d..4eeed9dca5494 100644 --- a/fs/fat/fat_test.c +++ b/fs/fat/fat_test.c @@ -20,6 +20,37 @@ static void fat_checksum_test(struct kunit *test) KUNIT_EXPECT_EQ(test, fat_checksum("ABCDEFGHA "), (u8)98); } +static void fat_clus_to_blknr_test(struct kunit *test) +{ + struct msdos_sb_info sbi = { + .sec_per_clus = 4, + .data_start = 100, + }; + + KUNIT_EXPECT_EQ(test, (sector_t)100, + fat_clus_to_blknr(&sbi, FAT_START_ENT)); + KUNIT_EXPECT_EQ(test, (sector_t)112, fat_clus_to_blknr(&sbi, 5)); +} + +static void fat_get_blknr_offset_test(struct kunit *test) +{ + struct msdos_sb_info sbi = { + .dir_per_block = 16, + .dir_per_block_bits = 4, + }; + + sector_t blknr; + int offset; + + fat_get_blknr_offset(&sbi, 0, &blknr, &offset); + KUNIT_EXPECT_EQ(test, (sector_t)0, blknr); + KUNIT_EXPECT_EQ(test, 0, offset); + + fat_get_blknr_offset(&sbi, (10 << 4) | 7, &blknr, &offset); + KUNIT_EXPECT_EQ(test, (sector_t)10, blknr); + KUNIT_EXPECT_EQ(test, 7, offset); +} + struct fat_timestamp_testcase { const char *name; struct timespec64 ts; @@ -341,6 +372,8 @@ static void fat_truncate_atime_test(struct kunit *test) static struct kunit_case fat_test_cases[] = { KUNIT_CASE(fat_checksum_test), + KUNIT_CASE(fat_clus_to_blknr_test), + KUNIT_CASE(fat_get_blknr_offset_test), KUNIT_CASE_PARAM(fat_time_fat2unix_test, fat_time_gen_params), KUNIT_CASE_PARAM(fat_time_unix2fat_test, fat_time_gen_params), KUNIT_CASE_PARAM(fat_time_unix2fat_clamp_test, diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 7df9921c1a389..d6e977ba65656 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -444,21 +444,26 @@ int ocfs2_truncate_file(struct inode *inode, struct ocfs2_dinode *fe = NULL; struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); - /* We trust di_bh because it comes from ocfs2_inode_lock(), which - * already validated it */ + /* + * On local mounts ocfs2_inode_lock_update() skips the inode + * refresh path, so truncation still needs to reject an inode + * state that no longer matches di_bh. + */ fe = (struct ocfs2_dinode *) di_bh->b_data; trace_ocfs2_truncate_file((unsigned long long)OCFS2_I(inode)->ip_blkno, (unsigned long long)le64_to_cpu(fe->i_size), (unsigned long long)new_i_size); - mlog_bug_on_msg(le64_to_cpu(fe->i_size) != i_size_read(inode), - "Inode %llu, inode i_size = %lld != di " - "i_size = %llu, i_flags = 0x%x\n", - (unsigned long long)OCFS2_I(inode)->ip_blkno, - i_size_read(inode), - (unsigned long long)le64_to_cpu(fe->i_size), - le32_to_cpu(fe->i_flags)); + if (unlikely(le64_to_cpu(fe->i_size) != i_size_read(inode))) { + status = ocfs2_error(inode->i_sb, + "Inode %llu has inconsistent i_size: inode = %lld, dinode = %llu, i_flags = 0x%x\n", + (unsigned long long)OCFS2_I(inode)->ip_blkno, + i_size_read(inode), + (unsigned long long)le64_to_cpu(fe->i_size), + le32_to_cpu(fe->i_flags)); + goto bail; + } if (new_i_size > le64_to_cpu(fe->i_size)) { trace_ocfs2_truncate_file_error( diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index a510a0eb1adcc..432eac01c1763 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -13,6 +13,7 @@ #include <linux/pagemap.h> #include <linux/quotaops.h> #include <linux/iversion.h> +#include <linux/fs_dirent.h> #include <asm/byteorder.h> @@ -64,7 +65,40 @@ static int ocfs2_filecheck_read_inode_block_full(struct inode *inode, static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, struct buffer_head *bh); static int ocfs2_filecheck_repair_inode_block(struct super_block *sb, - struct buffer_head *bh); + struct buffer_head *bh); + +static bool ocfs2_valid_inode_mode(umode_t mode) +{ + return fs_umode_to_ftype(mode) != FT_UNKNOWN; +} + +static bool ocfs2_dinode_has_unexpected_rdev(struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + + return !S_ISCHR(mode) && !S_ISBLK(mode) && di->id1.dev1.i_rdev != 0; +} + +static bool ocfs2_dinode_has_size_without_clusters(struct super_block *sb, + struct ocfs2_dinode *di) +{ + umode_t mode = le16_to_cpu(di->i_mode); + + if (le32_to_cpu(di->i_flags) & OCFS2_SYSTEM_FL) + return false; + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) + return false; + if (!le64_to_cpu(di->i_size) || le32_to_cpu(di->i_clusters)) + return false; + + if (S_ISDIR(mode)) + return true; + + return !ocfs2_sparse_alloc(OCFS2_SB(sb)) && S_ISREG(mode); +} void ocfs2_set_inode_flags(struct inode *inode) { @@ -1494,6 +1528,86 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + /* + * Reject dinodes whose i_mode does not name one of the seven + * canonical POSIX file types. ocfs2_populate_inode() copies + * i_mode verbatim into inode->i_mode and then dispatches via + * switch (mode & S_IFMT) to file/dir/symlink/special_file iops; + * an unrecognised type falls into ocfs2_special_file_iops with + * init_special_inode(), which interprets i_rdev. Constrain the + * type here so the dispatch only ever sees a value mkfs.ocfs2 / + * VFS can produce. + */ + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + goto bail; + } + + /* + * id1.dev1.i_rdev is the device-number arm of the id1 union and + * is only meaningful for character and block device inodes. For + * any other regular user-visible file type the on-disk value + * must be zero. ocfs2_populate_inode() currently runs + * + * inode->i_rdev = huge_decode_dev(le64_to_cpu(fe->id1.dev1.i_rdev)); + * + * unconditionally, before the S_IFMT switch decides whether the + * inode is a special file. As a result, an i_rdev value present + * on a non-device inode is silently published into the in-core + * inode; a subsequent forced re-read or in-core mode mutation + * (cluster peer with raw write access to the shared LUN, + * on-disk corruption, or a separately forged dinode) can then + * expose the attacker-controlled device number to + * init_special_inode() without ever showing an unusual i_mode + * at validation time. + * + * System inodes (OCFS2_SYSTEM_FL) legitimately use the bitmap1 + * and journal1 arms of the same union (allocator i_used / + * i_total counters and the journal ij_flags / + * ij_recovery_generation pair); those bytes are not an i_rdev + * and must not be checked here. Restrict the cross-check to + * non-system inodes, which is the full attacker-controllable + * surface. + */ + if (ocfs2_dinode_has_unexpected_rdev(di)) { + rc = ocfs2_error(sb, + "Invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + goto bail; + } + + /* + * Non-inline directories must not have i_size without allocated + * clusters: directory growth adds storage before advancing i_size, + * and readdir walks i_size block-by-block. A forged directory + * with zero clusters and a huge i_size would repeatedly fault on + * holes while advancing through the claimed size. + * + * Non-inline regular files have the same invariant on non-sparse + * volumes. Sparse regular files are different: truncate can + * legitimately grow i_size without allocating clusters, so keep + * the sparse-alloc carveout for S_IFREG only. System inodes and + * inline-data dinodes have their own storage rules. + */ + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + rc = ocfs2_error(sb, + "Invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + rc = ocfs2_error(sb, + "Invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + goto bail; + } + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) { struct ocfs2_inline_data *data = &di->id2.i_data; @@ -1624,6 +1738,40 @@ static int ocfs2_filecheck_validate_inode_block(struct super_block *sb, (unsigned long long)bh->b_blocknr, le32_to_cpu(di->i_fs_generation)); rc = -OCFS2_FILECHECK_ERR_GENERATION; + goto bail; + } + + if (!ocfs2_valid_inode_mode(le16_to_cpu(di->i_mode))) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: mode 0%o has unknown file type\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_unexpected_rdev(di)) { + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: non-device mode 0%o with i_rdev %llu\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(di->i_mode), + (unsigned long long)le64_to_cpu(di->id1.dev1.i_rdev)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; + goto bail; + } + + if (ocfs2_dinode_has_size_without_clusters(sb, di)) { + if (S_ISDIR(le16_to_cpu(di->i_mode))) + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: directory i_size %llu with i_clusters 0 and no inline-data flag\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + else + mlog(ML_ERROR, + "Filecheck: invalid dinode #%llu: regular file i_size %llu with i_clusters 0 and no inline-data flag on non-sparse volume\n", + (unsigned long long)bh->b_blocknr, + (unsigned long long)le64_to_cpu(di->i_size)); + rc = -OCFS2_FILECHECK_ERR_INVALIDINO; } bail: @@ -1812,4 +1960,3 @@ const struct ocfs2_caching_operations ocfs2_inode_caching_ops = { .co_io_lock = ocfs2_inode_cache_io_lock, .co_io_unlock = ocfs2_inode_cache_io_unlock, }; - diff --git a/fs/ocfs2/journal.c b/fs/ocfs2/journal.c index f9bf3bac085db..fc54cc798ce35 100644 --- a/fs/ocfs2/journal.c +++ b/fs/ocfs2/journal.c @@ -1022,11 +1022,8 @@ static int ocfs2_journal_toggle_dirty(struct ocfs2_super *osb, struct ocfs2_dinode *fe; fe = (struct ocfs2_dinode *)bh->b_data; - - /* The journal bh on the osb always comes from ocfs2_journal_init() - * and was validated there inside ocfs2_inode_lock_full(). It's a - * code bug if we mess it up. */ - BUG_ON(!OCFS2_IS_VALID_DINODE(fe)); + if (WARN_ON(!OCFS2_IS_VALID_DINODE(fe))) + return -EIO; flags = le32_to_cpu(fe->id1.journal1.ij_flags); if (dirty) diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h index 7b50e03dfa664..62cad6522c7a3 100644 --- a/fs/ocfs2/ocfs2.h +++ b/fs/ocfs2/ocfs2.h @@ -494,8 +494,6 @@ struct ocfs2_super struct rb_root osb_rf_lock_tree; struct ocfs2_refcount_tree *osb_ref_tree_lru; - struct mutex system_file_mutex; - /* * OCFS2 needs to schedule several different types of work which * require cluster locking, disk I/O, recovery waits, etc. Since these diff --git a/fs/ocfs2/quota_local.c b/fs/ocfs2/quota_local.c index 12cbb4fccda0d..f55810c59b1b1 100644 --- a/fs/ocfs2/quota_local.c +++ b/fs/ocfs2/quota_local.c @@ -302,7 +302,7 @@ static int ocfs2_add_recovery_chunk(struct super_block *sb, if (!rc) return -ENOMEM; rc->rc_chunk = chunk; - rc->rc_bitmap = kmalloc(sb->s_blocksize, GFP_NOFS); + rc->rc_bitmap = kzalloc(sb->s_blocksize, GFP_NOFS); if (!rc->rc_bitmap) { kfree(rc); return -ENOMEM; diff --git a/fs/ocfs2/super.c b/fs/ocfs2/super.c index b875f01c97564..6dd45c2153f88 100644 --- a/fs/ocfs2/super.c +++ b/fs/ocfs2/super.c @@ -1997,8 +1997,6 @@ static int ocfs2_initialize_super(struct super_block *sb, spin_lock_init(&osb->osb_xattr_lock); ocfs2_init_steal_slots(osb); - mutex_init(&osb->system_file_mutex); - atomic_set(&osb->alloc_stats.moves, 0); atomic_set(&osb->alloc_stats.local_data, 0); atomic_set(&osb->alloc_stats.bitmap_data, 0); diff --git a/fs/ocfs2/sysfile.c b/fs/ocfs2/sysfile.c index d53a6cc866bef..67e492f4b828b 100644 --- a/fs/ocfs2/sysfile.c +++ b/fs/ocfs2/sysfile.c @@ -98,11 +98,9 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, } else arr = get_local_system_inode(osb, type, slot); - mutex_lock(&osb->system_file_mutex); if (arr && ((inode = *arr) != NULL)) { /* get a ref in addition to the array ref */ inode = igrab(inode); - mutex_unlock(&osb->system_file_mutex); BUG_ON(!inode); return inode; @@ -112,11 +110,10 @@ struct inode *ocfs2_get_system_file_inode(struct ocfs2_super *osb, inode = _ocfs2_get_system_file_inode(osb, type, slot); /* add one more if putting into array for first time */ - if (arr && inode) { - *arr = igrab(inode); - BUG_ON(!*arr); + if (inode && arr && !*arr && !cmpxchg(&(*arr), NULL, inode)) { + inode = igrab(inode); + BUG_ON(!inode); } - mutex_unlock(&osb->system_file_mutex); return inode; } diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index 86cfd4c2adf92..fcddd3c13acdd 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -950,15 +950,51 @@ static int ocfs2_xattr_list_entries(struct inode *inode, return result; } +static int ocfs2_xattr_ibody_lookup_header(struct inode *inode, + struct ocfs2_dinode *di, + struct ocfs2_xattr_header **header) +{ + u16 xattr_count; + size_t max_entries; + u16 inline_size = le16_to_cpu(di->i_xattr_inline_size); + + if (inline_size > inode->i_sb->s_blocksize || + inline_size < sizeof(struct ocfs2_xattr_header)) { + ocfs2_error(inode->i_sb, + "Invalid xattr inline size %u in inode %llu\n", + inline_size, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + return -EFSCORRUPTED; + } + + *header = (struct ocfs2_xattr_header *) + ((void *)di + inode->i_sb->s_blocksize - inline_size); + + xattr_count = le16_to_cpu((*header)->xh_count); + max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / + sizeof(struct ocfs2_xattr_entry); + + if (xattr_count > max_entries) { + ocfs2_error(inode->i_sb, + "xattr entry count %u exceeds maximum %zu in inode %llu\n", + xattr_count, max_entries, + (unsigned long long)OCFS2_I(inode)->ip_blkno); + return -EFSCORRUPTED; + } + + return 0; +} + int ocfs2_has_inline_xattr_value_outside(struct inode *inode, struct ocfs2_dinode *di) { struct ocfs2_xattr_header *xh; + int ret; int i; - xh = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - - le16_to_cpu(di->i_xattr_inline_size)); + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xh); + if (ret) + return 1; for (i = 0; i < le16_to_cpu(xh->xh_count); i++) if (!ocfs2_xattr_is_local(&xh->xh_entries[i])) @@ -975,39 +1011,13 @@ static int ocfs2_xattr_ibody_list(struct inode *inode, struct ocfs2_xattr_header *header = NULL; struct ocfs2_inode_info *oi = OCFS2_I(inode); int ret = 0; - u16 xattr_count; - size_t max_entries; - u16 inline_size; if (!(oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL)) return ret; - inline_size = le16_to_cpu(di->i_xattr_inline_size); - - /* Validate inline size is reasonable */ - if (inline_size > inode->i_sb->s_blocksize || - inline_size < sizeof(struct ocfs2_xattr_header)) { - ocfs2_error(inode->i_sb, - "Invalid xattr inline size %u in inode %llu\n", - inline_size, - (unsigned long long)OCFS2_I(inode)->ip_blkno); - return -EFSCORRUPTED; - } - - header = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - inline_size); - - xattr_count = le16_to_cpu(header->xh_count); - max_entries = (inline_size - sizeof(struct ocfs2_xattr_header)) / - sizeof(struct ocfs2_xattr_entry); - - if (xattr_count > max_entries) { - ocfs2_error(inode->i_sb, - "xattr entry count %u exceeds maximum %zu in inode %llu\n", - xattr_count, max_entries, - (unsigned long long)OCFS2_I(inode)->ip_blkno); - return -EFSCORRUPTED; - } + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; ret = ocfs2_xattr_list_entries(inode, header, buffer, buffer_size); @@ -1200,8 +1210,9 @@ static int ocfs2_xattr_ibody_get(struct inode *inode, return -ENODATA; xs->end = (void *)di + inode->i_sb->s_blocksize; - xs->header = (struct ocfs2_xattr_header *) - (xs->end - le16_to_cpu(di->i_xattr_inline_size)); + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header); + if (ret) + return ret; xs->base = (void *)xs->header; xs->here = xs->header->xh_entries; @@ -2465,9 +2476,9 @@ static int ocfs2_xattr_ibody_remove(struct inode *inode, .vb_access = ocfs2_journal_access_di, }; - header = (struct ocfs2_xattr_header *) - ((void *)di + inode->i_sb->s_blocksize - - le16_to_cpu(di->i_xattr_inline_size)); + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; ret = ocfs2_remove_value_outside(inode, &vb, header, ref_ci, ref_root_bh); @@ -2726,12 +2737,14 @@ static int ocfs2_xattr_ibody_find(struct inode *inode, xs->xattr_bh = xs->inode_bh; xs->end = (void *)di + inode->i_sb->s_blocksize; - if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) - xs->header = (struct ocfs2_xattr_header *) - (xs->end - le16_to_cpu(di->i_xattr_inline_size)); - else + if (oi->ip_dyn_features & OCFS2_INLINE_XATTR_FL) { + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &xs->header); + if (ret) + return ret; + } else { xs->header = (struct ocfs2_xattr_header *) (xs->end - OCFS2_SB(inode->i_sb)->s_xattr_inline_size); + } xs->base = (void *)xs->header; xs->here = xs->header->xh_entries; @@ -6003,14 +6016,17 @@ static int ocfs2_xattr_inline_attach_refcount(struct inode *inode, struct ocfs2_cached_dealloc_ctxt *dealloc) { struct ocfs2_dinode *di = (struct ocfs2_dinode *)fe_bh->b_data; - struct ocfs2_xattr_header *header = (struct ocfs2_xattr_header *) - (fe_bh->b_data + inode->i_sb->s_blocksize - - le16_to_cpu(di->i_xattr_inline_size)); + struct ocfs2_xattr_header *header; + int ret; struct ocfs2_xattr_value_buf vb = { .vb_bh = fe_bh, .vb_access = ocfs2_journal_access_di, }; + ret = ocfs2_xattr_ibody_lookup_header(inode, di, &header); + if (ret) + return ret; + return ocfs2_xattr_attach_refcount_normal(inode, &vb, header, ref_ci, ref_root_bh, dealloc); } @@ -6495,12 +6511,10 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args) handle_t *handle; struct ocfs2_super *osb = OCFS2_SB(args->old_inode->i_sb); struct ocfs2_dinode *di = (struct ocfs2_dinode *)args->old_bh->b_data; - int inline_size = le16_to_cpu(di->i_xattr_inline_size); - int header_off = osb->sb->s_blocksize - inline_size; - struct ocfs2_xattr_header *xh = (struct ocfs2_xattr_header *) - (args->old_bh->b_data + header_off); - struct ocfs2_xattr_header *new_xh = (struct ocfs2_xattr_header *) - (args->new_bh->b_data + header_off); + int inline_size; + int header_off; + struct ocfs2_xattr_header *xh; + struct ocfs2_xattr_header *new_xh; struct ocfs2_alloc_context *meta_ac = NULL; struct ocfs2_inode_info *new_oi; struct ocfs2_dinode *new_di; @@ -6509,6 +6523,15 @@ static int ocfs2_reflink_xattr_inline(struct ocfs2_xattr_reflink *args) .vb_access = ocfs2_journal_access_di, }; + ret = ocfs2_xattr_ibody_lookup_header(args->old_inode, di, &xh); + if (ret) + goto out; + + inline_size = le16_to_cpu(di->i_xattr_inline_size); + header_off = osb->sb->s_blocksize - inline_size; + new_xh = (struct ocfs2_xattr_header *) + (args->new_bh->b_data + header_off); + ret = ocfs2_reflink_lock_xattr_allocators(osb, xh, args->ref_root_bh, &credits, &meta_ac); if (ret) { diff --git a/fs/proc/base.c b/fs/proc/base.c index d9acfa89c894b..49344de101582 100644 --- a/fs/proc/base.c +++ b/fs/proc/base.c @@ -3543,28 +3543,42 @@ out: struct tgid_iter { unsigned int tgid; struct task_struct *task; + struct pid_namespace *const pid_ns; }; -static struct tgid_iter next_tgid(struct pid_namespace *ns, struct tgid_iter iter) + +static struct tgid_iter +make_tgid_iter(unsigned int init_tgid, struct pid_namespace *pid_ns) { - struct pid *pid; + return (struct tgid_iter){ + .tgid = init_tgid - 1, + .pid_ns = pid_ns, + }; +} + +static bool next_tgid(struct tgid_iter *it) +{ + if (it->task) { + put_task_struct(it->task); + it->task = NULL; + } - if (iter.task) - put_task_struct(iter.task); rcu_read_lock(); -retry: - iter.task = NULL; - pid = find_ge_pid(iter.tgid, ns); - if (pid) { - iter.tgid = pid_nr_ns(pid, ns); - iter.task = pid_task(pid, PIDTYPE_TGID); - if (!iter.task) { - iter.tgid += 1; - goto retry; + while (1) { + it->tgid += 1; + const auto pid = find_ge_pid(it->tgid, it->pid_ns); + if (pid) { + it->tgid = pid_nr_ns(pid, it->pid_ns); + it->task = pid_task(pid, PIDTYPE_TGID); + if (it->task) { + get_task_struct(it->task); + rcu_read_unlock(); + return true; + } + } else { + rcu_read_unlock(); + return false; } - get_task_struct(iter.task); } - rcu_read_unlock(); - return iter; } #define TGID_OFFSET (FIRST_PROCESS_ENTRY + 2) @@ -3572,9 +3586,8 @@ retry: /* for the /proc/ directory itself, after non-process stuff has been done */ int proc_pid_readdir(struct file *file, struct dir_context *ctx) { - struct tgid_iter iter; struct proc_fs_info *fs_info = proc_sb_info(file_inode(file)->i_sb); - struct pid_namespace *ns = proc_pid_ns(file_inode(file)->i_sb); + struct pid_namespace *pid_ns = proc_pid_ns(file_inode(file)->i_sb); loff_t pos = ctx->pos; if (pos >= PID_MAX_LIMIT + TGID_OFFSET) @@ -3590,11 +3603,9 @@ int proc_pid_readdir(struct file *file, struct dir_context *ctx) return 0; ctx->pos = pos = pos + 1; } - iter.tgid = pos - TGID_OFFSET; - iter.task = NULL; - for (iter = next_tgid(ns, iter); - iter.task; - iter.tgid += 1, iter = next_tgid(ns, iter)) { + + auto iter = make_tgid_iter(pos - TGID_OFFSET, pid_ns); + while (next_tgid(&iter)) { char name[10 + 1]; unsigned int len; diff --git a/fs/proc/generic.c b/fs/proc/generic.c index 8bb81e58c9d8c..3063080f3bb2a 100644 --- a/fs/proc/generic.c +++ b/fs/proc/generic.c @@ -427,9 +427,13 @@ static struct proc_dir_entry *__proc_create(struct proc_dir_entry **parent, if (xlate_proc_name(name, parent, &fn) != 0) goto out; qstr.name = fn; - qstr.len = strlen(fn); - if (qstr.len == 0 || qstr.len >= 256) { - WARN(1, "name len %u\n", qstr.len); + qstr.len = strnlen(fn, NAME_MAX + 1); + if (qstr.len == 0) { + WARN(1, "empty name\n"); + return NULL; + } + if (qstr.len > NAME_MAX) { + WARN(1, "name too long\n"); return NULL; } if (qstr.len == 1 && fn[0] == '.') { diff --git a/include/asm-generic/uaccess.h b/include/asm-generic/uaccess.h index b276f783494c4..4569045e7139f 100644 --- a/include/asm-generic/uaccess.h +++ b/include/asm-generic/uaccess.h @@ -91,8 +91,7 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) memcpy((void __force *)to, from, n); return 0; } -#define INLINE_COPY_FROM_USER -#define INLINE_COPY_TO_USER +#define INLINE_COPY_USER #endif /* CONFIG_UACCESS_MEMCPY */ /* diff --git a/include/linux/cnt32_to_63.h b/include/linux/cnt32_to_63.h deleted file mode 100644 index 064428479f2d4..0000000000000 --- a/include/linux/cnt32_to_63.h +++ /dev/null @@ -1,104 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Extend a 32-bit counter to 63 bits - * - * Author: Nicolas Pitre - * Created: December 3, 2006 - * Copyright: MontaVista Software, Inc. - */ - -#ifndef __LINUX_CNT32_TO_63_H__ -#define __LINUX_CNT32_TO_63_H__ - -#include <linux/compiler.h> -#include <linux/types.h> -#include <asm/byteorder.h> - -/* this is used only to give gcc a clue about good code generation */ -union cnt32_to_63 { - struct { -#if defined(__LITTLE_ENDIAN) - u32 lo, hi; -#elif defined(__BIG_ENDIAN) - u32 hi, lo; -#endif - }; - u64 val; -}; - - -/** - * cnt32_to_63 - Expand a 32-bit counter to a 63-bit counter - * @cnt_lo: The low part of the counter - * - * Many hardware clock counters are only 32 bits wide and therefore have - * a relatively short period making wrap-arounds rather frequent. This - * is a problem when implementing sched_clock() for example, where a 64-bit - * non-wrapping monotonic value is expected to be returned. - * - * To overcome that limitation, let's extend a 32-bit counter to 63 bits - * in a completely lock free fashion. Bits 0 to 31 of the clock are provided - * by the hardware while bits 32 to 62 are stored in memory. The top bit in - * memory is used to synchronize with the hardware clock half-period. When - * the top bit of both counters (hardware and in memory) differ then the - * memory is updated with a new value, incrementing it when the hardware - * counter wraps around. - * - * Because a word store in memory is atomic then the incremented value will - * always be in synch with the top bit indicating to any potential concurrent - * reader if the value in memory is up to date or not with regards to the - * needed increment. And any race in updating the value in memory is harmless - * as the same value would simply be stored more than once. - * - * The restrictions for the algorithm to work properly are: - * - * 1) this code must be called at least once per each half period of the - * 32-bit counter; - * - * 2) this code must not be preempted for a duration longer than the - * 32-bit counter half period minus the longest period between two - * calls to this code; - * - * Those requirements ensure proper update to the state bit in memory. - * This is usually not a problem in practice, but if it is then a kernel - * timer should be scheduled to manage for this code to be executed often - * enough. - * - * And finally: - * - * 3) the cnt_lo argument must be seen as a globally incrementing value, - * meaning that it should be a direct reference to the counter data which - * can be evaluated according to a specific ordering within the macro, - * and not the result of a previous evaluation stored in a variable. - * - * For example, this is wrong: - * - * u32 partial = get_hw_count(); - * u64 full = cnt32_to_63(partial); - * return full; - * - * This is fine: - * - * u64 full = cnt32_to_63(get_hw_count()); - * return full; - * - * Note that the top bit (bit 63) in the returned value should be considered - * as garbage. It is not cleared here because callers are likely to use a - * multiplier on the returned value which can get rid of the top bit - * implicitly by making the multiplier even, therefore saving on a runtime - * clear-bit instruction. Otherwise caller must remember to clear the top - * bit explicitly. - */ -#define cnt32_to_63(cnt_lo) \ -({ \ - static u32 __m_cnt_hi; \ - union cnt32_to_63 __x; \ - __x.hi = __m_cnt_hi; \ - smp_rmb(); \ - __x.lo = (cnt_lo); \ - if (unlikely((s32)(__x.hi ^ __x.lo) < 0)) \ - __m_cnt_hi = __x.hi = (__x.hi ^ 0x80000000) + (__x.hi >> 31); \ - __x.val; \ -}) - -#endif diff --git a/include/linux/init.h b/include/linux/init.h index 40331923b9f4a..6326c61e2332d 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,7 +47,7 @@ #define __initdata __section(".init.data") #define __initconst __section(".init.rodata") #define __exitdata __section(".exit.data") -#define __exit_call __used __section(".exitcall.exit") +#define __exit_call __maybe_unused __section(".exitcall.exit") /* * modpost check for section mismatches during the kernel build. diff --git a/include/linux/kcov.h b/include/linux/kcov.h index 0143358874b07..895b761b2db15 100644 --- a/include/linux/kcov.h +++ b/include/linux/kcov.h @@ -21,8 +21,6 @@ enum kcov_mode { KCOV_MODE_TRACE_PC = 2, /* Collecting comparison operands mode. */ KCOV_MODE_TRACE_CMP = 3, - /* The process owns a KCOV remote reference. */ - KCOV_MODE_REMOTE = 4, }; #define KCOV_IN_CTXSW (1 << 30) @@ -43,11 +41,11 @@ do { \ /* See Documentation/dev-tools/kcov.rst for usage details. */ void kcov_remote_start(u64 handle); void kcov_remote_stop(void); -u64 kcov_common_handle(void); +struct kcov_common_handle_id kcov_common_handle(void); -static inline void kcov_remote_start_common(u64 id) +static inline void kcov_remote_start_common(struct kcov_common_handle_id id) { - kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id)); + kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id.val)); } static inline void kcov_remote_start_usb(u64 id) @@ -99,11 +97,11 @@ static inline void kcov_prepare_switch(struct task_struct *t) {} static inline void kcov_finish_switch(struct task_struct *t) {} static inline void kcov_remote_start(u64 handle) {} static inline void kcov_remote_stop(void) {} -static inline u64 kcov_common_handle(void) +static inline struct kcov_common_handle_id kcov_common_handle(void) { - return 0; + return (struct kcov_common_handle_id){}; } -static inline void kcov_remote_start_common(u64 id) {} +static inline void kcov_remote_start_common(struct kcov_common_handle_id id) {} static inline void kcov_remote_start_usb(u64 id) {} static inline void kcov_remote_start_usb_softirq(u64 id) {} static inline void kcov_remote_stop_softirq(void) {} diff --git a/include/linux/llist.h b/include/linux/llist.h index 607b2360c938c..8846b7709669f 100644 --- a/include/linux/llist.h +++ b/include/linux/llist.h @@ -26,8 +26,8 @@ * * | add | del_first | del_all * add | - | - | - - * del_first | | L | L - * del_all | | | - + * del_first | - | L | L + * del_all | - | - | - * * Where, a particular row's operation can happen concurrently with a column's * operation, with "-" being no lock needed, while "L" being lock is needed. diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index 2467b3be15c9e..b3bf9339cd86a 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -1,206 +1,24 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* -*- linux-c -*- ------------------------------------------------------- * +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved * - * Copyright 2003 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -#ifndef LINUX_RAID_RAID6_H -#define LINUX_RAID_RAID6_H - -#ifdef __KERNEL__ - -#include <linux/blkdev.h> -#include <linux/mm.h> - -/* This should be const but the raid6 code is too convoluted for that. */ -static inline void *raid6_get_zero_page(void) -{ - return page_address(ZERO_PAGE(0)); -} - -#else /* ! __KERNEL__ */ -/* Used for testing in user space */ - -#include <errno.h> -#include <inttypes.h> -#include <stddef.h> -#include <string.h> -#include <sys/mman.h> -#include <sys/time.h> -#include <sys/types.h> - -/* Not standard, but glibc defines it */ -#define BITS_PER_LONG __WORDSIZE - -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; - -#ifndef PAGE_SIZE -# define PAGE_SIZE 4096 -#endif -#ifndef PAGE_SHIFT -# define PAGE_SHIFT 12 -#endif -extern const char raid6_empty_zero_page[PAGE_SIZE]; - -#define __init -#define __exit -#ifndef __attribute_const__ -# define __attribute_const__ __attribute__((const)) -#endif -#define noinline __attribute__((noinline)) - -#define preempt_enable() -#define preempt_disable() -#define cpu_has_feature(x) 1 -#define enable_kernel_altivec() -#define disable_kernel_altivec() - -#undef EXPORT_SYMBOL -#define EXPORT_SYMBOL(sym) -#undef EXPORT_SYMBOL_GPL -#define EXPORT_SYMBOL_GPL(sym) -#define MODULE_LICENSE(licence) -#define MODULE_DESCRIPTION(desc) -#define subsys_initcall(x) -#define module_exit(x) - -#define IS_ENABLED(x) (x) -#define CONFIG_RAID6_PQ_BENCHMARK 1 -#endif /* __KERNEL__ */ - -/* Routine choices */ -struct raid6_calls { - void (*gen_syndrome)(int, size_t, void **); - void (*xor_syndrome)(int, int, int, size_t, void **); - int (*valid)(void); /* Returns 1 if this routine set is usable */ - const char *name; /* Name of this routine set */ - int priority; /* Relative priority ranking if non-zero */ -}; - -/* Selected algorithm */ -extern struct raid6_calls raid6_call; - -/* Various routine sets */ -extern const struct raid6_calls raid6_intx1; -extern const struct raid6_calls raid6_intx2; -extern const struct raid6_calls raid6_intx4; -extern const struct raid6_calls raid6_intx8; -extern const struct raid6_calls raid6_mmxx1; -extern const struct raid6_calls raid6_mmxx2; -extern const struct raid6_calls raid6_sse1x1; -extern const struct raid6_calls raid6_sse1x2; -extern const struct raid6_calls raid6_sse2x1; -extern const struct raid6_calls raid6_sse2x2; -extern const struct raid6_calls raid6_sse2x4; -extern const struct raid6_calls raid6_altivec1; -extern const struct raid6_calls raid6_altivec2; -extern const struct raid6_calls raid6_altivec4; -extern const struct raid6_calls raid6_altivec8; -extern const struct raid6_calls raid6_avx2x1; -extern const struct raid6_calls raid6_avx2x2; -extern const struct raid6_calls raid6_avx2x4; -extern const struct raid6_calls raid6_avx512x1; -extern const struct raid6_calls raid6_avx512x2; -extern const struct raid6_calls raid6_avx512x4; -extern const struct raid6_calls raid6_s390vx8; -extern const struct raid6_calls raid6_vpermxor1; -extern const struct raid6_calls raid6_vpermxor2; -extern const struct raid6_calls raid6_vpermxor4; -extern const struct raid6_calls raid6_vpermxor8; -extern const struct raid6_calls raid6_lsx; -extern const struct raid6_calls raid6_lasx; -extern const struct raid6_calls raid6_rvvx1; -extern const struct raid6_calls raid6_rvvx2; -extern const struct raid6_calls raid6_rvvx4; -extern const struct raid6_calls raid6_rvvx8; - -struct raid6_recov_calls { - void (*data2)(int, size_t, int, int, void **); - void (*datap)(int, size_t, int, void **); - int (*valid)(void); - const char *name; - int priority; -}; - -extern const struct raid6_recov_calls raid6_recov_intx1; -extern const struct raid6_recov_calls raid6_recov_ssse3; -extern const struct raid6_recov_calls raid6_recov_avx2; -extern const struct raid6_recov_calls raid6_recov_avx512; -extern const struct raid6_recov_calls raid6_recov_s390xc; -extern const struct raid6_recov_calls raid6_recov_neon; -extern const struct raid6_recov_calls raid6_recov_lsx; -extern const struct raid6_recov_calls raid6_recov_lasx; -extern const struct raid6_recov_calls raid6_recov_rvv; - -extern const struct raid6_calls raid6_neonx1; -extern const struct raid6_calls raid6_neonx2; -extern const struct raid6_calls raid6_neonx4; -extern const struct raid6_calls raid6_neonx8; - -/* Algorithm list */ -extern const struct raid6_calls * const raid6_algos[]; -extern const struct raid6_recov_calls *const raid6_recov_algos[]; -int raid6_select_algo(void); - -/* Return values from chk_syndrome */ -#define RAID6_OK 0 -#define RAID6_P_BAD 1 -#define RAID6_Q_BAD 2 -#define RAID6_PQ_BAD 3 - -/* Galois field tables */ -extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256))); -extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256))); -extern const u8 raid6_gfexp[256] __attribute__((aligned(256))); -extern const u8 raid6_gflog[256] __attribute__((aligned(256))); -extern const u8 raid6_gfinv[256] __attribute__((aligned(256))); -extern const u8 raid6_gfexi[256] __attribute__((aligned(256))); - -/* Recovery routines */ -extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb, - void **ptrs); -extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila, - void **ptrs); -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, - void **ptrs); - -/* Some definitions to allow code to be compiled for testing in userspace */ -#ifndef __KERNEL__ - -# define jiffies raid6_jiffies() -# define printk printf -# define pr_err(format, ...) fprintf(stderr, format, ## __VA_ARGS__) -# define pr_info(format, ...) fprintf(stdout, format, ## __VA_ARGS__) -# define GFP_KERNEL 0 -# define __get_free_pages(x, y) ((unsigned long)mmap(NULL, PAGE_SIZE << (y), \ - PROT_READ|PROT_WRITE, \ - MAP_PRIVATE|MAP_ANONYMOUS,\ - 0, 0)) -# define free_pages(x, y) munmap((void *)(x), PAGE_SIZE << (y)) + * Public interface to the RAID6 P/Q calculation and recovery library. + */ +#ifndef LINUX_RAID_PQ_H +#define LINUX_RAID_PQ_H -static inline void cpu_relax(void) -{ - /* Nothing */ -} +#include <linux/types.h> -#undef HZ -#define HZ 1000 -static inline uint32_t raid6_jiffies(void) -{ - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec*1000 + tv.tv_usec/1000; -} +#define RAID6_MIN_DISKS 4 -static inline void *raid6_get_zero_page(void) -{ - return raid6_empty_zero_page; -} +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs); +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs); +bool raid6_can_xor_syndrome(void); -#endif /* ! __KERNEL__ */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs); +void raid6_recov_datap(int disks, size_t bytes, int faila, + void **ptrs); -#endif /* LINUX_RAID_RAID6_H */ +#endif /* LINUX_RAID_PQ_H */ diff --git a/include/linux/raid/pq_tables.h b/include/linux/raid/pq_tables.h new file mode 100644 index 0000000000000..7b1ebe675677f --- /dev/null +++ b/include/linux/raid/pq_tables.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + * + * Galois field tables for the Linux RAID6 P/Q parity algorithm. + */ +#ifndef _LINUX_RAID_PQ_TABLES_H +#define _LINUX_RAID_PQ_TABLES_H + +#include <linux/types.h> + +extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256))); +extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256))); +extern const u8 raid6_gfexp[256] __attribute__((aligned(256))); +extern const u8 raid6_gflog[256] __attribute__((aligned(256))); +extern const u8 raid6_gfinv[256] __attribute__((aligned(256))); +extern const u8 raid6_gfexi[256] __attribute__((aligned(256))); + +#endif /* _LINUX_RAID_PQ_TABLES_H */ diff --git a/include/linux/sched.h b/include/linux/sched.h index ee06cba5c6f53..d71cec884a5d9 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1517,6 +1517,9 @@ struct task_struct { /* KCOV descriptor wired with this task or NULL: */ struct kcov *kcov; + /* KCOV descriptor for remote coverage collection from other tasks: */ + struct kcov *kcov_remote; + /* KCOV common handle for remote coverage collection: */ u64 kcov_handle; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index 2bcf78a4de7b9..a3fe418f7ced8 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -1082,9 +1082,7 @@ struct sk_buff { __u16 network_header; __u16 mac_header; -#ifdef CONFIG_KCOV - u64 kcov_handle; -#endif + struct kcov_common_handle_id kcov_handle; ); /* end headers group */ @@ -5437,20 +5435,14 @@ static inline void skb_reset_csum_not_inet(struct sk_buff *skb) } static inline void skb_set_kcov_handle(struct sk_buff *skb, - const u64 kcov_handle) + struct kcov_common_handle_id kcov_handle) { -#ifdef CONFIG_KCOV skb->kcov_handle = kcov_handle; -#endif } -static inline u64 skb_get_kcov_handle(struct sk_buff *skb) +static inline struct kcov_common_handle_id skb_get_kcov_handle(struct sk_buff *skb) { -#ifdef CONFIG_KCOV return skb->kcov_handle; -#else - return 0; -#endif } static inline void skb_mark_for_recycle(struct sk_buff *skb) diff --git a/include/linux/types.h b/include/linux/types.h index 608050dbca6a7..93166b0b0617a 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -224,6 +224,12 @@ struct ustat { char f_fpack[6]; }; +struct kcov_common_handle_id { +#ifdef CONFIG_KCOV + u64 val; +#endif +}; + /** * struct callback_head - callback structure for use with RCU and task_work * @next: next update requests in a list diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 56328601218c5..e0c3d6e29301d 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -84,7 +84,7 @@ * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) * that are used instead. Out of those, __... ones are inlined. Plain * copy_{to,from}_user() might or might not be inlined. If you want them - * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER. + * inlined, have asm/uaccess.h define INLINE_COPY_USER. * * NOTE: only copy_from_user() zero-pads the destination in case of short copy. * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything @@ -157,7 +157,7 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) } /* - * Architectures that #define INLINE_COPY_TO_USER use this function + * Architectures that #define INLINE_COPY_USER use this function * directly in the normal copy_to/from_user(), the other ones go * through an extern _copy_to/from_user(), which expands the same code * here. @@ -190,10 +190,6 @@ fail: memset(to + (n - res), 0, res); return res; } -#ifndef INLINE_COPY_FROM_USER -extern __must_check unsigned long -_copy_from_user(void *, const void __user *, unsigned long); -#endif static inline __must_check unsigned long _inline_copy_to_user(void __user *to, const void *from, unsigned long n) @@ -207,7 +203,13 @@ _inline_copy_to_user(void __user *to, const void *from, unsigned long n) } return n; } -#ifndef INLINE_COPY_TO_USER +#ifdef INLINE_COPY_USER +# define _copy_to_user _inline_copy_to_user +# define _copy_from_user _inline_copy_from_user +#else +extern __must_check unsigned long +_copy_from_user(void *, const void __user *, unsigned long); + extern __must_check unsigned long _copy_to_user(void __user *, const void *, unsigned long); #endif @@ -217,11 +219,7 @@ copy_from_user(void *to, const void __user *from, unsigned long n) { if (!check_copy_size(to, n, false)) return n; -#ifdef INLINE_COPY_FROM_USER - return _inline_copy_from_user(to, from, n); -#else return _copy_from_user(to, from, n); -#endif } static __always_inline unsigned long __must_check @@ -229,12 +227,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n) { if (!check_copy_size(from, n, true)) return n; - -#ifdef INLINE_COPY_TO_USER - return _inline_copy_to_user(to, from, n); -#else return _copy_to_user(to, from, n); -#endif } #ifndef copy_mc_to_kernel diff --git a/init/Kconfig b/init/Kconfig index 2937c4d308aec..624d935061904 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1050,14 +1050,14 @@ config PAGE_COUNTER bool config CGROUP_FAVOR_DYNMODS - bool "Favor dynamic modification latency reduction by default" - help - This option enables the "favordynmods" mount option by default - which reduces the latencies of dynamic cgroup modifications such - as task migrations and controller on/offs at the cost of making - hot path operations such as forks and exits more expensive. + bool "Favor dynamic modification latency reduction by default" + help + This option enables the "favordynmods" mount option by default + which reduces the latencies of dynamic cgroup modifications such + as task migrations and controller on/offs at the cost of making + hot path operations such as forks and exits more expensive. - Say N if unsure. + Say N if unsure. config MEMCG bool "Memory controller" @@ -1139,7 +1139,7 @@ config GROUP_SCHED_WEIGHT def_bool n config GROUP_SCHED_BANDWIDTH - def_bool n + def_bool n config FAIR_GROUP_SCHED bool "Group scheduling for SCHED_OTHER" @@ -1645,10 +1645,10 @@ config LD_ORPHAN_WARN depends on $(ld-option,--orphan-handling=error) config LD_ORPHAN_WARN_LEVEL - string - depends on LD_ORPHAN_WARN - default "error" if WERROR - default "warn" + string + depends on LD_ORPHAN_WARN + default "error" if WERROR + default "warn" config SYSCTL bool diff --git a/kernel/kcov.c b/kernel/kcov.c index 0b369e88c7c9b..fd25030307299 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -368,6 +368,7 @@ static void kcov_start(struct task_struct *t, struct kcov *kcov, WRITE_ONCE(t->kcov_mode, mode); } +/* operates on coverage-generator-owned fields */ static void kcov_stop(struct task_struct *t) { WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED); @@ -377,16 +378,17 @@ static void kcov_stop(struct task_struct *t) t->kcov_area = NULL; } +/* operates on coverage-generator-owned fields */ static void kcov_task_reset(struct task_struct *t) { kcov_stop(t); t->kcov_sequence = 0; - t->kcov_handle = 0; } void kcov_task_init(struct task_struct *t) { kcov_task_reset(t); + t->kcov_remote = NULL; t->kcov_handle = current->kcov_handle; } @@ -423,11 +425,14 @@ static void kcov_remote_reset(struct kcov *kcov) static void kcov_disable(struct task_struct *t, struct kcov *kcov) __must_hold(&kcov->lock) { - kcov_task_reset(t); - if (kcov->remote) + if (kcov->remote) { + t->kcov_handle = 0; + t->kcov_remote = NULL; kcov_remote_reset(kcov); - else + } else { + kcov_task_reset(t); kcov_reset(kcov); + } } static void kcov_get(struct kcov *kcov) @@ -453,41 +458,47 @@ void kcov_task_exit(struct task_struct *t) unsigned long flags; kcov = t->kcov; - if (kcov == NULL) - return; - - spin_lock_irqsave(&kcov->lock, flags); - kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); - /* - * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t, - * which comes down to: - * WARN_ON(!kcov->remote && kcov->t != t); - * - * For KCOV_REMOTE_ENABLE devices, the exiting task is either: - * - * 1. A remote task between kcov_remote_start() and kcov_remote_stop(). - * In this case we should print a warning right away, since a task - * shouldn't be exiting when it's in a kcov coverage collection - * section. Here t points to the task that is collecting remote - * coverage, and t->kcov->t points to the thread that created the - * kcov device. Which means that to detect this case we need to - * check that t != t->kcov->t, and this gives us the following: - * WARN_ON(kcov->remote && kcov->t != t); - * - * 2. The task that created kcov exiting without calling KCOV_DISABLE, - * and then again we make sure that t->kcov->t == t: - * WARN_ON(kcov->remote && kcov->t != t); - * - * By combining all three checks into one we get: - */ - if (WARN_ON(kcov->t != t)) { + if (kcov) { + spin_lock_irqsave(&kcov->lock, flags); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * This could be a remote task between kcov_remote_start() and + * kcov_remote_stop(). + * In this case we should print a warning right away, since a + * task shouldn't be exiting when it's in a kcov coverage + * collection section. + * + * Otherwise, this should be a task that created a local + * kcov instance and hasn't called KCOV_DISABLE. + * Make sure that t->kcov->t is consistent. + */ + if (WARN_ON(kcov->remote) || WARN_ON(kcov->t != t)) { + spin_unlock_irqrestore(&kcov->lock, flags); + return; + } + /* Just to not leave dangling references behind. */ + kcov_disable(t, kcov); spin_unlock_irqrestore(&kcov->lock, flags); - return; + kcov_put(kcov); + } + kcov = t->kcov_remote; + if (kcov) { + spin_lock_irqsave(&kcov->lock, flags); + kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t); + /* + * This is a KCOV_REMOTE_ENABLE device, and the task is the + * user task which has requested remote coverage collection. + * Make sure that t->kcov->t is consistent. + */ + if (WARN_ON(!kcov->remote) || WARN_ON(kcov->t != t)) { + spin_unlock_irqrestore(&kcov->lock, flags); + return; + } + /* Just to not leave dangling references behind. */ + kcov_disable(t, kcov); + spin_unlock_irqrestore(&kcov->lock, flags); + kcov_put(kcov); } - /* Just to not leave dangling references behind. */ - kcov_disable(t, kcov); - spin_unlock_irqrestore(&kcov->lock, flags); - kcov_put(kcov); } static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) @@ -629,9 +640,9 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, case KCOV_DISABLE: /* Disable coverage for the current task. */ unused = arg; - if (unused != 0 || current->kcov != kcov) - return -EINVAL; t = current; + if (unused != 0 || (kcov != t->kcov && kcov != t->kcov_remote)) + return -EINVAL; if (WARN_ON(kcov->t != t)) return -EINVAL; kcov_disable(t, kcov); @@ -641,7 +652,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, if (kcov->mode != KCOV_MODE_INIT || !kcov->area) return -EINVAL; t = current; - if (kcov->t != NULL || t->kcov != NULL) + if (kcov->t != NULL || t->kcov_remote != NULL) return -EBUSY; remote_arg = (struct kcov_remote_arg *)arg; mode = kcov_get_mode(remote_arg->trace_mode); @@ -651,8 +662,7 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, LONG_MAX / sizeof(unsigned long)) return -EINVAL; kcov->mode = mode; - t->kcov = kcov; - t->kcov_mode = KCOV_MODE_REMOTE; + t->kcov_remote = kcov; kcov->t = t; kcov->remote = true; kcov->remote_size = remote_arg->area_size; @@ -1083,11 +1093,11 @@ void kcov_remote_stop(void) EXPORT_SYMBOL(kcov_remote_stop); /* See the comment before kcov_remote_start() for usage details. */ -u64 kcov_common_handle(void) +struct kcov_common_handle_id kcov_common_handle(void) { if (!in_task()) - return 0; - return current->kcov_handle; + return (struct kcov_common_handle_id){ .val = 0 }; + return (struct kcov_common_handle_id){ .val = current->kcov_handle }; } EXPORT_SYMBOL(kcov_common_handle); diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 73bd6a6a78935..2cd0172d05160 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -210,13 +210,39 @@ static int fill_stats_for_pid(pid_t pid, struct taskstats *stats) return 0; } +static void tgid_stats_add_task(struct taskstats *stats, + struct task_struct *tsk, u64 now_ns) +{ + u64 delta, utime, stime; + + /* + * Each accounting subsystem calls its functions here to + * accumulate its per-task stats for tsk, into the per-tgid structure + * + * per-task-foo(stats, tsk); + */ + delayacct_add_tsk(stats, tsk); + + /* calculate task elapsed time in nsec */ + delta = now_ns - tsk->start_time; + /* Convert to micro seconds */ + do_div(delta, NSEC_PER_USEC); + stats->ac_etime += delta; + + task_cputime(tsk, &utime, &stime); + stats->ac_utime += div_u64(utime, NSEC_PER_USEC); + stats->ac_stime += div_u64(stime, NSEC_PER_USEC); + + stats->nvcsw += tsk->nvcsw; + stats->nivcsw += tsk->nivcsw; +} + static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) { struct task_struct *tsk, *first; unsigned long flags; int rc = -ESRCH; - u64 delta, utime, stime; - u64 start_time; + u64 now_ns; /* * Add additional stats from live tasks except zombie thread group @@ -233,30 +259,12 @@ static int fill_stats_for_tgid(pid_t tgid, struct taskstats *stats) else memset(stats, 0, sizeof(*stats)); - start_time = ktime_get_ns(); + now_ns = ktime_get_ns(); for_each_thread(first, tsk) { if (tsk->exit_state) continue; - /* - * Accounting subsystem can call its functions here to - * fill in relevant parts of struct taskstsats as follows - * - * per-task-foo(stats, tsk); - */ - delayacct_add_tsk(stats, tsk); - - /* calculate task elapsed time in nsec */ - delta = start_time - tsk->start_time; - /* Convert to micro seconds */ - do_div(delta, NSEC_PER_USEC); - stats->ac_etime += delta; - task_cputime(tsk, &utime, &stime); - stats->ac_utime += div_u64(utime, NSEC_PER_USEC); - stats->ac_stime += div_u64(stime, NSEC_PER_USEC); - - stats->nvcsw += tsk->nvcsw; - stats->nivcsw += tsk->nivcsw; + tgid_stats_add_task(stats, tsk, now_ns); } unlock_task_sighand(first, &flags); @@ -275,18 +283,14 @@ out: static void fill_tgid_exit(struct task_struct *tsk) { unsigned long flags; + u64 now_ns; spin_lock_irqsave(&tsk->sighand->siglock, flags); if (!tsk->signal->stats) goto ret; - /* - * Each accounting subsystem calls its functions here to - * accumalate its per-task stats for tsk, into the per-tgid structure - * - * per-task-foo(tsk->signal->stats, tsk); - */ - delayacct_add_tsk(tsk->signal->stats, tsk); + now_ns = ktime_get_ns(); + tgid_stats_add_task(tsk->signal->stats, tsk, now_ns); ret: spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return; diff --git a/lib/Kconfig b/lib/Kconfig index 00a9509636c18..f01a33e521e12 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -8,17 +8,6 @@ config BINARY_PRINTF menu "Library routines" -config RAID6_PQ - tristate - -config RAID6_PQ_BENCHMARK - bool "Automatically choose fastest RAID6 PQ functions" - depends on RAID6_PQ - default y - help - Benchmark all available RAID6 PQ functions on init and choose the - fastest one. - config LINEAR_RANGES tristate @@ -590,7 +579,7 @@ config OBJAGG config LWQ_TEST bool "Boot-time test for lwq queuing" help - Run boot-time test of light-weight queuing. + Run boot-time test of light-weight queuing. endmenu diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 8ff5adcfe1e0a..7ca468c7d81da 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2971,6 +2971,16 @@ config BITS_TEST If unsure, say N. +config SHDI3_KUNIT_TEST + tristate "KUnit test for __ashldi3(), __ashrdi3(), and __lshrdi3()" + depends on KUNIT + depends on ARM || XTENSA || MICROBLAZE || ((RISCV || SPARC) && !64BIT) + help + This builds the unit test for __ashldi3(), __ashrdi3(), and + __lshrdi3() helper functions used to implement 64-bit arithmetic + shift left, arithmetic shift right and logical shift right, + respectively, on a 32-bit CPUs. + config SLUB_KUNIT_TEST tristate "KUnit test for SLUB cache error detection" if !KUNIT_ALL_TESTS depends on SLUB_DEBUG && KUNIT diff --git a/lib/Makefile b/lib/Makefile index f33a24bf1c19a..6e72d2c1cce71 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -167,7 +167,6 @@ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/ obj-$(CONFIG_ZSTD_COMPRESS) += zstd/ obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/ obj-$(CONFIG_XZ_DEC) += xz/ -obj-$(CONFIG_RAID6_PQ) += raid6/ lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o diff --git a/lib/base64.c b/lib/base64.c index 41961a444028b..325c7332b0492 100644 --- a/lib/base64.c +++ b/lib/base64.c @@ -122,7 +122,7 @@ EXPORT_SYMBOL_GPL(base64_encode); * @src: the string to decode. Doesn't need to be NUL-terminated. * @srclen: the length of @src in bytes * @dst: (output) the decoded binary data - * @padding: whether to append '=' padding characters + * @padding: whether the input is expected to include '=' padding characters * @variant: which base64 variant to use * * Decodes a string using the selected Base64 variant. @@ -168,15 +168,16 @@ int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base6 return -1; val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6); - *bp++ = val >> 10; if (srclen == 2) { if (val & 0x800003ff) return -1; + *bp++ = val >> 10; } else { val |= base64_rev_tables[s[2]]; if (val & 0x80000003) return -1; + *bp++ = val >> 10; *bp++ = val >> 2; } return bp - dst; diff --git a/lib/bug.c b/lib/bug.c index 224f4cfa4aa31..f4a4df3991b09 100644 --- a/lib/bug.c +++ b/lib/bug.c @@ -1,41 +1,41 @@ // SPDX-License-Identifier: GPL-2.0 /* - Generic support for BUG() - - This respects the following config options: - - CONFIG_BUG - emit BUG traps. Nothing happens without this. - CONFIG_GENERIC_BUG - enable this code. - CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file - CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG - - CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable - (though they're generally always on). - - CONFIG_GENERIC_BUG is set by each architecture using this code. - - To use this, your architecture must: - - 1. Set up the config options: - - Enable CONFIG_GENERIC_BUG if CONFIG_BUG - - 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) - - Define HAVE_ARCH_BUG - - Implement BUG() to generate a faulting instruction - - NOTE: struct bug_entry does not have "file" or "line" entries - when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate - the values accordingly. - - 3. Implement the trap - - In the illegal instruction trap handler (typically), verify - that the fault was in kernel mode, and call report_bug() - - report_bug() will return whether it was a false alarm, a warning, - or an actual bug. - - You must implement the is_valid_bugaddr(bugaddr) callback which - returns true if the eip is a real kernel address, and it points - to the expected BUG trap instruction. - - Jeremy Fitzhardinge <jeremy@goop.org> 2006 + * Generic support for BUG() + * + * This respects the following config options: + * + * CONFIG_BUG - emit BUG traps. Nothing happens without this. + * CONFIG_GENERIC_BUG - enable this code. + * CONFIG_GENERIC_BUG_RELATIVE_POINTERS - use 32-bit relative pointers for bug_addr and file + * CONFIG_DEBUG_BUGVERBOSE - emit full file+line information for each BUG + * + * CONFIG_BUG and CONFIG_DEBUG_BUGVERBOSE are potentially user-settable + * (though they're generally always on). + * + * CONFIG_GENERIC_BUG is set by each architecture using this code. + * + * To use this, your architecture must: + * + * 1. Set up the config options: + * - Enable CONFIG_GENERIC_BUG if CONFIG_BUG + * + * 2. Implement BUG (and optionally BUG_ON, WARN, WARN_ON) + * - Define HAVE_ARCH_BUG + * - Implement BUG() to generate a faulting instruction + * - NOTE: struct bug_entry does not have "file" or "line" entries + * when CONFIG_DEBUG_BUGVERBOSE is not enabled, so you must generate + * the values accordingly. + * + * 3. Implement the trap + * - In the illegal instruction trap handler (typically), verify + * that the fault was in kernel mode, and call report_bug() + * - report_bug() will return whether it was a false alarm, a warning, + * or an actual bug. + * - You must implement the is_valid_bugaddr(bugaddr) callback which + * returns true if the eip is a real kernel address, and it points + * to the expected BUG trap instruction. + * + * Jeremy Fitzhardinge <jeremy@goop.org> 2006 */ #define pr_fmt(fmt) fmt @@ -71,7 +71,7 @@ static struct bug_entry *module_find_bug(unsigned long bugaddr) guard(rcu)(); list_for_each_entry_rcu(mod, &module_bug_list, bug_list) { - unsigned i; + unsigned int i; bug = mod->bug_table; for (i = 0; i < mod->num_bugs; ++i, ++bug) @@ -191,14 +191,14 @@ void __warn_printf(const char *fmt, struct pt_regs *regs) } #endif - printk("%s", fmt); + pr_warn("%s", fmt); } static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long bugaddr, struct pt_regs *regs) { bool warning, once, done, no_cut, has_args; const char *file, *fmt; - unsigned line; + unsigned int line; if (!bug) { if (!is_valid_bugaddr(bugaddr)) @@ -237,7 +237,7 @@ static enum bug_trap_type __report_bug(struct bug_entry *bug, unsigned long buga * extra debugging message it writes before triggering the handler. */ if (!no_cut) { - printk(KERN_DEFAULT CUT_HERE); + pr_info(CUT_HERE); __warn_printf(fmt, has_args ? regs : NULL); } diff --git a/lib/cmdline.c b/lib/cmdline.c index 90ed997d95701..16cce6621cec1 100644 --- a/lib/cmdline.c +++ b/lib/cmdline.c @@ -43,7 +43,7 @@ static int get_range(char **str, int *pint, int n) * When @pint is NULL the function can be used as a validator of * the current option in the string. * - * Return values: + * Return: * 0 - no int in string * 1 - int found, no subsequent comma * 2 - int found including a subsequent comma @@ -145,44 +145,54 @@ EXPORT_SYMBOL(get_options); * * Parses a string into a number. The number stored at @ptr is * potentially suffixed with K, M, G, T, P, E. + * + * Return: The value as recognized by simple_strtoull() multiplied + * by the value as specified by suffix, if any. */ unsigned long long memparse(const char *ptr, char **retptr) { char *endptr; /* local pointer to end of parsed string */ - unsigned long long ret = simple_strtoull(ptr, &endptr, 0); + unsigned int shl = 0; + /* Consume valid suffix even in case of overflow. */ switch (*endptr) { case 'E': case 'e': - ret <<= 10; + shl += 10; fallthrough; case 'P': case 'p': - ret <<= 10; + shl += 10; fallthrough; case 'T': case 't': - ret <<= 10; + shl += 10; fallthrough; case 'G': case 'g': - ret <<= 10; + shl += 10; fallthrough; case 'M': case 'm': - ret <<= 10; + shl += 10; fallthrough; case 'K': case 'k': - ret <<= 10; - endptr++; + shl += 10; fallthrough; default: break; } + if (shl && likely(ptr != endptr)) { + /* Have valid suffix with preceding number. */ + if (unlikely(check_shl_overflow(ret, shl, &ret))) + ret = ULLONG_MAX; + endptr++; + } + if (retptr) *retptr = endptr; @@ -198,7 +208,7 @@ EXPORT_SYMBOL(memparse); * This function parses a string containing a comma-separated list of * strings like a=b,c. * - * Return true if there's such option in the string, or return false. + * Return: True if there's such option in the string or false otherwise. */ bool parse_option_str(const char *str, const char *option) { diff --git a/lib/error-inject.c b/lib/error-inject.c index f3d1b70be605c..32f3d1ca9ea23 100644 --- a/lib/error-inject.c +++ b/lib/error-inject.c @@ -219,9 +219,9 @@ static int __init ei_debugfs_init(void) dir = debugfs_create_dir("error_injection", NULL); file = debugfs_create_file("list", 0444, dir, NULL, &ei_fops); - if (!file) { + if (IS_ERR(file)) { debugfs_remove(dir); - return -ENOMEM; + return PTR_ERR(file); } return 0; diff --git a/lib/kstrtox.c b/lib/kstrtox.c index 97be2a39f5371..edc4eb7c1bca1 100644 --- a/lib/kstrtox.c +++ b/lib/kstrtox.c @@ -39,25 +39,30 @@ const char *_parse_integer_fixup_radix(const char *s, unsigned int *base) return s; } -/* - * Convert non-negative integer string representation in explicitly given radix - * to an integer. A maximum of max_chars characters will be converted. +/** + * _parse_integer_limit - Convert integer string representation to an integer + * @s: Integer string representation + * @base: Radix + * @p: Where to store result + * @max_chars: Maximum amount of characters to convert + * + * Convert non-negative integer string representation in explicitly given + * radix to an integer. If overflow occurs, value at @p is set to ULLONG_MAX. * - * Return number of characters consumed maybe or-ed with overflow bit. - * If overflow occurs, result integer (incorrect) is still returned. + * This function is the workhorse of other string conversion functions and it + * is discouraged to use it explicitly. Consider kstrto*() family instead. * - * Don't you dare use this function. + * Return: Number of characters consumed, maybe ORed with overflow bit */ noinline unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned long long *p, size_t max_chars) { + unsigned int rv, overflow = 0; unsigned long long res; - unsigned int rv; res = 0; - rv = 0; - while (max_chars--) { + for (rv = 0; rv < max_chars; rv++, s++) { unsigned int c = *s; unsigned int lc = _tolower(c); unsigned int val; @@ -76,15 +81,17 @@ unsigned int _parse_integer_limit(const char *s, unsigned int base, unsigned lon * it in the max base we support (16) */ if (unlikely(res & (~0ull << 60))) { - if (res > div_u64(ULLONG_MAX - val, base)) - rv |= KSTRTOX_OVERFLOW; + if (check_mul_overflow(res, base, &res) || + check_add_overflow(res, val, &res)) { + res = ULLONG_MAX; + overflow = KSTRTOX_OVERFLOW; + } + } else { + res = res * base + val; } - res = res * base + val; - rv++; - s++; } *p = res; - return rv; + return rv | overflow; } noinline diff --git a/lib/nmi_backtrace.c b/lib/nmi_backtrace.c index 33c154264bfe2..a3bfa9360b23d 100644 --- a/lib/nmi_backtrace.c +++ b/lib/nmi_backtrace.c @@ -16,6 +16,7 @@ #include <linux/cpumask.h> #include <linux/delay.h> #include <linux/kprobes.h> +#include <linux/stringify.h> #include <linux/nmi.h> #include <linux/cpu.h> #include <linux/sched/debug.h> @@ -27,6 +28,8 @@ static DECLARE_BITMAP(backtrace_mask, NR_CPUS) __read_mostly; /* "in progress" flag of arch_trigger_cpumask_backtrace */ static unsigned long backtrace_flag; +#define NMI_BT_TIMEOUT_SEC 10 + /* * When raise() is called it will be passed a pointer to the * backtrace_mask. Architectures that call nmi_cpu_backtrace() @@ -68,14 +71,20 @@ void nmi_trigger_cpumask_backtrace(const cpumask_t *mask, raise(to_cpumask(backtrace_mask)); } - /* Wait for up to 10 seconds for all CPUs to do the backtrace */ - for (i = 0; i < 10 * 1000; i++) { + /* Wait for up to NMI_BT_TIMEOUT_SEC seconds for all CPUs to do the backtrace */ + for (i = 0; i < NMI_BT_TIMEOUT_SEC * 1000; i++) { if (cpumask_empty(to_cpumask(backtrace_mask))) break; mdelay(1); touch_softlockup_watchdog(); } - nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + + if (!cpumask_empty(to_cpumask(backtrace_mask))) { + pr_warn("After " __stringify(NMI_BT_TIMEOUT_SEC) " seconds, these CPUS still haven't responded to the NMI: %*pbl\n", + cpumask_pr_args(to_cpumask(backtrace_mask))); + + nmi_backtrace_stall_check(to_cpumask(backtrace_mask)); + } /* * Force flush any remote buffers that might be stuck in IRQ context diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig index 5ab2b0a7be4c6..978cd6ba08ac6 100644 --- a/lib/raid/Kconfig +++ b/lib/raid/Kconfig @@ -28,3 +28,36 @@ config XOR_KUNIT_TEST This is intended to help people writing architecture-specific optimized versions. If unsure, say N. + +config RAID6_PQ + tristate + +# selected by architectures that provide an optimized PQ implementation +config RAID6_PQ_ARCH + depends on RAID6_PQ + default y if KERNEL_MODE_NEON # arm32/arm64 + default y if LOONGARCH + default y if ALTIVEC # powerpc + default y if RISCV_ISA_V + default y if S390 + default y if X86 + bool + +config RAID6_PQ_KUNIT_TEST + tristate "KUnit tests for RAID6 PQ functions" if !KUNIT_ALL_TESTS + depends on KUNIT + depends on RAID6_PQ + default KUNIT_ALL_TESTS + help + Unit tests for the RAID6 PQ library functions. + + This is intended to help people writing architecture-specific + optimized versions. If unsure, say N. + +config RAID6_PQ_BENCHMARK + bool "Automatically choose fastest RAID6 PQ functions" + depends on RAID6_PQ + default y + help + Benchmark all available RAID6 PQ functions on init and choose the + fastest one. diff --git a/lib/raid/Makefile b/lib/raid/Makefile index 3540fe846dc42..6fc5eeb53df0f 100644 --- a/lib/raid/Makefile +++ b/lib/raid/Makefile @@ -1,3 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 -obj-y += xor/ +obj-y += xor/ raid6/ diff --git a/lib/raid6/.gitignore b/lib/raid/raid6/.gitignore index 6be57745afd12..6be57745afd12 100644 --- a/lib/raid6/.gitignore +++ b/lib/raid/raid6/.gitignore diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile new file mode 100644 index 0000000000000..038d6c74d1ba1 --- /dev/null +++ b/lib/raid/raid6/Makefile @@ -0,0 +1,126 @@ +# SPDX-License-Identifier: GPL-2.0 + +ccflags-y += -I $(src) + +ifeq ($(CONFIG_RAID6_PQ_ARCH),y) +CFLAGS_algos.o += -I$(src)/$(SRCARCH) +endif + +obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/ + +raid6_pq-y += algos.o tables.o + +# generic integer generation and recovery implementation +raid6_pq-y += int1.o int2.o int4.o int8.o +raid6_pq-y += recov.o + +# architecture-specific generation and recovery implementations: +raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += arm/neon.o \ + arm/neon1.o \ + arm/neon2.o \ + arm/neon4.o \ + arm/neon8.o \ + arm/recov_neon.o \ + arm/recov_neon_inner.o +raid6_pq-$(CONFIG_LOONGARCH) += loongarch/loongarch_simd.o \ + loongarch/recov_loongarch_simd.o +raid6_pq-$(CONFIG_ALTIVEC) += powerpc/altivec1.o \ + powerpc/altivec2.o \ + powerpc/altivec4.o \ + powerpc/altivec8.o \ + powerpc/vpermxor1.o \ + powerpc/vpermxor2.o \ + powerpc/vpermxor4.o \ + powerpc/vpermxor8.o +raid6_pq-$(CONFIG_RISCV_ISA_V) += riscv/rvv.o \ + riscv/recov_rvv.o +raid6_pq-$(CONFIG_S390) += s390/s390vx8.o \ + s390/recov_s390xc.o +ifeq ($(CONFIG_X86),y) +raid6_pq-$(CONFIG_X86_32) += x86/mmx.o \ + x86/sse1.o +endif +raid6_pq-$(CONFIG_X86) += x86/sse2.o \ + x86/avx2.o \ + x86/avx512.o \ + x86/recov_ssse3.o \ + x86/recov_avx2.o \ + x86/recov_avx512.o + +hostprogs += mktables + +CFLAGS_arm/neon1.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon2.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon4.o += $(CC_FLAGS_FPU) +CFLAGS_arm/neon8.o += $(CC_FLAGS_FPU) +CFLAGS_arm/recov_neon_inner.o += $(CC_FLAGS_FPU) +CFLAGS_REMOVE_arm/neon1.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon2.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon4.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/neon8.o += $(CC_FLAGS_NO_FPU) +CFLAGS_REMOVE_arm/recov_neon_inner.o += $(CC_FLAGS_NO_FPU) + +ifeq ($(CONFIG_ALTIVEC),y) +altivec_flags := -maltivec $(call cc-option,-mabi=altivec) +# Enable <altivec.h> +altivec_flags += -isystem $(shell $(CC) -print-file-name=include) + +CFLAGS_powerpc/altivec1.o += $(altivec_flags) +CFLAGS_powerpc/altivec2.o += $(altivec_flags) +CFLAGS_powerpc/altivec4.o += $(altivec_flags) +CFLAGS_powerpc/altivec8.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor1.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor2.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor4.o += $(altivec_flags) +CFLAGS_powerpc/vpermxor8.o += $(altivec_flags) + +ifdef CONFIG_CC_IS_CLANG +# clang ppc port does not yet support -maltivec when -msoft-float is +# enabled. A future release of clang will resolve this +# https://llvm.org/pr31177 +CFLAGS_REMOVE_powerpc/altivec1.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec2.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec4.o += -msoft-float +CFLAGS_REMOVE_powerpc/altivec8.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor1.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor2.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor4.o += -msoft-float +CFLAGS_REMOVE_powerpc/vpermxor8.o += -msoft-float +endif # CONFIG_CC_IS_CLANG +endif # CONFIG_ALTIVEC + +quiet_cmd_mktable = TABLE $@ + cmd_mktable = $(obj)/mktables > $@ + +targets += tables.c +$(obj)/tables.c: $(obj)/mktables FORCE + $(call if_changed,mktable) + +quiet_cmd_unroll = UNROLL $@ + cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ + +targets += int1.c int2.c int4.c int8.c +$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += arm/neon1.c arm/neon2.c arm/neon4.c arm/neon8.c +$(obj)/arm/neon%.c: $(src)/arm/neon.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/altivec1.c \ + powerpc/altivec2.c \ + powerpc/altivec4.c \ + powerpc/altivec8.c +$(obj)/powerpc/altivec%.c: $(src)/powerpc/altivec.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += powerpc/vpermxor1.c \ + powerpc/vpermxor2.c \ + powerpc/vpermxor4.c \ + powerpc/vpermxor8.c +$(obj)/powerpc/vpermxor%.c: $(src)/powerpc/vpermxor.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) + +targets += s390/s390vx8.c +$(obj)/s390/s390vx%.c: $(src)/s390/s390vx.uc $(src)/unroll.awk FORCE + $(call if_changed,unroll) diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c new file mode 100644 index 0000000000000..a600d58536729 --- /dev/null +++ b/lib/raid/raid6/algos.c @@ -0,0 +1,377 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002 H. Peter Anvin - All Rights Reserved + * + * Algorithm list and algorithm selection for RAID-6 + */ + +#include <linux/module.h> +#include <linux/gfp.h> +#include <linux/raid/pq.h> +#include <linux/static_call.h> +#include <kunit/visibility.h> +#include "algos.h" + +#define RAID6_MAX_ALGOS 16 +static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; +static unsigned int raid6_nr_algos; +static const struct raid6_recov_calls *raid6_recov_algo; + +/* Selected algorithm */ +DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); +DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); +DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); + +/** + * raid6_gen_syndrome - generate RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to + * @ptrs[@disks - 3]. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. + * + * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying + * algorithm. + */ +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + + static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_gen_syndrome); + +/** + * raid6_xor_syndrome - update RAID6 P/Q parity + * @disks: number of "disks" to operate on including parity + * @start: first index into @disk to update + * @stop: last index into @disk to update + * @bytes: length in bytes of each vector + * @ptrs: @disks size array of memory pointers + * + * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and + * @ptrs[@disks - 1] respectively for the memory pointed to by + * @ptrs[@start..@stop]. + * + * This is used to update parity in place using the following sequence: + * + * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. + * 2) update the the data in @ptrs[@start..@stop]. + * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. + * + * Data between @start and @stop that is not changed should be filled + * with a pointer to the kernel zero page. + * + * @disks must be at least 4, and the memory pointed to by each member of @ptrs + * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of + * 512. @stop must be larger or equal to @start. + */ +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(disks < RAID6_MIN_DISKS); + WARN_ON_ONCE(stop < start); + + static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_xor_syndrome); + +/* + * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used + * + * Returns %true if raid6_can_xor_syndrome() can be used, else %false. + */ +bool raid6_can_xor_syndrome(void) +{ + return !!static_call_query(raid6_xor_syndrome_impl); +} +EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); + +/** + * raid6_recov_2data - recover two missing data disks + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: first failed data disk index + * @failb: second failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the + * data in the remaining disks and the two parities pointed to by the other + * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks + * and the two parities. @faila must be smaller than @failb. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + WARN_ON_ONCE(failb <= faila); + + static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_2data); + +/** + * raid6_recov_datap - recover a missing data disk and missing P-parity + * @disks: number of "disks" to operate on including parity + * @bytes: length in bytes of each vector + * @faila: failed data disk index + * @ptrs: @disks size array of memory pointers + * + * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in + * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity + * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks + * includes the data disks and the two parities. + * + * Memory pointed to by each pointer in @ptrs must be page aligned and is + * limited to %PAGE_SIZE. + */ +void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) +{ + WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); + WARN_ON_ONCE(bytes & 511); + WARN_ON_ONCE(bytes > PAGE_SIZE); + + static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); +} +EXPORT_SYMBOL_GPL(raid6_recov_datap); + +#define RAID6_TIME_JIFFIES_LG2 4 +#define RAID6_TEST_DISKS 8 +#define RAID6_TEST_DISKS_ORDER 3 + +static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], + const int disks) +{ + /* work on the second half of the disks */ + int start = (disks >> 1) - 1, stop = disks - 3; + const struct raid6_calls *best = NULL; + unsigned long bestgenperf = 0; + unsigned int i; + + for (i = 0; i < raid6_nr_algos; i++) { + const struct raid6_calls *algo = raid6_algos[i]; + unsigned long perf = 0, j0, j1; + + preempt_disable(); + j0 = jiffies; + while ((j1 = jiffies) == j0) + cpu_relax(); + while (time_before(jiffies, + j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { + algo->gen_syndrome(disks, PAGE_SIZE, *dptrs); + perf++; + } + preempt_enable(); + + if (perf > bestgenperf) { + bestgenperf = perf; + best = algo; + } + pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, + (perf * HZ * (disks-2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + } + + if (!best) { + pr_err("raid6: Yikes! No algorithm found!\n"); + return -EINVAL; + } + + static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); + + pr_info("raid6: using algorithm %s gen() %ld MB/s\n", + best->name, + (bestgenperf * HZ * (disks - 2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); + + if (best->xor_syndrome) { + unsigned long perf = 0, j0, j1; + + preempt_disable(); + j0 = jiffies; + while ((j1 = jiffies) == j0) + cpu_relax(); + while (time_before(jiffies, + j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { + best->xor_syndrome(disks, start, stop, + PAGE_SIZE, *dptrs); + perf++; + } + preempt_enable(); + + pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", + (perf * HZ * (disks - 2)) >> + (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); + } + + return 0; +} + + +/* Try to pick the best algorithm */ +/* This code uses the gfmul table as convenient data set to abuse */ + +static int __init raid6_select_algo(void) +{ + const int disks = RAID6_TEST_DISKS; + char *disk_ptr, *p; + void *dptrs[RAID6_TEST_DISKS]; + int i, cycle; + int error; + + if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { + pr_info("raid6: skipped pq benchmark and selected %s\n", + raid6_algos[raid6_nr_algos - 1]->name); + static_call_update(raid6_gen_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->gen_syndrome); + static_call_update(raid6_xor_syndrome_impl, + raid6_algos[raid6_nr_algos - 1]->xor_syndrome); + return 0; + } + + /* prepare the buffer and fill it circularly with gfmul table */ + disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); + if (!disk_ptr) { + pr_err("raid6: Yikes! No memory available.\n"); + return -ENOMEM; + } + + p = disk_ptr; + for (i = 0; i < disks; i++) + dptrs[i] = p + PAGE_SIZE * i; + + cycle = ((disks - 2) * PAGE_SIZE) / 65536; + for (i = 0; i < cycle; i++) { + memcpy(p, raid6_gfmul, 65536); + p += 65536; + } + + if ((disks - 2) * PAGE_SIZE % 65536) + memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); + + /* select raid gen_syndrome function */ + error = raid6_choose_gen(&dptrs, disks); + + free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); + + return error; +} + +/* + * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled + * implementation should be registered last so it will be selected when the + * boot-time benchmark is disabled. + */ +void __init raid6_algo_add(const struct raid6_calls *algo) +{ + if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) + return; + raid6_algos[raid6_nr_algos++] = algo; +} + +void __init raid6_algo_add_default(void) +{ + raid6_algo_add(&raid6_intx1); + raid6_algo_add(&raid6_intx2); + raid6_algo_add(&raid6_intx4); + raid6_algo_add(&raid6_intx8); +} + +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) +{ + if (WARN_ON_ONCE(raid6_recov_algo)) + return; + raid6_recov_algo = algo; +} + +#ifdef CONFIG_RAID6_PQ_ARCH +#include "pq_arch.h" +#else +static inline void arch_raid6_init(void) +{ + raid6_algo_add_default(); +} +#endif /* CONFIG_RAID6_PQ_ARCH */ + +static int __init raid6_init(void) +{ + /* + * Architectures providing arch_raid6_init must add all PQ generation + * algorithms they want to consider in arch_raid6_init(), including + * the generic ones using raid6_algo_add_default() if wanted. + */ + arch_raid6_init(); + + /* + * Architectures don't have to set a recovery algorithm, we'll just pick + * the generic integer one if none was set. + */ + if (!raid6_recov_algo) + raid6_recov_algo = &raid6_recov_intx1; + static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); + static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); + pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); + + return raid6_select_algo(); +} + +static void __exit raid6_exit(void) +{ +} + +subsys_initcall(raid6_init); +module_exit(raid6_exit); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); + +#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) +const struct raid6_calls *raid6_algo_find(unsigned int idx) +{ + if (idx >= raid6_nr_algos) { + /* + * Always include the simplest generic integer implementation in + * the unit tests as a baseline. + */ + if (idx == raid6_nr_algos && + raid6_algos[0] != &raid6_intx1) + return &raid6_intx1; + return NULL; + } + return raid6_algos[idx]; +} +EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); + +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) +{ + switch (idx) { + case 0: + /* always test the generic integer implementation */ + return &raid6_recov_intx1; + case 1: + /* test the optimized implementation if there is one */ + if (raid6_recov_algo != &raid6_recov_intx1) + return raid6_recov_algo; + return NULL; + default: + return NULL; + } +} +EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); +#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h new file mode 100644 index 0000000000000..43f636be183f6 --- /dev/null +++ b/lib/raid/raid6/algos.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + */ +#ifndef _PQ_IMPL_H +#define _PQ_IMPL_H + +#include <linux/init.h> +#include <linux/raid/pq_tables.h> + +/* Routine choices */ +struct raid6_calls { + const char *name; + void (*gen_syndrome)(int disks, size_t bytes, void **ptrs); + void (*xor_syndrome)(int disks, int start, int stop, size_t bytes, + void **ptrs); +}; + +struct raid6_recov_calls { + const char *name; + void (*data2)(int disks, size_t bytes, int faila, int failb, + void **ptrs); + void (*datap)(int disks, size_t bytes, int faila, void **ptrs); +}; + +void __init raid6_algo_add(const struct raid6_calls *algo); +void __init raid6_algo_add_default(void); +void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo); + +/* for the kunit test */ +const struct raid6_calls *raid6_algo_find(unsigned int idx); +const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx); + +/* generic implementations */ +extern const struct raid6_calls raid6_intx1; +extern const struct raid6_calls raid6_intx2; +extern const struct raid6_calls raid6_intx4; +extern const struct raid6_calls raid6_intx8; +extern const struct raid6_recov_calls raid6_recov_intx1; + +#endif /* _PQ_IMPL_H */ diff --git a/lib/raid6/neon.c b/lib/raid/raid6/arm/neon.c index 6d9474ce6da91..af90869aaffc1 100644 --- a/lib/raid6/neon.c +++ b/lib/raid/raid6/arm/neon.c @@ -1,18 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* - * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics + * RAID6 syndrome calculation using ARM NEON intrinsics * * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org> */ -#include <linux/raid/pq.h> - -#ifdef __KERNEL__ #include <asm/simd.h> -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif +#include "algos.h" /* * There are 2 reasons these wrappers are kept in a separate compilation unit @@ -46,18 +40,11 @@ start, stop, (unsigned long)bytes, ptrs);\ } \ struct raid6_calls const raid6_neonx ## _n = { \ - raid6_neon ## _n ## _gen_syndrome, \ - raid6_neon ## _n ## _xor_syndrome, \ - raid6_have_neon, \ - "neonx" #_n, \ - 0 \ + .gen_syndrome = raid6_neon ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_neon ## _n ## _xor_syndrome, \ + .name = "neonx" #_n, \ } -static int raid6_have_neon(void) -{ - return cpu_has_neon(); -} - RAID6_NEON_WRAPPER(1); RAID6_NEON_WRAPPER(2); RAID6_NEON_WRAPPER(4); diff --git a/lib/raid6/neon.h b/lib/raid/raid6/arm/neon.h index 2ca41ee9b4996..2ca41ee9b4996 100644 --- a/lib/raid6/neon.h +++ b/lib/raid/raid6/arm/neon.h diff --git a/lib/raid6/neon.uc b/lib/raid/raid6/arm/neon.uc index 355270af0cd61..14a9fc2c60fa2 100644 --- a/lib/raid6/neon.uc +++ b/lib/raid/raid6/arm/neon.uc @@ -25,7 +25,7 @@ */ #include <arm_neon.h> -#include "neon.h" +#include "arm/neon.h" typedef uint8x16_t unative_t; diff --git a/lib/raid/raid6/arm/pq_arch.h b/lib/raid/raid6/arm/pq_arch.h new file mode 100644 index 0000000000000..3f876ea6749cc --- /dev/null +++ b/lib/raid/raid6/arm/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/neon.h> + +extern const struct raid6_calls raid6_neonx1; +extern const struct raid6_calls raid6_neonx2; +extern const struct raid6_calls raid6_neonx4; +extern const struct raid6_calls raid6_neonx8; +extern const struct raid6_recov_calls raid6_recov_neon; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (cpu_has_neon()) { + raid6_algo_add(&raid6_neonx1); + raid6_algo_add(&raid6_neonx2); + raid6_algo_add(&raid6_neonx4); + raid6_algo_add(&raid6_neonx8); + raid6_recov_algo_add(&raid6_recov_neon); + } +} diff --git a/lib/raid6/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c index 9d99aeabd31a9..1524050d09b7f 100644 --- a/lib/raid6/recov_neon.c +++ b/lib/raid/raid6/arm/recov_neon.c @@ -4,20 +4,11 @@ * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> */ +#include <linux/mm.h> #include <linux/raid/pq.h> - -#ifdef __KERNEL__ #include <asm/simd.h> -#include "neon.h" -#else -#define scoped_ksimd() -#define cpu_has_neon() (1) -#endif - -static int raid6_has_neon(void) -{ - return cpu_has_neon(); -} +#include "algos.h" +#include "arm/neon.h" static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -35,13 +26,13 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +63,10 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -91,7 +82,5 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_neon = { .data2 = raid6_2data_recov_neon, .datap = raid6_datap_recov_neon, - .valid = raid6_has_neon, .name = "neon", - .priority = 10, }; diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid/raid6/arm/recov_neon_inner.c index f9e7e8f5a1510..53c355efa7ff6 100644 --- a/lib/raid6/recov_neon_inner.c +++ b/lib/raid/raid6/arm/recov_neon_inner.c @@ -5,7 +5,7 @@ */ #include <arm_neon.h> -#include "neon.h" +#include "arm/neon.h" #ifdef CONFIG_ARM /* diff --git a/lib/raid/raid6/arm64/pq_arch.h b/lib/raid/raid6/arm64/pq_arch.h new file mode 100644 index 0000000000000..27ff564d7594d --- /dev/null +++ b/lib/raid/raid6/arm64/pq_arch.h @@ -0,0 +1 @@ +#include "arm/pq_arch.h" diff --git a/lib/raid6/int.uc b/lib/raid/raid6/int.uc index 1ba56c3fa4825..e63bd5a9c2ed1 100644 --- a/lib/raid6/int.uc +++ b/lib/raid/raid6/int.uc @@ -18,7 +18,7 @@ * This file is postprocessed using unroll.awk */ -#include <linux/raid/pq.h> +#include "algos.h" /* * This is the C data type to use @@ -139,9 +139,7 @@ static void raid6_int$#_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_intx$# = { - raid6_int$#_gen_syndrome, - raid6_int$#_xor_syndrome, - NULL, /* always valid */ - "int" NSTRING "x$#", - 0 + .gen_syndrome = raid6_int$#_gen_syndrome, + .xor_syndrome = raid6_int$#_xor_syndrome, + .name = "int" NSTRING "x$#", }; diff --git a/lib/raid6/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c index aa5d9f924ca39..c1eb53fafd27e 100644 --- a/lib/raid6/loongarch_simd.c +++ b/lib/raid/raid6/loongarch/loongarch_simd.c @@ -9,8 +9,9 @@ * Copyright 2002-2004 H. Peter Anvin */ -#include <linux/raid/pq.h> -#include "loongarch.h" +#include <asm/cpu-features.h> +#include <asm/fpu.h> +#include "algos.h" /* * The vector algorithms are currently priority 0, which means the generic @@ -25,11 +26,6 @@ #ifdef CONFIG_CPU_HAS_LSX #define NSIZE 16 -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -243,11 +239,9 @@ static void raid6_lsx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lsx = { - raid6_lsx_gen_syndrome, - raid6_lsx_xor_syndrome, - raid6_has_lsx, - "lsx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lsx_gen_syndrome, + .xor_syndrome = raid6_lsx_xor_syndrome, + .name = "lsx", }; #undef NSIZE @@ -256,11 +250,6 @@ const struct raid6_calls raid6_lsx = { #ifdef CONFIG_CPU_HAS_LASX #define NSIZE 32 -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -412,11 +401,9 @@ static void raid6_lasx_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_lasx = { - raid6_lasx_gen_syndrome, - raid6_lasx_xor_syndrome, - raid6_has_lasx, - "lasx", - .priority = 0 /* see the comment near the top of the file for reason */ + .gen_syndrome = raid6_lasx_gen_syndrome, + .xor_syndrome = raid6_lasx_xor_syndrome, + .name = "lasx", }; #undef NSIZE #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid/raid6/loongarch/pq_arch.h b/lib/raid/raid6/loongarch/pq_arch.h new file mode 100644 index 0000000000000..ae443a4d7b696 --- /dev/null +++ b/lib/raid/raid6/loongarch/pq_arch.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cpu-features.h> + +extern const struct raid6_calls raid6_lsx; +extern const struct raid6_calls raid6_lasx; + +extern const struct raid6_recov_calls raid6_recov_lsx; +extern const struct raid6_recov_calls raid6_recov_lasx; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_algo_add(&raid6_lsx); + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_algo_add(&raid6_lasx); + + if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx) + raid6_recov_algo_add(&raid6_recov_lasx); + else if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx) + raid6_recov_algo_add(&raid6_recov_lsx); +} diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c index 93dc515997a14..87a2313bbb4f7 100644 --- a/lib/raid6/recov_loongarch_simd.c +++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c @@ -10,8 +10,11 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "loongarch.h" +#include <asm/cpu-features.h> +#include <asm/fpu.h> +#include "algos.h" /* * Unlike with the syndrome calculation algorithms, there's no boot-time @@ -21,11 +24,6 @@ */ #ifdef CONFIG_CPU_HAS_LSX -static int raid6_has_lsx(void) -{ - return cpu_has_lsx; -} - static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -42,13 +40,13 @@ static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -197,10 +195,10 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -288,18 +286,11 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lsx = { .data2 = raid6_2data_recov_lsx, .datap = raid6_datap_recov_lsx, - .valid = raid6_has_lsx, .name = "lsx", - .priority = 1, }; #endif /* CONFIG_CPU_HAS_LSX */ #ifdef CONFIG_CPU_HAS_LASX -static int raid6_has_lasx(void) -{ - return cpu_has_lasx; -} - static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, int failb, void **ptrs) { @@ -316,13 +307,13 @@ static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -436,10 +427,10 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -506,8 +497,6 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_lasx = { .data2 = raid6_2data_recov_lasx, .datap = raid6_datap_recov_lasx, - .valid = raid6_has_lasx, .name = "lasx", - .priority = 2, }; #endif /* CONFIG_CPU_HAS_LASX */ diff --git a/lib/raid6/mktables.c b/lib/raid/raid6/mktables.c index 3be03793237c2..b6327b562fdb4 100644 --- a/lib/raid6/mktables.c +++ b/lib/raid/raid6/mktables.c @@ -1,15 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * mktables.c + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved * - * Make RAID-6 tables. This is a host user space program to be run at - * compile time. + * Make RAID-6 tables. This is a host user space program to be run at compile + * time. */ #include <stdio.h> @@ -56,10 +50,8 @@ int main(int argc, char *argv[]) uint8_t v; uint8_t exptbl[256], invtbl[256]; - printf("#ifdef __KERNEL__\n"); printf("#include <linux/export.h>\n"); - printf("#endif\n"); - printf("#include <linux/raid/pq.h>\n"); + printf("#include \"algos.h\"\n"); /* Compute multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -76,9 +68,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfmul);\n"); - printf("#endif\n"); /* Compute vector multiplication table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -101,9 +91,7 @@ int main(int argc, char *argv[]) printf("\t},\n"); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_vgfmul);\n"); - printf("#endif\n"); /* Compute power-of-2 table (exponent) */ v = 1; @@ -120,9 +108,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexp);\n"); - printf("#endif\n"); /* Compute log-of-2 table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -140,9 +126,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gflog);\n"); - printf("#endif\n"); /* Compute inverse table x^-1 == x^254 */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -155,9 +139,7 @@ int main(int argc, char *argv[]) } } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfinv);\n"); - printf("#endif\n"); /* Compute inv(2^x + 1) (exponent-xor-inverse) table */ printf("\nconst u8 __attribute__((aligned(256)))\n" @@ -169,9 +151,7 @@ int main(int argc, char *argv[]) (j == 7) ? '\n' : ' '); } printf("};\n"); - printf("#ifdef __KERNEL__\n"); printf("EXPORT_SYMBOL(raid6_gfexi);\n"); - printf("#endif\n"); return 0; } diff --git a/lib/raid6/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc index d20ed0d114111..c5429fb71dd6f 100644 --- a/lib/raid6/altivec.uc +++ b/lib/raid/raid6/powerpc/altivec.uc @@ -22,15 +22,11 @@ * bracked this with preempt_disable/enable or in a lock) */ -#include <linux/raid/pq.h> - -#ifdef CONFIG_ALTIVEC +#include "algos.h" #include <altivec.h> -#ifdef __KERNEL__ -# include <asm/cputable.h> -# include <asm/switch_to.h> -#endif /* __KERNEL__ */ +#include <asm/cputable.h> +#include <asm/switch_to.h> /* * This is the C data type to use. We use a vector of @@ -108,25 +104,7 @@ static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec(void); -#if $# == 1 -int raid6_have_altivec(void) -{ - /* This assumes either all CPUs have Altivec or none does */ -# ifdef __KERNEL__ - return cpu_has_feature(CPU_FTR_ALTIVEC); -# else - return 1; -# endif -} -#endif - const struct raid6_calls raid6_altivec$# = { - raid6_altivec$#_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_altivec, - "altivecx$#", - 0 + .gen_syndrome = raid6_altivec$#_gen_syndrome, + .name = "altivecx$#", }; - -#endif /* CONFIG_ALTIVEC */ diff --git a/lib/raid/raid6/powerpc/pq_arch.h b/lib/raid/raid6/powerpc/pq_arch.h new file mode 100644 index 0000000000000..ea1878777ff29 --- /dev/null +++ b/lib/raid/raid6/powerpc/pq_arch.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cputable.h> + +extern const struct raid6_calls raid6_altivec1; +extern const struct raid6_calls raid6_altivec2; +extern const struct raid6_calls raid6_altivec4; +extern const struct raid6_calls raid6_altivec8; +extern const struct raid6_calls raid6_vpermxor1; +extern const struct raid6_calls raid6_vpermxor2; +extern const struct raid6_calls raid6_vpermxor4; +extern const struct raid6_calls raid6_vpermxor8; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + + /* This assumes either all CPUs have Altivec or none does */ + if (cpu_has_feature(CPU_FTR_ALTIVEC)) { + raid6_algo_add(&raid6_altivec1); + raid6_algo_add(&raid6_altivec2); + raid6_algo_add(&raid6_altivec4); + raid6_algo_add(&raid6_altivec8); + } + if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && + cpu_has_feature(CPU_FTR_ARCH_207S)) { + raid6_algo_add(&raid6_vpermxor1); + raid6_algo_add(&raid6_vpermxor2); + raid6_algo_add(&raid6_vpermxor4); + raid6_algo_add(&raid6_vpermxor8); + } +} diff --git a/lib/raid6/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc index 1bfb127fbfe81..e8964361aaefb 100644 --- a/lib/raid6/vpermxor.uc +++ b/lib/raid/raid6/powerpc/vpermxor.uc @@ -20,15 +20,11 @@ * This instruction was introduced in POWER8 - ISA v2.07. */ -#include <linux/raid/pq.h> -#ifdef CONFIG_ALTIVEC - #include <altivec.h> #include <asm/ppc-opcode.h> -#ifdef __KERNEL__ #include <asm/cputable.h> #include <asm/switch_to.h> -#endif +#include "algos.h" typedef vector unsigned char unative_t; #define NSIZE sizeof(unative_t) @@ -80,26 +76,7 @@ static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs) preempt_enable(); } -int raid6_have_altivec_vpermxor(void); -#if $# == 1 -int raid6_have_altivec_vpermxor(void) -{ - /* Check if arch has both altivec and the vpermxor instructions */ -# ifdef __KERNEL__ - return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) && - cpu_has_feature(CPU_FTR_ARCH_207S)); -# else - return 1; -#endif - -} -#endif - const struct raid6_calls raid6_vpermxor$# = { - raid6_vpermxor$#_gen_syndrome, - NULL, - raid6_have_altivec_vpermxor, - "vpermxor$#", - 0 + .gen_syndrome = raid6_vpermxor$#_gen_syndrome, + .name = "vpermxor$#", }; -#endif diff --git a/lib/raid6/recov.c b/lib/raid/raid6/recov.c index b5e47c008b41b..3fa53bc3fde4d 100644 --- a/lib/raid6/recov.c +++ b/lib/raid/raid6/recov.c @@ -1,19 +1,15 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/recov.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * RAID-6 data recovery in dual failure mode. In single failure mode, - * use the RAID-5 algorithm (or, in the case of Q failure, just reconstruct - * the syndrome.) + * RAID-6 data recovery in dual failure mode. In single failure mode, use the + * RAID-5 algorithm (or, in the case of Q failure, just reconstruct the + * syndrome.) */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" /* Recover two failed data blocks. */ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, @@ -31,13 +27,13 @@ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -72,10 +68,10 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -95,41 +91,5 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_intx1 = { .data2 = raid6_2data_recov_intx1, .datap = raid6_datap_recov_intx1, - .valid = NULL, .name = "intx1", - .priority = 0, }; - -#ifndef __KERNEL__ -/* Testing only */ - -/* Recover two failed blocks. */ -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs) -{ - if ( faila > failb ) { - int tmp = faila; - faila = failb; - failb = tmp; - } - - if ( failb == disks-1 ) { - if ( faila == disks-2 ) { - /* P+Q failure. Just rebuild the syndrome. */ - raid6_call.gen_syndrome(disks, bytes, ptrs); - } else { - /* data+Q failure. Reconstruct data from P, - then rebuild syndrome. */ - /* NOT IMPLEMENTED - equivalent to RAID-5 */ - } - } else { - if ( failb == disks-2 ) { - /* data+P failure. */ - raid6_datap_recov(disks, bytes, faila, ptrs); - } else { - /* data+data failure. */ - raid6_2data_recov(disks, bytes, faila, failb, ptrs); - } - } -} - -#endif diff --git a/lib/raid/raid6/riscv/pq_arch.h b/lib/raid/raid6/riscv/pq_arch.h new file mode 100644 index 0000000000000..82f1a188f8c4f --- /dev/null +++ b/lib/raid/raid6/riscv/pq_arch.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/vector.h> + +extern const struct raid6_calls raid6_rvvx1; +extern const struct raid6_calls raid6_rvvx2; +extern const struct raid6_calls raid6_rvvx4; +extern const struct raid6_calls raid6_rvvx8; +extern const struct raid6_recov_calls raid6_recov_rvv; + +static __always_inline void __init arch_raid6_init(void) +{ + raid6_algo_add_default(); + if (has_vector()) { + raid6_algo_add(&raid6_rvvx1); + raid6_algo_add(&raid6_rvvx2); + raid6_algo_add(&raid6_rvvx4); + raid6_algo_add(&raid6_rvvx8); + raid6_recov_algo_add(&raid6_recov_rvv); + } +} diff --git a/lib/raid6/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c index 40c393206b6a1..2305940276ddf 100644 --- a/lib/raid6/recov_rvv.c +++ b/lib/raid/raid6/riscv/recov_rvv.c @@ -4,7 +4,9 @@ * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn> */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" #include "rvv.h" static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, @@ -158,13 +160,13 @@ static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, * delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +198,10 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, * Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks - 1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -216,7 +218,5 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_rvv = { .data2 = raid6_2data_recov_rvv, .datap = raid6_datap_recov_rvv, - .valid = rvv_has_vector, .name = "rvv", - .priority = 1, }; diff --git a/lib/raid6/rvv.c b/lib/raid/raid6/riscv/rvv.c index 75c9dafedb284..75c9dafedb284 100644 --- a/lib/raid6/rvv.c +++ b/lib/raid/raid6/riscv/rvv.c diff --git a/lib/raid6/rvv.h b/lib/raid/raid6/riscv/rvv.h index 6d0708a2c8a4b..df0e3637cae8b 100644 --- a/lib/raid6/rvv.h +++ b/lib/raid/raid6/riscv/rvv.h @@ -2,27 +2,11 @@ /* * Copyright 2024 Institute of Software, CAS. * - * raid6/rvv.h - * * Definitions for RISC-V RAID-6 code */ -#ifdef __KERNEL__ #include <asm/vector.h> -#else -#define kernel_vector_begin() -#define kernel_vector_end() -#include <sys/auxv.h> -#include <asm/hwcap.h> -#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V) -#endif - -#include <linux/raid/pq.h> - -static int rvv_has_vector(void) -{ - return has_vector(); -} +#include "algos.h" #define RAID6_RVV_WRAPPER(_n) \ static void raid6_rvv ## _n ## _gen_syndrome(int disks, \ @@ -48,9 +32,7 @@ static int rvv_has_vector(void) kernel_vector_end(); \ } \ struct raid6_calls const raid6_rvvx ## _n = { \ - raid6_rvv ## _n ## _gen_syndrome, \ - raid6_rvv ## _n ## _xor_syndrome, \ - rvv_has_vector, \ - "rvvx" #_n, \ - 0 \ + .gen_syndrome = raid6_rvv ## _n ## _gen_syndrome, \ + .xor_syndrome = raid6_rvv ## _n ## _xor_syndrome, \ + .name = "rvvx" #_n, \ } diff --git a/lib/raid/raid6/s390/pq_arch.h b/lib/raid/raid6/s390/pq_arch.h new file mode 100644 index 0000000000000..95d14c3423068 --- /dev/null +++ b/lib/raid/raid6/s390/pq_arch.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <linux/cpufeature.h> + +extern const struct raid6_calls raid6_s390vx8; +extern const struct raid6_recov_calls raid6_recov_s390xc; + +static __always_inline void __init arch_raid6_init(void) +{ + if (cpu_has_vx()) + raid6_algo_add(&raid6_s390vx8); + else + raid6_algo_add_default(); + raid6_recov_algo_add(&raid6_recov_s390xc); +} diff --git a/lib/raid6/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c index 487018f811924..08d56896e5eab 100644 --- a/lib/raid6/recov_s390xc.c +++ b/lib/raid/raid6/s390/recov_s390xc.c @@ -6,7 +6,9 @@ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> +#include "algos.h" static inline void xor_block(u8 *p1, u8 *p2) { @@ -34,13 +36,13 @@ static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -81,10 +83,10 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -110,7 +112,5 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_s390xc = { .data2 = raid6_2data_recov_s390xc, .datap = raid6_datap_recov_s390xc, - .valid = NULL, .name = "s390xc", - .priority = 1, }; diff --git a/lib/raid6/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc index 8aa53eb2f3958..e5cf9054be2a2 100644 --- a/lib/raid6/s390vx.uc +++ b/lib/raid/raid6/s390/s390vx.uc @@ -12,8 +12,8 @@ */ #include <linux/cpufeature.h> -#include <linux/raid/pq.h> #include <asm/fpu.h> +#include "algos.h" #define NSIZE 16 @@ -121,15 +121,8 @@ static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(&vxstate, KERNEL_VXR); } -static int raid6_s390vx$#_valid(void) -{ - return cpu_has_vx(); -} - const struct raid6_calls raid6_s390vx$# = { - raid6_s390vx$#_gen_syndrome, - raid6_s390vx$#_xor_syndrome, - raid6_s390vx$#_valid, - "vx128x$#", - 1 + .gen_syndrome = raid6_s390vx$#_gen_syndrome, + .xor_syndrome = raid6_s390vx$#_xor_syndrome, + .name = "vx128x$#", }; diff --git a/lib/raid/raid6/tests/Makefile b/lib/raid/raid6/tests/Makefile new file mode 100644 index 0000000000000..87a001b228474 --- /dev/null +++ b/lib/raid/raid6/tests/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += raid6_kunit.o diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c new file mode 100644 index 0000000000000..9f3e671a12241 --- /dev/null +++ b/lib/raid/raid6/tests/raid6_kunit.c @@ -0,0 +1,321 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved + * + * Test RAID-6 recovery algorithms. + */ + +#include <kunit/test.h> +#include <linux/prandom.h> +#include <linux/vmalloc.h> +#include <linux/raid/pq.h> +#include "../algos.h" + +MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING"); + +#define RAID6_KUNIT_SEED 42 +#define RAID6_KUNIT_NUM_TEST_ITERS 10 +#define RAID6_KUNIT_MAX_BUFFERS 64 /* Including P and Q */ +#define RAID6_KUNIT_MAX_FAILURES 2 +#define RAID6_KUNIT_MAX_BYTES PAGE_SIZE + +static struct rnd_state rng; +static void *test_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *aligned_buffers[RAID6_KUNIT_MAX_BUFFERS]; +static void *test_recov_buffers[RAID6_KUNIT_MAX_FAILURES]; +static size_t test_buflen; + +struct test_args { + unsigned int recov_idx; + const struct raid6_recov_calls *recov; + unsigned int gen_idx; + const struct raid6_calls *gen; +}; + +static struct test_args args; + +static u32 rand32(void) +{ + return prandom_u32_state(&rng); +} + +/* Generate a random length that is a multiple of 512. */ +static unsigned int random_length(unsigned int max_length) +{ + return round_up((rand32() % max_length) + 1, 512); +} + +static unsigned int random_nr_buffers(void) +{ + return (rand32() % (RAID6_KUNIT_MAX_BUFFERS - (RAID6_MIN_DISKS - 1))) + + RAID6_MIN_DISKS; +} + +/* Generate a random alignment that is a multiple of 64. */ +static unsigned int random_alignment(unsigned int max_alignment) +{ + if (max_alignment == 0) + return 0; + return (rand32() % (max_alignment + 1)) & ~63; +} + +static void makedata(int start, int stop) +{ + int i; + + for (i = start; i <= stop; i++) + prandom_bytes_state(&rng, test_buffers[i], test_buflen); +} + +static char member_type(unsigned int nr_buffers, int d) +{ + if (d == nr_buffers - 2) + return 'P'; + if (d == nr_buffers - 1) + return 'Q'; + return 'D'; +} + +static void test_recover_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int faila, int failb) +{ + const struct test_args *ta = test->param_value; + void *dataptrs[RAID6_KUNIT_MAX_BUFFERS]; + int i; + + if (faila > failb) + swap(faila, failb); + + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + memset(test_recov_buffers[i], 0xf0, test_buflen); + + memcpy(dataptrs, aligned_buffers, sizeof(dataptrs)); + dataptrs[faila] = test_recov_buffers[0]; + dataptrs[failb] = test_recov_buffers[1]; + + if (failb == nr_buffers - 1) { + /* + * We don't implement the data+Q failure scenario, since it + * is equivalent to a RAID-5 failure (XOR, then recompute Q). + */ + if (WARN_ON_ONCE(faila != nr_buffers - 2)) + return; + + /* P+Q failure. Just rebuild the syndrome. */ + ta->gen->gen_syndrome(nr_buffers, len, dataptrs); + } else if (failb == nr_buffers - 2) { + /* data+P failure. */ + ta->recov->datap(nr_buffers, len, faila, dataptrs); + } else { + /* data+data failure. */ + ta->recov->data2(nr_buffers, len, faila, failb, dataptrs); + } + + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[faila], dataptrs[faila], + len, + "faila miscompared: %3d[%c] buffers %u len %u (failb=%3d[%c])\n", + faila, member_type(nr_buffers, faila), + nr_buffers, len, + failb, member_type(nr_buffers, failb)); + KUNIT_EXPECT_MEMEQ_MSG(test, aligned_buffers[failb], dataptrs[failb], + len, + "failb miscompared: %3d[%c] buffers %u len %u (faila=%3d[%c])\n", + failb, member_type(nr_buffers, failb), + nr_buffers, len, + faila, member_type(nr_buffers, faila)); +} + +static void test_recover(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + unsigned int nr_data = nr_buffers - 2; + int iterations, i; + + /* Test P+Q recovery */ + test_recover_one(test, nr_buffers, len, nr_data, nr_buffers - 1); + + /* Test data+P recovery */ + for (i = 0; i < nr_buffers - 2; i++) + test_recover_one(test, nr_buffers, len, i, nr_data); + + /* Double data failure is impossible with a single data disk */ + if (nr_data == 1) + return; + + /* Test data+data recovery using random sampling */ + iterations = nr_buffers * 2; /* should provide good enough coverage */ + for (i = 0; i < iterations; i++) { + int faila = rand32() % nr_data, failb; + + do { + failb = rand32() % nr_data; + } while (failb == faila); + + test_recover_one(test, nr_buffers, len, faila, failb); + } +} + +/* Simulate rmw run */ +static void test_rmw_one(struct kunit *test, unsigned int nr_buffers, + unsigned int len, int p1, int p2) +{ + const struct test_args *ta = test->param_value; + + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + makedata(p1, p2); + ta->gen->xor_syndrome(nr_buffers, p1, p2, len, aligned_buffers); + test_recover(test, nr_buffers, len); +} + +static void test_rmw(struct kunit *test, unsigned int nr_buffers, + unsigned int len) +{ + int iterations = nr_buffers / 2, i; + + for (i = 0; i < iterations; i++) { + int p1 = rand32() % (nr_buffers - 2); + int p2 = rand32() % (nr_buffers - 2); + + if (p2 < p1) + swap(p1, p2); + test_rmw_one(test, nr_buffers, len, p1, p2); + } +} + +static void raid6_test_one(struct kunit *test) +{ + const struct test_args *ta = test->param_value; + unsigned int nr_buffers = random_nr_buffers(); + unsigned int len = random_length(RAID6_KUNIT_MAX_BYTES); + unsigned int max_alignment; + int i; + + /* Nuke syndromes */ + memset(test_buffers[nr_buffers - 2], 0xee, test_buflen); + memset(test_buffers[nr_buffers - 1], 0xee, test_buflen); + + /* + * If we're not using the entire buffer size, inject randomize alignment + * into the buffer. + */ + max_alignment = RAID6_KUNIT_MAX_BYTES - len; + if (rand32() % 2 == 0) { + /* Use random alignments mod 64 */ + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + + random_alignment(max_alignment); + } else { + /* Go up to the guard page, to catch buffer overreads */ + unsigned int align = test_buflen - len; + + for (i = 0; i < nr_buffers; i++) + aligned_buffers[i] = test_buffers[i] + align; + } + + /* Generate assumed good syndrome */ + ta->gen->gen_syndrome(nr_buffers, len, aligned_buffers); + + test_recover(test, nr_buffers, len); + + if (ta->gen->xor_syndrome) + test_rmw(test, nr_buffers, len); +} + +static void raid6_test(struct kunit *test) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_NUM_TEST_ITERS; i++) + raid6_test_one(test); +} + +static const void *raid6_gen_params(struct kunit *test, const void *prev, + char *desc) +{ + if (!prev) { + memset(&args, 0, sizeof(args)); +next_algo: + args.recov_idx = 0; + args.gen = raid6_algo_find(args.gen_idx); + if (!args.gen) + return NULL; + } + + if (args.recov) + args.recov_idx++; + args.recov = raid6_recov_algo_find(args.recov_idx); + if (!args.recov) { + args.gen_idx++; + goto next_algo; + } + + snprintf(desc, KUNIT_PARAM_DESC_SIZE, "gen=%s recov=%s", + args.gen->name, args.recov->name); + return &args; +} + +static struct kunit_case raid6_test_cases[] = { + KUNIT_CASE_PARAM(raid6_test, raid6_gen_params), + {}, +}; + +static int raid6_suite_init(struct kunit_suite *suite) +{ + int i; + + prandom_seed_state(&rng, RAID6_KUNIT_SEED); + + /* + * Allocate the test buffer using vmalloc() with a page-aligned length + * so that it is immediately followed by a guard page. This allows + * buffer overreads to be detected, even in assembly code. + */ + test_buflen = round_up(RAID6_KUNIT_MAX_BYTES, PAGE_SIZE); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) { + test_recov_buffers[i] = vmalloc(test_buflen); + if (!test_recov_buffers[i]) + goto out_free_recov_buffers; + } + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) { + test_buffers[i] = vmalloc(test_buflen); + if (!test_buffers[i]) + goto out_free_buffers; + } + + makedata(0, RAID6_KUNIT_MAX_BUFFERS - 1); + + return 0; + +out_free_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); +out_free_recov_buffers: + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); + return -ENOMEM; +} + +static void raid6_suite_exit(struct kunit_suite *suite) +{ + int i; + + for (i = 0; i < RAID6_KUNIT_MAX_BUFFERS; i++) + vfree(test_buffers[i]); + memset(test_buffers, 0, sizeof(test_buffers)); + for (i = 0; i < RAID6_KUNIT_MAX_FAILURES; i++) + vfree(test_recov_buffers[i]); + memset(test_recov_buffers, 0, sizeof(test_recov_buffers)); +} + +static struct kunit_suite raid6_test_suite = { + .name = "raid6", + .test_cases = raid6_test_cases, + .suite_init = raid6_suite_init, + .suite_exit = raid6_suite_exit, +}; +kunit_test_suite(raid6_test_suite); + +MODULE_DESCRIPTION("Unit test for the RAID P/Q library functions"); +MODULE_LICENSE("GPL"); diff --git a/lib/raid6/unroll.awk b/lib/raid/raid6/unroll.awk index 0809805a7e231..0809805a7e231 100644 --- a/lib/raid6/unroll.awk +++ b/lib/raid/raid6/unroll.awk diff --git a/lib/raid6/avx2.c b/lib/raid/raid6/x86/avx2.c index 059024234dce1..7d829c669ea79 100644 --- a/lib/raid6/avx2.c +++ b/lib/raid/raid6/x86/avx2.c @@ -1,20 +1,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright (C) 2012 Intel Corporation - * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com> +/* + * Copyright (C) 2012 Intel Corporation + * Author: Yuanhan Liu <yuanhan.liu@linux.intel.com> * - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- */ - -/* * AVX2 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_avx2_constants { u64 x1d[4]; @@ -23,11 +19,6 @@ static const struct raid6_avx2_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); -} - /* * Plain AVX2 implementation */ @@ -128,11 +119,9 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x1 = { - raid6_avx21_gen_syndrome, - raid6_avx21_xor_syndrome, - raid6_have_avx2, - "avx2x1", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx21_gen_syndrome, + .xor_syndrome = raid6_avx21_xor_syndrome, + .name = "avx2x1", }; /* @@ -258,11 +247,9 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x2 = { - raid6_avx22_gen_syndrome, - raid6_avx22_xor_syndrome, - raid6_have_avx2, - "avx2x2", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx22_gen_syndrome, + .xor_syndrome = raid6_avx22_xor_syndrome, + .name = "avx2x2", }; #ifdef CONFIG_X86_64 @@ -461,10 +448,8 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx2x4 = { - raid6_avx24_gen_syndrome, - raid6_avx24_xor_syndrome, - raid6_have_avx2, - "avx2x4", - .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx24_gen_syndrome, + .xor_syndrome = raid6_avx24_xor_syndrome, + .name = "avx2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid6/avx512.c b/lib/raid/raid6/x86/avx512.c index 009bd0adeebf0..e671eb5bde63e 100644 --- a/lib/raid6/avx512.c +++ b/lib/raid/raid6/x86/avx512.c @@ -1,24 +1,19 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- -------------------------------------------------------- - * - * Copyright (C) 2016 Intel Corporation +/* + * Copyright (C) 2016 Intel Corporation * - * Author: Gayatri Kammela <gayatri.kammela@intel.com> - * Author: Megha Dey <megha.dey@linux.intel.com> + * Author: Gayatri Kammela <gayatri.kammela@intel.com> + * Author: Megha Dey <megha.dey@linux.intel.com> * - * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved - * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved + * Based on avx2.c: Copyright 2012 Yuanhan Liu All Rights Reserved + * Based on sse2.c: Copyright 2002 H. Peter Anvin - All Rights Reserved * - * ----------------------------------------------------------------------- - */ - -/* * AVX512 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_avx512_constants { u64 x1d[8]; @@ -29,16 +24,6 @@ static const struct raid6_avx512_constants { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,}, }; -static int raid6_have_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} - static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs) { u8 **dptr = (u8 **)ptrs; @@ -156,11 +141,9 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x1 = { - raid6_avx5121_gen_syndrome, - raid6_avx5121_xor_syndrome, - raid6_have_avx512, - "avx512x1", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5121_gen_syndrome, + .xor_syndrome = raid6_avx5121_xor_syndrome, + .name = "avx512x1", }; /* @@ -313,11 +296,9 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_avx512x2 = { - raid6_avx5122_gen_syndrome, - raid6_avx5122_xor_syndrome, - raid6_have_avx512, - "avx512x2", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5122_gen_syndrome, + .xor_syndrome = raid6_avx5122_xor_syndrome, + .name = "avx512x2", }; #ifdef CONFIG_X86_64 @@ -551,10 +532,8 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop, kernel_fpu_end(); } const struct raid6_calls raid6_avx512x4 = { - raid6_avx5124_gen_syndrome, - raid6_avx5124_xor_syndrome, - raid6_have_avx512, - "avx512x4", - .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */ + .gen_syndrome = raid6_avx5124_gen_syndrome, + .xor_syndrome = raid6_avx5124_xor_syndrome, + .name = "avx512x4", }; #endif diff --git a/lib/raid6/mmx.c b/lib/raid/raid6/x86/mmx.c index 3a5bf53a297b4..afa82536142da 100644 --- a/lib/raid6/mmx.c +++ b/lib/raid/raid6/x86/mmx.c @@ -1,20 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/mmx.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * MMX implementation of RAID-6 syndrome functions + * MMX implementation of RAID-6 syndrome functions. */ -#ifdef CONFIG_X86_32 - -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" /* Shared with raid6/sse1.c */ const struct raid6_mmx_constants { @@ -23,12 +16,6 @@ const struct raid6_mmx_constants { 0x1d1d1d1d1d1d1d1dULL, }; -static int raid6_have_mmx(void) -{ - /* Not really "boot_cpu" but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX); -} - /* * Plain MMX implementation */ @@ -70,11 +57,8 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx1 = { - raid6_mmx1_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx1", - 0 + .gen_syndrome = raid6_mmx1_gen_syndrome, + .name = "mmxx1", }; /* @@ -129,11 +113,6 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_mmxx2 = { - raid6_mmx2_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_mmx, - "mmxx2", - 0 + .gen_syndrome = raid6_mmx2_gen_syndrome, + .name = "mmxx2", }; - -#endif diff --git a/lib/raid/raid6/x86/pq_arch.h b/lib/raid/raid6/x86/pq_arch.h new file mode 100644 index 0000000000000..02f8843b05372 --- /dev/null +++ b/lib/raid/raid6/x86/pq_arch.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ + +#include <asm/cpufeature.h> + +extern const struct raid6_calls raid6_mmxx1; +extern const struct raid6_calls raid6_mmxx2; +extern const struct raid6_calls raid6_sse1x1; +extern const struct raid6_calls raid6_sse1x2; +extern const struct raid6_calls raid6_sse2x1; +extern const struct raid6_calls raid6_sse2x2; +extern const struct raid6_calls raid6_sse2x4; +extern const struct raid6_calls raid6_avx2x1; +extern const struct raid6_calls raid6_avx2x2; +extern const struct raid6_calls raid6_avx2x4; +extern const struct raid6_calls raid6_avx512x1; +extern const struct raid6_calls raid6_avx512x2; +extern const struct raid6_calls raid6_avx512x4; + +extern const struct raid6_recov_calls raid6_recov_ssse3; +extern const struct raid6_recov_calls raid6_recov_avx2; +extern const struct raid6_recov_calls raid6_recov_avx512; + +static inline int raid6_has_avx512(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_AVX) && + boot_cpu_has(X86_FEATURE_AVX512F) && + boot_cpu_has(X86_FEATURE_AVX512BW) && + boot_cpu_has(X86_FEATURE_AVX512VL) && + boot_cpu_has(X86_FEATURE_AVX512DQ); +} + +static inline bool raid6_has_avx2(void) +{ + return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX); +} + +static inline bool raid6_has_ssse3(void) +{ + return boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2) && + boot_cpu_has(X86_FEATURE_SSSE3); +} + +static inline bool raid6_has_sse2(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + boot_cpu_has(X86_FEATURE_FXSR) && + boot_cpu_has(X86_FEATURE_XMM) && + boot_cpu_has(X86_FEATURE_XMM2); +} + +static inline bool raid6_has_sse1_or_mmxext(void) +{ + return boot_cpu_has(X86_FEATURE_MMX) && + (boot_cpu_has(X86_FEATURE_XMM) || + boot_cpu_has(X86_FEATURE_MMXEXT)); +} + +static __always_inline void __init arch_raid6_init(void) +{ + if (raid6_has_avx2()) { + raid6_algo_add(&raid6_avx2x1); + raid6_algo_add(&raid6_avx2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx2x4); + if (raid6_has_avx512()) { + raid6_algo_add(&raid6_avx512x1); + raid6_algo_add(&raid6_avx512x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_avx512x4); + } + } else if (IS_ENABLED(CONFIG_X86_64) || raid6_has_sse2()) { + /* x86_64 can assume SSE2 as baseline */ + raid6_algo_add(&raid6_sse2x1); + raid6_algo_add(&raid6_sse2x2); + if (IS_ENABLED(CONFIG_X86_64)) + raid6_algo_add(&raid6_sse2x4); + } else { + raid6_algo_add_default(); + if (raid6_has_sse1_or_mmxext()) { + raid6_algo_add(&raid6_sse1x1); + raid6_algo_add(&raid6_sse1x2); + } else if (boot_cpu_has(X86_FEATURE_MMX)) { + raid6_algo_add(&raid6_mmxx1); + raid6_algo_add(&raid6_mmxx2); + } + } + + if (raid6_has_avx512()) + raid6_recov_algo_add(&raid6_recov_avx512); + else if (raid6_has_avx2()) + raid6_recov_algo_add(&raid6_recov_avx2); + else if (raid6_has_ssse3()) + raid6_recov_algo_add(&raid6_recov_ssse3); +} diff --git a/lib/raid6/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c index 97d598d2535ca..a714a780a2d8f 100644 --- a/lib/raid6/recov_avx2.c +++ b/lib/raid/raid6/x86/recov_avx2.c @@ -4,14 +4,10 @@ * Author: Jim Kukunas <james.t.kukunas@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_avx2(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -28,13 +24,13 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -196,10 +192,10 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -303,11 +299,9 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx2 = { .data2 = raid6_2data_recov_avx2, .datap = raid6_datap_recov_avx2, - .valid = raid6_has_avx2, #ifdef CONFIG_X86_64 .name = "avx2x2", #else .name = "avx2x1", #endif - .priority = 2, }; diff --git a/lib/raid6/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c index 7986120ca4442..ec72d5a30c01e 100644 --- a/lib/raid6/recov_avx512.c +++ b/lib/raid/raid6/x86/recov_avx512.c @@ -6,18 +6,10 @@ * Author: Megha Dey <megha.dey@linux.intel.com> */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_avx512(void) -{ - return boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_AVX) && - boot_cpu_has(X86_FEATURE_AVX512F) && - boot_cpu_has(X86_FEATURE_AVX512BW) && - boot_cpu_has(X86_FEATURE_AVX512VL) && - boot_cpu_has(X86_FEATURE_AVX512DQ); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -37,13 +29,13 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila, */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -238,10 +230,10 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -367,11 +359,9 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_avx512 = { .data2 = raid6_2data_recov_avx512, .datap = raid6_datap_recov_avx512, - .valid = raid6_has_avx512, #ifdef CONFIG_X86_64 .name = "avx512x2", #else .name = "avx512x1", #endif - .priority = 3, }; diff --git a/lib/raid6/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c index 2e849185c32b3..700bd2c865ece 100644 --- a/lib/raid6/recov_ssse3.c +++ b/lib/raid/raid6/x86/recov_ssse3.c @@ -3,15 +3,10 @@ * Copyright (C) 2012 Intel Corporation */ +#include <linux/mm.h> #include <linux/raid/pq.h> -#include "x86.h" - -static int raid6_has_ssse3(void) -{ - return boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2) && - boot_cpu_has(X86_FEATURE_SSSE3); -} +#include <asm/fpu/api.h> +#include "algos.h" static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, int failb, void **ptrs) @@ -30,13 +25,13 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila, Use the dead data pages as temporary storage for delta p and delta q */ dp = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-2] = dp; dq = (u8 *)ptrs[failb]; - ptrs[failb] = raid6_get_zero_page(); + ptrs[failb] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dp; @@ -203,10 +198,10 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, /* Compute syndrome with zero for the missing data page Use the dead data page as temporary storage for delta q */ dq = (u8 *)ptrs[faila]; - ptrs[faila] = raid6_get_zero_page(); + ptrs[faila] = page_address(ZERO_PAGE(0)); ptrs[disks-1] = dq; - raid6_call.gen_syndrome(disks, bytes, ptrs); + raid6_gen_syndrome(disks, bytes, ptrs); /* Restore pointer table */ ptrs[faila] = dq; @@ -318,11 +313,9 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila, const struct raid6_recov_calls raid6_recov_ssse3 = { .data2 = raid6_2data_recov_ssse3, .datap = raid6_datap_recov_ssse3, - .valid = raid6_has_ssse3, #ifdef CONFIG_X86_64 .name = "ssse3x2", #else .name = "ssse3x1", #endif - .priority = 1, }; diff --git a/lib/raid6/sse1.c b/lib/raid/raid6/x86/sse1.c index 692fa3a93bf0b..f4b260df522a3 100644 --- a/lib/raid6/sse1.c +++ b/lib/raid/raid6/x86/sse1.c @@ -1,39 +1,24 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse1.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * - * SSE-1/MMXEXT implementation of RAID-6 syndrome functions + * SSE-1/MMXEXT implementation of RAID-6 syndrome functions. * - * This is really an MMX implementation, but it requires SSE-1 or - * AMD MMXEXT for prefetch support and a few other features. The - * support for nontemporal memory accesses is enough to make this - * worthwhile as a separate implementation. + * This is really an MMX implementation, but it requires SSE-1 or AMD MMXEXT for + * prefetch support and a few other features. The support for nontemporal + * memory accesses is enough to make this worthwhile as a separate + * implementation. */ -#ifdef CONFIG_X86_32 - -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" /* Defined in raid6/mmx.c */ extern const struct raid6_mmx_constants { u64 x1d; } raid6_mmx_constants; -static int raid6_have_sse1_or_mmxext(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - (boot_cpu_has(X86_FEATURE_XMM) || - boot_cpu_has(X86_FEATURE_MMXEXT)); -} - /* * Plain SSE1 implementation */ @@ -86,11 +71,8 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x1 = { - raid6_sse11_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse11_gen_syndrome, + .name = "sse1x1", }; /* @@ -149,11 +131,6 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs) } const struct raid6_calls raid6_sse1x2 = { - raid6_sse12_gen_syndrome, - NULL, /* XOR not yet implemented */ - raid6_have_sse1_or_mmxext, - "sse1x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse12_gen_syndrome, + .name = "sse1x2", }; - -#endif diff --git a/lib/raid6/sse2.c b/lib/raid/raid6/x86/sse2.c index 2930220249c90..43b09ce58270a 100644 --- a/lib/raid6/sse2.c +++ b/lib/raid/raid6/x86/sse2.c @@ -1,19 +1,13 @@ // SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - /* - * raid6/sse2.c + * Copyright 2002 H. Peter Anvin - All Rights Reserved * * SSE-2 implementation of RAID-6 syndrome functions - * */ -#include <linux/raid/pq.h> -#include "x86.h" +#include <asm/cpufeature.h> +#include <asm/fpu/api.h> +#include "algos.h" static const struct raid6_sse_constants { u64 x1d[2]; @@ -21,15 +15,6 @@ static const struct raid6_sse_constants { { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, }; -static int raid6_have_sse2(void) -{ - /* Not really boot_cpu but "all_cpus" */ - return boot_cpu_has(X86_FEATURE_MMX) && - boot_cpu_has(X86_FEATURE_FXSR) && - boot_cpu_has(X86_FEATURE_XMM) && - boot_cpu_has(X86_FEATURE_XMM2); -} - /* * Plain SSE2 implementation */ @@ -133,11 +118,9 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x1 = { - raid6_sse21_gen_syndrome, - raid6_sse21_xor_syndrome, - raid6_have_sse2, - "sse2x1", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse21_gen_syndrome, + .xor_syndrome = raid6_sse21_xor_syndrome, + .name = "sse2x1", }; /* @@ -263,11 +246,9 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop, } const struct raid6_calls raid6_sse2x2 = { - raid6_sse22_gen_syndrome, - raid6_sse22_xor_syndrome, - raid6_have_sse2, - "sse2x2", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse22_gen_syndrome, + .xor_syndrome = raid6_sse22_xor_syndrome, + .name = "sse2x2", }; #ifdef CONFIG_X86_64 @@ -470,11 +451,9 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop, const struct raid6_calls raid6_sse2x4 = { - raid6_sse24_gen_syndrome, - raid6_sse24_xor_syndrome, - raid6_have_sse2, - "sse2x4", - 1 /* Has cache hints */ + .gen_syndrome = raid6_sse24_gen_syndrome, + .xor_syndrome = raid6_sse24_xor_syndrome, + .name = "sse2x4", }; #endif /* CONFIG_X86_64 */ diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile deleted file mode 100644 index 5be0a4e60ab1e..0000000000000 --- a/lib/raid6/Makefile +++ /dev/null @@ -1,83 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -obj-$(CONFIG_RAID6_PQ) += raid6_pq.o - -raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \ - int8.o - -raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o -raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o -raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o -raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o -raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o - -hostprogs += mktables - -ifeq ($(CONFIG_ALTIVEC),y) -altivec_flags := -maltivec $(call cc-option,-mabi=altivec) -# Enable <altivec.h> -altivec_flags += -isystem $(shell $(CC) -print-file-name=include) - -ifdef CONFIG_CC_IS_CLANG -# clang ppc port does not yet support -maltivec when -msoft-float is -# enabled. A future release of clang will resolve this -# https://llvm.org/pr31177 -CFLAGS_REMOVE_altivec1.o += -msoft-float -CFLAGS_REMOVE_altivec2.o += -msoft-float -CFLAGS_REMOVE_altivec4.o += -msoft-float -CFLAGS_REMOVE_altivec8.o += -msoft-float -CFLAGS_REMOVE_vpermxor1.o += -msoft-float -CFLAGS_REMOVE_vpermxor2.o += -msoft-float -CFLAGS_REMOVE_vpermxor4.o += -msoft-float -CFLAGS_REMOVE_vpermxor8.o += -msoft-float -endif -endif - -quiet_cmd_unroll = UNROLL $@ - cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@ - -targets += int1.c int2.c int4.c int8.c -$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_altivec1.o += $(altivec_flags) -CFLAGS_altivec2.o += $(altivec_flags) -CFLAGS_altivec4.o += $(altivec_flags) -CFLAGS_altivec8.o += $(altivec_flags) -targets += altivec1.c altivec2.c altivec4.c altivec8.c -$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_vpermxor1.o += $(altivec_flags) -CFLAGS_vpermxor2.o += $(altivec_flags) -CFLAGS_vpermxor4.o += $(altivec_flags) -CFLAGS_vpermxor8.o += $(altivec_flags) -targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c -$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -CFLAGS_neon1.o += $(CC_FLAGS_FPU) -CFLAGS_neon2.o += $(CC_FLAGS_FPU) -CFLAGS_neon4.o += $(CC_FLAGS_FPU) -CFLAGS_neon8.o += $(CC_FLAGS_FPU) -CFLAGS_recov_neon_inner.o += $(CC_FLAGS_FPU) -CFLAGS_REMOVE_neon1.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU) -CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU) -targets += neon1.c neon2.c neon4.c neon8.c -$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -targets += s390vx8.c -$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE - $(call if_changed,unroll) - -quiet_cmd_mktable = TABLE $@ - cmd_mktable = $(obj)/mktables > $@ - -targets += tables.c -$(obj)/tables.c: $(obj)/mktables FORCE - $(call if_changed,mktable) diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c deleted file mode 100644 index 799e0e5eac26d..0000000000000 --- a/lib/raid6/algos.c +++ /dev/null @@ -1,291 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/algos.c - * - * Algorithm list and algorithm selection for RAID-6 - */ - -#include <linux/raid/pq.h> -#ifndef __KERNEL__ -#include <sys/mman.h> -#include <stdio.h> -#else -#include <linux/module.h> -#include <linux/gfp.h> -#endif - -struct raid6_calls raid6_call; -EXPORT_SYMBOL_GPL(raid6_call); - -const struct raid6_calls * const raid6_algos[] = { -#if defined(__i386__) && !defined(__arch_um__) - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x2, - &raid6_sse2x1, - &raid6_sse1x2, - &raid6_sse1x1, - &raid6_mmxx2, - &raid6_mmxx1, -#endif -#if defined(__x86_64__) && !defined(__arch_um__) - &raid6_avx512x4, - &raid6_avx512x2, - &raid6_avx512x1, - &raid6_avx2x4, - &raid6_avx2x2, - &raid6_avx2x1, - &raid6_sse2x4, - &raid6_sse2x2, - &raid6_sse2x1, -#endif -#ifdef CONFIG_ALTIVEC - &raid6_vpermxor8, - &raid6_vpermxor4, - &raid6_vpermxor2, - &raid6_vpermxor1, - &raid6_altivec8, - &raid6_altivec4, - &raid6_altivec2, - &raid6_altivec1, -#endif -#if defined(CONFIG_S390) - &raid6_s390vx8, -#endif -#ifdef CONFIG_KERNEL_MODE_NEON - &raid6_neonx8, - &raid6_neonx4, - &raid6_neonx2, - &raid6_neonx1, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_rvvx1, - &raid6_rvvx2, - &raid6_rvvx4, - &raid6_rvvx8, -#endif - &raid6_intx8, - &raid6_intx4, - &raid6_intx2, - &raid6_intx1, - NULL -}; - -void (*raid6_2data_recov)(int, size_t, int, int, void **); -EXPORT_SYMBOL_GPL(raid6_2data_recov); - -void (*raid6_datap_recov)(int, size_t, int, void **); -EXPORT_SYMBOL_GPL(raid6_datap_recov); - -const struct raid6_recov_calls *const raid6_recov_algos[] = { -#ifdef CONFIG_X86 - &raid6_recov_avx512, - &raid6_recov_avx2, - &raid6_recov_ssse3, -#endif -#ifdef CONFIG_S390 - &raid6_recov_s390xc, -#endif -#if defined(CONFIG_KERNEL_MODE_NEON) - &raid6_recov_neon, -#endif -#ifdef CONFIG_LOONGARCH -#ifdef CONFIG_CPU_HAS_LASX - &raid6_recov_lasx, -#endif -#ifdef CONFIG_CPU_HAS_LSX - &raid6_recov_lsx, -#endif -#endif -#ifdef CONFIG_RISCV_ISA_V - &raid6_recov_rvv, -#endif - &raid6_recov_intx1, - NULL -}; - -#ifdef __KERNEL__ -#define RAID6_TIME_JIFFIES_LG2 4 -#else -/* Need more time to be stable in userspace */ -#define RAID6_TIME_JIFFIES_LG2 9 -#define time_before(x, y) ((x) < (y)) -#endif - -#define RAID6_TEST_DISKS 8 -#define RAID6_TEST_DISKS_ORDER 3 - -static inline const struct raid6_recov_calls *raid6_choose_recov(void) -{ - const struct raid6_recov_calls *const *algo; - const struct raid6_recov_calls *best; - - for (best = NULL, algo = raid6_recov_algos; *algo; algo++) - if (!best || (*algo)->priority > best->priority) - if (!(*algo)->valid || (*algo)->valid()) - best = *algo; - - if (best) { - raid6_2data_recov = best->data2; - raid6_datap_recov = best->datap; - - pr_info("raid6: using %s recovery algorithm\n", best->name); - } else - pr_err("raid6: Yikes! No recovery algorithm found!\n"); - - return best; -} - -static inline const struct raid6_calls *raid6_choose_gen( - void *(*const dptrs)[RAID6_TEST_DISKS], const int disks) -{ - unsigned long perf, bestgenperf, j0, j1; - int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ - const struct raid6_calls *const *algo; - const struct raid6_calls *best; - - for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { - if (!best || (*algo)->priority >= best->priority) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - best = *algo; - break; - } - - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { - (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - if (perf > bestgenperf) { - bestgenperf = perf; - best = *algo; - } - pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, - (perf * HZ * (disks-2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - } - } - - if (!best) { - pr_err("raid6: Yikes! No algorithm found!\n"); - goto out; - } - - raid6_call = *best; - - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) { - pr_info("raid6: skipped pq benchmark and selected %s\n", - best->name); - goto out; - } - - pr_info("raid6: using algorithm %s gen() %ld MB/s\n", - best->name, - (bestgenperf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); - - if (best->xor_syndrome) { - perf = 0; - - preempt_disable(); - j0 = jiffies; - while ((j1 = jiffies) == j0) - cpu_relax(); - while (time_before(jiffies, - j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { - best->xor_syndrome(disks, start, stop, - PAGE_SIZE, *dptrs); - perf++; - } - preempt_enable(); - - pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", - (perf * HZ * (disks - 2)) >> - (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); - } - -out: - return best; -} - - -/* Try to pick the best algorithm */ -/* This code uses the gfmul table as convenient data set to abuse */ - -int __init raid6_select_algo(void) -{ - const int disks = RAID6_TEST_DISKS; - - const struct raid6_calls *gen_best; - const struct raid6_recov_calls *rec_best; - char *disk_ptr, *p; - void *dptrs[RAID6_TEST_DISKS]; - int i, cycle; - - /* prepare the buffer and fill it circularly with gfmul table */ - disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); - if (!disk_ptr) { - pr_err("raid6: Yikes! No memory available.\n"); - return -ENOMEM; - } - - p = disk_ptr; - for (i = 0; i < disks; i++) - dptrs[i] = p + PAGE_SIZE * i; - - cycle = ((disks - 2) * PAGE_SIZE) / 65536; - for (i = 0; i < cycle; i++) { - memcpy(p, raid6_gfmul, 65536); - p += 65536; - } - - if ((disks - 2) * PAGE_SIZE % 65536) - memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); - - /* select raid gen_syndrome function */ - gen_best = raid6_choose_gen(&dptrs, disks); - - /* select raid recover functions */ - rec_best = raid6_choose_recov(); - - free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); - - return gen_best && rec_best ? 0 : -EINVAL; -} - -static void raid6_exit(void) -{ - do { } while (0); -} - -subsys_initcall(raid6_select_algo); -module_exit(raid6_exit); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h deleted file mode 100644 index acfc33ce70562..0000000000000 --- a/lib/raid6/loongarch.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2023 WANG Xuerui <git@xen0n.name> - * - * raid6/loongarch.h - * - * Definitions common to LoongArch RAID-6 code only - */ - -#ifndef _LIB_RAID6_LOONGARCH_H -#define _LIB_RAID6_LOONGARCH_H - -#ifdef __KERNEL__ - -#include <asm/cpu-features.h> -#include <asm/fpu.h> - -#else /* for user-space testing */ - -#include <sys/auxv.h> - -/* have to supply these defines for glibc 2.37- and musl */ -#ifndef HWCAP_LOONGARCH_LSX -#define HWCAP_LOONGARCH_LSX (1 << 4) -#endif -#ifndef HWCAP_LOONGARCH_LASX -#define HWCAP_LOONGARCH_LASX (1 << 5) -#endif - -#define kernel_fpu_begin() -#define kernel_fpu_end() - -#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX) -#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX) - -#endif /* __KERNEL__ */ - -#endif /* _LIB_RAID6_LOONGARCH_H */ diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore deleted file mode 100644 index 1b68a77f348f6..0000000000000 --- a/lib/raid6/test/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/int.uc -/neon.uc -/raid6test diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile deleted file mode 100644 index 09bbe2b14cceb..0000000000000 --- a/lib/raid6/test/Makefile +++ /dev/null @@ -1,156 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -# -# This is a simple Makefile to test some of the RAID-6 code -# from userspace. -# - -pound := \# - -# Adjust as desired -CC = gcc -OPTFLAGS = -O2 -CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS) -LD = ld -AWK = awk -f -AR = ar -RANLIB = ranlib -OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o - -ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/) -ifeq ($(ARCH),i386) - CFLAGS += -DCONFIG_X86_32 - IS_X86 = yes -endif -ifeq ($(ARCH),x86_64) - CFLAGS += -DCONFIG_X86_64 - IS_X86 = yes -endif - -ifeq ($(ARCH),arm) - CFLAGS += -I../../../arch/arm/include -mfpu=neon - HAS_NEON = yes -endif -ifeq ($(ARCH),aarch64) - CFLAGS += -I../../../arch/arm64/include - HAS_NEON = yes -endif - -ifeq ($(findstring riscv,$(ARCH)),riscv) - CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1 - HAS_RVV = yes -endif - -ifeq ($(findstring ppc,$(ARCH)),ppc) - CFLAGS += -I../../../arch/powerpc/include - HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\ - gcc -c -x c - >/dev/null && rm ./-.o && echo yes) -endif - -ifeq ($(ARCH),loongarch64) - CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1 - CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1) - CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \ - gcc -c -x assembler - >/dev/null 2>&1 && \ - rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1) -endif - -ifeq ($(IS_X86),yes) - OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o - CFLAGS += -DCONFIG_X86 -else ifeq ($(HAS_NEON),yes) - OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o - CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1 -else ifeq ($(HAS_ALTIVEC),yes) - CFLAGS += -DCONFIG_ALTIVEC - OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \ - vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o -else ifeq ($(ARCH),loongarch64) - OBJS += loongarch_simd.o recov_loongarch_simd.o -else ifeq ($(HAS_RVV),yes) - OBJS += rvv.o recov_rvv.o - CFLAGS += -DCONFIG_RISCV_ISA_V=1 -endif - -.c.o: - $(CC) $(CFLAGS) -c -o $@ $< - -%.c: ../%.c - cp -f $< $@ - -%.uc: ../%.uc - cp -f $< $@ - -all: raid6.a raid6test - -raid6.a: $(OBJS) - rm -f $@ - $(AR) cq $@ $^ - $(RANLIB) $@ - -raid6test: test.c raid6.a - $(CC) $(CFLAGS) -o raid6test $^ - -neon1.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < neon.uc > $@ - -neon2.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < neon.uc > $@ - -neon4.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < neon.uc > $@ - -neon8.c: neon.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < neon.uc > $@ - -altivec1.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@ - -altivec2.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < altivec.uc > $@ - -altivec4.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < altivec.uc > $@ - -altivec8.c: altivec.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < altivec.uc > $@ - -vpermxor1.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < vpermxor.uc > $@ - -vpermxor2.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < vpermxor.uc > $@ - -vpermxor4.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < vpermxor.uc > $@ - -vpermxor8.c: vpermxor.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < vpermxor.uc > $@ - -int1.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=1 < int.uc > $@ - -int2.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=2 < int.uc > $@ - -int4.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=4 < int.uc > $@ - -int8.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=8 < int.uc > $@ - -int16.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=16 < int.uc > $@ - -int32.c: int.uc ../unroll.awk - $(AWK) ../unroll.awk -vN=32 < int.uc > $@ - -tables.c: mktables - ./mktables > tables.c - -clean: - rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c vpermxor*.c neon*.c tables.c raid6test - -spotless: clean - rm -f *~ diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c deleted file mode 100644 index 841a55242abaa..0000000000000 --- a/lib/raid6/test/test.c +++ /dev/null @@ -1,152 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* -*- linux-c -*- ------------------------------------------------------- * - * - * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6test.c - * - * Test RAID-6 recovery with various algorithms - */ - -#include <stdlib.h> -#include <stdio.h> -#include <string.h> -#include <linux/raid/pq.h> - -#define NDISKS 16 /* Including P and Q */ - -const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -char *dataptrs[NDISKS]; -char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); -char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE))); - -static void makedata(int start, int stop) -{ - int i, j; - - for (i = start; i <= stop; i++) { - for (j = 0; j < PAGE_SIZE; j++) - data[i][j] = rand(); - - dataptrs[i] = data[i]; - } -} - -static char disk_type(int d) -{ - switch (d) { - case NDISKS-2: - return 'P'; - case NDISKS-1: - return 'Q'; - default: - return 'D'; - } -} - -static int test_disks(int i, int j) -{ - int erra, errb; - - memset(recovi, 0xf0, PAGE_SIZE); - memset(recovj, 0xba, PAGE_SIZE); - - dataptrs[i] = recovi; - dataptrs[j] = recovj; - - raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs); - - erra = memcmp(data[i], recovi, PAGE_SIZE); - errb = memcmp(data[j], recovj, PAGE_SIZE); - - if (i < NDISKS-2 && j == NDISKS-1) { - /* We don't implement the DQ failure scenario, since it's - equivalent to a RAID-5 failure (XOR, then recompute Q) */ - erra = errb = 0; - } else { - printf("algo=%-8s faila=%3d(%c) failb=%3d(%c) %s\n", - raid6_call.name, - i, disk_type(i), - j, disk_type(j), - (!erra && !errb) ? "OK" : - !erra ? "ERRB" : - !errb ? "ERRA" : "ERRAB"); - } - - dataptrs[i] = data[i]; - dataptrs[j] = data[j]; - - return erra || errb; -} - -int main(int argc, char *argv[]) -{ - const struct raid6_calls *const *algo; - const struct raid6_recov_calls *const *ra; - int i, j, p1, p2; - int err = 0; - - makedata(0, NDISKS-1); - - for (ra = raid6_recov_algos; *ra; ra++) { - if ((*ra)->valid && !(*ra)->valid()) - continue; - - raid6_2data_recov = (*ra)->data2; - raid6_datap_recov = (*ra)->datap; - - printf("using recovery %s\n", (*ra)->name); - - for (algo = raid6_algos; *algo; algo++) { - if ((*algo)->valid && !(*algo)->valid()) - continue; - - raid6_call = **algo; - - /* Nuke syndromes */ - memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE); - - /* Generate assumed good syndrome */ - raid6_call.gen_syndrome(NDISKS, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - - if (!raid6_call.xor_syndrome) - continue; - - for (p1 = 0; p1 < NDISKS-2; p1++) - for (p2 = p1; p2 < NDISKS-2; p2++) { - - /* Simulate rmw run */ - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - makedata(p1, p2); - raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE, - (void **)&dataptrs); - - for (i = 0; i < NDISKS-1; i++) - for (j = i+1; j < NDISKS; j++) - err += test_disks(i, j); - } - - } - printf("\n"); - } - - printf("\n"); - /* Pick the best algorithm test */ - raid6_select_algo(); - - if (err) - printf("\n*** ERRORS FOUND ***\n"); - - return err; -} diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h deleted file mode 100644 index 9a6ff37115e71..0000000000000 --- a/lib/raid6/x86.h +++ /dev/null @@ -1,75 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* ----------------------------------------------------------------------- * - * - * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -/* - * raid6/x86.h - * - * Definitions common to x86 and x86-64 RAID-6 code only - */ - -#ifndef LINUX_RAID_RAID6X86_H -#define LINUX_RAID_RAID6X86_H - -#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__) - -#ifdef __KERNEL__ /* Real code */ - -#include <asm/fpu/api.h> - -#else /* Dummy code for user space testing */ - -static inline void kernel_fpu_begin(void) -{ -} - -static inline void kernel_fpu_end(void) -{ -} - -#define __aligned(x) __attribute__((aligned(x))) - -#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ -#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions - * (fast save and restore) */ -#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */ -#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */ -#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */ -#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */ -#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */ -#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */ -#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */ -#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular) - * Instructions - */ -#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular) - * Instructions - */ -#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length) - * Extensions - */ -#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ - -/* Should work well enough on modern CPUs for testing */ -static inline int boot_cpu_has(int flag) -{ - u32 eax, ebx, ecx, edx; - - eax = (flag & 0x100) ? 7 : - (flag & 0x20) ? 0x80000001 : 1; - ecx = 0; - - asm volatile("cpuid" - : "+a" (eax), "=b" (ebx), "=d" (edx), "+c" (ecx)); - - return ((flag & 0x100 ? ebx : - (flag & 0x80) ? ecx : edx) >> (flag & 31)) & 1; -} - -#endif /* ndef __KERNEL__ */ - -#endif -#endif diff --git a/lib/seq_buf.c b/lib/seq_buf.c index f3f3436d60a94..b59488fa8135c 100644 --- a/lib/seq_buf.c +++ b/lib/seq_buf.c @@ -298,6 +298,7 @@ int seq_buf_putmem_hex(struct seq_buf *s, const void *mem, } return 0; } +EXPORT_SYMBOL_GPL(seq_buf_putmem_hex); /** * seq_buf_path - copy a path into the sequence buffer diff --git a/lib/string.c b/lib/string.c index b632c71df1a50..1f9297e9776a9 100644 --- a/lib/string.c +++ b/lib/string.c @@ -21,6 +21,7 @@ #include <linux/errno.h> #include <linux/limits.h> #include <linux/linkage.h> +#include <linux/minmax.h> #include <linux/stddef.h> #include <linux/string.h> #include <linux/types.h> @@ -125,11 +126,8 @@ ssize_t sized_strscpy(char *dest, const char *src, size_t count) * If src is unaligned, don't cross a page boundary, * since we don't know if the next page is mapped. */ - if ((long)src & (sizeof(long) - 1)) { - size_t limit = PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)); - if (limit < max) - max = limit; - } + if ((long)src & (sizeof(long) - 1)) + max = min(PAGE_SIZE - ((long)src & (PAGE_SIZE - 1)), max); #else /* If src or dest is unaligned, don't do word-at-a-time. */ if (((long) dest | (long) src) & (sizeof(long) - 1)) diff --git a/lib/test-kstrtox.c b/lib/test-kstrtox.c index ee87fef66cb58..811128d0df16f 100644 --- a/lib/test-kstrtox.c +++ b/lib/test-kstrtox.c @@ -198,6 +198,7 @@ static void __init test_kstrtoull_fail(void) {"10000000000000000000000000000000000000000000000000000000000000000", 2}, {"2000000000000000000000", 8}, {"18446744073709551616", 10}, + {"569202370375329612767", 10}, {"10000000000000000", 16}, /* negative */ {"-0", 0}, @@ -275,9 +276,11 @@ static void __init test_kstrtoll_fail(void) {"9223372036854775809", 10}, {"18446744073709551614", 10}, {"18446744073709551615", 10}, + {"569202370375329612767", 10}, {"-9223372036854775809", 10}, {"-18446744073709551614", 10}, {"-18446744073709551615", 10}, + {"-569202370375329612767", 10}, /* sign is first character if any */ {"-+1", 0}, {"-+1", 8}, @@ -334,6 +337,7 @@ static void __init test_kstrtou64_fail(void) {"-1", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, }; TEST_FAIL(kstrtou64, u64, "%llu", test_u64_fail); } @@ -386,6 +390,8 @@ static void __init test_kstrtos64_fail(void) {"18446744073709551615", 10}, {"18446744073709551616", 10}, {"18446744073709551617", 10}, + {"569202370375329612767", 10}, + {"-569202370375329612767", 10}, }; TEST_FAIL(kstrtos64, s64, "%lld", test_s64_fail); } diff --git a/lib/tests/Makefile b/lib/tests/Makefile index 7e9c2fa52e35a..4ead57602eac4 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -8,6 +8,7 @@ obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o obj-$(CONFIG_BITOPS_KUNIT) += bitops_kunit.o obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_BITS_TEST) += test_bits.o +obj-$(CONFIG_SHDI3_KUNIT_TEST) += shdi3_kunit.o obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o obj-$(CONFIG_CHECKSUM_KUNIT) += checksum_kunit.o obj-$(CONFIG_CMDLINE_KUNIT_TEST) += cmdline_kunit.o diff --git a/lib/tests/cmdline_kunit.c b/lib/tests/cmdline_kunit.c index c1602f797637b..3f61ff8d31782 100644 --- a/lib/tests/cmdline_kunit.c +++ b/lib/tests/cmdline_kunit.c @@ -6,6 +6,7 @@ #include <kunit/test.h> #include <linux/kernel.h> #include <linux/random.h> +#include <linux/sizes.h> #include <linux/string.h> static const char *cmdline_test_strings[] = { @@ -139,11 +140,128 @@ static void cmdline_test_range(struct kunit *test) } while (++i < ARRAY_SIZE(cmdline_test_range_strings)); } +static void cmdline_test_next_arg_quoted_value(struct kunit *test) +{ + char in[] = "foo=\"bar baz\" qux=1"; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar baz"); + KUNIT_EXPECT_STREQ(test, next, "qux=1"); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "qux"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "1"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_bare_quote_regression(struct kunit *test) +{ + char in[] = "foo=bar \""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "foo"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "bar"); + KUNIT_EXPECT_STREQ(test, next, "\""); + + /* This hits the i == 0 quoted-token case fixed by 9847f21225c4. */ + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, ""); + KUNIT_EXPECT_PTR_EQ(test, val, NULL); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +static void cmdline_test_next_arg_mixed_tokens(struct kunit *test) +{ + char in[] = "bbb= jjj kkk=\"a=b\""; + char *next, *param, *val; + + next = next_arg(in, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "bbb"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, ""); + KUNIT_EXPECT_STREQ(test, next, "jjj kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "jjj"); + KUNIT_EXPECT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, next, "kkk=\"a=b\""); + + next = next_arg(next, ¶m, &val); + KUNIT_EXPECT_STREQ(test, param, "kkk"); + KUNIT_ASSERT_NOT_NULL(test, val); + KUNIT_EXPECT_STREQ(test, val, "a=b"); + KUNIT_EXPECT_STREQ(test, next, ""); +} + +struct cmdline_test_memparse_entry { + const char *input; + const char *unrecognized; + unsigned long long result; +}; + +static const struct cmdline_test_memparse_entry testdata[] = { + { "0", "", 0ULL }, + { "1", "", 1ULL }, + { "a", "a", 0ULL }, + { "k", "k", 0ULL }, + { "E", "E", 0ULL }, + { "0xb", "", 11ULL }, + { "0xz", "x", 0ULL }, + { "1234", "", 1234ULL }, + { "04567", "", 2423ULL }, + { "0x9876", "", 39030LL }, + { "05678", "8", 375ULL }, + { "0xabcdefz", "z", 11259375ULL }, + { "0cdba", "c", 0ULL }, + { "4K", "", SZ_4K }, + { "0x10k@0xaaaabbbb", "@", SZ_16K }, + { "32M", "", SZ_32M }, + { "067m:foo", ":", 55 * SZ_1M }, + { "2G;bar=baz", ";", SZ_2G }, + { "07gz", "z", 7ULL * SZ_1G }, + { "3T+data", "+", 3 * SZ_1T }, + { "04t,ro", ",", SZ_4T }, + { "012p", "", 11258999068426240ULL }, + { "7P,sync", ",", 7881299347898368ULL }, + { "0x2e", "", 46ULL }, + { "2E and more", " ", 2305843009213693952ULL }, + { "18446744073709551615", "", ULLONG_MAX }, + { "0xffffffffffffffff0", "", ULLONG_MAX }, + { "1111111111111111111T", "", ULLONG_MAX }, + { "222222222222222222222G", "", ULLONG_MAX }, + { "3333333333333333333333M", "", ULLONG_MAX }, +}; + +static void cmdline_test_memparse(struct kunit *test) +{ + const struct cmdline_test_memparse_entry *e; + unsigned long long ret; + char *retptr; + + for (e = testdata; e < testdata + ARRAY_SIZE(testdata); e++) { + ret = memparse(e->input, &retptr); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when parsing '%s'", e->input); + KUNIT_EXPECT_EQ_MSG(test, *retptr, *e->unrecognized, + " when parsing '%s'", e->input); + } +} + static struct kunit_case cmdline_test_cases[] = { KUNIT_CASE(cmdline_test_noint), KUNIT_CASE(cmdline_test_lead_int), KUNIT_CASE(cmdline_test_tail_int), KUNIT_CASE(cmdline_test_range), + KUNIT_CASE(cmdline_test_next_arg_quoted_value), + KUNIT_CASE(cmdline_test_next_arg_bare_quote_regression), + KUNIT_CASE(cmdline_test_next_arg_mixed_tokens), + KUNIT_CASE(cmdline_test_memparse), {} }; diff --git a/lib/tests/seq_buf_kunit.c b/lib/tests/seq_buf_kunit.c index 8a01579a978e6..eb466386bbefb 100644 --- a/lib/tests/seq_buf_kunit.c +++ b/lib/tests/seq_buf_kunit.c @@ -184,6 +184,38 @@ static void seq_buf_get_buf_commit_test(struct kunit *test) KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s)); } +static void seq_buf_putmem_hex_test(struct kunit *test) +{ + DECLARE_SEQ_BUF(s, 24); + const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +#ifdef __BIG_ENDIAN + const char *expected = "0001020304050607 0809 "; +#else + const char *expected = "0706050403020100 0908 "; +#endif + + KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), 0); + KUNIT_EXPECT_FALSE(test, seq_buf_has_overflowed(&s)); + KUNIT_EXPECT_EQ(test, seq_buf_used(&s), strlen(expected)); + KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected); +} + +static void seq_buf_putmem_hex_overflow_test(struct kunit *test) +{ + DECLARE_SEQ_BUF(s, 20); + const u8 data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; +#ifdef __BIG_ENDIAN + const char *expected = "0001020304050607 "; +#else + const char *expected = "0706050403020100 "; +#endif + + KUNIT_EXPECT_EQ(test, seq_buf_putmem_hex(&s, data, sizeof(data)), -1); + KUNIT_EXPECT_TRUE(test, seq_buf_has_overflowed(&s)); + KUNIT_EXPECT_EQ(test, seq_buf_used(&s), 20); + KUNIT_EXPECT_STREQ(test, seq_buf_str(&s), expected); +} + static struct kunit_case seq_buf_test_cases[] = { KUNIT_CASE(seq_buf_init_test), KUNIT_CASE(seq_buf_declare_test), @@ -194,6 +226,8 @@ static struct kunit_case seq_buf_test_cases[] = { KUNIT_CASE(seq_buf_printf_test), KUNIT_CASE(seq_buf_printf_overflow_test), KUNIT_CASE(seq_buf_get_buf_commit_test), + KUNIT_CASE(seq_buf_putmem_hex_test), + KUNIT_CASE(seq_buf_putmem_hex_overflow_test), {} }; diff --git a/lib/tests/shdi3_kunit.c b/lib/tests/shdi3_kunit.c new file mode 100644 index 0000000000000..44f65e66512b5 --- /dev/null +++ b/lib/tests/shdi3_kunit.c @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: GPL-2.0-or-later OR Apache-2.0 +/* + * Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3(). + */ + +#include <linux/array_size.h> +#include <linux/module.h> +#include <linux/libgcc.h> +#include <kunit/test.h> + +struct shdi3_test_entry { + long long input; + int shift; + long long result; +}; + +static const struct shdi3_test_entry ashldi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashldi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x2468ACF13579BDELL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BCLL }, + { 0x0123456789ABCDEFLL, 3, 0x91A2B3C4D5E6F78LL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDEF0LL }, + { 0x0123456789ABCDEFLL, 28, 0x789ABCDEF0000000LL }, + { 0x0123456789ABCDEFLL, 29, 0xF13579BDE0000000LL }, + { 0x0123456789ABCDEFLL, 30, 0xE26AF37BC0000000LL }, + { 0x0123456789ABCDEFLL, 31, 0xC4D5E6F780000000LL }, + { 0x0123456789ABCDEFLL, 32, 0x89ABCDEF00000000LL }, + { 0x0123456789ABCDEFLL, 33, 0x13579BDE00000000LL }, + { 0x0123456789ABCDEFLL, 34, 0x26AF37BC00000000LL }, + { 0x0123456789ABCDEFLL, 35, 0x4D5E6F7800000000LL }, + { 0x0123456789ABCDEFLL, 36, 0x9ABCDEF000000000LL }, + { 0x0123456789ABCDEFLL, 60, 0xF000000000000000LL }, + { 0x0123456789ABCDEFLL, 61, 0xE000000000000000LL }, + { 0x0123456789ABCDEFLL, 62, 0xC000000000000000LL }, + { 0x0123456789ABCDEFLL, 63, 0x8000000000000000LL }, +}; + +static void shdi3_test_ashldi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashldi3_testdata; + e < ashldi3_testdata + ARRAY_SIZE(ashldi3_testdata); e++) { + ret = __ashldi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashldi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry ashrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/ashrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0xFF6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0xFFB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0xFFDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFFFFFFFFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0xFFFFFFFFF6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0xFFFFFFFFFB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0xFFFFFFFFFDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFFFFFFFFFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0xFFFFFFFFFF6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0xFFFFFFFFFFB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0xFFFFFFFFFFDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFFFFFFFFFFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xFFFFFFFFFFFFFFFALL }, + { 0xAEDCBA9876543210LL, 61, 0xFFFFFFFFFFFFFFFDLL }, + { 0xAEDCBA9876543210LL, 62, 0xFFFFFFFFFFFFFFFELL }, + { 0xAEDCBA9876543210LL, 63, 0xFFFFFFFFFFFFFFFFLL }, +}; + +static void shdi3_test_ashrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = ashrdi3_testdata; + e < ashrdi3_testdata + ARRAY_SIZE(ashrdi3_testdata); e++) { + ret = __ashrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __ashrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static const struct shdi3_test_entry lshrdi3_testdata[] = { + /* https://github.com/llvm/llvm-project/compiler-rt/test/builtins/Unit/lshrdi3_test.c */ + { 0x0123456789ABCDEFLL, 0, 0x123456789ABCDEFLL }, + { 0x0123456789ABCDEFLL, 1, 0x91A2B3C4D5E6F7LL }, + { 0x0123456789ABCDEFLL, 2, 0x48D159E26AF37BLL }, + { 0x0123456789ABCDEFLL, 3, 0x2468ACF13579BDLL }, + { 0x0123456789ABCDEFLL, 4, 0x123456789ABCDELL }, + { 0x0123456789ABCDEFLL, 28, 0x12345678LL }, + { 0x0123456789ABCDEFLL, 29, 0x91A2B3CLL }, + { 0x0123456789ABCDEFLL, 30, 0x48D159ELL }, + { 0x0123456789ABCDEFLL, 31, 0x2468ACFLL }, + { 0x0123456789ABCDEFLL, 32, 0x1234567LL }, + { 0x0123456789ABCDEFLL, 33, 0x91A2B3LL }, + { 0x0123456789ABCDEFLL, 34, 0x48D159LL }, + { 0x0123456789ABCDEFLL, 35, 0x2468ACLL }, + { 0x0123456789ABCDEFLL, 36, 0x123456LL }, + { 0x0123456789ABCDEFLL, 60, 0 }, + { 0x0123456789ABCDEFLL, 61, 0 }, + { 0x0123456789ABCDEFLL, 62, 0 }, + { 0x0123456789ABCDEFLL, 63, 0 }, + { 0xFEDCBA9876543210LL, 0, 0xFEDCBA9876543210LL }, + { 0xFEDCBA9876543210LL, 1, 0x7F6E5D4C3B2A1908LL }, + { 0xFEDCBA9876543210LL, 2, 0x3FB72EA61D950C84LL }, + { 0xFEDCBA9876543210LL, 3, 0x1FDB97530ECA8642LL }, + { 0xFEDCBA9876543210LL, 4, 0xFEDCBA987654321LL }, + { 0xFEDCBA9876543210LL, 28, 0xFEDCBA987LL }, + { 0xFEDCBA9876543210LL, 29, 0x7F6E5D4C3LL }, + { 0xFEDCBA9876543210LL, 30, 0x3FB72EA61LL }, + { 0xFEDCBA9876543210LL, 31, 0x1FDB97530LL }, + { 0xFEDCBA9876543210LL, 32, 0xFEDCBA98LL }, + { 0xFEDCBA9876543210LL, 33, 0x7F6E5D4CLL }, + { 0xFEDCBA9876543210LL, 34, 0x3FB72EA6LL }, + { 0xFEDCBA9876543210LL, 35, 0x1FDB9753LL }, + { 0xFEDCBA9876543210LL, 36, 0xFEDCBA9LL }, + { 0xAEDCBA9876543210LL, 60, 0xALL }, + { 0xAEDCBA9876543210LL, 61, 0x5LL }, + { 0xAEDCBA9876543210LL, 62, 0x2LL }, + { 0xAEDCBA9876543210LL, 63, 0x1LL }, +}; + +static void shdi3_test_lshrdi3(struct kunit *test) +{ + const struct shdi3_test_entry *e; + long long ret; + + for (e = lshrdi3_testdata; + e < lshrdi3_testdata + ARRAY_SIZE(lshrdi3_testdata); e++) { + ret = __lshrdi3(e->input, e->shift); + KUNIT_EXPECT_EQ_MSG(test, ret, e->result, + " when evaluating __lshrdi3(%lld, %d)", + e->input, e->shift); + } +} + +static struct kunit_case shdi3_test_cases[] = { + KUNIT_CASE(shdi3_test_ashldi3), + KUNIT_CASE(shdi3_test_ashrdi3), + KUNIT_CASE(shdi3_test_lshrdi3), + {} +}; + +static struct kunit_suite shdi3_test_suite = { + .name = "shdi3", + .test_cases = shdi3_test_cases, +}; +kunit_test_suite(shdi3_test_suite); + +MODULE_DESCRIPTION("Test cases for __ashldi3(), __ashrdi3(), and __lshrdi3()"); +MODULE_LICENSE("GPL"); diff --git a/lib/tests/string_helpers_kunit.c b/lib/tests/string_helpers_kunit.c index c853046183d24..9fbe91079c7ed 100644 --- a/lib/tests/string_helpers_kunit.c +++ b/lib/tests/string_helpers_kunit.c @@ -5,11 +5,16 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <kunit/test.h> + #include <linux/array_size.h> -#include <linux/kernel.h> +#include <linux/bug.h> +#include <linux/limits.h> +#include <linux/module.h> #include <linux/random.h> -#include <linux/string.h> +#include <linux/slab.h> +#include <linux/sprintf.h> #include <linux/string_helpers.h> +#include <linux/types.h> static void test_string_check_buf(struct kunit *test, const char *name, unsigned int flags, @@ -601,6 +606,11 @@ static void test_unescape(struct kunit *test) test_string_unescape(test, "unescape", i, false); test_string_unescape(test, "unescape inplace", get_random_u32_below(UNESCAPE_ALL_MASK + 1), true); +} + +static void test_escape(struct kunit *test) +{ + unsigned int i; /* Without dictionary */ for (i = 0; i < ESCAPE_ALL_MASK + 1; i++) @@ -615,6 +625,7 @@ static struct kunit_case string_helpers_test_cases[] = { KUNIT_CASE(test_get_size), KUNIT_CASE(test_upper_lower), KUNIT_CASE(test_unescape), + KUNIT_CASE(test_escape), {} }; diff --git a/lib/tests/uuid_kunit.c b/lib/tests/uuid_kunit.c index de71b2649dac6..2ef64fbe67d6f 100644 --- a/lib/tests/uuid_kunit.c +++ b/lib/tests/uuid_kunit.c @@ -86,11 +86,67 @@ static void uuid_test_uuid_invalid(struct kunit *test) } } +/* + * RFC 4122 section 4.4 says random UUIDs/GUIDs (version 4) must have: + * - version 4 in the high nibble of the version byte, + * - variant DCE 1.1 (binary 10x) in the high bits of byte 8. + * + * The version byte is byte 6 in the "wire" uuid_t layout and byte 7 in + * the byte-swapped guid_t layout. + */ +static void uuid_test_uuid_gen(struct kunit *test) +{ + uuid_t u; + + for (unsigned int i = 0; i < 8; i++) { + uuid_gen(&u); + KUNIT_EXPECT_EQ(test, u.b[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, u.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_guid_gen(struct kunit *test) +{ + guid_t g; + + for (unsigned int i = 0; i < 8; i++) { + guid_gen(&g); + KUNIT_EXPECT_EQ(test, g.b[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, g.b[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_uuid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_uuid(buf); + KUNIT_EXPECT_EQ(test, buf[6] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + +static void uuid_test_generate_random_guid(struct kunit *test) +{ + unsigned char buf[16]; + + for (unsigned int i = 0; i < 8; i++) { + generate_random_guid(buf); + KUNIT_EXPECT_EQ(test, buf[7] & 0xf0, 0x40); + KUNIT_EXPECT_EQ(test, buf[8] & 0xc0, 0x80); + } +} + static struct kunit_case uuid_test_cases[] = { KUNIT_CASE(uuid_test_guid_valid), KUNIT_CASE(uuid_test_uuid_valid), KUNIT_CASE(uuid_test_guid_invalid), KUNIT_CASE(uuid_test_uuid_invalid), + KUNIT_CASE(uuid_test_uuid_gen), + KUNIT_CASE(uuid_test_guid_gen), + KUNIT_CASE(uuid_test_generate_random_uuid), + KUNIT_CASE(uuid_test_generate_random_guid), {}, }; diff --git a/lib/usercopy.c b/lib/usercopy.c index b00a3a957de6b..e2f0bf104a591 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,15 +12,13 @@ /* out-of-line parts */ -#if !defined(INLINE_COPY_FROM_USER) +#if !defined(INLINE_COPY_USER) unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) { return _inline_copy_from_user(to, from, n); } EXPORT_SYMBOL(_copy_from_user); -#endif -#if !defined(INLINE_COPY_TO_USER) unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { return _inline_copy_to_user(to, from, n); diff --git a/mm/Kconfig b/mm/Kconfig index e8bf1e9e6ad90..6c2217ea35232 100644 --- a/mm/Kconfig +++ b/mm/Kconfig @@ -1303,7 +1303,7 @@ config ARCH_HAS_PTE_SPECIAL bool config MAPPING_DIRTY_HELPERS - bool + bool config KMAP_LOCAL bool @@ -1434,7 +1434,7 @@ config ARCH_HAS_USER_SHADOW_STACK bool help The architecture has hardware support for userspace shadow call - stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss). + stacks (eg, x86 CET, arm64 GCS or RISC-V Zicfiss). config HAVE_ARCH_TLB_REMOVE_TABLE def_bool n diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 655dc5ce32409..ee6ae01de5aef 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -77,6 +77,11 @@ static int param_set_sample_interval(const char *val, const struct kernel_param WRITE_ONCE(kfence_enabled, false); } + if (num && kasan_hw_tags_enabled()) { + pr_info("disabled as KASAN HW tags are enabled\n"); + return -EINVAL; + } + *((unsigned long *)kp->arg) = num; if (num && !READ_ONCE(kfence_enabled) && system_state != SYSTEM_BOOTING) diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c index d9625b9ee0466..6e59cc9c665cc 100644 --- a/rust/helpers/uaccess.c +++ b/rust/helpers/uaccess.c @@ -14,7 +14,7 @@ rust_helper_copy_to_user(void __user *to, const void *from, unsigned long n) return copy_to_user(to, from, n); } -#ifdef INLINE_COPY_FROM_USER +#ifdef INLINE_COPY_USER __rust_helper unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) { diff --git a/scripts/bloat-o-meter b/scripts/bloat-o-meter index 9b4fb996d95bd..5868a8b11b0f9 100755 --- a/scripts/bloat-o-meter +++ b/scripts/bloat-o-meter @@ -43,6 +43,7 @@ def getsizes(file, format): if name.startswith("__se_compat_sys"): continue if name.startswith("__addressable_"): continue if name.startswith("__noinstr_text_start"): continue + if name.startswith("_sdata"): continue if name == "linux_banner": continue if name == "vermagic": continue # statics and some other optimizations adds random .NUMBER diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 0492d6afc9a1f..3727156e4ccad 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -57,8 +57,12 @@ my %ignore_type = (); my @ignore = (); my $help = 0; my $configuration_file = ".checkpatch.conf"; +my $def_configuration_dirs_help = '.:$HOME:.scripts'; +(my $def_configuration_dirs = $def_configuration_dirs_help) =~ s/\$(\w+)/$ENV{$1}/g; +my $env_config_dir = 'CHECKPATCH_CONFIG_DIR'; my $max_line_length = 100; my $ignore_perl_version = 0; +my $spdx_cxx_comments = 0; my $minimum_perl_version = 5.10.0; my $min_conf_desc_length = 4; my $spelling_file = "$D/spelling.txt"; @@ -135,6 +139,10 @@ Options: file. It's your fault if there's no backup or git --ignore-perl-version override checking of perl version. expect runtime errors. + --spdx-cxx-comments don't force C comments (/* */) for SPDX license + (required by old toolchains), allow also C++ + comments (//). + NOTE: it should *not* be used for Linux mainline. --codespell Use the codespell dictionary for spelling/typos (default:$codespellfile) --codespellfile Use this codespell dictionary @@ -146,6 +154,11 @@ Options: -h, --help, --version display this help and exit When FILE is - read standard input. + +CONFIGURATION FILE +Default configuration options can be stored in $configuration_file, +search path: '$def_configuration_dirs_help' or in a directory specified by +\$$env_config_dir environment variable (fallback to the default search path). EOM exit($exitcode); @@ -237,7 +250,7 @@ sub list_types { exit($exitcode); } -my $conf = which_conf($configuration_file); +my $conf = which_conf($configuration_file, $env_config_dir, $def_configuration_dirs); if (-f $conf) { my @conf_args; open(my $conffile, '<', "$conf") @@ -339,6 +352,7 @@ GetOptions( 'fix!' => \$fix, 'fix-inplace!' => \$fix_inplace, 'ignore-perl-version!' => \$ignore_perl_version, + 'spdx-cxx-comments!' => \$spdx_cxx_comments, 'debug=s' => \%debug, 'test-only=s' => \$tst_only, 'codespell!' => \$codespell, @@ -1531,9 +1545,15 @@ sub which { } sub which_conf { - my ($conf) = @_; + my ($conf, $env_key, $paths) = @_; + my $env_dir = $ENV{$env_key}; + + if (defined($env_dir) && $env_dir ne "") { + return "$env_dir/$conf" if (-e "$env_dir/$conf"); + warn "$P: Can't find a readable $conf in '$env_dir', falling back to default search paths\n"; + } - foreach my $path (split(/:/, ".:$ENV{HOME}:.scripts")) { + foreach my $path (split(/:/, $paths)) { if (-e "$path/$conf") { return "$path/$conf"; } @@ -3801,26 +3821,33 @@ sub process { $checklicenseline = 2; } elsif ($rawline =~ /^\+/) { my $comment = ""; - if ($realfile =~ /\.(h|s|S)$/) { - $comment = '/*'; - } elsif ($realfile =~ /\.(c|rs|dts|dtsi)$/) { + if ($realfile =~ /\.(c|rs|dts|dtsi)$/) { $comment = '//'; } elsif (($checklicenseline == 2) || $realfile =~ /\.(sh|pl|py|awk|tc|yaml)$/) { $comment = '#'; } elsif ($realfile =~ /\.rst$/) { $comment = '..'; } + my $pattern = qr{\Q$comment\E}; + if ($realfile =~ /\.(h|s|S)$/) { + $comment = '/*'; + $pattern = qr{/\*}; + if ($spdx_cxx_comments) { + $comment = '// or /*'; + $pattern = qr{//|/\*}; + } + } # check SPDX comment style for .[chsS] files if ($realfile =~ /\.[chsS]$/ && $rawline =~ /SPDX-License-Identifier:/ && - $rawline !~ m@^\+\s*\Q$comment\E\s*@) { + $rawline !~ m@^\+\s*$pattern\s*@) { WARN("SPDX_LICENSE_TAG", "Improper SPDX comment style for '$realfile', please use '$comment' instead\n" . $herecurr); } if ($comment !~ /^$/ && - $rawline !~ m@^\+\Q$comment\E SPDX-License-Identifier: @) { + $rawline !~ m@^\+$pattern SPDX-License-Identifier: @) { WARN("SPDX_LICENSE_TAG", "Missing or malformed SPDX-License-Identifier tag in line $checklicenseline\n" . $herecurr); } elsif ($rawline =~ /(SPDX-License-Identifier: .*)/) { @@ -4156,7 +4183,7 @@ sub process { $pl =~ s/\b(?:$Attribute|$Sparse)\b//g; if (($pl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $pl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define $pl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros @@ -4170,7 +4197,7 @@ sub process { # looks like a declaration !($sl =~ /^\+\s+$Declare\s*$Ident\s*[=,;:\[]/ || # function pointer declarations - $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident\s*\)\s*[=,;:\[\(]/ || + $sl =~ /^\+\s+$Declare\s*\(\s*\*\s*$Ident(?:\s*\[\s*(?:$Ident|$Constant)?\s*\])?\s*\)\s*[=,;:\[\(]/ || # foo bar; where foo is some local typedef or #define $sl =~ /^\+\s+$Ident(?:\s+|\s*\*\s*)$Ident\s*[=,;\[]/ || # known declaration macros diff --git a/scripts/get_maintainer.pl b/scripts/get_maintainer.pl index f0ca0db6ddc27..16b80a700d4ac 100755 --- a/scripts/get_maintainer.pl +++ b/scripts/get_maintainer.pl @@ -21,6 +21,7 @@ use Cwd; use File::Find; use File::Spec::Functions; use open qw(:std :encoding(UTF-8)); +use JSON::PP; my $cur_path = fastgetcwd() . '/'; my $lk_path = "./"; @@ -68,6 +69,7 @@ my $pattern_depth = 0; my $self_test = undef; my $version = 0; my $help = 0; +my $json = 0; my $find_maintainer_files = 0; my $maintainer_path; my $vcs_used = 0; @@ -285,6 +287,7 @@ if (!GetOptions( 'find-maintainer-files' => \$find_maintainer_files, 'mpath|maintainer-path=s' => \$maintainer_path, 'self-test:s' => \$self_test, + 'json!' => \$json, 'v|version' => \$version, 'h|help|usage' => \$help, )) { @@ -650,39 +653,48 @@ my %deduplicate_name_hash = (); my %deduplicate_address_hash = (); my @maintainers = get_maintainers(); -if (@maintainers) { - @maintainers = merge_email(@maintainers); - output(@maintainers); -} - -if ($scm) { - @scm = uniq(@scm); - output(@scm); -} -if ($output_substatus) { - @substatus = uniq(@substatus); - output(@substatus); -} - -if ($status) { - @status = uniq(@status); - output(@status); -} +@maintainers = merge_email(@maintainers) if (@maintainers); +@scm = uniq(@scm) if ($scm); +@substatus = uniq(@substatus) if ($output_substatus); +@status = uniq(@status) if ($status); +@subsystem = uniq(@subsystem) if ($subsystem); +@web = uniq(@web) if ($web); +@bug = uniq(@bug) if ($bug); -if ($subsystem) { - @subsystem = uniq(@subsystem); - output(@subsystem); -} +if ($json) { + my @json_maintainers; + for my $m (@maintainers) { + my ($addr, $role); + if ($output_roles && $m =~ /^(.*?)\s+\((.+)\)\s*$/) { + $addr = $1; + $role = $2; + } else { + $addr = $m; + } + my ($name, $email_addr) = parse_email($addr); + my %entry = (name => $name, email => $email_addr); + $entry{role} = $role if (defined $role && $role ne ''); + push(@json_maintainers, \%entry); + } -if ($web) { - @web = uniq(@web); - output(@web); -} + my %result = (maintainers => \@json_maintainers); + $result{scm} = \@scm if ($scm); + $result{status} = \@status if ($status); + $result{subsystem} = \@subsystem if ($subsystem); + $result{web} = \@web if ($web); + $result{bug} = \@bug if ($bug); -if ($bug) { - @bug = uniq(@bug); - output(@bug); + my $json_encoder = JSON::PP->new->canonical->utf8; + print($json_encoder->encode(\%result) . "\n"); +} else { + output(@maintainers) if (@maintainers); + output(@scm) if ($scm); + output(@substatus) if ($output_substatus); + output(@status) if ($status); + output(@subsystem) if ($subsystem); + output(@web) if ($web); + output(@bug) if ($bug); } exit($exit); @@ -1104,6 +1116,7 @@ Output type options: --separator [, ] => separator for multiple entries on 1 line using --separator also sets --nomultiline if --separator is not [, ] --multiline => print 1 entry per line + --json => output results as JSON Other options: --pattern-depth => Number of pattern directory traversals (default: 0 (all)) diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 368a622ca0273..caa5fe9dd5734 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -241,13 +241,7 @@ static const char *format_timespec(struct __kernel_timespec *ts) if (localtime_r(&time_sec, &tm_info) == NULL) return "N/A"; - snprintf(buffer, sizeof(buffer), "%04d-%02d-%02dT%02d:%02d:%02d", - tm_info.tm_year + 1900, - tm_info.tm_mon + 1, - tm_info.tm_mday, - tm_info.tm_hour, - tm_info.tm_min, - tm_info.tm_sec); + strftime(buffer, sizeof(buffer), "%Y-%m-%dT%H:%M:%S", &tm_info); return buffer; } diff --git a/tools/testing/selftests/acct/.gitignore b/tools/testing/selftests/acct/.gitignore index 7e78aac190386..9e9c61c5bfd6f 100644 --- a/tools/testing/selftests/acct/.gitignore +++ b/tools/testing/selftests/acct/.gitignore @@ -1,3 +1,4 @@ acct_syscall +taskstats_fill_stats_tgid config -process_log
\ No newline at end of file +process_log diff --git a/tools/testing/selftests/acct/Makefile b/tools/testing/selftests/acct/Makefile index 7e025099cf657..083cab5ddb72c 100644 --- a/tools/testing/selftests/acct/Makefile +++ b/tools/testing/selftests/acct/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 TEST_GEN_PROGS := acct_syscall +TEST_GEN_PROGS += taskstats_fill_stats_tgid + CFLAGS += -Wall +LDLIBS += -lpthread -include ../lib.mk
\ No newline at end of file +include ../lib.mk diff --git a/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c new file mode 100644 index 0000000000000..d6cab4ae26f28 --- /dev/null +++ b/tools/testing/selftests/acct/taskstats_fill_stats_tgid.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE + +#include <errno.h> +#include <linux/genetlink.h> +#include <linux/netlink.h> +#include <linux/taskstats.h> +#include <pthread.h> +#include <stdbool.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <time.h> +#include <unistd.h> + +#include "kselftest.h" + +#ifndef NLA_ALIGN +#define NLA_ALIGNTO 4 +#define NLA_ALIGN(len) (((len) + NLA_ALIGNTO - 1) & ~(NLA_ALIGNTO - 1)) +#define NLA_HDRLEN ((int)NLA_ALIGN(sizeof(struct nlattr))) +#endif + +#define BUSY_NS (200ULL * 1000 * 1000) + +struct worker_ctx { + pthread_mutex_t lock; + pthread_cond_t cond; + bool ready; + bool release; +}; + +static unsigned long busy_sink; + +static void *taskstats_nla_data(const struct nlattr *na) +{ + return (void *)((char *)na + NLA_HDRLEN); +} + +static bool taskstats_nla_ok(const struct nlattr *na, int remaining) +{ + return remaining >= (int)sizeof(*na) && + na->nla_len >= sizeof(*na) && + na->nla_len <= remaining; +} + +static struct nlattr *taskstats_nla_next(const struct nlattr *na, int *remaining) +{ + int aligned_len = NLA_ALIGN(na->nla_len); + + *remaining -= aligned_len; + return (struct nlattr *)((char *)na + aligned_len); +} + +static uint64_t timespec_diff_ns(const struct timespec *start, + const struct timespec *end) +{ + return (uint64_t)(end->tv_sec - start->tv_sec) * 1000000000ULL + + (uint64_t)(end->tv_nsec - start->tv_nsec); +} + +static void burn_cpu_for_ns(uint64_t runtime_ns) +{ + struct timespec start, now; + unsigned long acc = 0; + + if (clock_gettime(CLOCK_MONOTONIC, &start)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + + do { + for (int i = 0; i < 100000; i++) + acc += i; + if (clock_gettime(CLOCK_MONOTONIC, &now)) { + perror("clock_gettime"); + exit(EXIT_FAILURE); + } + } while (timespec_diff_ns(&start, &now) < runtime_ns); + + busy_sink = acc; +} + +static int netlink_open(void) +{ + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + .nl_pid = getpid(), + }; + int fd; + + fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_GENERIC); + if (fd < 0) + return -errno; + + if (bind(fd, (struct sockaddr *)&addr, sizeof(addr)) < 0) { + int err = -errno; + + close(fd); + return err; + } + + return fd; +} + +static int send_request(int fd, void *buf, size_t len) +{ + struct sockaddr_nl addr = { + .nl_family = AF_NETLINK, + }; + + if (sendto(fd, buf, len, 0, (struct sockaddr *)&addr, sizeof(addr)) < 0) + return -errno; + + return 0; +} + +static int get_family_id(int fd, const char *name) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + char resp[8192]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + int len; + int rem; + int ret; + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = GENL_ID_CTRL; + req.nlh.nlmsg_flags = NLM_F_REQUEST; + req.nlh.nlmsg_seq = 1; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = CTRL_CMD_GETFAMILY; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = CTRL_ATTR_FAMILY_NAME; + na->nla_len = NLA_HDRLEN + strlen(name) + 1; + memcpy(taskstats_nla_data(na), name, strlen(name) + 1); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + ret = send_request(fd, &req, req.nlh.nlmsg_len); + if (ret) + return ret; + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error ? err->error : -ENOENT; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (taskstats_nla_ok(na, rem)) { + if (na->nla_type == CTRL_ATTR_FAMILY_ID) + return *(uint16_t *)taskstats_nla_data(na); + na = taskstats_nla_next(na, &rem); + } + } + + return -ENOENT; +} + +static int get_taskstats(int fd, int family_id, uint16_t attr_type, uint32_t id, + struct taskstats *stats) +{ + struct { + struct nlmsghdr nlh; + struct genlmsghdr genl; + char buf[256]; + } req = { 0 }; + char resp[16384]; + struct nlmsghdr *nlh; + struct genlmsghdr *genl; + struct nlattr *na; + struct nlattr *nested; + int len; + int rem; + int nrem; + int ret; + + memset(stats, 0, sizeof(*stats)); + + req.nlh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); + req.nlh.nlmsg_type = family_id; + req.nlh.nlmsg_flags = NLM_F_REQUEST; + req.nlh.nlmsg_seq = 2; + req.nlh.nlmsg_pid = getpid(); + + req.genl.cmd = TASKSTATS_CMD_GET; + req.genl.version = 1; + + na = (struct nlattr *)((char *)&req + NLMSG_ALIGN(req.nlh.nlmsg_len)); + na->nla_type = attr_type; + na->nla_len = NLA_HDRLEN + sizeof(id); + memcpy(taskstats_nla_data(na), &id, sizeof(id)); + req.nlh.nlmsg_len = NLMSG_ALIGN(req.nlh.nlmsg_len) + NLA_ALIGN(na->nla_len); + + ret = send_request(fd, &req, req.nlh.nlmsg_len); + if (ret) + return ret; + + len = recv(fd, resp, sizeof(resp), 0); + if (len < 0) + return -errno; + + for (nlh = (struct nlmsghdr *)resp; NLMSG_OK(nlh, len); + nlh = NLMSG_NEXT(nlh, len)) { + if (nlh->nlmsg_type == NLMSG_ERROR) { + struct nlmsgerr *err = NLMSG_DATA(nlh); + + return err->error ? err->error : -ENOENT; + } + + genl = (struct genlmsghdr *)NLMSG_DATA(nlh); + rem = nlh->nlmsg_len - NLMSG_HDRLEN - GENL_HDRLEN; + na = (struct nlattr *)((char *)genl + GENL_HDRLEN); + while (taskstats_nla_ok(na, rem)) { + if (na->nla_type == TASKSTATS_TYPE_AGGR_PID || + na->nla_type == TASKSTATS_TYPE_AGGR_TGID) { + nested = (struct nlattr *)taskstats_nla_data(na); + nrem = na->nla_len - NLA_HDRLEN; + while (taskstats_nla_ok(nested, nrem)) { + if (nested->nla_type == TASKSTATS_TYPE_STATS) { + memcpy(stats, taskstats_nla_data(nested), + sizeof(*stats)); + return 0; + } + nested = taskstats_nla_next(nested, &nrem); + } + } + na = taskstats_nla_next(na, &rem); + } + } + + return -ENOENT; +} + +static uint64_t cpu_total(const struct taskstats *stats) +{ + return (uint64_t)stats->ac_utime + (uint64_t)stats->ac_stime; +} + +static void print_stats(const char *label, const struct taskstats *stats) +{ + ksft_print_msg("%s: cpu_total=%llu nvcsw=%llu nivcsw=%llu\n", + label, (unsigned long long)cpu_total(stats), + (unsigned long long)stats->nvcsw, + (unsigned long long)stats->nivcsw); +} + +static void *worker_thread(void *arg) +{ + struct worker_ctx *ctx = arg; + + burn_cpu_for_ns(BUSY_NS); + + pthread_mutex_lock(&ctx->lock); + ctx->ready = true; + pthread_cond_broadcast(&ctx->cond); + while (!ctx->release) + pthread_cond_wait(&ctx->cond, &ctx->lock); + pthread_mutex_unlock(&ctx->lock); + + return NULL; +} + +int main(void) +{ + struct worker_ctx ctx = { + .lock = PTHREAD_MUTEX_INITIALIZER, + .cond = PTHREAD_COND_INITIALIZER, + }; + struct taskstats before, after; + pthread_t thread; + pid_t tgid = getpid(); + int family_id; + int fd; + int ret; + + ksft_print_header(); + ksft_set_plan(1); + + if (geteuid()) + ksft_exit_skip("taskstats_fill_stats_tgid needs root\n"); + + fd = netlink_open(); + if (fd < 0) + ksft_exit_skip("failed to open generic netlink socket: %s\n", + strerror(-fd)); + + family_id = get_family_id(fd, TASKSTATS_GENL_NAME); + if (family_id < 0) + ksft_exit_skip("taskstats generic netlink family unavailable: %s\n", + strerror(-family_id)); + + /* Create worker thread that burns 200ms of CPU */ + if (pthread_create(&thread, NULL, worker_thread, &ctx) != 0) + ksft_exit_fail_msg("pthread_create failed: %s\n", strerror(errno)); + + /* Wait for worker to finish generating activity */ + pthread_mutex_lock(&ctx.lock); + while (!ctx.ready) + pthread_cond_wait(&ctx.cond, &ctx.lock); + pthread_mutex_unlock(&ctx.lock); + + /* + * Snapshot A: TGID stats while worker is alive and sleeping. + * Contains main thread + worker contributions. + */ + ret = get_taskstats(fd, family_id, TASKSTATS_CMD_ATTR_TGID, tgid, &before); + if (ret) + ksft_exit_fail_msg("TGID query before exit failed: %s\n", + strerror(-ret)); + + /* Release worker so it can exit, then join (deterministic wait). + * + * Kernel exit path ordering guarantees: + * do_exit() + * taskstats_exit() -> fill_tgid_exit() (accumulates worker into signal->stats) + * exit_notify() (releases the thread) + * do_task_dead() -> __schedule() (wakes joiner) + * + * So pthread_join() returns only after fill_tgid_exit() has completed. + */ + pthread_mutex_lock(&ctx.lock); + ctx.release = true; + pthread_cond_broadcast(&ctx.cond); + pthread_mutex_unlock(&ctx.lock); + + pthread_join(thread, NULL); + + /* + * Snapshot B: TGID stats after worker has exited. + * fill_stats_for_tgid() does: + * memcpy(signal->stats) <- includes fill_tgid_exit accumulation + * + scan live threads <- only main thread now + */ + ret = get_taskstats(fd, family_id, TASKSTATS_CMD_ATTR_TGID, tgid, &after); + if (ret) + ksft_exit_fail_msg("TGID query after exit failed: %s\n", + strerror(-ret)); + + print_stats("TGID before worker exit", &before); + print_stats("TGID after worker exit", &after); + + /* + * The worker burned 200ms of CPU before the first snapshot. + * If the kernel correctly retained its contribution via + * fill_tgid_exit(), then the TGID CPU total after exit must be at + * least as large as the TGID CPU total before exit. + */ + ksft_test_result(cpu_total(&after) >= cpu_total(&before), + "TGID CPU stats should not regress after thread exit\n"); + + close(fd); + ksft_finished(); + return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; +} diff --git a/tools/testing/selftests/filelock/.gitignore b/tools/testing/selftests/filelock/.gitignore new file mode 100644 index 0000000000000..825e899a121bc --- /dev/null +++ b/tools/testing/selftests/filelock/.gitignore @@ -0,0 +1 @@ +ofdlocks diff --git a/tools/testing/selftests/filelock/ofdlocks.c b/tools/testing/selftests/filelock/ofdlocks.c index ff8d47fc373ad..68bac28b234b7 100644 --- a/tools/testing/selftests/filelock/ofdlocks.c +++ b/tools/testing/selftests/filelock/ofdlocks.c @@ -16,7 +16,7 @@ static int lock_set(int fd, struct flock *fl) fl->l_whence = SEEK_SET; ret = fcntl(fd, F_OFD_SETLK, fl); if (ret) - perror("fcntl()"); + ksft_perror("fcntl()"); return ret; } @@ -28,7 +28,7 @@ static int lock_get(int fd, struct flock *fl) fl->l_whence = SEEK_SET; ret = fcntl(fd, F_OFD_GETLK, fl); if (ret) - perror("fcntl()"); + ksft_perror("fcntl()"); return ret; } @@ -39,94 +39,82 @@ int main(void) int fd = open("/tmp/aa", O_RDWR | O_CREAT | O_EXCL, 0600); int fd2 = open("/tmp/aa", O_RDONLY); + ksft_print_header(); + ksft_set_plan(4); + unlink("/tmp/aa"); assert(fd != -1); assert(fd2 != -1); - ksft_print_msg("[INFO] opened fds %i %i\n", fd, fd2); + ksft_print_msg("opened fds %i %i\n", fd, fd2); /* Set some read lock */ fl.l_type = F_RDLCK; fl.l_start = 5; fl.l_len = 3; rc = lock_set(fd, &fl); - if (rc == 0) { - ksft_print_msg - ("[SUCCESS] set OFD read lock on first fd\n"); - } else { - ksft_print_msg("[FAIL] to set OFD read lock on first fd\n"); - return -1; - } + ksft_test_result(rc == 0, "set OFD read lock on first fd\n"); + if (rc != 0) + ksft_finished(); + /* Make sure read locks do not conflict on different fds. */ fl.l_type = F_RDLCK; fl.l_start = 5; fl.l_len = 1; rc = lock_get(fd2, &fl); if (rc != 0) - return -1; - if (fl.l_type != F_UNLCK) { - ksft_print_msg("[FAIL] read locks conflicted\n"); - return -1; - } + ksft_finished(); + if (fl.l_type != F_UNLCK) + ksft_exit_fail_msg("read locks conflicted\n"); + /* Make sure read/write locks do conflict on different fds. */ fl.l_type = F_WRLCK; fl.l_start = 5; fl.l_len = 1; rc = lock_get(fd2, &fl); if (rc != 0) - return -1; - if (fl.l_type != F_UNLCK) { - ksft_print_msg - ("[SUCCESS] read and write locks conflicted\n"); - } else { - ksft_print_msg - ("[SUCCESS] read and write locks not conflicted\n"); - return -1; - } + ksft_finished(); + ksft_test_result(fl.l_type != F_UNLCK, + "read and write locks conflicted\n"); + if (fl.l_type == F_UNLCK) + ksft_finished(); + /* Get info about the lock on first fd. */ fl.l_type = F_UNLCK; fl.l_start = 5; fl.l_len = 1; rc = lock_get(fd, &fl); - if (rc != 0) { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK not supported\n"); - return -1; - } - if (fl.l_type != F_UNLCK) { - ksft_print_msg - ("[SUCCESS] F_UNLCK test returns: locked, type %i pid %i len %zi\n", - fl.l_type, fl.l_pid, fl.l_len); - } else { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK did not return lock info\n"); - return -1; - } + if (rc != 0) + ksft_exit_fail_msg("F_OFD_GETLK with F_UNLCK not supported\n"); + ksft_test_result(fl.l_type != F_UNLCK, + "F_OFD_GETLK with F_UNLCK returned lock info\n"); + if (fl.l_type == F_UNLCK) + ksft_exit_fail(); + ksft_print_msg("F_UNLCK test returns: locked, type %i pid %i len %zi\n", + fl.l_type, fl.l_pid, fl.l_len); + /* Try the same but by locking everything by len==0. */ fl2.l_type = F_UNLCK; fl2.l_start = 0; fl2.l_len = 0; rc = lock_get(fd, &fl2); - if (rc != 0) { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK not supported\n"); - return -1; - } + if (rc != 0) + ksft_exit_fail_msg + ("F_OFD_GETLK with F_UNLCK not supported\n"); + ksft_test_result(memcmp(&fl, &fl2, sizeof(fl)) == 0, + "F_UNLCK with len==0 returned the same\n"); if (memcmp(&fl, &fl2, sizeof(fl))) { - ksft_print_msg - ("[FAIL] F_UNLCK test returns: locked, type %i pid %i len %zi\n", + ksft_exit_fail_msg + ("F_UNLCK test returns: locked, type %i pid %i len %zi\n", fl.l_type, fl.l_pid, fl.l_len); - return -1; } - ksft_print_msg("[SUCCESS] F_UNLCK with len==0 returned the same\n"); + /* Get info about the lock on second fd - no locks on it. */ fl.l_type = F_UNLCK; fl.l_start = 0; fl.l_len = 0; lock_get(fd2, &fl); - if (fl.l_type != F_UNLCK) { - ksft_print_msg - ("[FAIL] F_OFD_GETLK with F_UNLCK return lock info from another fd\n"); - return -1; - } - return 0; + ksft_test_result(fl.l_type == F_UNLCK, + "F_OFD_GETLK with F_UNLCK return lock info from another fd\n"); + + ksft_finished(); } diff --git a/tools/testing/selftests/perf_events/watermark_signal.c b/tools/testing/selftests/perf_events/watermark_signal.c index 0f64b9b170813..a84709cabd8be 100644 --- a/tools/testing/selftests/perf_events/watermark_signal.c +++ b/tools/testing/selftests/perf_events/watermark_signal.c @@ -102,7 +102,7 @@ TEST(watermark_signal) } p = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); - if (p == NULL) { + if (p == MAP_FAILED) { perror("mmap"); goto cleanup; } |