diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-06-16 05:08:13 +0530 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-06-16 05:08:13 +0530 |
| commit | 25a01b5155d207e72bdd31b138406f37788403cb (patch) | |
| tree | d4b2c03475c65c0ef1ee88acfd671581ea96bbc4 /arch | |
| parent | 44308fbe8feb0861053b9173e3fda2849944b355 (diff) | |
| parent | 37540b8c287fc817bdbd0c62bb75ad6eab0e5d03 (diff) | |
| download | ath-25a01b5155d207e72bdd31b138406f37788403cb.tar.gz | |
Merge tag 's390-7.2-1' of gitolite.kernel.org:pub/scm/linux/kernel/git/s390/linux
Pull s390 updates from Alexander Gordeev:
- Use CIO device online variable instead of the internal FSM state to
determine device availability during purge operations
- Remove extra check of task_stack_page() because try_get_task_stack()
already takes care of that when reading /proc/<pid>/wchan
- Allow user-space to use the new SCLP action qualifier 4 for to
provide NVMe SMART log data to the platform.
- Send AP CHANGE uevents on successful bind and successful association
to notify user-space about SE operations on AP queue devices
- Add an s390dbf kernel parameter to configure debug log levels and
area sizes during early boot
- On arm64 the empty zero page is going to be mapped read-only. Do the
same for s390 with an explicit set_memory_ro() call
- Improve s390-specific bcr_serialize() and cpu_relax() implementations
- Remove all unused variables to avoid allmodconfig W=1 build fails
with latest clang-23
- Cleanup default Kconfig values for s390 selftests
- Add a s390-tod trace clock to allow comparing trace timestamps
between different systems or virtual machines on s390
- Remove the s390 implementation of strlcat() in favor of the generic
variant
- Make consistent the calling order between
page_table_check_pte_clear() and secure page conversion across all
code paths
- Rearrange some fields within AP and zcrypt structs to reduce memory
consumption and unused holes
- Shorten GR_NUM and VX_NUM macros and move them to a separate header
- Replace __get_free_page() with kmalloc() in few sources
- Introduce an infrastructure for more efficient this_cpu operations.
Eliminate conditional branches when PREEMPT_NONE is removed
- Enable Rust support
- Use z10 as minimum architecture level, similar to the boot code, to
enforce a defined architecture level set
- Improve and convert various mem*() helper functions to C. For that
add .noinstr.text section to avoid orphaned warnings from the linker
- Fix the function pointer type in __ret_from_fork() to correct the
indirect call to match kernel thread return type of int
- Revert support for DCACHE_WORD_ACCESS to avoid an endless exception
loop on read from donated Ultravisor pages at unaligned addresses
* tag 's390-7.2-1' of gitolite.kernel.org:pub/scm/linux/kernel/git/s390/linux: (52 commits)
s390: Revert support for DCACHE_WORD_ACCESS
s390/process: Fix kernel thread function pointer type
s390/tishift: Convert __ashlti3(), __ashrti3(), __lshrti3() to C
s390/memmove: Optimize backward copy case
s390/string: Convert memset(16|32|64)() to C
s390/string: Convert memcpy() to C
s390/string: Convert memset() to C
s390/string: Convert memmove() to C
s390/string: Add -ffreestanding compile option to string.o
s390: Add .noinstr.text to boot and purgatory linker scripts
s390/purgatory: Enforce z10 minimum architecture level
s390: Enable Rust support
s390/cmpxchg: Fix KASAN stack-out-of-bounds in atomic helpers
rust: helpers: Add memchr wrapper for string operations
rust/bindgen_parameters: Mark s390 types as opaque to prevent repr conflicts
s390/jump_label: Implement ARCH_STATIC_BRANCH_JUMP_ASM and ARCH_STATIC_BRANCH_ASM macros
s390/bug: Provide ARCH_WARN_ASM for Rust WARN/BUG support
s390/ap: Fix locking issue in SE bind and associate sysfs functions
s390/percpu: Provide arch_this_cpu_write() implementation
s390/percpu: Provide arch_this_cpu_read() implementation
...
Diffstat (limited to 'arch')
46 files changed, 909 insertions, 656 deletions
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index 81185b19e1067..84404e6778d50 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -163,7 +163,6 @@ config S390 select ARCH_WANTS_THP_SWAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 - select DCACHE_WORD_ACCESS if !KMSAN select DYNAMIC_FTRACE if FUNCTION_TRACER select FUNCTION_ALIGNMENT_8B if CC_IS_GCC select FUNCTION_ALIGNMENT_16B if !CC_IS_GCC @@ -175,6 +174,7 @@ config S390 select GENERIC_GETTIMEOFDAY select GENERIC_SMP_IDLE_THREAD select GENERIC_IOREMAP if PCI + select GLOB select HAVE_ALIGNED_STRUCT_PAGE select HAVE_ARCH_AUDITSYSCALL select HAVE_ARCH_GET_SECUREBOOT @@ -244,6 +244,7 @@ config S390 select HAVE_RELIABLE_STACKTRACE select HAVE_RETHOOK select HAVE_RSEQ + select HAVE_RUST if !EXPOLINE select HAVE_SAMPLE_FTRACE_DIRECT select HAVE_SAMPLE_FTRACE_DIRECT_MULTI select HAVE_SETUP_PER_CPU_AREA @@ -995,9 +996,8 @@ config S390_MODULES_SANITY_TEST_HELPERS menu "Selftests" config S390_UNWIND_SELFTEST - def_tristate n + def_tristate KUNIT_ALL_TESTS depends on KUNIT - default KUNIT_ALL_TESTS prompt "Test unwind functions" help This option enables s390 specific stack unwinder testing kernel @@ -1007,7 +1007,7 @@ config S390_UNWIND_SELFTEST Say N if you are unsure. config S390_KPROBES_SANITY_TEST - def_tristate n + def_tristate KUNIT_ALL_TESTS prompt "Enable s390 specific kprobes tests" depends on KPROBES depends on KUNIT @@ -1019,9 +1019,8 @@ config S390_KPROBES_SANITY_TEST Say N if you are unsure. config S390_MODULES_SANITY_TEST - def_tristate n + def_tristate KUNIT_ALL_TESTS depends on KUNIT && m - default KUNIT_ALL_TESTS prompt "Enable s390 specific modules tests" select S390_MODULES_SANITY_TEST_HELPERS help diff --git a/arch/s390/Makefile b/arch/s390/Makefile index 297976b410886..8b712cd85fcd3 100644 --- a/arch/s390/Makefile +++ b/arch/s390/Makefile @@ -35,25 +35,31 @@ KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO),-g) KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_DEBUG_INFO_DWARF4), $(call cc-option, -gdwarf-4,)) KBUILD_CFLAGS_DECOMPRESSOR += $(if $(CONFIG_CC_NO_ARRAY_BOUNDS),-Wno-array-bounds) +KBUILD_RUSTFLAGS += --target=s390x-unknown-none-softfloat -Zpacked-stack -Ctarget-feature=+backchain + UTS_MACHINE := s390x STACK_SIZE := $(if $(CONFIG_KASAN),65536,$(if $(CONFIG_KMSAN),65536,16384)) CHECKFLAGS += -D__s390__ -D__s390x__ export LD_BFD -mflags-$(CONFIG_MARCH_Z10) := -march=z10 -mflags-$(CONFIG_MARCH_Z196) := -march=z196 -mflags-$(CONFIG_MARCH_ZEC12) := -march=zEC12 -mflags-$(CONFIG_MARCH_Z13) := -march=z13 -mflags-$(CONFIG_MARCH_Z14) := -march=z14 -mflags-$(CONFIG_MARCH_Z15) := -march=z15 -mflags-$(CONFIG_MARCH_Z16) := -march=z16 -mflags-$(CONFIG_MARCH_Z17) := -march=z17 +march-name-$(CONFIG_MARCH_Z10) := z10 +march-name-$(CONFIG_MARCH_Z196) := z196 +march-name-$(CONFIG_MARCH_ZEC12) := zEC12 +march-name-$(CONFIG_MARCH_Z13) := z13 +march-name-$(CONFIG_MARCH_Z14) := z14 +march-name-$(CONFIG_MARCH_Z15) := z15 +march-name-$(CONFIG_MARCH_Z16) := z16 +march-name-$(CONFIG_MARCH_Z17) := z17 + +mflags := -march=$(march-name-y) + +export CC_FLAGS_MARCH := $(mflags) -export CC_FLAGS_MARCH := $(mflags-y) +aflags-y += $(mflags) +cflags-y += $(mflags) -aflags-y += $(mflags-y) -cflags-y += $(mflags-y) +KBUILD_RUSTFLAGS += -Ctarget-cpu=$(march-name-y) cflags-$(CONFIG_MARCH_Z10_TUNE) += -mtune=z10 cflags-$(CONFIG_MARCH_Z196_TUNE) += -mtune=z196 diff --git a/arch/s390/appldata/appldata_base.c b/arch/s390/appldata/appldata_base.c index edbedbb2a773a..9cba4633c3f3b 100644 --- a/arch/s390/appldata/appldata_base.c +++ b/arch/s390/appldata/appldata_base.c @@ -51,7 +51,6 @@ static int appldata_timer_handler(const struct ctl_table *ctl, int write, static int appldata_interval_handler(const struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); -static struct ctl_table_header *appldata_sysctl_header; static const struct ctl_table appldata_table[] = { { .procname = "timer", @@ -406,7 +405,7 @@ static int __init appldata_init(void) appldata_wq = alloc_ordered_workqueue("appldata", 0); if (!appldata_wq) return -ENOMEM; - appldata_sysctl_header = register_sysctl(appldata_proc_name, appldata_table); + register_sysctl(appldata_proc_name, appldata_table); return 0; } diff --git a/arch/s390/boot/Makefile b/arch/s390/boot/Makefile index a1e719a79d38c..10b75e053a6f6 100644 --- a/arch/s390/boot/Makefile +++ b/arch/s390/boot/Makefile @@ -25,8 +25,13 @@ KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) CFLAGS_sclp_early_core.o += -I$(srctree)/drivers/s390/char +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o = -ffreestanding + obj-y := head.o als.o startup.o physmem_info.o ipl_parm.o ipl_report.o vmem.o -obj-y += string.o ebcdic.o sclp_early_core.o mem.o ipl_vmparm.o cmdline.o +obj-y += string.o ebcdic.o sclp_early_core.o ipl_vmparm.o cmdline.o obj-y += version.o pgm_check.o ctype.o ipl_data.o relocs.o alternative.o obj-y += uv.o printk.o trampoline.o obj-$(CONFIG_RANDOMIZE_BASE) += kaslr.o diff --git a/arch/s390/boot/mem.S b/arch/s390/boot/mem.S deleted file mode 100644 index b33463633f03e..0000000000000 --- a/arch/s390/boot/mem.S +++ /dev/null @@ -1,2 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#include "../lib/mem.S" diff --git a/arch/s390/boot/string.c b/arch/s390/boot/string.c index bd68161434a60..e4ad196cb7200 100644 --- a/arch/s390/boot/string.c +++ b/arch/s390/boot/string.c @@ -43,11 +43,7 @@ ssize_t sized_strscpy(char *dst, const char *src, size_t count) void *memset64(uint64_t *s, uint64_t v, size_t count) { - uint64_t *xs = s; - - while (count--) - *xs++ = v; - return s; + return __memset64(s, v, count * sizeof(v)); } char *skip_spaces(const char *str) diff --git a/arch/s390/boot/vmlinux.lds.S b/arch/s390/boot/vmlinux.lds.S index 070bc18babd0e..d44964592541e 100644 --- a/arch/s390/boot/vmlinux.lds.S +++ b/arch/s390/boot/vmlinux.lds.S @@ -31,6 +31,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + *(.noinstr.text) INIT_TEXT _etext = . ; } diff --git a/arch/s390/include/asm/asm-extable.h b/arch/s390/include/asm/asm-extable.h index d23ea0c94e4ea..99748c20e7671 100644 --- a/arch/s390/include/asm/asm-extable.h +++ b/arch/s390/include/asm/asm-extable.h @@ -12,7 +12,6 @@ #define EX_TYPE_UA_FAULT 3 #define EX_TYPE_UA_LOAD_REG 5 #define EX_TYPE_UA_LOAD_REGPAIR 6 -#define EX_TYPE_ZEROPAD 7 #define EX_TYPE_FPC 8 #define EX_TYPE_UA_MVCOS_TO 9 #define EX_TYPE_UA_MVCOS_FROM 10 @@ -80,9 +79,6 @@ #define EX_TABLE_UA_LOAD_REGPAIR(_fault, _target, _regerr, _regzero) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_UA_LOAD_REGPAIR, _regerr, _regzero, 0) -#define EX_TABLE_ZEROPAD(_fault, _target, _regdata, _regaddr) \ - __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_ZEROPAD, _regdata, _regaddr, 0) - #define EX_TABLE_FPC(_fault, _target) \ __EX_TABLE(__ex_table, _fault, _target, EX_TYPE_FPC, __stringify(%%r0), __stringify(%%r0), 0) diff --git a/arch/s390/include/asm/asm-prototypes.h b/arch/s390/include/asm/asm-prototypes.h index 7bd1801cf241c..d4da4436d02be 100644 --- a/arch/s390/include/asm/asm-prototypes.h +++ b/arch/s390/include/asm/asm-prototypes.h @@ -8,8 +8,4 @@ #include <asm/nospec-branch.h> #include <asm-generic/asm-prototypes.h> -__int128_t __ashlti3(__int128_t a, int b); -__int128_t __ashrti3(__int128_t a, int b); -__int128_t __lshrti3(__int128_t a, int b); - #endif /* _ASM_S390_PROTOTYPES_H */ diff --git a/arch/s390/include/asm/barrier.h b/arch/s390/include/asm/barrier.h index dad02f5b3c8d3..d1e4201f38506 100644 --- a/arch/s390/include/asm/barrier.h +++ b/arch/s390/include/asm/barrier.h @@ -8,6 +8,7 @@ #ifndef __ASM_BARRIER_H #define __ASM_BARRIER_H +#include <asm/alternative.h> #include <asm/march.h> /* @@ -16,16 +17,11 @@ * to devices. */ -#ifdef MARCH_HAS_Z196_FEATURES -/* Fast-BCR without checkpoint synchronization */ -#define __ASM_BCR_SERIALIZE "bcr 14,0" -#else -#define __ASM_BCR_SERIALIZE "bcr 15,0" -#endif - static __always_inline void bcr_serialize(void) { - asm volatile(__ASM_BCR_SERIALIZE : : : "memory"); + asm_inline volatile( + ALTERNATIVE("bcr 15,0", "bcr 14,0", ALT_FACILITY(45)) + : : : "memory"); } #define __mb() bcr_serialize() diff --git a/arch/s390/include/asm/bug.h b/arch/s390/include/asm/bug.h index 50a270edb0203..2010342c97b1b 100644 --- a/arch/s390/include/asm/bug.h +++ b/arch/s390/include/asm/bug.h @@ -4,6 +4,7 @@ #include <linux/compiler.h> #include <linux/const.h> +#include <linux/stringify.h> #define MONCODE_BUG _AC(0, U) #define MONCODE_BUG_ARG _AC(1, U) @@ -121,6 +122,17 @@ do { \ #define HAVE_ARCH_BUG_FORMAT #define HAVE_ARCH_BUG_FORMAT_ARGS +#define ARCH_WARN_ASM(file, line, flags, size) \ + ".section .rodata.str,\"aMS\",@progbits,1\n" \ + "9:\n" \ + ".asciz \"\"\n" /* Empty string for compatibility */ \ + ".previous\n" \ + "0:\n" \ + __stringify(mc 0(%r0),0) "\n" \ + __BUG_ENTRY("9b", file, line, flags, size) + +#define ARCH_WARN_REACHABLE + #endif /* CONFIG_BUG && CONFIG_CC_HAS_ASM_IMMEDIATE_STRINGS */ #endif /* __ASSEMBLER__ */ diff --git a/arch/s390/include/asm/cmpxchg.h b/arch/s390/include/asm/cmpxchg.h index 008357996262d..e6ac55cf3c172 100644 --- a/arch/s390/include/asm/cmpxchg.h +++ b/arch/s390/include/asm/cmpxchg.h @@ -35,7 +35,7 @@ static __always_inline u64 __csg_asm(u64 ptr, u64 old, u64 new) return old; } -static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) +static __no_sanitize_or_inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) { union { u8 b[4]; @@ -58,7 +58,7 @@ static inline u8 __arch_cmpxchg1(u64 ptr, u8 old, u8 new) return old; } -static inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) +static __no_sanitize_or_inline u16 __arch_cmpxchg2(u64 ptr, u16 old, u16 new) { union { u16 b[2]; @@ -173,7 +173,7 @@ static __always_inline u64 __arch_cmpxchg(u64 ptr, u64 old, u64 new, int size) void __xchg_called_with_bad_pointer(void); -static inline u8 __arch_xchg1(u64 ptr, u8 x) +static __no_sanitize_or_inline u8 __arch_xchg1(u64 ptr, u8 x) { int shift = (3 ^ (ptr & 3)) << 3; u32 mask, old, new; @@ -188,7 +188,7 @@ static inline u8 __arch_xchg1(u64 ptr, u8 x) return old >> shift; } -static inline u16 __arch_xchg2(u64 ptr, u16 x) +static __no_sanitize_or_inline u16 __arch_xchg2(u64 ptr, u16 x) { int shift = (2 ^ (ptr & 2)) << 3; u32 mask, old, new; diff --git a/arch/s390/include/asm/debug.h b/arch/s390/include/asm/debug.h index c5440b3ee53d7..39d484c597748 100644 --- a/arch/s390/include/asm/debug.h +++ b/arch/s390/include/asm/debug.h @@ -490,6 +490,7 @@ arch_initcall(VNAME(var, reg)) __DEFINE_STATIC_AREA(var); \ static debug_info_t __refdata var = \ __DEBUG_INFO_INIT(var, (name), (buf_size)); \ +static debug_info_t __used __section(".s390dbf_info") *VNAME(var, info) = &var; \ __REGISTER_STATIC_DEBUG_INFO(var, name, pages, nr_areas, view) void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas); diff --git a/arch/s390/include/asm/entry-percpu.h b/arch/s390/include/asm/entry-percpu.h new file mode 100644 index 0000000000000..4ef47265cfce8 --- /dev/null +++ b/arch/s390/include/asm/entry-percpu.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef ARCH_S390_ENTRY_PERCPU_H +#define ARCH_S390_ENTRY_PERCPU_H + +#include <linux/kprobes.h> +#include <linux/percpu.h> +#include <asm/lowcore.h> +#include <asm/ptrace.h> +#include <asm/asm-offsets.h> + +static __always_inline void percpu_entry(struct pt_regs *regs) +{ + struct lowcore *lc = get_lowcore(); + + if (user_mode(regs)) + return; + regs->cpu = lc->cpu_nr; + regs->percpu_register = lc->percpu_register; + lc->percpu_register = 0; +} + +static __always_inline bool percpu_code_check(struct pt_regs *regs) +{ + unsigned int insn, disp; + struct kprobe *p; + + if (likely(user_mode(regs) || !regs->percpu_register)) + return false; + /* + * Within a percpu code section - check if the percpu base register + * needs to be updated. This is the case if the PSW does not point to + * the ADD instruction within the section. + * - AG %rx,percpu_offset_in_lowcore(%r0,%r0) + * which adds the percpu offset to the percpu base register. + */ + lockdep_assert_preemption_disabled(); +again: + insn = READ_ONCE(*(u16 *)psw_bits(regs->psw).ia); + if (unlikely(insn == BREAKPOINT_INSTRUCTION)) { + p = get_kprobe((void *)psw_bits(regs->psw).ia); + /* + * If the kprobe is concurrently removed on a different CPU + * it might not be found anymore. However text must have + * been restored - try again. + */ + if (!p) + goto again; + insn = p->opcode; + } + if ((insn & 0xff0f) != 0xe300) + return true; + disp = offsetof(struct lowcore, percpu_offset); + if (machine_has_relocated_lowcore()) + disp += LOWCORE_ALT_ADDRESS; + insn = (disp & 0xff000) >> 4 | (disp & 0x00fff) << 16 | 0x8; + if (*(u32 *)(psw_bits(regs->psw).ia + 2) != insn) + return true; + return false; +} + +static __always_inline void percpu_exit(struct pt_regs *regs, bool needs_fixup) +{ + struct lowcore *lc = get_lowcore(); + unsigned char reg; + + if (user_mode(regs)) + return; + reg = regs->percpu_register; + lc->percpu_register = reg; + if (likely(!needs_fixup)) + return; + /* Check if process has been migrated to a different CPU. */ + if (regs->cpu == lc->cpu_nr) + return; + /* Fixup percpu base register */ + regs->gprs[reg] -= __per_cpu_offset[regs->cpu]; + regs->gprs[reg] += lc->percpu_offset; +} + +#endif diff --git a/arch/s390/include/asm/fpu-insn-asm.h b/arch/s390/include/asm/fpu-insn-asm.h index cc0468fdf2d07..4a1bfc0f578e0 100644 --- a/arch/s390/include/asm/fpu-insn-asm.h +++ b/arch/s390/include/asm/fpu-insn-asm.h @@ -16,181 +16,9 @@ #error only <asm/fpu-insn.h> can be included directly #endif -#ifdef __ASSEMBLER__ - -/* Macros to generate vector instruction byte code */ +#include <asm/insn-common-asm.h> -/* GR_NUM - Retrieve general-purpose register number - * - * @opd: Operand to store register number - * @r64: String designation register in the format "%rN" - */ -.macro GR_NUM opd gr - \opd = 255 - .ifc \gr,%r0 - \opd = 0 - .endif - .ifc \gr,%r1 - \opd = 1 - .endif - .ifc \gr,%r2 - \opd = 2 - .endif - .ifc \gr,%r3 - \opd = 3 - .endif - .ifc \gr,%r4 - \opd = 4 - .endif - .ifc \gr,%r5 - \opd = 5 - .endif - .ifc \gr,%r6 - \opd = 6 - .endif - .ifc \gr,%r7 - \opd = 7 - .endif - .ifc \gr,%r8 - \opd = 8 - .endif - .ifc \gr,%r9 - \opd = 9 - .endif - .ifc \gr,%r10 - \opd = 10 - .endif - .ifc \gr,%r11 - \opd = 11 - .endif - .ifc \gr,%r12 - \opd = 12 - .endif - .ifc \gr,%r13 - \opd = 13 - .endif - .ifc \gr,%r14 - \opd = 14 - .endif - .ifc \gr,%r15 - \opd = 15 - .endif - .if \opd == 255 - \opd = \gr - .endif -.endm - -/* VX_NUM - Retrieve vector register number - * - * @opd: Operand to store register number - * @vxr: String designation register in the format "%vN" - * - * The vector register number is used for as input number to the - * instruction and, as well as, to compute the RXB field of the - * instruction. - */ -.macro VX_NUM opd vxr - \opd = 255 - .ifc \vxr,%v0 - \opd = 0 - .endif - .ifc \vxr,%v1 - \opd = 1 - .endif - .ifc \vxr,%v2 - \opd = 2 - .endif - .ifc \vxr,%v3 - \opd = 3 - .endif - .ifc \vxr,%v4 - \opd = 4 - .endif - .ifc \vxr,%v5 - \opd = 5 - .endif - .ifc \vxr,%v6 - \opd = 6 - .endif - .ifc \vxr,%v7 - \opd = 7 - .endif - .ifc \vxr,%v8 - \opd = 8 - .endif - .ifc \vxr,%v9 - \opd = 9 - .endif - .ifc \vxr,%v10 - \opd = 10 - .endif - .ifc \vxr,%v11 - \opd = 11 - .endif - .ifc \vxr,%v12 - \opd = 12 - .endif - .ifc \vxr,%v13 - \opd = 13 - .endif - .ifc \vxr,%v14 - \opd = 14 - .endif - .ifc \vxr,%v15 - \opd = 15 - .endif - .ifc \vxr,%v16 - \opd = 16 - .endif - .ifc \vxr,%v17 - \opd = 17 - .endif - .ifc \vxr,%v18 - \opd = 18 - .endif - .ifc \vxr,%v19 - \opd = 19 - .endif - .ifc \vxr,%v20 - \opd = 20 - .endif - .ifc \vxr,%v21 - \opd = 21 - .endif - .ifc \vxr,%v22 - \opd = 22 - .endif - .ifc \vxr,%v23 - \opd = 23 - .endif - .ifc \vxr,%v24 - \opd = 24 - .endif - .ifc \vxr,%v25 - \opd = 25 - .endif - .ifc \vxr,%v26 - \opd = 26 - .endif - .ifc \vxr,%v27 - \opd = 27 - .endif - .ifc \vxr,%v28 - \opd = 28 - .endif - .ifc \vxr,%v29 - \opd = 29 - .endif - .ifc \vxr,%v30 - \opd = 30 - .endif - .ifc \vxr,%v31 - \opd = 31 - .endif - .if \opd == 255 - \opd = \vxr - .endif -.endm +#ifdef __ASSEMBLER__ /* RXB - Compute most significant bit used vector registers * diff --git a/arch/s390/include/asm/fpu-insn.h b/arch/s390/include/asm/fpu-insn.h index 96727f3bd0dce..ae8b7033cfd2c 100644 --- a/arch/s390/include/asm/fpu-insn.h +++ b/arch/s390/include/asm/fpu-insn.h @@ -15,6 +15,7 @@ #include <linux/kmsan.h> #include <asm/asm-extable.h> +asm(".include \"asm/insn-common-asm.h\"\n"); asm(".include \"asm/fpu-insn-asm.h\"\n"); /* diff --git a/arch/s390/include/asm/insn-common-asm.h b/arch/s390/include/asm/insn-common-asm.h new file mode 100644 index 0000000000000..fd9b3cacb7c54 --- /dev/null +++ b/arch/s390/include/asm/insn-common-asm.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Assembler helper macros to generate .byte/.word code for instructions + * that are unknown to older binutils versions. + */ + +#ifndef __ASM_S390_INSN_COMMON_ASM_H +#define __ASM_S390_INSN_COMMON_ASM_H + +#ifdef __ASSEMBLER__ + +/* + * GR_NUM - Retrieve general-purpose register number + * + * @opd: Operand to store register number + * @gr: String designation register in the format "%rN" + */ +.macro GR_NUM opd gr + \opd = 255 + .irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 + .ifc \gr,%r\rs + \opd = \rs + .endif + .endr + .if \opd == 255 + \opd = \gr + .endif +.endm + +/* + * VX_NUM - Retrieve vector register number + * + * @opd: Operand to store register number + * @vxr: String designation register in the format "%vN" + * + * The vector register number is used for as input number to the + * instruction and, as well as, to compute the RXB field of the + * instruction. + */ +.macro VX_NUM opd vxr + \opd = 255 + .irp vs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 + .ifc \vxr,%v\vs + \opd = \vs + .endif + .endr + .if \opd == 255 + \opd = \vxr + .endif +.endm + +#endif /* __ASSEMBLER__ */ +#endif /* __ASM_S390_INSN_COMMON_ASM_H */ diff --git a/arch/s390/include/asm/jump_label.h b/arch/s390/include/asm/jump_label.h index d9cbc18f6b2ee..0e28c90dc2425 100644 --- a/arch/s390/include/asm/jump_label.h +++ b/arch/s390/include/asm/jump_label.h @@ -19,19 +19,29 @@ #define JUMP_LABEL_STATIC_KEY_CONSTRAINT "jdd" #endif +#define ARCH_JUMP_TABLE_ENTRY(key, label, local_label) \ + ".pushsection __jump_table,\"aw\"\n" \ + ".balign 8\n" \ + ".long " local_label "-.," label "-.\n" \ + ".quad " key "-.\n" \ + ".popsection\n" + /* * We use a brcl 0,<offset> instruction for jump labels so it * can be easily distinguished from a hotpatch generated instruction. */ +#define ARCH_STATIC_BRANCH_ASM(key, label) \ + "0: brcl 0," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + +#define ARCH_STATIC_BRANCH_JUMP_ASM(key, label) \ + "0: brcl 15," label "\n" \ + ARCH_JUMP_TABLE_ENTRY(key, label, "0b") + static __always_inline bool arch_static_branch(struct static_key *key, bool branch) { - asm goto("0: brcl 0,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; @@ -39,13 +49,8 @@ label: static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch) { - asm goto("0: brcl 15,%l[label]\n" - ".pushsection __jump_table,\"aw\"\n" - ".balign 8\n" - ".long 0b-.,%l[label]-.\n" - ".quad %0+%1-.\n" - ".popsection\n" - : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); + asm goto(ARCH_STATIC_BRANCH_JUMP_ASM("%0+%1", "%l[label]") + : : JUMP_LABEL_STATIC_KEY_CONSTRAINT (key), "i" (branch) : : label); return false; label: return true; diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h index 50ffe75adeb47..cd1ddfdb5d35d 100644 --- a/arch/s390/include/asm/lowcore.h +++ b/arch/s390/include/asm/lowcore.h @@ -165,7 +165,8 @@ struct lowcore { __u32 spinlock_index; /* 0x03b0 */ __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */ __u64 percpu_offset; /* 0x03b8 */ - __u8 pad_0x03c0[0x0400-0x03c0]; /* 0x03c0 */ + __u8 percpu_register; /* 0x03c0 */ + __u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */ __u32 return_lpswe; /* 0x0400 */ __u32 return_mcck_lpswe; /* 0x0404 */ diff --git a/arch/s390/include/asm/percpu.h b/arch/s390/include/asm/percpu.h index b18a96f3a3345..1d955dd0defa3 100644 --- a/arch/s390/include/asm/percpu.h +++ b/arch/s390/include/asm/percpu.h @@ -60,6 +60,68 @@ #define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) #define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |) +/* + * Macros to be used for percpu code section based on atomic instructions. + * + * Avoid the need to use preempt_disable() / preempt_disable() pairs and the + * conditional preempt_schedule_notrace() function calls which come with + * this. The idea is that this_cpu operations based on atomic instructions are + * guarded with mviy instructions: + * + * - The first mviy instruction writes the register number, which contains the + * percpu address variable to lowcore. This also indicates that a percpu + * code section is executed. + * + * - The first mviy instruction following the mviy instruction must be the ag + * instruction which adds the percpu offset to the percpu address register. + * + * - Afterwards the atomic percpu operation follows. + * + * - Then a second mviy instruction writes a zero to lowcore, which indicates + * the end of the percpu code section. + * + * - In case of an interrupt/exception/nmi the register number which was + * written to lowcore is copied to the exception frame (pt_regs), and a zero + * is written to lowcore. + * + * - On return to the previous context it is checked if a percpu code section + * was executed (saved register number not zero), and if the process was + * migrated to a different cpu. If the percpu offset was already added to + * the percpu address register (instruction address does _not_ point to the + * ag instruction) the content of the percpu address register is adjusted so + * it points to percpu variable of the new cpu. + * + * Inline assemblies making use of this typically have a code sequence like: + * + * MVIY_PERCPU(...) <- start of percpu code section + * AG_ALT(...) <- add percpu offset; must be the second instruction + * atomic_op <- atomic op + * MVIY_ALT(...) <- end of percpu code section + */ + +#define MVIY_PERCPU(disp, dispalt, reg) \ + ".macro GEN_MVIY disp reg\n" \ + ".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \ + " .ifc \\reg,%%r\\rs\n" \ + " mviy \\disp(%%r0),\\rs\n" \ + " .endif\n" \ + ".endr\n" \ + ".endm\n" \ + ALTERNATIVE("GEN_MVIY " __stringify(disp) " " __stringify(reg) "\n", \ + "GEN_MVIY " __stringify(dispalt) " " __stringify(reg) "\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) \ + ".purgem GEN_MVIY\n" + +#define MVIY_ALT(disp, dispalt) \ + ALTERNATIVE(" mviy " disp "(%%r0),0\n", \ + " mviy " dispalt "(%%r0),0\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + +#define AG_ALT(disp, dispalt, reg) \ + ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \ + " ag " reg ", " dispalt "(%%r0)\n", \ + ALT_FEATURE(MFEATURE_LOWCORE)) + #ifndef MARCH_HAS_Z196_FEATURES #define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +) @@ -73,46 +135,79 @@ #else /* MARCH_HAS_Z196_FEATURES */ -#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ -{ \ - typedef typeof(pcp) pcp_op_T__; \ - pcp_op_T__ val__ = (val); \ - pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - if (__builtin_constant_p(val__) && \ - ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ - asm volatile( \ - op2 " %[ptr__],%[val__]" \ - : [ptr__] "+Q" (*ptr__) \ - : [val__] "i" ((szcast)val__) \ - : "cc"); \ - } else { \ - asm volatile( \ - op1 " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ - : "cc"); \ - } \ - preempt_enable_notrace(); \ -} +#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ val__ = (val); \ + pcp_op_T__ old__, *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + if (__builtin_constant_p(val__) && \ + ((szcast)val__ > -129) && ((szcast)val__ < 128)) { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op2 " 0(%[ptr__]),%[val__]\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "i" ((szcast)val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } else { \ + asm volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op1 " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + } \ +} while (0) #define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int) #define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long) #define arch_this_cpu_add_return(pcp, val, op) \ ({ \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ old__ + val__; \ }) @@ -120,19 +215,31 @@ #define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag") #define arch_this_cpu_to_op(pcp, val, op) \ -{ \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ typedef typeof(pcp) pcp_op_T__; \ pcp_op_T__ val__ = (val); \ pcp_op_T__ old__, *ptr__; \ - preempt_disable_notrace(); \ - ptr__ = raw_cpu_ptr(&(pcp)); \ - asm volatile( \ - op " %[old__],%[val__],%[ptr__]" \ - : [old__] "=d" (old__), [ptr__] "+Q" (*ptr__) \ - : [val__] "d" (val__) \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[old__],%[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [old__] "=&d" (old__), \ + [ptr__] "+&a" (ptr__), "+m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ : "cc"); \ - preempt_enable_notrace(); \ -} +} while (0) #define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan") #define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang") @@ -141,6 +248,67 @@ #endif /* MARCH_HAS_Z196_FEATURES */ +#define arch_this_cpu_read(pcp, op) \ +({ \ + unsigned long lc_pcpr, lc_pcpo, res__; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__; \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[res__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (*ptr__), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ + (pcp_op_T__)res__; \ +}) + +#define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc") +#define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh") +#define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf") +#define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg") + +#define arch_this_cpu_write(pcp, val, op) \ +do { \ + unsigned long lc_pcpr, lc_pcpo; \ + typedef typeof(pcp) pcp_op_T__; \ + pcp_op_T__ *ptr__, val__ = (val); \ + \ + lc_pcpr = offsetof(struct lowcore, percpu_register); \ + lc_pcpo = offsetof(struct lowcore, percpu_offset); \ + ptr__ = PERCPU_PTR(&(pcp)); \ + asm_inline volatile( \ + MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\ + AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \ + op " %[val__],0(%[ptr__])\n" \ + MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \ + : [ptr__] "+&a" (ptr__), "=m" (*ptr__), \ + "=m" (((struct lowcore *)0)->percpu_register) \ + : [val__] "d" (val__), \ + [disppcpr] "i" (lc_pcpr), \ + [disppcpo] "i" (lc_pcpo), \ + [dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \ + [dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \ + "m" (((struct lowcore *)0)->percpu_offset) \ + : "cc"); \ +} while (0) + +#define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc") +#define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth") +#define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st") +#define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg") + #define arch_this_cpu_cmpxchg(pcp, oval, nval) \ ({ \ typedef typeof(pcp) pcp_op_T__; \ diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 2c6cee8241e04..3197b8b372a2a 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1189,10 +1189,10 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm, pte_t res; res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(mm) && pte_present(res)) WARN_ON_ONCE(uv_convert_from_secure_pte(res)); - page_table_check_pte_clear(mm, addr, res); return res; } @@ -1208,10 +1208,10 @@ static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, pte_t res; res = ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); + page_table_check_pte_clear(vma->vm_mm, addr, res); /* At this point the reference through the mapping is still present */ if (mm_is_protected(vma->vm_mm) && pte_present(res)) WARN_ON_ONCE(uv_convert_from_secure_pte(res)); - page_table_check_pte_clear(vma->vm_mm, addr, res); return res; } @@ -1235,26 +1235,23 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, } else { res = ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); } - page_table_check_pte_clear(mm, addr, res); - - /* Nothing to do */ - if (!mm_is_protected(mm) || !pte_present(res)) - return res; - /* - * At this point the reference through the mapping is still present. - * The notifier should have destroyed all protected vCPUs at this - * point, so the destroy should be successful. - */ - if (full && !uv_destroy_pte(res)) - return res; - /* - * If something went wrong and the page could not be destroyed, or - * if this is not a mm teardown, the slower export is used as - * fallback instead. If even that fails, print a warning and leak - * the page, to avoid crashing the whole system. - */ - WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + /* At this point the reference through the mapping is still present */ + if (mm_is_protected(mm) && pte_present(res)) { + /* + * The notifier should have destroyed all protected vCPUs at + * this point, so the destroy should be successful. + */ + if (full && !uv_destroy_pte(res)) + return res; + /* + * If something went wrong and the page could not be destroyed, + * or if this is not a mm teardown, the slower export is used + * as fallback instead. If even that fails, print a warning and + * leak the page, to avoid crashing the whole system. + */ + WARN_ON_ONCE(uv_convert_from_secure_pte(res)); + } return res; } diff --git a/arch/s390/include/asm/processor.h b/arch/s390/include/asm/processor.h index 78195ee5e99f6..ecd3341686eb6 100644 --- a/arch/s390/include/asm/processor.h +++ b/arch/s390/include/asm/processor.h @@ -33,6 +33,7 @@ #include <linux/irqflags.h> #include <linux/instruction_pointer.h> #include <linux/bitops.h> +#include <asm/vdso/processor.h> #include <asm/fpu-types.h> #include <asm/cpu.h> #include <asm/page.h> @@ -282,8 +283,6 @@ static __always_inline unsigned short stap(void) return cpu_address; } -#define cpu_relax() barrier() - #define ECAG_CACHE_ATTRIBUTE 0 #define ECAG_CPU_ATTRIBUTE 1 diff --git a/arch/s390/include/asm/ptrace.h b/arch/s390/include/asm/ptrace.h index aaceb1d9110a4..495e310c3d6d7 100644 --- a/arch/s390/include/asm/ptrace.h +++ b/arch/s390/include/asm/ptrace.h @@ -134,6 +134,8 @@ struct pt_regs { }; unsigned long flags; unsigned long last_break; + unsigned int cpu; + unsigned char percpu_register; }; /* diff --git a/arch/s390/include/asm/sclp.h b/arch/s390/include/asm/sclp.h index 0f184dbdbe5e0..d928a9ddfe402 100644 --- a/arch/s390/include/asm/sclp.h +++ b/arch/s390/include/asm/sclp.h @@ -20,6 +20,7 @@ #define SCLP_ERRNOTIFY_AQ_REPAIR 1 #define SCLP_ERRNOTIFY_AQ_INFO_LOG 2 #define SCLP_ERRNOTIFY_AQ_OPTICS_DATA 3 +#define SCLP_ERRNOTIFY_AQ_NVME_SMART_LOG 4 #ifndef __ASSEMBLER__ #include <linux/uio.h> diff --git a/arch/s390/include/asm/string.h b/arch/s390/include/asm/string.h index 238e721e5a22c..378f85304ef5e 100644 --- a/arch/s390/include/asm/string.h +++ b/arch/s390/include/asm/string.h @@ -26,7 +26,6 @@ void *memmove(void *dest, const void *src, size_t n); #define __HAVE_ARCH_MEMSCAN /* inline & arch function */ #define __HAVE_ARCH_STRCAT /* inline & arch function */ #define __HAVE_ARCH_STRCMP /* arch function */ -#define __HAVE_ARCH_STRLCAT /* arch function */ #define __HAVE_ARCH_STRLEN /* inline & arch function */ #define __HAVE_ARCH_STRNCAT /* arch function */ #define __HAVE_ARCH_STRNLEN /* inline & arch function */ @@ -38,7 +37,6 @@ void *memmove(void *dest, const void *src, size_t n); /* Prototypes for non-inlined arch strings functions. */ int memcmp(const void *s1, const void *s2, size_t n); int strcmp(const char *s1, const char *s2); -size_t strlcat(char *dest, const char *src, size_t n); char *strncat(char *dest, const char *src, size_t n); char *strstr(const char *s1, const char *s2); #endif /* !defined(CONFIG_KASAN) && !defined(CONFIG_KMSAN) */ diff --git a/arch/s390/include/asm/trace_clock.h b/arch/s390/include/asm/trace_clock.h new file mode 100644 index 0000000000000..273e05cbdae02 --- /dev/null +++ b/arch/s390/include/asm/trace_clock.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_S390_TRACE_CLOCK_H +#define _ASM_S390_TRACE_CLOCK_H + +#include <linux/compiler.h> +#include <linux/types.h> + +u64 notrace trace_clock_s390_tod(void); + +#define ARCH_TRACE_CLOCKS \ + { trace_clock_s390_tod, "s390-tod", .in_ns = 0 }, + +#endif /* _ASM_S390_TRACE_CLOCK_H */ diff --git a/arch/s390/include/asm/vdso/processor.h b/arch/s390/include/asm/vdso/processor.h index cfcc3e117c4c9..6775621e5a5a3 100644 --- a/arch/s390/include/asm/vdso/processor.h +++ b/arch/s390/include/asm/vdso/processor.h @@ -2,6 +2,8 @@ #ifndef __ASM_VDSO_PROCESSOR_H #define __ASM_VDSO_PROCESSOR_H -#define cpu_relax() barrier() +#include <asm/barrier.h> + +#define cpu_relax() bcr_serialize() #endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/s390/include/asm/word-at-a-time.h b/arch/s390/include/asm/word-at-a-time.h index eaa19dee76994..e9287036392d0 100644 --- a/arch/s390/include/asm/word-at-a-time.h +++ b/arch/s390/include/asm/word-at-a-time.h @@ -4,7 +4,6 @@ #include <linux/bitops.h> #include <linux/wordpart.h> -#include <asm/asm-extable.h> #include <asm/bitsperlong.h> struct word_at_a_time { @@ -41,25 +40,4 @@ static inline unsigned long zero_bytemask(unsigned long data) return ~1UL << data; } -/* - * Load an unaligned word from kernel space. - * - * In the (very unlikely) case of the word being a page-crosser - * and the next page not being mapped, take the exception and - * return zeroes in the non-existing part. - */ -static inline unsigned long load_unaligned_zeropad(const void *addr) -{ - unsigned long data; - - asm_inline volatile( - "0: lg %[data],0(%[addr])\n" - "1: nopr %%r7\n" - EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) - EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) - : [data] "=d" (data) - : [addr] "a" (addr), "m" (*(unsigned long *)addr)); - return data; -} - #endif /* _ASM_WORD_AT_A_TIME_H */ diff --git a/arch/s390/kernel/Makefile b/arch/s390/kernel/Makefile index 137e4793ec117..6c88476d79a31 100644 --- a/arch/s390/kernel/Makefile +++ b/arch/s390/kernel/Makefile @@ -62,6 +62,7 @@ obj-$(CONFIG_KPROBES) += mcount.o obj-$(CONFIG_RETHOOK) += rethook.o obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o obj-$(CONFIG_FUNCTION_TRACER) += mcount.o +obj-$(CONFIG_TRACING) += trace_clock.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o relocate_kernel.o obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 7650f2adb5cf8..dbf430f479bdf 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -26,6 +26,8 @@ #include <linux/math.h> #include <linux/minmax.h> #include <linux/debugfs.h> +#include <linux/glob.h> +#include <linux/stringify.h> #include <asm/debug.h> @@ -154,6 +156,7 @@ static unsigned int __used debug_feature_version = __DEBUG_FEATURE_VERSION; static debug_info_t *debug_area_first; static debug_info_t *debug_area_last; static DEFINE_MUTEX(debug_mutex); +extern debug_info_t *__s390dbf_info[], *__s390dbf_info_end[]; static int initialized; static int debug_critical; @@ -168,7 +171,91 @@ static const struct file_operations debug_file_ops = { static struct dentry *debug_debugfs_root_entry; +/* List of debug area parameters to override */ +#define PARAM_UNSET -2 +#define PARAM_NUM 16 +static struct debug_param_t { + char name[DEBUG_MAX_NAME_LEN + 1]; + int level; + int pages; +} debug_param[PARAM_NUM]; +static int debug_param_num; + /* functions */ +static void debug_get_param(const char *name, int *level, int *pages) +{ + struct debug_param_t *p; + int i; + + for (i = 0; i < debug_param_num; i++) { + p = &debug_param[i]; + if (!glob_match(p->name, name)) + continue; + if (level && p->level != PARAM_UNSET) { + pr_info("%s: override level to %d\n", name, p->level); + *level = p->level; + } + if (pages && p->pages != PARAM_UNSET) { + pr_info("%s: override pages to %d\n", name, p->pages); + *pages = p->pages; + } + } +} + +#define LVL_LEN 10 +#define LVL_FMT "%" __stringify(LVL_LEN) "[^:]" +#define NAME_FMT "%" __stringify(DEBUG_MAX_NAME_LEN) "[^:]" + +static bool __init s390dbf_parse_one(const char *arg, struct debug_param_t *param) +{ + struct debug_param_t p = { { 0 }, PARAM_UNSET, PARAM_UNSET }; + char level[LVL_LEN + 1] = { 0 }; + + /* arg: <name|pattern>:[<level>|-]:[<pages>] */ + if (sscanf(arg, NAME_FMT ":" LVL_FMT ":%d", p.name, level, &p.pages) > 1) { + if (strcmp(level, "-") == 0) + p.level = DEBUG_OFF_LEVEL; + else if (kstrtoint(level, 0, &p.level) != 0) + return false; + } else if (sscanf(arg, NAME_FMT "::%d", p.name, &p.pages) != 2) { + return false; + } + + if (p.level != PARAM_UNSET && p.level != DEBUG_OFF_LEVEL && + (p.level < 0 || p.level > DEBUG_MAX_LEVEL)) + return false; + if (p.pages != PARAM_UNSET && p.pages < 0) + return false; + *param = p; + + return true; +} + +static int __init s390dbf_parse(char *arg) +{ + debug_info_t **id; + int i, rc = 0; + + while (arg && debug_param_num < PARAM_NUM) { + if (s390dbf_parse_one(arg, &debug_param[debug_param_num])) + debug_param_num++; + else + rc = -EINVAL; + arg = strchr(arg, ','); + if (arg) + arg++; + } + + /* + * Apply level to static debug areas, delay buffer size changes until + * regular memory allocations are possible. + */ + for (i = 0, id = __s390dbf_info; &id[i] < __s390dbf_info_end; i++) + debug_get_param(id[i]->name, &id[i]->level, NULL); + + return rc; +} +early_param("s390dbf", s390dbf_parse); /* * debug_areas_alloc @@ -305,10 +392,11 @@ static void debug_info_free(debug_info_t *db_info) static debug_info_t *debug_info_create(const char *name, int pages_per_area, int nr_areas, int buf_size, umode_t mode) { + int level = DEBUG_DEFAULT_LEVEL; debug_info_t *rc; - rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, - DEBUG_DEFAULT_LEVEL, ALL_AREAS); + debug_get_param(name, &level, &pages_per_area); + rc = debug_info_alloc(name, pages_per_area, nr_areas, buf_size, level, ALL_AREAS); if (!rc) goto out; @@ -872,6 +960,7 @@ void debug_register_static(debug_info_t *id, int pages_per_area, int nr_areas) return; } + debug_get_param(id->name, &id->level, &pages_per_area); copy = debug_info_alloc("", pages_per_area, nr_areas, id->buf_size, id->level, ALL_AREAS); if (!copy) { @@ -975,16 +1064,7 @@ static int debug_set_size(debug_info_t *id, int nr_areas, int pages_per_area) return 0; } -/** - * debug_set_level() - Sets new actual debug level if new_level is valid. - * - * @id: handle for debug log - * @new_level: new debug level - * - * Return: - * none - */ -void debug_set_level(debug_info_t *id, int new_level) +static void _debug_set_level(debug_info_t *id, int new_level) { unsigned long flags; @@ -1003,6 +1083,23 @@ void debug_set_level(debug_info_t *id, int new_level) id->level = new_level; raw_spin_unlock_irqrestore(&id->lock, flags); } + +/** + * debug_set_level() - Sets new actual debug level if new_level is valid. + * + * @id: handle for debug log + * @new_level: new debug level + * + * Return: + * none + */ +void debug_set_level(debug_info_t *id, int new_level) +{ + /* Level specified via kernel parameter takes precedence */ + debug_get_param(id->name, &new_level, NULL); + + _debug_set_level(id, new_level); +} EXPORT_SYMBOL(debug_set_level); /* @@ -1137,8 +1234,6 @@ static const struct ctl_table s390dbf_table[] = { }, }; -static struct ctl_table_header *s390dbf_sysctl_header; - /** * debug_stop_all() - stops the debug feature if stopping is allowed. * @@ -1529,7 +1624,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view, goto out; } if (str[0] == '-') { - debug_set_level(id, DEBUG_OFF_LEVEL); + _debug_set_level(id, DEBUG_OFF_LEVEL); rc = user_len; goto free_str; } else { @@ -1539,7 +1634,7 @@ static int debug_input_level_fn(debug_info_t *id, struct debug_view *view, pr_warn("%s is not a valid level for a debug feature\n", str); rc = -EINVAL; } else { - debug_set_level(id, new_level); + _debug_set_level(id, new_level); rc = user_len; } free_str: @@ -1728,7 +1823,7 @@ EXPORT_SYMBOL(debug_sprintf_format_fn); */ static int __init debug_init(void) { - s390dbf_sysctl_header = register_sysctl("s390dbf", s390dbf_table); + register_sysctl("s390dbf", s390dbf_table); mutex_lock(&debug_mutex); debug_debugfs_root_entry = debugfs_create_dir(DEBUG_DIR_ROOT, NULL); initialized = 1; diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index d10a17e6531da..efb988833c88d 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -33,6 +33,7 @@ #include <asm/softirq_stack.h> #include <asm/vtime.h> #include <asm/asm.h> +#include <asm/entry-percpu.h> #include "entry.h" DEFINE_PER_CPU_SHARED_ALIGNED(struct irq_stat, irq_stat); @@ -142,10 +143,13 @@ static int irq_pending(struct pt_regs *regs) void noinstr do_io_irq(struct pt_regs *regs) { - irqentry_state_t state = irqentry_enter(regs); - struct pt_regs *old_regs = set_irq_regs(regs); - bool from_idle; + bool from_idle, percpu_needs_fixup; + struct pt_regs *old_regs; + irqentry_state_t state; + percpu_entry(regs); + state = irqentry_enter(regs); + old_regs = set_irq_regs(regs); from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) update_timer_idle(); @@ -170,21 +174,25 @@ void noinstr do_io_irq(struct pt_regs *regs) do_irq_async(regs, IO_INTERRUPT); } while (machine_is_lpar() && irq_pending(regs)); + percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); - set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); + percpu_exit(regs, percpu_needs_fixup); } void noinstr do_ext_irq(struct pt_regs *regs) { - irqentry_state_t state = irqentry_enter(regs); - struct pt_regs *old_regs = set_irq_regs(regs); - bool from_idle; + bool from_idle, percpu_needs_fixup; + struct pt_regs *old_regs; + irqentry_state_t state; + percpu_entry(regs); + state = irqentry_enter(regs); + old_regs = set_irq_regs(regs); from_idle = test_and_clear_cpu_flag(CIF_ENABLED_WAIT); if (from_idle) update_timer_idle(); @@ -206,12 +214,14 @@ void noinstr do_ext_irq(struct pt_regs *regs) do_irq_async(regs, EXT_INTERRUPT); + percpu_needs_fixup = percpu_code_check(regs); irq_exit_rcu(); set_irq_regs(old_regs); irqentry_exit(regs, state); if (from_idle) regs->psw.mask &= ~(PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_WAIT); + percpu_exit(regs, percpu_needs_fixup); } static void show_msi_interrupt(struct seq_file *p, int irq) diff --git a/arch/s390/kernel/nmi.c b/arch/s390/kernel/nmi.c index 94fbfad49f620..e17a59d4d5a41 100644 --- a/arch/s390/kernel/nmi.c +++ b/arch/s390/kernel/nmi.c @@ -22,6 +22,7 @@ #include <linux/module.h> #include <linux/sched/signal.h> #include <linux/kvm_host.h> +#include <asm/entry-percpu.h> #include <asm/lowcore.h> #include <asm/ctlreg.h> #include <asm/fpu.h> @@ -363,6 +364,7 @@ NOKPROBE_SYMBOL(s390_backup_mcck_info); */ void notrace s390_do_machine_check(struct pt_regs *regs) { + bool percpu_needs_fixup; static int ipd_count; static DEFINE_SPINLOCK(ipd_lock); static unsigned long long last_ipd; @@ -374,6 +376,7 @@ void notrace s390_do_machine_check(struct pt_regs *regs) unsigned long mcck_dam_code; int mcck_pending = 0; + percpu_entry(regs); irq_state = irqentry_nmi_enter(regs); if (user_mode(regs)) @@ -495,7 +498,9 @@ void notrace s390_do_machine_check(struct pt_regs *regs) if (mcck_pending) schedule_mcck_handler(); + percpu_needs_fixup = percpu_code_check(regs); irqentry_nmi_exit(regs, irq_state); + percpu_exit(regs, percpu_needs_fixup); } NOKPROBE_SYMBOL(s390_do_machine_check); diff --git a/arch/s390/kernel/process.c b/arch/s390/kernel/process.c index 0df95dcb21012..416650ae4871c 100644 --- a/arch/s390/kernel/process.c +++ b/arch/s390/kernel/process.c @@ -50,7 +50,7 @@ void ret_from_fork(void) asm("ret_from_fork"); void __ret_from_fork(struct task_struct *prev, struct pt_regs *regs) { - void (*func)(void *arg); + int (*func)(void *arg); schedule_tail(prev); @@ -203,9 +203,6 @@ unsigned long __get_wchan(struct task_struct *p) struct unwind_state state; unsigned long ip = 0; - if (!task_stack_page(p)) - return 0; - if (!try_get_task_stack(p)) return 0; diff --git a/arch/s390/kernel/trace_clock.c b/arch/s390/kernel/trace_clock.c new file mode 100644 index 0000000000000..9ca568058cbce --- /dev/null +++ b/arch/s390/kernel/trace_clock.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/trace_clock.h> +#include <linux/timex.h> +/* + * trace_clock_s390_tod(): trace clock based on the s390 TOD clock + * + * Unlike the other clocks, this is not in nanoseconds. + */ +u64 notrace trace_clock_s390_tod(void) +{ + return get_tod_clock(); +} diff --git a/arch/s390/kernel/traps.c b/arch/s390/kernel/traps.c index 1b5c6fc431cc1..564403496a7ce 100644 --- a/arch/s390/kernel/traps.c +++ b/arch/s390/kernel/traps.c @@ -24,6 +24,7 @@ #include <linux/entry-common.h> #include <linux/kmsan.h> #include <linux/bug.h> +#include <asm/entry-percpu.h> #include <asm/asm-extable.h> #include <asm/irqflags.h> #include <asm/ptrace.h> @@ -329,6 +330,7 @@ static void (*pgm_check_table[128])(struct pt_regs *regs); void noinstr __do_pgm_check(struct pt_regs *regs) { struct lowcore *lc = get_lowcore(); + bool percpu_needs_fixup; irqentry_state_t state; unsigned int trapnr; union teid teid; @@ -349,6 +351,7 @@ void noinstr __do_pgm_check(struct pt_regs *regs) current->thread.gmap_int_code = regs->int_code & 0xffff; return; } + percpu_entry(regs); state = irqentry_enter(regs); if (user_mode(regs)) { update_timer_sys(); @@ -385,7 +388,9 @@ void noinstr __do_pgm_check(struct pt_regs *regs) pgm_check_table[trapnr](regs); out: local_irq_disable(); + percpu_needs_fixup = percpu_code_check(regs); irqentry_exit(regs, state); + percpu_exit(regs, percpu_needs_fixup); } /* diff --git a/arch/s390/kernel/vmlinux.lds.S b/arch/s390/kernel/vmlinux.lds.S index 2b62395e35bfb..1f0e71c58eb9c 100644 --- a/arch/s390/kernel/vmlinux.lds.S +++ b/arch/s390/kernel/vmlinux.lds.S @@ -159,6 +159,13 @@ SECTIONS } #endif + . = ALIGN(8); + .s390dbf_info : { + __s390dbf_info = .; + *(.s390dbf_info) + __s390dbf_info_end = .; + } + /* * Table with the patch locations to undo expolines */ diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile index 2bf47204f6abd..aa6cc6a1fe881 100644 --- a/arch/s390/lib/Makefile +++ b/arch/s390/lib/Makefile @@ -3,9 +3,13 @@ # Makefile for s390-specific library files.. # +# string.o implements standard library functions like memset/memcpy etc. +# Use -ffreestanding to ensure that the compiler does not try to "optimize" +# them into calls to themselves. +CFLAGS_string.o = -ffreestanding + lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o lib-y += csum-partial.o -obj-y += mem.o lib-$(CONFIG_KPROBES) += probes.o lib-$(CONFIG_UPROBES) += probes.o obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o diff --git a/arch/s390/lib/mem.S b/arch/s390/lib/mem.S deleted file mode 100644 index d026debf250c7..0000000000000 --- a/arch/s390/lib/mem.S +++ /dev/null @@ -1,192 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * String handling functions. - * - * Copyright IBM Corp. 2012 - */ - -#include <linux/export.h> -#include <linux/linkage.h> -#include <asm/nospec-insn.h> - - GEN_BR_THUNK %r14 - -/* - * void *memmove(void *dest, const void *src, size_t n) - */ -SYM_FUNC_START(__memmove) - ltgr %r4,%r4 - lgr %r1,%r2 - jz .Lmemmove_exit - aghi %r4,-1 - clgr %r2,%r3 - jnh .Lmemmove_forward - la %r5,1(%r4,%r3) - clgr %r2,%r5 - jl .Lmemmove_reverse -.Lmemmove_forward: - srlg %r0,%r4,8 - ltgr %r0,%r0 - jz .Lmemmove_forward_remainder -.Lmemmove_forward_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brctg %r0,.Lmemmove_forward_loop -.Lmemmove_forward_remainder: - exrl %r4,.Lmemmove_mvc -.Lmemmove_exit: - BR_EX %r14 -.Lmemmove_reverse: - ic %r0,0(%r4,%r3) - stc %r0,0(%r4,%r1) - brctg %r4,.Lmemmove_reverse - ic %r0,0(%r4,%r3) - stc %r0,0(%r4,%r1) - BR_EX %r14 -.Lmemmove_mvc: - mvc 0(1,%r1),0(%r3) -SYM_FUNC_END(__memmove) -EXPORT_SYMBOL(__memmove) - -SYM_FUNC_ALIAS(memmove, __memmove) -EXPORT_SYMBOL(memmove) - -/* - * memset implementation - * - * This code corresponds to the C construct below. We do distinguish - * between clearing (c == 0) and setting a memory array (c != 0) simply - * because nearly all memset invocations in the kernel clear memory and - * the xc instruction is preferred in such cases. - * - * void *memset(void *s, int c, size_t n) - * { - * if (likely(c == 0)) - * return __builtin_memset(s, 0, n); - * return __builtin_memset(s, c, n); - * } - */ -SYM_FUNC_START(__memset) - ltgr %r4,%r4 - jz .Lmemset_exit - ltgr %r3,%r3 - jnz .Lmemset_fill - aghi %r4,-1 - srlg %r3,%r4,8 - ltgr %r3,%r3 - lgr %r1,%r2 - jz .Lmemset_clear_remainder -.Lmemset_clear_loop: - xc 0(256,%r1),0(%r1) - la %r1,256(%r1) - brctg %r3,.Lmemset_clear_loop -.Lmemset_clear_remainder: - exrl %r4,.Lmemset_xc -.Lmemset_exit: - BR_EX %r14 -.Lmemset_fill: - cghi %r4,1 - lgr %r1,%r2 - je .Lmemset_fill_exit - aghi %r4,-2 - srlg %r5,%r4,8 - ltgr %r5,%r5 - jz .Lmemset_fill_remainder -.Lmemset_fill_loop: - stc %r3,0(%r1) - mvc 1(255,%r1),0(%r1) - la %r1,256(%r1) - brctg %r5,.Lmemset_fill_loop -.Lmemset_fill_remainder: - stc %r3,0(%r1) - exrl %r4,.Lmemset_mvc - BR_EX %r14 -.Lmemset_fill_exit: - stc %r3,0(%r1) - BR_EX %r14 -.Lmemset_xc: - xc 0(1,%r1),0(%r1) -.Lmemset_mvc: - mvc 1(1,%r1),0(%r1) -SYM_FUNC_END(__memset) -EXPORT_SYMBOL(__memset) - -SYM_FUNC_ALIAS(memset, __memset) -EXPORT_SYMBOL(memset) - -/* - * memcpy implementation - * - * void *memcpy(void *dest, const void *src, size_t n) - */ -SYM_FUNC_START(__memcpy) - ltgr %r4,%r4 - jz .Lmemcpy_exit - aghi %r4,-1 - srlg %r5,%r4,8 - ltgr %r5,%r5 - lgr %r1,%r2 - jnz .Lmemcpy_loop -.Lmemcpy_remainder: - exrl %r4,.Lmemcpy_mvc -.Lmemcpy_exit: - BR_EX %r14 -.Lmemcpy_loop: - mvc 0(256,%r1),0(%r3) - la %r1,256(%r1) - la %r3,256(%r3) - brctg %r5,.Lmemcpy_loop - j .Lmemcpy_remainder -.Lmemcpy_mvc: - mvc 0(1,%r1),0(%r3) -SYM_FUNC_END(__memcpy) -EXPORT_SYMBOL(__memcpy) - -SYM_FUNC_ALIAS(memcpy, __memcpy) -EXPORT_SYMBOL(memcpy) - -/* - * __memset16/32/64 - * - * void *__memset16(uint16_t *s, uint16_t v, size_t count) - * void *__memset32(uint32_t *s, uint32_t v, size_t count) - * void *__memset64(uint64_t *s, uint64_t v, size_t count) - */ -.macro __MEMSET bits,bytes,insn -SYM_FUNC_START(__memset\bits) - ltgr %r4,%r4 - jz .L__memset_exit\bits - cghi %r4,\bytes - je .L__memset_store\bits - aghi %r4,-(\bytes+1) - srlg %r5,%r4,8 - ltgr %r5,%r5 - lgr %r1,%r2 - jz .L__memset_remainder\bits -.L__memset_loop\bits: - \insn %r3,0(%r1) - mvc \bytes(256-\bytes,%r1),0(%r1) - la %r1,256(%r1) - brctg %r5,.L__memset_loop\bits -.L__memset_remainder\bits: - \insn %r3,0(%r1) - exrl %r4,.L__memset_mvc\bits - BR_EX %r14 -.L__memset_store\bits: - \insn %r3,0(%r2) -.L__memset_exit\bits: - BR_EX %r14 -.L__memset_mvc\bits: - mvc \bytes(1,%r1),0(%r1) -SYM_FUNC_END(__memset\bits) -.endm - -__MEMSET 16,2,sth -EXPORT_SYMBOL(__memset16) - -__MEMSET 32,4,st -EXPORT_SYMBOL(__memset32) - -__MEMSET 64,8,stg -EXPORT_SYMBOL(__memset64) diff --git a/arch/s390/lib/string.c b/arch/s390/lib/string.c index 757f589601981..32e0e6b1e6239 100644 --- a/arch/s390/lib/string.c +++ b/arch/s390/lib/string.c @@ -15,8 +15,218 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/export.h> +#include <asm/facility.h> #include <asm/asm.h> +#define SYMBOL_FUNCTION_ALIAS(alias, name) \ +asm(".globl " __stringify(alias) "\n\t" \ + ".set " __stringify(alias) "," __stringify(name)) + +#ifdef __HAVE_ARCH_MEMMOVE +noinstr void *__memmove(void *dest, const void *src, size_t n) +{ + const char *s = src; + char *d = dest; + + if (!n) + return dest; + if ((d <= s || d >= s + n)) { + /* Forward copy */ + while (n >= 256) { + asm volatile( + " mvc 0(256,%[d]),0(%[s])\n" + : + : [d] "a" (d), [s] "a" (s) + : "memory"); + d += 256; + s += 256; + n -= 256; + } + if (n) { + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: mvc 0(1,%[d]),0(%[s])\n" + "1:" + : + : [d] "a" (d), [s] "a" (s), [n] "a" (n - 1) + : "memory"); + } + return dest; + } + /* Backward copy */ + if (test_facility(61)) { + /* Use mvcrl instruction if available */ + while (n >= 256) { + asm volatile( + " lghi %%r0,255\n" + " .insn sse,0xe50a00000000,%[d],%[s]\n" + : [d] "=Q" (*(d + n - 256)) + : [s] "Q" (*(s + n - 256)) + : "0", "memory"); + n -= 256; + } + if (n) { + asm volatile( + " lgr %%r0,%[n]\n" + " .insn sse,0xe50a00000000,%[d],%[s]\n" + : [d] "=Q" (*d) + : [s] "Q" (*s), [n] "d" (n - 1) + : "0", "memory"); + } + } else { + while (n--) + d[n] = s[n]; + } + return dest; +} +SYMBOL_FUNCTION_ALIAS(memmove, __memmove); +EXPORT_SYMBOL(__memmove); +EXPORT_SYMBOL(memmove); +#endif + +#ifdef __HAVE_ARCH_MEMSET +noinstr void *__memset(void *s, int c, size_t n) +{ + char *xs = s; + + if (!n) + return s; + if (!c) { + /* Clear memory */ + while (n >= 256) { + asm volatile( + " xc 0(256,%[xs]),0(%[xs])" + : + : [xs] "a" (xs) + : "cc", "memory"); + xs += 256; + n -= 256; + } + if (!n) + return s; + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: xc 0(1,%[xs]),0(%[xs])\n" + "1:" + : + : [xs] "a" (xs), [n] "a" (n - 1) + : "cc", "memory"); + } else { + /* Fill memory */ + while (n >= 256) { + *xs = c; + asm volatile( + " mvc 1(255,%[xs]),0(%[xs])" + : + : [xs] "a" (xs) + : "memory"); + xs += 256; + n -= 256; + } + if (!n) + return s; + *xs = c; + if (n == 1) + return s; + asm volatile( + " exrl %[n],0f\n" + " j 1f\n" + "0: mvc 1(1,%[xs]),0(%[xs])\n" + "1:" + : + : [xs] "a" (xs), [n] "a" (n - 2) + : "memory"); + } + return s; +} +SYMBOL_FUNCTION_ALIAS(memset, __memset); +EXPORT_SYMBOL(__memset); +EXPORT_SYMBOL(memset); +#endif + +#ifdef __HAVE_ARCH_MEMCPY +noinstr void *__memcpy(void *dest, const void *src, size_t n) +{ + void *d = dest; + + if (!n) + return d; + while (n >= 256) { + asm volatile( + " mvc 0(256,%[dest]),0(%[src])" + : + : [dest] "a" (dest), [src] "a" (src) + : "memory"); + dest += 256; + src += 256; + n -= 256; + } + if (!n) + return d; + asm volatile( + " exrl %[n],1f\n" + " j 2f\n" + "1: mvc 0(1,%[dest]),0(%[src])\n" + "2:" + : + : [dest] "a" (dest), [src] "a" (src), [n] "a" (n - 1) + : "memory"); + return d; +} +SYMBOL_FUNCTION_ALIAS(memcpy, __memcpy); +EXPORT_SYMBOL(__memcpy); +EXPORT_SYMBOL(memcpy); +#endif + +#define DEFINE_MEMSET(_bits, _bytes, _type) \ +void *__memset##_bits(_type *s, _type v, size_t n) \ +{ \ + _type *xs = s; \ + \ + if (!n) \ + return s; \ + while (n >= 256) { \ + *xs = v; \ + asm volatile( \ + " mvc %[_b](256-%[_b],%[xs]),0(%[xs])\n" \ + : \ + : [xs] "a" (xs), [_b] "i" (_bytes) \ + : "memory"); \ + xs = (_type *)((char *)xs + 256); \ + n -= 256; \ + } \ + if (!n) \ + return s; \ + *xs = v; \ + if (n == _bytes) \ + return s; \ + n -= _bytes + 1; \ + asm volatile( \ + " exrl %[n],1f\n" \ + " j 2f\n" \ + "1: mvc %[_b](1,%[xs]),0(%[xs])\n" \ + "2:" \ + : \ + : [n] "a" (n), [xs] "a" (xs), [_b] "i" (_bytes) \ + : "memory"); \ + return s; \ +} \ +EXPORT_SYMBOL(__memset##_bits) + +#ifdef __HAVE_ARCH_MEMSET16 +DEFINE_MEMSET(16, 2, uint16_t); +#endif + +#ifdef __HAVE_ARCH_MEMSET32 +DEFINE_MEMSET(32, 4, uint32_t); +#endif + +#ifdef __HAVE_ARCH_MEMSET64 +DEFINE_MEMSET(64, 8, uint64_t); +#endif + /* * Helper functions to find the end of a string */ @@ -105,32 +315,6 @@ EXPORT_SYMBOL(strcat); #endif /** - * strlcat - Append a length-limited, %NUL-terminated string to another - * @dest: The string to be appended to - * @src: The string to append to it - * @n: The size of the destination buffer. - */ -#ifdef __HAVE_ARCH_STRLCAT -size_t strlcat(char *dest, const char *src, size_t n) -{ - size_t dsize = __strend(dest) - dest; - size_t len = __strend(src) - src; - size_t res = dsize + len; - - if (dsize < n) { - dest += dsize; - n -= dsize; - if (len >= n) - len = n - 1; - dest[len] = '\0'; - memcpy(dest, src, len); - } - return res; -} -EXPORT_SYMBOL(strlcat); -#endif - -/** * strncat - Append a length-limited, %NUL-terminated string to another * @dest: The string to be appended to * @src: The string to append to it diff --git a/arch/s390/lib/tishift.S b/arch/s390/lib/tishift.S deleted file mode 100644 index 96214f51f49b8..0000000000000 --- a/arch/s390/lib/tishift.S +++ /dev/null @@ -1,63 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#include <linux/export.h> -#include <linux/linkage.h> -#include <asm/nospec-insn.h> - - .section .noinstr.text, "ax" - - GEN_BR_THUNK %r14 - -SYM_FUNC_START(__ashlti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - srlg %r3,%r1,0(%r3) - sllg %r0,%r0,0(%r4) - sllg %r1,%r1,0(%r4) - ogr %r0,%r3 - j 1f -0: sllg %r0,%r1,-64(%r4) - lghi %r1,0 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__ashlti3) -EXPORT_SYMBOL(__ashlti3) - -SYM_FUNC_START(__ashrti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - sllg %r3,%r0,0(%r3) - srlg %r1,%r1,0(%r4) - srag %r0,%r0,0(%r4) - ogr %r1,%r3 - j 1f -0: srag %r1,%r0,-64(%r4) - srag %r0,%r0,63 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__ashrti3) -EXPORT_SYMBOL(__ashrti3) - -SYM_FUNC_START(__lshrti3) - lmg %r0,%r1,0(%r3) - cije %r4,0,1f - lhi %r3,64 - sr %r3,%r4 - jnh 0f - sllg %r3,%r0,0(%r3) - srlg %r1,%r1,0(%r4) - srlg %r0,%r0,0(%r4) - ogr %r1,%r3 - j 1f -0: srlg %r1,%r0,-64(%r4) - lghi %r0,0 -1: stmg %r0,%r1,0(%r2) - BR_EX %r14 -SYM_FUNC_END(__lshrti3) -EXPORT_SYMBOL(__lshrti3) diff --git a/arch/s390/lib/tishift.c b/arch/s390/lib/tishift.c new file mode 100644 index 0000000000000..bb16cf639af3c --- /dev/null +++ b/arch/s390/lib/tishift.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/export.h> +#include <linux/types.h> +#include "tishift.h" + +union ti { + __int128_t val; + struct { + u64 high; + u64 low; + }; +}; + +noinstr __int128_t __ashlti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.high = (ti.high << shift) | (ti.low >> (64 - shift)); + ti.low = ti.low << shift; + } else { + ti.high = ti.low << (shift - 64); + ti.low = 0; + } + return ti.val; +} +EXPORT_SYMBOL(__ashlti3); + +noinstr __int128_t __ashrti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.low = (ti.low >> shift) | (ti.high << (64 - shift)); + ti.high = (int64_t)ti.high >> shift; + } else { + ti.low = (int64_t)ti.high >> (shift - 64); + ti.high = (int64_t)ti.high >> 63; + } + return ti.val; +} +EXPORT_SYMBOL(__ashrti3); + +noinstr __int128_t __lshrti3(__int128_t a, int shift) +{ + union ti ti = { .val = a }; + + if (!shift) + return ti.val; + if (shift < 64) { + ti.low = (ti.low >> shift) | (ti.high << (64 - shift)); + ti.high = ti.high >> shift; + } else { + ti.low = ti.high >> (shift - 64); + ti.high = 0; + } + return ti.val; +} +EXPORT_SYMBOL(__lshrti3); diff --git a/arch/s390/lib/tishift.h b/arch/s390/lib/tishift.h new file mode 100644 index 0000000000000..43a9b8c8e545d --- /dev/null +++ b/arch/s390/lib/tishift.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _S390_LIB_TISHIFT_H +#define _S390_LIB_TISHIFT_H + +__int128_t __ashlti3(__int128_t a, int b); +__int128_t __ashrti3(__int128_t a, int b); +__int128_t __lshrti3(__int128_t a, int b); + +#endif /* _S390_LIB_TISHIFT_H */ diff --git a/arch/s390/mm/extable.c b/arch/s390/mm/extable.c index 7498e858c4019..063b4346742d9 100644 --- a/arch/s390/mm/extable.c +++ b/arch/s390/mm/extable.c @@ -50,22 +50,6 @@ static bool ex_handler_ua_load_reg(const struct exception_table_entry *ex, return true; } -static bool ex_handler_zeropad(const struct exception_table_entry *ex, struct pt_regs *regs) -{ - unsigned int reg_addr = FIELD_GET(EX_DATA_REG_ADDR, ex->data); - unsigned int reg_data = FIELD_GET(EX_DATA_REG_ERR, ex->data); - unsigned long data, addr, offset; - - addr = regs->gprs[reg_addr]; - offset = addr & (sizeof(unsigned long) - 1); - addr &= ~(sizeof(unsigned long) - 1); - data = *(unsigned long *)addr; - data <<= BITS_PER_BYTE * offset; - regs->gprs[reg_data] = data; - regs->psw.addr = extable_fixup(ex); - return true; -} - static bool ex_handler_fpc(const struct exception_table_entry *ex, struct pt_regs *regs) { fpu_sfpc(0); @@ -134,8 +118,6 @@ bool fixup_exception(struct pt_regs *regs) return ex_handler_ua_load_reg(ex, false, regs); case EX_TYPE_UA_LOAD_REGPAIR: return ex_handler_ua_load_reg(ex, true, regs); - case EX_TYPE_ZEROPAD: - return ex_handler_zeropad(ex, regs); case EX_TYPE_FPC: return ex_handler_fpc(ex, regs); case EX_TYPE_UA_MVCOS_TO: diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c index 1f72efc2a579f..36bd9530db528 100644 --- a/arch/s390/mm/init.c +++ b/arch/s390/mm/init.c @@ -84,6 +84,8 @@ void __init arch_setup_zero_pages(void) empty_zero_page = (unsigned long)memblock_alloc_or_panic(PAGE_SIZE << order, PAGE_SIZE); zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK; + + set_memory_ro(empty_zero_page, 1UL << order); } void __init arch_zone_limits_init(unsigned long *max_zone_pfns) diff --git a/arch/s390/purgatory/Makefile b/arch/s390/purgatory/Makefile index 95a8ac45b67e6..e74410bb1b884 100644 --- a/arch/s390/purgatory/Makefile +++ b/arch/s390/purgatory/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 -purgatory-y := head.o purgatory.o string.o sha256.o mem.o +purgatory-y := head.o purgatory.o string.o sha256.o targets += $(purgatory-y) purgatory.lds purgatory purgatory.chk purgatory.ro PURGATORY_OBJS = $(addprefix $(obj)/,$(purgatory-y)) @@ -10,8 +10,7 @@ $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE CFLAGS_sha256.o := -D__NO_FORTIFY -$(obj)/mem.o: $(srctree)/arch/s390/lib/mem.S FORCE - $(call if_changed_rule,as_o_S) +CC_FLAGS_MARCH_MINIMUM := -march=z10 KBUILD_CFLAGS := $(CC_FLAGS_DIALECT) -fno-strict-aliasing -Wall -Wstrict-prototypes KBUILD_CFLAGS += -Wno-pointer-sign -Wno-sign-compare @@ -19,12 +18,12 @@ KBUILD_CFLAGS += -fno-zero-initialized-in-bss -fno-builtin -ffreestanding KBUILD_CFLAGS += -Os -m64 -msoft-float -fno-common KBUILD_CFLAGS += -fno-stack-protector KBUILD_CFLAGS += -DDISABLE_BRANCH_PROFILING -KBUILD_CFLAGS += -D__DISABLE_EXPORTS +KBUILD_CFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS KBUILD_CFLAGS += $(CLANG_FLAGS) KBUILD_CFLAGS += $(call cc-option,-fno-PIE) KBUILD_CFLAGS += $(call cc-option, -Wno-default-const-init-unsafe) KBUILD_AFLAGS := $(filter-out -DCC_USING_EXPOLINE,$(KBUILD_AFLAGS)) -KBUILD_AFLAGS += -D__DISABLE_EXPORTS +KBUILD_AFLAGS += $(CC_FLAGS_MARCH_MINIMUM) -D__DISABLE_EXPORTS # Since we link purgatory with -r unresolved symbols are not checked, so we # also link a purgatory.chk binary without -r to check for unresolved symbols. diff --git a/arch/s390/purgatory/purgatory.lds.S b/arch/s390/purgatory/purgatory.lds.S index 482eb4fbcef19..387d0db4085f5 100644 --- a/arch/s390/purgatory/purgatory.lds.S +++ b/arch/s390/purgatory/purgatory.lds.S @@ -19,6 +19,7 @@ SECTIONS _text = .; /* Text */ *(.text) *(.text.*) + *(.noinstr.text) _etext = . ; } .rodata : { |