1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/module.h>
3 #include <linux/cpu.h>
4 #include <linux/smp.h>
5 #include <asm/text-patching.h>
6 #include <asm/alternative.h>
7 #include <asm/facility.h>
8 #include <asm/nospec-branch.h>
9 
10 #define MAX_PATCH_LEN (255 - 1)
11 
12 static int __initdata_or_module alt_instr_disabled;
13 
disable_alternative_instructions(char * str)14 static int __init disable_alternative_instructions(char *str)
15 {
16 	alt_instr_disabled = 1;
17 	return 0;
18 }
19 
20 early_param("noaltinstr", disable_alternative_instructions);
21 
22 struct brcl_insn {
23 	u16 opc;
24 	s32 disp;
25 } __packed;
26 
27 static u16 __initdata_or_module nop16 = 0x0700;
28 static u32 __initdata_or_module nop32 = 0x47000000;
29 static struct brcl_insn __initdata_or_module nop48 = {
30 	0xc004, 0
31 };
32 
33 static const void *nops[] __initdata_or_module = {
34 	&nop16,
35 	&nop32,
36 	&nop48
37 };
38 
add_jump_padding(void * insns,unsigned int len)39 static void __init_or_module add_jump_padding(void *insns, unsigned int len)
40 {
41 	struct brcl_insn brcl = {
42 		0xc0f4,
43 		len / 2
44 	};
45 
46 	memcpy(insns, &brcl, sizeof(brcl));
47 	insns += sizeof(brcl);
48 	len -= sizeof(brcl);
49 
50 	while (len > 0) {
51 		memcpy(insns, &nop16, 2);
52 		insns += 2;
53 		len -= 2;
54 	}
55 }
56 
add_padding(void * insns,unsigned int len)57 static void __init_or_module add_padding(void *insns, unsigned int len)
58 {
59 	if (len > 6)
60 		add_jump_padding(insns, len);
61 	else if (len >= 2)
62 		memcpy(insns, nops[len / 2 - 1], len);
63 }
64 
__apply_alternatives(struct alt_instr * start,struct alt_instr * end)65 static void __init_or_module __apply_alternatives(struct alt_instr *start,
66 						  struct alt_instr *end)
67 {
68 	struct alt_instr *a;
69 	u8 *instr, *replacement;
70 	u8 insnbuf[MAX_PATCH_LEN];
71 
72 	/*
73 	 * The scan order should be from start to end. A later scanned
74 	 * alternative code can overwrite previously scanned alternative code.
75 	 */
76 	for (a = start; a < end; a++) {
77 		int insnbuf_sz = 0;
78 
79 		instr = (u8 *)&a->instr_offset + a->instr_offset;
80 		replacement = (u8 *)&a->repl_offset + a->repl_offset;
81 
82 		if (!__test_facility(a->facility, alt_stfle_fac_list))
83 			continue;
84 
85 		if (unlikely(a->instrlen % 2 || a->replacementlen % 2)) {
86 			WARN_ONCE(1, "cpu alternatives instructions length is "
87 				     "odd, skipping patching\n");
88 			continue;
89 		}
90 
91 		memcpy(insnbuf, replacement, a->replacementlen);
92 		insnbuf_sz = a->replacementlen;
93 
94 		if (a->instrlen > a->replacementlen) {
95 			add_padding(insnbuf + a->replacementlen,
96 				    a->instrlen - a->replacementlen);
97 			insnbuf_sz += a->instrlen - a->replacementlen;
98 		}
99 
100 		s390_kernel_write(instr, insnbuf, insnbuf_sz);
101 	}
102 }
103 
apply_alternatives(struct alt_instr * start,struct alt_instr * end)104 void __init_or_module apply_alternatives(struct alt_instr *start,
105 					 struct alt_instr *end)
106 {
107 	if (!alt_instr_disabled)
108 		__apply_alternatives(start, end);
109 }
110 
111 extern struct alt_instr __alt_instructions[], __alt_instructions_end[];
apply_alternative_instructions(void)112 void __init apply_alternative_instructions(void)
113 {
114 	apply_alternatives(__alt_instructions, __alt_instructions_end);
115 }
116 
do_sync_core(void * info)117 static void do_sync_core(void *info)
118 {
119 	sync_core();
120 }
121 
text_poke_sync(void)122 void text_poke_sync(void)
123 {
124 	on_each_cpu(do_sync_core, NULL, 1);
125 }
126 
text_poke_sync_lock(void)127 void text_poke_sync_lock(void)
128 {
129 	cpus_read_lock();
130 	text_poke_sync();
131 	cpus_read_unlock();
132 }
133