1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * FPU register's regset abstraction, for ptrace, core dumps, etc.
4  */
5 #include <linux/sched/task_stack.h>
6 #include <linux/vmalloc.h>
7 
8 #include <asm/fpu/api.h>
9 #include <asm/fpu/signal.h>
10 #include <asm/fpu/regset.h>
11 
12 #include "context.h"
13 #include "internal.h"
14 #include "legacy.h"
15 #include "xstate.h"
16 
17 /*
18  * The xstateregs_active() routine is the same as the regset_fpregs_active() routine,
19  * as the "regset->n" for the xstate regset will be updated based on the feature
20  * capabilities supported by the xsave.
21  */
regset_fpregs_active(struct task_struct * target,const struct user_regset * regset)22 int regset_fpregs_active(struct task_struct *target, const struct user_regset *regset)
23 {
24 	return regset->n;
25 }
26 
regset_xregset_fpregs_active(struct task_struct * target,const struct user_regset * regset)27 int regset_xregset_fpregs_active(struct task_struct *target, const struct user_regset *regset)
28 {
29 	if (boot_cpu_has(X86_FEATURE_FXSR))
30 		return regset->n;
31 	else
32 		return 0;
33 }
34 
35 /*
36  * The regset get() functions are invoked from:
37  *
38  *   - coredump to dump the current task's fpstate. If the current task
39  *     owns the FPU then the memory state has to be synchronized and the
40  *     FPU register state preserved. Otherwise fpstate is already in sync.
41  *
42  *   - ptrace to dump fpstate of a stopped task, in which case the registers
43  *     have already been saved to fpstate on context switch.
44  */
sync_fpstate(struct fpu * fpu)45 static void sync_fpstate(struct fpu *fpu)
46 {
47 	if (fpu == &current->thread.fpu)
48 		fpu_sync_fpstate(fpu);
49 }
50 
51 /*
52  * Invalidate cached FPU registers before modifying the stopped target
53  * task's fpstate.
54  *
55  * This forces the target task on resume to restore the FPU registers from
56  * modified fpstate. Otherwise the task might skip the restore and operate
57  * with the cached FPU registers which discards the modifications.
58  */
fpu_force_restore(struct fpu * fpu)59 static void fpu_force_restore(struct fpu *fpu)
60 {
61 	/*
62 	 * Only stopped child tasks can be used to modify the FPU
63 	 * state in the fpstate buffer:
64 	 */
65 	WARN_ON_FPU(fpu == &current->thread.fpu);
66 
67 	__fpu_invalidate_fpregs_state(fpu);
68 }
69 
xfpregs_get(struct task_struct * target,const struct user_regset * regset,struct membuf to)70 int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
71 		struct membuf to)
72 {
73 	struct fpu *fpu = &target->thread.fpu;
74 
75 	if (!cpu_feature_enabled(X86_FEATURE_FXSR))
76 		return -ENODEV;
77 
78 	sync_fpstate(fpu);
79 
80 	if (!use_xsave()) {
81 		return membuf_write(&to, &fpu->fpstate->regs.fxsave,
82 				    sizeof(fpu->fpstate->regs.fxsave));
83 	}
84 
85 	copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_FX);
86 	return 0;
87 }
88 
xfpregs_set(struct task_struct * target,const struct user_regset * regset,unsigned int pos,unsigned int count,const void * kbuf,const void __user * ubuf)89 int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
90 		unsigned int pos, unsigned int count,
91 		const void *kbuf, const void __user *ubuf)
92 {
93 	struct fpu *fpu = &target->thread.fpu;
94 	struct user32_fxsr_struct newstate;
95 	int ret;
96 
97 	BUILD_BUG_ON(sizeof(newstate) != sizeof(struct fxregs_state));
98 
99 	if (!cpu_feature_enabled(X86_FEATURE_FXSR))
100 		return -ENODEV;
101 
102 	/* No funny business with partial or oversized writes is permitted. */
103 	if (pos != 0 || count != sizeof(newstate))
104 		return -EINVAL;
105 
106 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &newstate, 0, -1);
107 	if (ret)
108 		return ret;
109 
110 	/* Do not allow an invalid MXCSR value. */
111 	if (newstate.mxcsr & ~mxcsr_feature_mask)
112 		return -EINVAL;
113 
114 	fpu_force_restore(fpu);
115 
116 	/* Copy the state  */
117 	memcpy(&fpu->fpstate->regs.fxsave, &newstate, sizeof(newstate));
118 
119 	/* Clear xmm8..15 */
120 	BUILD_BUG_ON(sizeof(fpu->__fpstate.regs.fxsave.xmm_space) != 16 * 16);
121 	memset(&fpu->fpstate->regs.fxsave.xmm_space[8], 0, 8 * 16);
122 
123 	/* Mark FP and SSE as in use when XSAVE is enabled */
124 	if (use_xsave())
125 		fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FPSSE;
126 
127 	return 0;
128 }
129 
xstateregs_get(struct task_struct * target,const struct user_regset * regset,struct membuf to)130 int xstateregs_get(struct task_struct *target, const struct user_regset *regset,
131 		struct membuf to)
132 {
133 	if (!cpu_feature_enabled(X86_FEATURE_XSAVE))
134 		return -ENODEV;
135 
136 	sync_fpstate(&target->thread.fpu);
137 
138 	copy_xstate_to_uabi_buf(to, target, XSTATE_COPY_XSAVE);
139 	return 0;
140 }
141 
xstateregs_set(struct task_struct * target,const struct user_regset * regset,unsigned int pos,unsigned int count,const void * kbuf,const void __user * ubuf)142 int xstateregs_set(struct task_struct *target, const struct user_regset *regset,
143 		  unsigned int pos, unsigned int count,
144 		  const void *kbuf, const void __user *ubuf)
145 {
146 	struct fpu *fpu = &target->thread.fpu;
147 	struct xregs_state *tmpbuf = NULL;
148 	int ret;
149 
150 	if (!cpu_feature_enabled(X86_FEATURE_XSAVE))
151 		return -ENODEV;
152 
153 	/*
154 	 * A whole standard-format XSAVE buffer is needed:
155 	 */
156 	if (pos != 0 || count != fpu_user_cfg.max_size)
157 		return -EFAULT;
158 
159 	if (!kbuf) {
160 		tmpbuf = vmalloc(count);
161 		if (!tmpbuf)
162 			return -ENOMEM;
163 
164 		if (copy_from_user(tmpbuf, ubuf, count)) {
165 			ret = -EFAULT;
166 			goto out;
167 		}
168 	}
169 
170 	fpu_force_restore(fpu);
171 	ret = copy_uabi_from_kernel_to_xstate(fpu->fpstate, kbuf ?: tmpbuf);
172 
173 out:
174 	vfree(tmpbuf);
175 	return ret;
176 }
177 
178 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
179 
180 /*
181  * FPU tag word conversions.
182  */
183 
twd_i387_to_fxsr(unsigned short twd)184 static inline unsigned short twd_i387_to_fxsr(unsigned short twd)
185 {
186 	unsigned int tmp; /* to avoid 16 bit prefixes in the code */
187 
188 	/* Transform each pair of bits into 01 (valid) or 00 (empty) */
189 	tmp = ~twd;
190 	tmp = (tmp | (tmp>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
191 	/* and move the valid bits to the lower byte. */
192 	tmp = (tmp | (tmp >> 1)) & 0x3333; /* 00VV00VV00VV00VV */
193 	tmp = (tmp | (tmp >> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
194 	tmp = (tmp | (tmp >> 4)) & 0x00ff; /* 00000000VVVVVVVV */
195 
196 	return tmp;
197 }
198 
199 #define FPREG_ADDR(f, n)	((void *)&(f)->st_space + (n) * 16)
200 #define FP_EXP_TAG_VALID	0
201 #define FP_EXP_TAG_ZERO		1
202 #define FP_EXP_TAG_SPECIAL	2
203 #define FP_EXP_TAG_EMPTY	3
204 
twd_fxsr_to_i387(struct fxregs_state * fxsave)205 static inline u32 twd_fxsr_to_i387(struct fxregs_state *fxsave)
206 {
207 	struct _fpxreg *st;
208 	u32 tos = (fxsave->swd >> 11) & 7;
209 	u32 twd = (unsigned long) fxsave->twd;
210 	u32 tag;
211 	u32 ret = 0xffff0000u;
212 	int i;
213 
214 	for (i = 0; i < 8; i++, twd >>= 1) {
215 		if (twd & 0x1) {
216 			st = FPREG_ADDR(fxsave, (i - tos) & 7);
217 
218 			switch (st->exponent & 0x7fff) {
219 			case 0x7fff:
220 				tag = FP_EXP_TAG_SPECIAL;
221 				break;
222 			case 0x0000:
223 				if (!st->significand[0] &&
224 				    !st->significand[1] &&
225 				    !st->significand[2] &&
226 				    !st->significand[3])
227 					tag = FP_EXP_TAG_ZERO;
228 				else
229 					tag = FP_EXP_TAG_SPECIAL;
230 				break;
231 			default:
232 				if (st->significand[3] & 0x8000)
233 					tag = FP_EXP_TAG_VALID;
234 				else
235 					tag = FP_EXP_TAG_SPECIAL;
236 				break;
237 			}
238 		} else {
239 			tag = FP_EXP_TAG_EMPTY;
240 		}
241 		ret |= tag << (2 * i);
242 	}
243 	return ret;
244 }
245 
246 /*
247  * FXSR floating point environment conversions.
248  */
249 
__convert_from_fxsr(struct user_i387_ia32_struct * env,struct task_struct * tsk,struct fxregs_state * fxsave)250 static void __convert_from_fxsr(struct user_i387_ia32_struct *env,
251 				struct task_struct *tsk,
252 				struct fxregs_state *fxsave)
253 {
254 	struct _fpreg *to = (struct _fpreg *) &env->st_space[0];
255 	struct _fpxreg *from = (struct _fpxreg *) &fxsave->st_space[0];
256 	int i;
257 
258 	env->cwd = fxsave->cwd | 0xffff0000u;
259 	env->swd = fxsave->swd | 0xffff0000u;
260 	env->twd = twd_fxsr_to_i387(fxsave);
261 
262 #ifdef CONFIG_X86_64
263 	env->fip = fxsave->rip;
264 	env->foo = fxsave->rdp;
265 	/*
266 	 * should be actually ds/cs at fpu exception time, but
267 	 * that information is not available in 64bit mode.
268 	 */
269 	env->fcs = task_pt_regs(tsk)->cs;
270 	if (tsk == current) {
271 		savesegment(ds, env->fos);
272 	} else {
273 		env->fos = tsk->thread.ds;
274 	}
275 	env->fos |= 0xffff0000;
276 #else
277 	env->fip = fxsave->fip;
278 	env->fcs = (u16) fxsave->fcs | ((u32) fxsave->fop << 16);
279 	env->foo = fxsave->foo;
280 	env->fos = fxsave->fos;
281 #endif
282 
283 	for (i = 0; i < 8; ++i)
284 		memcpy(&to[i], &from[i], sizeof(to[0]));
285 }
286 
287 void
convert_from_fxsr(struct user_i387_ia32_struct * env,struct task_struct * tsk)288 convert_from_fxsr(struct user_i387_ia32_struct *env, struct task_struct *tsk)
289 {
290 	__convert_from_fxsr(env, tsk, &tsk->thread.fpu.fpstate->regs.fxsave);
291 }
292 
convert_to_fxsr(struct fxregs_state * fxsave,const struct user_i387_ia32_struct * env)293 void convert_to_fxsr(struct fxregs_state *fxsave,
294 		     const struct user_i387_ia32_struct *env)
295 
296 {
297 	struct _fpreg *from = (struct _fpreg *) &env->st_space[0];
298 	struct _fpxreg *to = (struct _fpxreg *) &fxsave->st_space[0];
299 	int i;
300 
301 	fxsave->cwd = env->cwd;
302 	fxsave->swd = env->swd;
303 	fxsave->twd = twd_i387_to_fxsr(env->twd);
304 	fxsave->fop = (u16) ((u32) env->fcs >> 16);
305 #ifdef CONFIG_X86_64
306 	fxsave->rip = env->fip;
307 	fxsave->rdp = env->foo;
308 	/* cs and ds ignored */
309 #else
310 	fxsave->fip = env->fip;
311 	fxsave->fcs = (env->fcs & 0xffff);
312 	fxsave->foo = env->foo;
313 	fxsave->fos = env->fos;
314 #endif
315 
316 	for (i = 0; i < 8; ++i)
317 		memcpy(&to[i], &from[i], sizeof(from[0]));
318 }
319 
fpregs_get(struct task_struct * target,const struct user_regset * regset,struct membuf to)320 int fpregs_get(struct task_struct *target, const struct user_regset *regset,
321 	       struct membuf to)
322 {
323 	struct fpu *fpu = &target->thread.fpu;
324 	struct user_i387_ia32_struct env;
325 	struct fxregs_state fxsave, *fx;
326 
327 	sync_fpstate(fpu);
328 
329 	if (!cpu_feature_enabled(X86_FEATURE_FPU))
330 		return fpregs_soft_get(target, regset, to);
331 
332 	if (!cpu_feature_enabled(X86_FEATURE_FXSR)) {
333 		return membuf_write(&to, &fpu->fpstate->regs.fsave,
334 				    sizeof(struct fregs_state));
335 	}
336 
337 	if (use_xsave()) {
338 		struct membuf mb = { .p = &fxsave, .left = sizeof(fxsave) };
339 
340 		/* Handle init state optimized xstate correctly */
341 		copy_xstate_to_uabi_buf(mb, target, XSTATE_COPY_FP);
342 		fx = &fxsave;
343 	} else {
344 		fx = &fpu->fpstate->regs.fxsave;
345 	}
346 
347 	__convert_from_fxsr(&env, target, fx);
348 	return membuf_write(&to, &env, sizeof(env));
349 }
350 
fpregs_set(struct task_struct * target,const struct user_regset * regset,unsigned int pos,unsigned int count,const void * kbuf,const void __user * ubuf)351 int fpregs_set(struct task_struct *target, const struct user_regset *regset,
352 	       unsigned int pos, unsigned int count,
353 	       const void *kbuf, const void __user *ubuf)
354 {
355 	struct fpu *fpu = &target->thread.fpu;
356 	struct user_i387_ia32_struct env;
357 	int ret;
358 
359 	/* No funny business with partial or oversized writes is permitted. */
360 	if (pos != 0 || count != sizeof(struct user_i387_ia32_struct))
361 		return -EINVAL;
362 
363 	if (!cpu_feature_enabled(X86_FEATURE_FPU))
364 		return fpregs_soft_set(target, regset, pos, count, kbuf, ubuf);
365 
366 	ret = user_regset_copyin(&pos, &count, &kbuf, &ubuf, &env, 0, -1);
367 	if (ret)
368 		return ret;
369 
370 	fpu_force_restore(fpu);
371 
372 	if (cpu_feature_enabled(X86_FEATURE_FXSR))
373 		convert_to_fxsr(&fpu->fpstate->regs.fxsave, &env);
374 	else
375 		memcpy(&fpu->fpstate->regs.fsave, &env, sizeof(env));
376 
377 	/*
378 	 * Update the header bit in the xsave header, indicating the
379 	 * presence of FP.
380 	 */
381 	if (cpu_feature_enabled(X86_FEATURE_XSAVE))
382 		fpu->fpstate->regs.xsave.header.xfeatures |= XFEATURE_MASK_FP;
383 
384 	return 0;
385 }
386 
387 #endif	/* CONFIG_X86_32 || CONFIG_IA32_EMULATION */
388