1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Access kernel or user memory without faulting.
4  */
5 #include <linux/export.h>
6 #include <linux/mm.h>
7 #include <linux/uaccess.h>
8 
copy_from_kernel_nofault_allowed(const void * unsafe_src,size_t size)9 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
10 		size_t size)
11 {
12 	return true;
13 }
14 
15 #ifdef HAVE_GET_KERNEL_NOFAULT
16 
17 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label)	\
18 	while (len >= sizeof(type)) {					\
19 		__get_kernel_nofault(dst, src, type, err_label);		\
20 		dst += sizeof(type);					\
21 		src += sizeof(type);					\
22 		len -= sizeof(type);					\
23 	}
24 
copy_from_kernel_nofault(void * dst,const void * src,size_t size)25 long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
26 {
27 	unsigned long align = 0;
28 
29 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
30 		align = (unsigned long)dst | (unsigned long)src;
31 
32 	if (!copy_from_kernel_nofault_allowed(src, size))
33 		return -ERANGE;
34 
35 	pagefault_disable();
36 	if (!(align & 7))
37 		copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
38 	if (!(align & 3))
39 		copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
40 	if (!(align & 1))
41 		copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
42 	copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
43 	pagefault_enable();
44 	return 0;
45 Efault:
46 	pagefault_enable();
47 	return -EFAULT;
48 }
49 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
50 
51 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label)	\
52 	while (len >= sizeof(type)) {					\
53 		__put_kernel_nofault(dst, src, type, err_label);		\
54 		dst += sizeof(type);					\
55 		src += sizeof(type);					\
56 		len -= sizeof(type);					\
57 	}
58 
copy_to_kernel_nofault(void * dst,const void * src,size_t size)59 long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
60 {
61 	unsigned long align = 0;
62 
63 	if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
64 		align = (unsigned long)dst | (unsigned long)src;
65 
66 	pagefault_disable();
67 	if (!(align & 7))
68 		copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
69 	if (!(align & 3))
70 		copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
71 	if (!(align & 1))
72 		copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
73 	copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
74 	pagefault_enable();
75 	return 0;
76 Efault:
77 	pagefault_enable();
78 	return -EFAULT;
79 }
80 
strncpy_from_kernel_nofault(char * dst,const void * unsafe_addr,long count)81 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
82 {
83 	const void *src = unsafe_addr;
84 
85 	if (unlikely(count <= 0))
86 		return 0;
87 	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
88 		return -ERANGE;
89 
90 	pagefault_disable();
91 	do {
92 		__get_kernel_nofault(dst, src, u8, Efault);
93 		dst++;
94 		src++;
95 	} while (dst[-1] && src - unsafe_addr < count);
96 	pagefault_enable();
97 
98 	dst[-1] = '\0';
99 	return src - unsafe_addr;
100 Efault:
101 	pagefault_enable();
102 	dst[-1] = '\0';
103 	return -EFAULT;
104 }
105 #else /* HAVE_GET_KERNEL_NOFAULT */
106 /**
107  * copy_from_kernel_nofault(): safely attempt to read from kernel-space
108  * @dst: pointer to the buffer that shall take the data
109  * @src: address to read from
110  * @size: size of the data chunk
111  *
112  * Safely read from kernel address @src to the buffer at @dst.  If a kernel
113  * fault happens, handle that and return -EFAULT.  If @src is not a valid kernel
114  * address, return -ERANGE.
115  *
116  * We ensure that the copy_from_user is executed in atomic context so that
117  * do_page_fault() doesn't attempt to take mmap_lock.  This makes
118  * copy_from_kernel_nofault() suitable for use within regions where the caller
119  * already holds mmap_lock, or other locks which nest inside mmap_lock.
120  */
copy_from_kernel_nofault(void * dst,const void * src,size_t size)121 long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
122 {
123 	long ret;
124 	mm_segment_t old_fs = get_fs();
125 
126 	if (!copy_from_kernel_nofault_allowed(src, size))
127 		return -ERANGE;
128 
129 	set_fs(KERNEL_DS);
130 	pagefault_disable();
131 	ret = __copy_from_user_inatomic(dst, (__force const void __user *)src,
132 			size);
133 	pagefault_enable();
134 	set_fs(old_fs);
135 
136 	if (ret)
137 		return -EFAULT;
138 	return 0;
139 }
140 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
141 
142 /**
143  * copy_to_kernel_nofault(): safely attempt to write to a location
144  * @dst: address to write to
145  * @src: pointer to the data that shall be written
146  * @size: size of the data chunk
147  *
148  * Safely write to address @dst from the buffer at @src.  If a kernel fault
149  * happens, handle that and return -EFAULT.
150  */
copy_to_kernel_nofault(void * dst,const void * src,size_t size)151 long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
152 {
153 	long ret;
154 	mm_segment_t old_fs = get_fs();
155 
156 	set_fs(KERNEL_DS);
157 	pagefault_disable();
158 	ret = __copy_to_user_inatomic((__force void __user *)dst, src, size);
159 	pagefault_enable();
160 	set_fs(old_fs);
161 
162 	if (ret)
163 		return -EFAULT;
164 	return 0;
165 }
166 
167 /**
168  * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe
169  *				 address.
170  * @dst:   Destination address, in kernel space.  This buffer must be at
171  *         least @count bytes long.
172  * @unsafe_addr: Unsafe address.
173  * @count: Maximum number of bytes to copy, including the trailing NUL.
174  *
175  * Copies a NUL-terminated string from unsafe address to kernel buffer.
176  *
177  * On success, returns the length of the string INCLUDING the trailing NUL.
178  *
179  * If access fails, returns -EFAULT (some data may have been copied and the
180  * trailing NUL added).  If @unsafe_addr is not a valid kernel address, return
181  * -ERANGE.
182  *
183  * If @count is smaller than the length of the string, copies @count-1 bytes,
184  * sets the last byte of @dst buffer to NUL and returns @count.
185  */
strncpy_from_kernel_nofault(char * dst,const void * unsafe_addr,long count)186 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
187 {
188 	mm_segment_t old_fs = get_fs();
189 	const void *src = unsafe_addr;
190 	long ret;
191 
192 	if (unlikely(count <= 0))
193 		return 0;
194 	if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
195 		return -ERANGE;
196 
197 	set_fs(KERNEL_DS);
198 	pagefault_disable();
199 
200 	do {
201 		ret = __get_user(*dst++, (const char __user __force *)src++);
202 	} while (dst[-1] && ret == 0 && src - unsafe_addr < count);
203 
204 	dst[-1] = '\0';
205 	pagefault_enable();
206 	set_fs(old_fs);
207 
208 	return ret ? -EFAULT : src - unsafe_addr;
209 }
210 #endif /* HAVE_GET_KERNEL_NOFAULT */
211 
212 /**
213  * copy_from_user_nofault(): safely attempt to read from a user-space location
214  * @dst: pointer to the buffer that shall take the data
215  * @src: address to read from. This must be a user address.
216  * @size: size of the data chunk
217  *
218  * Safely read from user address @src to the buffer at @dst. If a kernel fault
219  * happens, handle that and return -EFAULT.
220  */
copy_from_user_nofault(void * dst,const void __user * src,size_t size)221 long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
222 {
223 	long ret = -EFAULT;
224 	mm_segment_t old_fs = force_uaccess_begin();
225 
226 	if (access_ok(src, size)) {
227 		pagefault_disable();
228 		ret = __copy_from_user_inatomic(dst, src, size);
229 		pagefault_enable();
230 	}
231 	force_uaccess_end(old_fs);
232 
233 	if (ret)
234 		return -EFAULT;
235 	return 0;
236 }
237 EXPORT_SYMBOL_GPL(copy_from_user_nofault);
238 
239 /**
240  * copy_to_user_nofault(): safely attempt to write to a user-space location
241  * @dst: address to write to
242  * @src: pointer to the data that shall be written
243  * @size: size of the data chunk
244  *
245  * Safely write to address @dst from the buffer at @src.  If a kernel fault
246  * happens, handle that and return -EFAULT.
247  */
copy_to_user_nofault(void __user * dst,const void * src,size_t size)248 long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
249 {
250 	long ret = -EFAULT;
251 	mm_segment_t old_fs = force_uaccess_begin();
252 
253 	if (access_ok(dst, size)) {
254 		pagefault_disable();
255 		ret = __copy_to_user_inatomic(dst, src, size);
256 		pagefault_enable();
257 	}
258 	force_uaccess_end(old_fs);
259 
260 	if (ret)
261 		return -EFAULT;
262 	return 0;
263 }
264 EXPORT_SYMBOL_GPL(copy_to_user_nofault);
265 
266 /**
267  * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
268  *				address.
269  * @dst:   Destination address, in kernel space.  This buffer must be at
270  *         least @count bytes long.
271  * @unsafe_addr: Unsafe user address.
272  * @count: Maximum number of bytes to copy, including the trailing NUL.
273  *
274  * Copies a NUL-terminated string from unsafe user address to kernel buffer.
275  *
276  * On success, returns the length of the string INCLUDING the trailing NUL.
277  *
278  * If access fails, returns -EFAULT (some data may have been copied
279  * and the trailing NUL added).
280  *
281  * If @count is smaller than the length of the string, copies @count-1 bytes,
282  * sets the last byte of @dst buffer to NUL and returns @count.
283  */
strncpy_from_user_nofault(char * dst,const void __user * unsafe_addr,long count)284 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
285 			      long count)
286 {
287 	mm_segment_t old_fs;
288 	long ret;
289 
290 	if (unlikely(count <= 0))
291 		return 0;
292 
293 	old_fs = force_uaccess_begin();
294 	pagefault_disable();
295 	ret = strncpy_from_user(dst, unsafe_addr, count);
296 	pagefault_enable();
297 	force_uaccess_end(old_fs);
298 
299 	if (ret >= count) {
300 		ret = count;
301 		dst[ret - 1] = '\0';
302 	} else if (ret > 0) {
303 		ret++;
304 	}
305 
306 	return ret;
307 }
308 
309 /**
310  * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
311  * @unsafe_addr: The string to measure.
312  * @count: Maximum count (including NUL)
313  *
314  * Get the size of a NUL-terminated string in user space without pagefault.
315  *
316  * Returns the size of the string INCLUDING the terminating NUL.
317  *
318  * If the string is too long, returns a number larger than @count. User
319  * has to check the return value against "> count".
320  * On exception (or invalid count), returns 0.
321  *
322  * Unlike strnlen_user, this can be used from IRQ handler etc. because
323  * it disables pagefaults.
324  */
strnlen_user_nofault(const void __user * unsafe_addr,long count)325 long strnlen_user_nofault(const void __user *unsafe_addr, long count)
326 {
327 	mm_segment_t old_fs;
328 	int ret;
329 
330 	old_fs = force_uaccess_begin();
331 	pagefault_disable();
332 	ret = strnlen_user(unsafe_addr, count);
333 	pagefault_enable();
334 	force_uaccess_end(old_fs);
335 
336 	return ret;
337 }
338