1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5
6#include <linux/linkage.h>
7#include <linux/const.h>
8#include <asm/assembler.h>
9#include <asm/page.h>
10#include <asm/cpufeature.h>
11#include <asm/alternative.h>
12
13/*
14 * Copy a page from src to dest (both are page aligned)
15 *
16 * Parameters:
17 *	x0 - dest
18 *	x1 - src
19 */
20SYM_FUNC_START_PI(copy_page)
21alternative_if ARM64_HAS_NO_HW_PREFETCH
22	// Prefetch three cache lines ahead.
23	prfm	pldl1strm, [x1, #128]
24	prfm	pldl1strm, [x1, #256]
25	prfm	pldl1strm, [x1, #384]
26alternative_else_nop_endif
27
28	ldp	x2, x3, [x1]
29	ldp	x4, x5, [x1, #16]
30	ldp	x6, x7, [x1, #32]
31	ldp	x8, x9, [x1, #48]
32	ldp	x10, x11, [x1, #64]
33	ldp	x12, x13, [x1, #80]
34	ldp	x14, x15, [x1, #96]
35	ldp	x16, x17, [x1, #112]
36
37	add	x0, x0, #256
38	add	x1, x1, #128
391:
40	tst	x0, #(PAGE_SIZE - 1)
41
42alternative_if ARM64_HAS_NO_HW_PREFETCH
43	prfm	pldl1strm, [x1, #384]
44alternative_else_nop_endif
45
46	stnp	x2, x3, [x0, #-256]
47	ldp	x2, x3, [x1]
48	stnp	x4, x5, [x0, #16 - 256]
49	ldp	x4, x5, [x1, #16]
50	stnp	x6, x7, [x0, #32 - 256]
51	ldp	x6, x7, [x1, #32]
52	stnp	x8, x9, [x0, #48 - 256]
53	ldp	x8, x9, [x1, #48]
54	stnp	x10, x11, [x0, #64 - 256]
55	ldp	x10, x11, [x1, #64]
56	stnp	x12, x13, [x0, #80 - 256]
57	ldp	x12, x13, [x1, #80]
58	stnp	x14, x15, [x0, #96 - 256]
59	ldp	x14, x15, [x1, #96]
60	stnp	x16, x17, [x0, #112 - 256]
61	ldp	x16, x17, [x1, #112]
62
63	add	x0, x0, #128
64	add	x1, x1, #128
65
66	b.ne	1b
67
68	stnp	x2, x3, [x0, #-256]
69	stnp	x4, x5, [x0, #16 - 256]
70	stnp	x6, x7, [x0, #32 - 256]
71	stnp	x8, x9, [x0, #48 - 256]
72	stnp	x10, x11, [x0, #64 - 256]
73	stnp	x12, x13, [x0, #80 - 256]
74	stnp	x14, x15, [x0, #96 - 256]
75	stnp	x16, x17, [x0, #112 - 256]
76
77	ret
78SYM_FUNC_END_PI(copy_page)
79EXPORT_SYMBOL(copy_page)
80