1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * McKinley-optimized version of copy_page().
4 *
5 * Copyright (C) 2002 Hewlett-Packard Co
6 *	David Mosberger <davidm@hpl.hp.com>
7 *
8 * Inputs:
9 *	in0:	address of target page
10 *	in1:	address of source page
11 * Output:
12 *	no return value
13 *
14 * General idea:
15 *	- use regular loads and stores to prefetch data to avoid consuming M-slot just for
16 *	  lfetches => good for in-cache performance
17 *	- avoid l2 bank-conflicts by not storing into the same 16-byte bank within a single
18 *	  cycle
19 *
20 * Principle of operation:
21 *	First, note that L1 has a line-size of 64 bytes and L2 a line-size of 128 bytes.
22 *	To avoid secondary misses in L2, we prefetch both source and destination with a line-size
23 *	of 128 bytes.  When both of these lines are in the L2 and the first half of the
24 *	source line is in L1, we start copying the remaining words.  The second half of the
25 *	source line is prefetched in an earlier iteration, so that by the time we start
26 *	accessing it, it's also present in the L1.
27 *
28 *	We use a software-pipelined loop to control the overall operation.  The pipeline
29 *	has 2*PREFETCH_DIST+K stages.  The first PREFETCH_DIST stages are used for prefetching
30 *	source cache-lines.  The second PREFETCH_DIST stages are used for prefetching destination
31 *	cache-lines, the last K stages are used to copy the cache-line words not copied by
32 *	the prefetches.  The four relevant points in the pipelined are called A, B, C, D:
33 *	p[A] is TRUE if a source-line should be prefetched, p[B] is TRUE if a destination-line
34 *	should be prefetched, p[C] is TRUE if the second half of an L2 line should be brought
35 *	into L1D and p[D] is TRUE if a cacheline needs to be copied.
36 *
37 *	This all sounds very complicated, but thanks to the modulo-scheduled loop support,
38 *	the resulting code is very regular and quite easy to follow (once you get the idea).
39 *
40 *	As a secondary optimization, the first 2*PREFETCH_DIST iterations are implemented
41 *	as the separate .prefetch_loop.  Logically, this loop performs exactly like the
42 *	main-loop (.line_copy), but has all known-to-be-predicated-off instructions removed,
43 *	so that each loop iteration is faster (again, good for cached case).
44 *
45 *	When reading the code, it helps to keep the following picture in mind:
46 *
47 *	       word 0 word 1
48 *            +------+------+---
49 *	      |	v[x] | 	t1  | ^
50 *	      |	t2   |	t3  | |
51 *	      |	t4   |	t5  | |
52 *	      |	t6   |	t7  | | 128 bytes
53 *     	      |	n[y] | 	t9  | |	(L2 cache line)
54 *	      |	t10  | 	t11 | |
55 *	      |	t12  | 	t13 | |
56 *	      |	t14  | 	t15 | v
57 *	      +------+------+---
58 *
59 *	Here, v[x] is copied by the (memory) prefetch.  n[y] is loaded at p[C]
60 *	to fetch the second-half of the L2 cache line into L1, and the tX words are copied in
61 *	an order that avoids bank conflicts.
62 */
63#include <asm/asmmacro.h>
64#include <asm/page.h>
65#include <asm/export.h>
66
67#define PREFETCH_DIST	8		// McKinley sustains 16 outstanding L2 misses (8 ld, 8 st)
68
69#define src0		r2
70#define src1		r3
71#define dst0		r9
72#define dst1		r10
73#define src_pre_mem	r11
74#define dst_pre_mem	r14
75#define src_pre_l2	r15
76#define dst_pre_l2	r16
77#define t1		r17
78#define t2		r18
79#define t3		r19
80#define t4		r20
81#define t5		t1	// alias!
82#define t6		t2	// alias!
83#define t7		t3	// alias!
84#define t9		t5	// alias!
85#define t10		t4	// alias!
86#define t11		t7	// alias!
87#define t12		t6	// alias!
88#define t14		t10	// alias!
89#define t13		r21
90#define t15		r22
91
92#define saved_lc	r23
93#define saved_pr	r24
94
95#define	A	0
96#define B	(PREFETCH_DIST)
97#define C	(B + PREFETCH_DIST)
98#define D	(C + 3)
99#define N	(D + 1)
100#define Nrot	((N + 7) & ~7)
101
102GLOBAL_ENTRY(copy_page)
103	.prologue
104	alloc r8 = ar.pfs, 2, Nrot-2, 0, Nrot
105
106	.rotr v[2*PREFETCH_DIST], n[D-C+1]
107	.rotp p[N]
108
109	.save ar.lc, saved_lc
110	mov saved_lc = ar.lc
111	.save pr, saved_pr
112	mov saved_pr = pr
113	.body
114
115	mov src_pre_mem = in1
116	mov pr.rot = 0x10000
117	mov ar.ec = 1				// special unrolled loop
118
119	mov dst_pre_mem = in0
120	mov ar.lc = 2*PREFETCH_DIST - 1
121
122	add src_pre_l2 = 8*8, in1
123	add dst_pre_l2 = 8*8, in0
124	add src0 = 8, in1			// first t1 src
125	add src1 = 3*8, in1			// first t3 src
126	add dst0 = 8, in0			// first t1 dst
127	add dst1 = 3*8, in0			// first t3 dst
128	mov t1 = (PAGE_SIZE/128) - (2*PREFETCH_DIST) - 1
129	nop.m 0
130	nop.i 0
131	;;
132	// same as .line_copy loop, but with all predicated-off instructions removed:
133.prefetch_loop:
134(p[A])	ld8 v[A] = [src_pre_mem], 128		// M0
135(p[B])	st8 [dst_pre_mem] = v[B], 128		// M2
136	br.ctop.sptk .prefetch_loop
137	;;
138	cmp.eq p16, p0 = r0, r0			// reset p16 to 1 (br.ctop cleared it to zero)
139	mov ar.lc = t1				// with 64KB pages, t1 is too big to fit in 8 bits!
140	mov ar.ec = N				// # of stages in pipeline
141	;;
142.line_copy:
143(p[D])	ld8 t2 = [src0], 3*8			// M0
144(p[D])	ld8 t4 = [src1], 3*8			// M1
145(p[B])	st8 [dst_pre_mem] = v[B], 128		// M2 prefetch dst from memory
146(p[D])	st8 [dst_pre_l2] = n[D-C], 128		// M3 prefetch dst from L2
147	;;
148(p[A])	ld8 v[A] = [src_pre_mem], 128		// M0 prefetch src from memory
149(p[C])	ld8 n[0] = [src_pre_l2], 128		// M1 prefetch src from L2
150(p[D])	st8 [dst0] =  t1, 8			// M2
151(p[D])	st8 [dst1] =  t3, 8			// M3
152	;;
153(p[D])	ld8  t5 = [src0], 8
154(p[D])	ld8  t7 = [src1], 3*8
155(p[D])	st8 [dst0] =  t2, 3*8
156(p[D])	st8 [dst1] =  t4, 3*8
157	;;
158(p[D])	ld8  t6 = [src0], 3*8
159(p[D])	ld8 t10 = [src1], 8
160(p[D])	st8 [dst0] =  t5, 8
161(p[D])	st8 [dst1] =  t7, 3*8
162	;;
163(p[D])	ld8  t9 = [src0], 3*8
164(p[D])	ld8 t11 = [src1], 3*8
165(p[D])	st8 [dst0] =  t6, 3*8
166(p[D])	st8 [dst1] = t10, 8
167	;;
168(p[D])	ld8 t12 = [src0], 8
169(p[D])	ld8 t14 = [src1], 8
170(p[D])	st8 [dst0] =  t9, 3*8
171(p[D])	st8 [dst1] = t11, 3*8
172	;;
173(p[D])	ld8 t13 = [src0], 4*8
174(p[D])	ld8 t15 = [src1], 4*8
175(p[D])	st8 [dst0] = t12, 8
176(p[D])	st8 [dst1] = t14, 8
177	;;
178(p[D-1])ld8  t1 = [src0], 8
179(p[D-1])ld8  t3 = [src1], 8
180(p[D])	st8 [dst0] = t13, 4*8
181(p[D])	st8 [dst1] = t15, 4*8
182	br.ctop.sptk .line_copy
183	;;
184	mov ar.lc = saved_lc
185	mov pr = saved_pr, -1
186	br.ret.sptk.many rp
187END(copy_page)
188EXPORT_SYMBOL(copy_page)
189