1 /*
2  * Copyright 2017 Red Hat Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  */
22 #include "vmm.h"
23 
24 #include <subdev/fb.h>
25 #include <subdev/ltc.h>
26 #include <subdev/timer.h>
27 
28 #include <nvif/if900d.h>
29 #include <nvif/unpack.h>
30 
31 static inline void
gf100_vmm_pgt_pte(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map,u64 addr)32 gf100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
33 		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map, u64 addr)
34 {
35 	u64 base = (addr >> 8) | map->type;
36 	u64 data = base;
37 
38 	if (map->ctag && !(map->next & (1ULL << 44))) {
39 		while (ptes--) {
40 			data = base | ((map->ctag >> 1) << 44);
41 			if (!(map->ctag++ & 1))
42 				data |= BIT_ULL(60);
43 
44 			VMM_WO064(pt, vmm, ptei++ * 8, data);
45 			base += map->next;
46 		}
47 	} else {
48 		map->type += ptes * map->ctag;
49 
50 		while (ptes--) {
51 			VMM_WO064(pt, vmm, ptei++ * 8, data);
52 			data += map->next;
53 		}
54 	}
55 }
56 
57 void
gf100_vmm_pgt_sgl(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)58 gf100_vmm_pgt_sgl(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
59 		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
60 {
61 	VMM_MAP_ITER_SGL(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
62 }
63 
64 void
gf100_vmm_pgt_dma(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)65 gf100_vmm_pgt_dma(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
66 		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
67 {
68 	if (map->page->shift == PAGE_SHIFT) {
69 		VMM_SPAM(vmm, "DMAA %08x %08x PTE(s)", ptei, ptes);
70 		nvkm_kmap(pt->memory);
71 		while (ptes--) {
72 			const u64 data = (*map->dma++ >> 8) | map->type;
73 			VMM_WO064(pt, vmm, ptei++ * 8, data);
74 			map->type += map->ctag;
75 		}
76 		nvkm_done(pt->memory);
77 		return;
78 	}
79 
80 	VMM_MAP_ITER_DMA(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
81 }
82 
83 void
gf100_vmm_pgt_mem(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes,struct nvkm_vmm_map * map)84 gf100_vmm_pgt_mem(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
85 		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
86 {
87 	VMM_MAP_ITER_MEM(vmm, pt, ptei, ptes, map, gf100_vmm_pgt_pte);
88 }
89 
90 void
gf100_vmm_pgt_unmap(struct nvkm_vmm * vmm,struct nvkm_mmu_pt * pt,u32 ptei,u32 ptes)91 gf100_vmm_pgt_unmap(struct nvkm_vmm *vmm,
92 		    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
93 {
94 	VMM_FO064(pt, vmm, ptei * 8, 0ULL, ptes);
95 }
96 
97 const struct nvkm_vmm_desc_func
98 gf100_vmm_pgt = {
99 	.unmap = gf100_vmm_pgt_unmap,
100 	.mem = gf100_vmm_pgt_mem,
101 	.dma = gf100_vmm_pgt_dma,
102 	.sgl = gf100_vmm_pgt_sgl,
103 };
104 
105 void
gf100_vmm_pgd_pde(struct nvkm_vmm * vmm,struct nvkm_vmm_pt * pgd,u32 pdei)106 gf100_vmm_pgd_pde(struct nvkm_vmm *vmm, struct nvkm_vmm_pt *pgd, u32 pdei)
107 {
108 	struct nvkm_vmm_pt *pgt = pgd->pde[pdei];
109 	struct nvkm_mmu_pt *pd = pgd->pt[0];
110 	struct nvkm_mmu_pt *pt;
111 	u64 data = 0;
112 
113 	if ((pt = pgt->pt[0])) {
114 		switch (nvkm_memory_target(pt->memory)) {
115 		case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 0; break;
116 		case NVKM_MEM_TARGET_HOST: data |= 2ULL << 0;
117 			data |= BIT_ULL(35); /* VOL */
118 			break;
119 		case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 0; break;
120 		default:
121 			WARN_ON(1);
122 			return;
123 		}
124 		data |= pt->addr >> 8;
125 	}
126 
127 	if ((pt = pgt->pt[1])) {
128 		switch (nvkm_memory_target(pt->memory)) {
129 		case NVKM_MEM_TARGET_VRAM: data |= 1ULL << 32; break;
130 		case NVKM_MEM_TARGET_HOST: data |= 2ULL << 32;
131 			data |= BIT_ULL(34); /* VOL */
132 			break;
133 		case NVKM_MEM_TARGET_NCOH: data |= 3ULL << 32; break;
134 		default:
135 			WARN_ON(1);
136 			return;
137 		}
138 		data |= pt->addr << 24;
139 	}
140 
141 	nvkm_kmap(pd->memory);
142 	VMM_WO064(pd, vmm, pdei * 8, data);
143 	nvkm_done(pd->memory);
144 }
145 
146 const struct nvkm_vmm_desc_func
147 gf100_vmm_pgd = {
148 	.unmap = gf100_vmm_pgt_unmap,
149 	.pde = gf100_vmm_pgd_pde,
150 };
151 
152 static const struct nvkm_vmm_desc
153 gf100_vmm_desc_17_12[] = {
154 	{ SPT, 15, 8, 0x1000, &gf100_vmm_pgt },
155 	{ PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
156 	{}
157 };
158 
159 static const struct nvkm_vmm_desc
160 gf100_vmm_desc_17_17[] = {
161 	{ LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
162 	{ PGD, 13, 8, 0x1000, &gf100_vmm_pgd },
163 	{}
164 };
165 
166 static const struct nvkm_vmm_desc
167 gf100_vmm_desc_16_12[] = {
168 	{ SPT, 14, 8, 0x1000, &gf100_vmm_pgt },
169 	{ PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
170 	{}
171 };
172 
173 static const struct nvkm_vmm_desc
174 gf100_vmm_desc_16_16[] = {
175 	{ LPT, 10, 8, 0x1000, &gf100_vmm_pgt },
176 	{ PGD, 14, 8, 0x1000, &gf100_vmm_pgd },
177 	{}
178 };
179 
180 void
gf100_vmm_invalidate_pdb(struct nvkm_vmm * vmm,u64 addr)181 gf100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr)
182 {
183 	struct nvkm_device *device = vmm->mmu->subdev.device;
184 	nvkm_wr32(device, 0x100cb8, addr);
185 }
186 
187 void
gf100_vmm_invalidate(struct nvkm_vmm * vmm,u32 type)188 gf100_vmm_invalidate(struct nvkm_vmm *vmm, u32 type)
189 {
190 	struct nvkm_device *device = vmm->mmu->subdev.device;
191 	struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
192 	u64 addr = 0;
193 
194 	mutex_lock(&vmm->mmu->mutex);
195 	/* Looks like maybe a "free flush slots" counter, the
196 	 * faster you write to 0x100cbc to more it decreases.
197 	 */
198 	nvkm_msec(device, 2000,
199 		if (nvkm_rd32(device, 0x100c80) & 0x00ff0000)
200 			break;
201 	);
202 
203 	if (!(type & 0x00000002) /* ALL_PDB. */) {
204 		switch (nvkm_memory_target(pd->memory)) {
205 		case NVKM_MEM_TARGET_VRAM: addr |= 0x00000000; break;
206 		case NVKM_MEM_TARGET_HOST: addr |= 0x00000002; break;
207 		case NVKM_MEM_TARGET_NCOH: addr |= 0x00000003; break;
208 		default:
209 			WARN_ON(1);
210 			break;
211 		}
212 		addr |= (vmm->pd->pt[0]->addr >> 12) << 4;
213 
214 		vmm->func->invalidate_pdb(vmm, addr);
215 	}
216 
217 	nvkm_wr32(device, 0x100cbc, 0x80000000 | type);
218 
219 	/* Wait for flush to be queued? */
220 	nvkm_msec(device, 2000,
221 		if (nvkm_rd32(device, 0x100c80) & 0x00008000)
222 			break;
223 	);
224 	mutex_unlock(&vmm->mmu->mutex);
225 }
226 
227 void
gf100_vmm_flush(struct nvkm_vmm * vmm,int depth)228 gf100_vmm_flush(struct nvkm_vmm *vmm, int depth)
229 {
230 	u32 type = 0x00000001; /* PAGE_ALL */
231 	if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
232 		type |= 0x00000004; /* HUB_ONLY */
233 	gf100_vmm_invalidate(vmm, type);
234 }
235 
236 int
gf100_vmm_valid(struct nvkm_vmm * vmm,void * argv,u32 argc,struct nvkm_vmm_map * map)237 gf100_vmm_valid(struct nvkm_vmm *vmm, void *argv, u32 argc,
238 		struct nvkm_vmm_map *map)
239 {
240 	const enum nvkm_memory_target target = nvkm_memory_target(map->memory);
241 	const struct nvkm_vmm_page *page = map->page;
242 	const bool gm20x = page->desc->func->sparse != NULL;
243 	union {
244 		struct gf100_vmm_map_vn vn;
245 		struct gf100_vmm_map_v0 v0;
246 	} *args = argv;
247 	struct nvkm_device *device = vmm->mmu->subdev.device;
248 	struct nvkm_memory *memory = map->memory;
249 	u8  kind, kind_inv, priv, ro, vol;
250 	int kindn, aper, ret = -ENOSYS;
251 	const u8 *kindm;
252 
253 	map->next = (1 << page->shift) >> 8;
254 	map->type = map->ctag = 0;
255 
256 	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
257 		vol  = !!args->v0.vol;
258 		ro   = !!args->v0.ro;
259 		priv = !!args->v0.priv;
260 		kind =   args->v0.kind;
261 	} else
262 	if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
263 		vol  = target == NVKM_MEM_TARGET_HOST;
264 		ro   = 0;
265 		priv = 0;
266 		kind = 0x00;
267 	} else {
268 		VMM_DEBUG(vmm, "args");
269 		return ret;
270 	}
271 
272 	aper = vmm->func->aper(target);
273 	if (WARN_ON(aper < 0))
274 		return aper;
275 
276 	kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);
277 	if (kind >= kindn || kindm[kind] == kind_inv) {
278 		VMM_DEBUG(vmm, "kind %02x", kind);
279 		return -EINVAL;
280 	}
281 
282 	if (kindm[kind] != kind) {
283 		u32 comp = (page->shift == 16 && !gm20x) ? 16 : 17;
284 		u32 tags = ALIGN(nvkm_memory_size(memory), 1 << 17) >> comp;
285 		if (aper != 0 || !(page->type & NVKM_VMM_PAGE_COMP)) {
286 			VMM_DEBUG(vmm, "comp %d %02x", aper, page->type);
287 			return -EINVAL;
288 		}
289 
290 		ret = nvkm_memory_tags_get(memory, device, tags,
291 					   nvkm_ltc_tags_clear,
292 					   &map->tags);
293 		if (ret) {
294 			VMM_DEBUG(vmm, "comp %d", ret);
295 			return ret;
296 		}
297 
298 		if (map->tags->mn) {
299 			u64 tags = map->tags->mn->offset + (map->offset >> 17);
300 			if (page->shift == 17 || !gm20x) {
301 				map->type |= tags << 44;
302 				map->ctag |= 1ULL << 44;
303 				map->next |= 1ULL << 44;
304 			} else {
305 				map->ctag |= tags << 1 | 1;
306 			}
307 		} else {
308 			kind = kindm[kind];
309 		}
310 	}
311 
312 	map->type |= BIT(0);
313 	map->type |= (u64)priv << 1;
314 	map->type |= (u64)  ro << 2;
315 	map->type |= (u64) vol << 32;
316 	map->type |= (u64)aper << 33;
317 	map->type |= (u64)kind << 36;
318 	return 0;
319 }
320 
321 int
gf100_vmm_aper(enum nvkm_memory_target target)322 gf100_vmm_aper(enum nvkm_memory_target target)
323 {
324 	switch (target) {
325 	case NVKM_MEM_TARGET_VRAM: return 0;
326 	case NVKM_MEM_TARGET_HOST: return 2;
327 	case NVKM_MEM_TARGET_NCOH: return 3;
328 	default:
329 		return -EINVAL;
330 	}
331 }
332 
333 void
gf100_vmm_part(struct nvkm_vmm * vmm,struct nvkm_memory * inst)334 gf100_vmm_part(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
335 {
336 	nvkm_fo64(inst, 0x0200, 0x00000000, 2);
337 }
338 
339 int
gf100_vmm_join_(struct nvkm_vmm * vmm,struct nvkm_memory * inst,u64 base)340 gf100_vmm_join_(struct nvkm_vmm *vmm, struct nvkm_memory *inst, u64 base)
341 {
342 	struct nvkm_mmu_pt *pd = vmm->pd->pt[0];
343 
344 	switch (nvkm_memory_target(pd->memory)) {
345 	case NVKM_MEM_TARGET_VRAM: base |= 0ULL << 0; break;
346 	case NVKM_MEM_TARGET_HOST: base |= 2ULL << 0;
347 		base |= BIT_ULL(2) /* VOL. */;
348 		break;
349 	case NVKM_MEM_TARGET_NCOH: base |= 3ULL << 0; break;
350 	default:
351 		WARN_ON(1);
352 		return -EINVAL;
353 	}
354 	base |= pd->addr;
355 
356 	nvkm_kmap(inst);
357 	nvkm_wo64(inst, 0x0200, base);
358 	nvkm_wo64(inst, 0x0208, vmm->limit - 1);
359 	nvkm_done(inst);
360 	return 0;
361 }
362 
363 int
gf100_vmm_join(struct nvkm_vmm * vmm,struct nvkm_memory * inst)364 gf100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
365 {
366 	return gf100_vmm_join_(vmm, inst, 0);
367 }
368 
369 static const struct nvkm_vmm_func
370 gf100_vmm_17 = {
371 	.join = gf100_vmm_join,
372 	.part = gf100_vmm_part,
373 	.aper = gf100_vmm_aper,
374 	.valid = gf100_vmm_valid,
375 	.flush = gf100_vmm_flush,
376 	.invalidate_pdb = gf100_vmm_invalidate_pdb,
377 	.page = {
378 		{ 17, &gf100_vmm_desc_17_17[0], NVKM_VMM_PAGE_xVxC },
379 		{ 12, &gf100_vmm_desc_17_12[0], NVKM_VMM_PAGE_xVHx },
380 		{}
381 	}
382 };
383 
384 static const struct nvkm_vmm_func
385 gf100_vmm_16 = {
386 	.join = gf100_vmm_join,
387 	.part = gf100_vmm_part,
388 	.aper = gf100_vmm_aper,
389 	.valid = gf100_vmm_valid,
390 	.flush = gf100_vmm_flush,
391 	.invalidate_pdb = gf100_vmm_invalidate_pdb,
392 	.page = {
393 		{ 16, &gf100_vmm_desc_16_16[0], NVKM_VMM_PAGE_xVxC },
394 		{ 12, &gf100_vmm_desc_16_12[0], NVKM_VMM_PAGE_xVHx },
395 		{}
396 	}
397 };
398 
399 int
gf100_vmm_new_(const struct nvkm_vmm_func * func_16,const struct nvkm_vmm_func * func_17,struct nvkm_mmu * mmu,bool managed,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)400 gf100_vmm_new_(const struct nvkm_vmm_func *func_16,
401 	       const struct nvkm_vmm_func *func_17,
402 	       struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
403 	       void *argv, u32 argc, struct lock_class_key *key,
404 	       const char *name, struct nvkm_vmm **pvmm)
405 {
406 	switch (mmu->subdev.device->fb->page) {
407 	case 16: return nv04_vmm_new_(func_16, mmu, 0, managed, addr, size,
408 				      argv, argc, key, name, pvmm);
409 	case 17: return nv04_vmm_new_(func_17, mmu, 0, managed, addr, size,
410 				      argv, argc, key, name, pvmm);
411 	default:
412 		WARN_ON(1);
413 		return -EINVAL;
414 	}
415 }
416 
417 int
gf100_vmm_new(struct nvkm_mmu * mmu,bool managed,u64 addr,u64 size,void * argv,u32 argc,struct lock_class_key * key,const char * name,struct nvkm_vmm ** pvmm)418 gf100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
419 	      void *argv, u32 argc, struct lock_class_key *key,
420 	      const char *name, struct nvkm_vmm **pvmm)
421 {
422 	return gf100_vmm_new_(&gf100_vmm_16, &gf100_vmm_17, mmu, managed, addr,
423 			      size, argv, argc, key, name, pvmm);
424 }
425