1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2019, The Linux Foundation. All rights reserved.
4 */
5
6 #include <linux/acpi.h>
7 #include <linux/adreno-smmu-priv.h>
8 #include <linux/of_device.h>
9 #include <linux/qcom_scm.h>
10
11 #include "arm-smmu.h"
12
13 struct qcom_smmu {
14 struct arm_smmu_device smmu;
15 bool bypass_quirk;
16 u8 bypass_cbndx;
17 u32 stall_enabled;
18 };
19
to_qcom_smmu(struct arm_smmu_device * smmu)20 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
21 {
22 return container_of(smmu, struct qcom_smmu, smmu);
23 }
24
qcom_adreno_smmu_write_sctlr(struct arm_smmu_device * smmu,int idx,u32 reg)25 static void qcom_adreno_smmu_write_sctlr(struct arm_smmu_device *smmu, int idx,
26 u32 reg)
27 {
28 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
29
30 /*
31 * On the GPU device we want to process subsequent transactions after a
32 * fault to keep the GPU from hanging
33 */
34 reg |= ARM_SMMU_SCTLR_HUPCF;
35
36 if (qsmmu->stall_enabled & BIT(idx))
37 reg |= ARM_SMMU_SCTLR_CFCFG;
38
39 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
40 }
41
qcom_adreno_smmu_get_fault_info(const void * cookie,struct adreno_smmu_fault_info * info)42 static void qcom_adreno_smmu_get_fault_info(const void *cookie,
43 struct adreno_smmu_fault_info *info)
44 {
45 struct arm_smmu_domain *smmu_domain = (void *)cookie;
46 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
47 struct arm_smmu_device *smmu = smmu_domain->smmu;
48
49 info->fsr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSR);
50 info->fsynr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR0);
51 info->fsynr1 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_FSYNR1);
52 info->far = arm_smmu_cb_readq(smmu, cfg->cbndx, ARM_SMMU_CB_FAR);
53 info->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(cfg->cbndx));
54 info->ttbr0 = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_TTBR0);
55 info->contextidr = arm_smmu_cb_read(smmu, cfg->cbndx, ARM_SMMU_CB_CONTEXTIDR);
56 }
57
qcom_adreno_smmu_set_stall(const void * cookie,bool enabled)58 static void qcom_adreno_smmu_set_stall(const void *cookie, bool enabled)
59 {
60 struct arm_smmu_domain *smmu_domain = (void *)cookie;
61 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
62 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu_domain->smmu);
63
64 if (enabled)
65 qsmmu->stall_enabled |= BIT(cfg->cbndx);
66 else
67 qsmmu->stall_enabled &= ~BIT(cfg->cbndx);
68 }
69
qcom_adreno_smmu_resume_translation(const void * cookie,bool terminate)70 static void qcom_adreno_smmu_resume_translation(const void *cookie, bool terminate)
71 {
72 struct arm_smmu_domain *smmu_domain = (void *)cookie;
73 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
74 struct arm_smmu_device *smmu = smmu_domain->smmu;
75 u32 reg = 0;
76
77 if (terminate)
78 reg |= ARM_SMMU_RESUME_TERMINATE;
79
80 arm_smmu_cb_write(smmu, cfg->cbndx, ARM_SMMU_CB_RESUME, reg);
81 }
82
83 #define QCOM_ADRENO_SMMU_GPU_SID 0
84
qcom_adreno_smmu_is_gpu_device(struct device * dev)85 static bool qcom_adreno_smmu_is_gpu_device(struct device *dev)
86 {
87 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
88 int i;
89
90 /*
91 * The GPU will always use SID 0 so that is a handy way to uniquely
92 * identify it and configure it for per-instance pagetables
93 */
94 for (i = 0; i < fwspec->num_ids; i++) {
95 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
96
97 if (sid == QCOM_ADRENO_SMMU_GPU_SID)
98 return true;
99 }
100
101 return false;
102 }
103
qcom_adreno_smmu_get_ttbr1_cfg(const void * cookie)104 static const struct io_pgtable_cfg *qcom_adreno_smmu_get_ttbr1_cfg(
105 const void *cookie)
106 {
107 struct arm_smmu_domain *smmu_domain = (void *)cookie;
108 struct io_pgtable *pgtable =
109 io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
110 return &pgtable->cfg;
111 }
112
113 /*
114 * Local implementation to configure TTBR0 with the specified pagetable config.
115 * The GPU driver will call this to enable TTBR0 when per-instance pagetables
116 * are active
117 */
118
qcom_adreno_smmu_set_ttbr0_cfg(const void * cookie,const struct io_pgtable_cfg * pgtbl_cfg)119 static int qcom_adreno_smmu_set_ttbr0_cfg(const void *cookie,
120 const struct io_pgtable_cfg *pgtbl_cfg)
121 {
122 struct arm_smmu_domain *smmu_domain = (void *)cookie;
123 struct io_pgtable *pgtable = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
124 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
125 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
126
127 /* The domain must have split pagetables already enabled */
128 if (cb->tcr[0] & ARM_SMMU_TCR_EPD1)
129 return -EINVAL;
130
131 /* If the pagetable config is NULL, disable TTBR0 */
132 if (!pgtbl_cfg) {
133 /* Do nothing if it is already disabled */
134 if ((cb->tcr[0] & ARM_SMMU_TCR_EPD0))
135 return -EINVAL;
136
137 /* Set TCR to the original configuration */
138 cb->tcr[0] = arm_smmu_lpae_tcr(&pgtable->cfg);
139 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
140 } else {
141 u32 tcr = cb->tcr[0];
142
143 /* Don't call this again if TTBR0 is already enabled */
144 if (!(cb->tcr[0] & ARM_SMMU_TCR_EPD0))
145 return -EINVAL;
146
147 tcr |= arm_smmu_lpae_tcr(pgtbl_cfg);
148 tcr &= ~(ARM_SMMU_TCR_EPD0 | ARM_SMMU_TCR_EPD1);
149
150 cb->tcr[0] = tcr;
151 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
152 cb->ttbr[0] |= FIELD_PREP(ARM_SMMU_TTBRn_ASID, cb->cfg->asid);
153 }
154
155 arm_smmu_write_context_bank(smmu_domain->smmu, cb->cfg->cbndx);
156
157 return 0;
158 }
159
qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,int start)160 static int qcom_adreno_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
161 struct arm_smmu_device *smmu,
162 struct device *dev, int start)
163 {
164 int count;
165
166 /*
167 * Assign context bank 0 to the GPU device so the GPU hardware can
168 * switch pagetables
169 */
170 if (qcom_adreno_smmu_is_gpu_device(dev)) {
171 start = 0;
172 count = 1;
173 } else {
174 start = 1;
175 count = smmu->num_context_banks;
176 }
177
178 return __arm_smmu_alloc_bitmap(smmu->context_map, start, count);
179 }
180
qcom_adreno_can_do_ttbr1(struct arm_smmu_device * smmu)181 static bool qcom_adreno_can_do_ttbr1(struct arm_smmu_device *smmu)
182 {
183 const struct device_node *np = smmu->dev->of_node;
184
185 if (of_device_is_compatible(np, "qcom,msm8996-smmu-v2"))
186 return false;
187
188 return true;
189 }
190
qcom_adreno_smmu_init_context(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg,struct device * dev)191 static int qcom_adreno_smmu_init_context(struct arm_smmu_domain *smmu_domain,
192 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
193 {
194 struct adreno_smmu_priv *priv;
195
196 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
197
198 /* Only enable split pagetables for the GPU device (SID 0) */
199 if (!qcom_adreno_smmu_is_gpu_device(dev))
200 return 0;
201
202 /*
203 * All targets that use the qcom,adreno-smmu compatible string *should*
204 * be AARCH64 stage 1 but double check because the arm-smmu code assumes
205 * that is the case when the TTBR1 quirk is enabled
206 */
207 if (qcom_adreno_can_do_ttbr1(smmu_domain->smmu) &&
208 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) &&
209 (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64))
210 pgtbl_cfg->quirks |= IO_PGTABLE_QUIRK_ARM_TTBR1;
211
212 /*
213 * Initialize private interface with GPU:
214 */
215
216 priv = dev_get_drvdata(dev);
217 priv->cookie = smmu_domain;
218 priv->get_ttbr1_cfg = qcom_adreno_smmu_get_ttbr1_cfg;
219 priv->set_ttbr0_cfg = qcom_adreno_smmu_set_ttbr0_cfg;
220 priv->get_fault_info = qcom_adreno_smmu_get_fault_info;
221 priv->set_stall = qcom_adreno_smmu_set_stall;
222 priv->resume_translation = qcom_adreno_smmu_resume_translation;
223
224 return 0;
225 }
226
227 static const struct of_device_id qcom_smmu_client_of_match[] __maybe_unused = {
228 { .compatible = "qcom,adreno" },
229 { .compatible = "qcom,mdp4" },
230 { .compatible = "qcom,mdss" },
231 { .compatible = "qcom,sc7180-mdss" },
232 { .compatible = "qcom,sc7180-mss-pil" },
233 { .compatible = "qcom,sc7280-mdss" },
234 { .compatible = "qcom,sc7280-mss-pil" },
235 { .compatible = "qcom,sc8180x-mdss" },
236 { .compatible = "qcom,sdm845-mdss" },
237 { .compatible = "qcom,sdm845-mss-pil" },
238 { }
239 };
240
qcom_smmu_init_context(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg,struct device * dev)241 static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain,
242 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
243 {
244 smmu_domain->cfg.flush_walk_prefer_tlbiasid = true;
245
246 return 0;
247 }
248
qcom_smmu_cfg_probe(struct arm_smmu_device * smmu)249 static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu)
250 {
251 unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1);
252 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
253 u32 reg;
254 u32 smr;
255 int i;
256
257 /*
258 * With some firmware versions writes to S2CR of type FAULT are
259 * ignored, and writing BYPASS will end up written as FAULT in the
260 * register. Perform a write to S2CR to detect if this is the case and
261 * if so reserve a context bank to emulate bypass streams.
262 */
263 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, S2CR_TYPE_BYPASS) |
264 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, 0xff) |
265 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, S2CR_PRIVCFG_DEFAULT);
266 arm_smmu_gr0_write(smmu, last_s2cr, reg);
267 reg = arm_smmu_gr0_read(smmu, last_s2cr);
268 if (FIELD_GET(ARM_SMMU_S2CR_TYPE, reg) != S2CR_TYPE_BYPASS) {
269 qsmmu->bypass_quirk = true;
270 qsmmu->bypass_cbndx = smmu->num_context_banks - 1;
271
272 set_bit(qsmmu->bypass_cbndx, smmu->context_map);
273
274 arm_smmu_cb_write(smmu, qsmmu->bypass_cbndx, ARM_SMMU_CB_SCTLR, 0);
275
276 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, CBAR_TYPE_S1_TRANS_S2_BYPASS);
277 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(qsmmu->bypass_cbndx), reg);
278 }
279
280 for (i = 0; i < smmu->num_mapping_groups; i++) {
281 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
282
283 if (FIELD_GET(ARM_SMMU_SMR_VALID, smr)) {
284 /* Ignore valid bit for SMR mask extraction. */
285 smr &= ~ARM_SMMU_SMR_VALID;
286 smmu->smrs[i].id = FIELD_GET(ARM_SMMU_SMR_ID, smr);
287 smmu->smrs[i].mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
288 smmu->smrs[i].valid = true;
289
290 smmu->s2crs[i].type = S2CR_TYPE_BYPASS;
291 smmu->s2crs[i].privcfg = S2CR_PRIVCFG_DEFAULT;
292 smmu->s2crs[i].cbndx = 0xff;
293 }
294 }
295
296 return 0;
297 }
298
qcom_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)299 static void qcom_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
300 {
301 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
302 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
303 u32 cbndx = s2cr->cbndx;
304 u32 type = s2cr->type;
305 u32 reg;
306
307 if (qsmmu->bypass_quirk) {
308 if (type == S2CR_TYPE_BYPASS) {
309 /*
310 * Firmware with quirky S2CR handling will substitute
311 * BYPASS writes with FAULT, so point the stream to the
312 * reserved context bank and ask for translation on the
313 * stream
314 */
315 type = S2CR_TYPE_TRANS;
316 cbndx = qsmmu->bypass_cbndx;
317 } else if (type == S2CR_TYPE_FAULT) {
318 /*
319 * Firmware with quirky S2CR handling will ignore FAULT
320 * writes, so trick it to write FAULT by asking for a
321 * BYPASS.
322 */
323 type = S2CR_TYPE_BYPASS;
324 cbndx = 0xff;
325 }
326 }
327
328 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, type) |
329 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, cbndx) |
330 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
331 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
332 }
333
qcom_smmu_def_domain_type(struct device * dev)334 static int qcom_smmu_def_domain_type(struct device *dev)
335 {
336 const struct of_device_id *match =
337 of_match_device(qcom_smmu_client_of_match, dev);
338
339 return match ? IOMMU_DOMAIN_IDENTITY : 0;
340 }
341
qcom_sdm845_smmu500_reset(struct arm_smmu_device * smmu)342 static int qcom_sdm845_smmu500_reset(struct arm_smmu_device *smmu)
343 {
344 int ret;
345
346 /*
347 * To address performance degradation in non-real time clients,
348 * such as USB and UFS, turn off wait-for-safe on sdm845 based boards,
349 * such as MTP and db845, whose firmwares implement secure monitor
350 * call handlers to turn on/off the wait-for-safe logic.
351 */
352 ret = qcom_scm_qsmmu500_wait_safe_toggle(0);
353 if (ret)
354 dev_warn(smmu->dev, "Failed to turn off SAFE logic\n");
355
356 return ret;
357 }
358
qcom_smmu500_reset(struct arm_smmu_device * smmu)359 static int qcom_smmu500_reset(struct arm_smmu_device *smmu)
360 {
361 const struct device_node *np = smmu->dev->of_node;
362
363 arm_mmu500_reset(smmu);
364
365 if (of_device_is_compatible(np, "qcom,sdm845-smmu-500"))
366 return qcom_sdm845_smmu500_reset(smmu);
367
368 return 0;
369 }
370
371 static const struct arm_smmu_impl qcom_smmu_impl = {
372 .init_context = qcom_smmu_init_context,
373 .cfg_probe = qcom_smmu_cfg_probe,
374 .def_domain_type = qcom_smmu_def_domain_type,
375 .reset = qcom_smmu500_reset,
376 .write_s2cr = qcom_smmu_write_s2cr,
377 };
378
379 static const struct arm_smmu_impl qcom_adreno_smmu_impl = {
380 .init_context = qcom_adreno_smmu_init_context,
381 .def_domain_type = qcom_smmu_def_domain_type,
382 .reset = qcom_smmu500_reset,
383 .alloc_context_bank = qcom_adreno_smmu_alloc_context_bank,
384 .write_sctlr = qcom_adreno_smmu_write_sctlr,
385 };
386
qcom_smmu_create(struct arm_smmu_device * smmu,const struct arm_smmu_impl * impl)387 static struct arm_smmu_device *qcom_smmu_create(struct arm_smmu_device *smmu,
388 const struct arm_smmu_impl *impl)
389 {
390 struct qcom_smmu *qsmmu;
391
392 /* Check to make sure qcom_scm has finished probing */
393 if (!qcom_scm_is_available())
394 return ERR_PTR(-EPROBE_DEFER);
395
396 qsmmu = devm_krealloc(smmu->dev, smmu, sizeof(*qsmmu), GFP_KERNEL);
397 if (!qsmmu)
398 return ERR_PTR(-ENOMEM);
399
400 qsmmu->smmu.impl = impl;
401
402 return &qsmmu->smmu;
403 }
404
405 static const struct of_device_id __maybe_unused qcom_smmu_impl_of_match[] = {
406 { .compatible = "qcom,msm8998-smmu-v2" },
407 { .compatible = "qcom,qcm2290-smmu-500" },
408 { .compatible = "qcom,sc7180-smmu-500" },
409 { .compatible = "qcom,sc7280-smmu-500" },
410 { .compatible = "qcom,sc8180x-smmu-500" },
411 { .compatible = "qcom,sdm630-smmu-v2" },
412 { .compatible = "qcom,sdm845-smmu-500" },
413 { .compatible = "qcom,sm6125-smmu-500" },
414 { .compatible = "qcom,sm6350-smmu-500" },
415 { .compatible = "qcom,sm8150-smmu-500" },
416 { .compatible = "qcom,sm8250-smmu-500" },
417 { .compatible = "qcom,sm8350-smmu-500" },
418 { }
419 };
420
421 #ifdef CONFIG_ACPI
422 static struct acpi_platform_list qcom_acpi_platlist[] = {
423 { "LENOVO", "CB-01 ", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
424 { "QCOM ", "QCOMEDK2", 0x8180, ACPI_SIG_IORT, equal, "QCOM SMMU" },
425 { }
426 };
427 #endif
428
qcom_smmu_impl_init(struct arm_smmu_device * smmu)429 struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu)
430 {
431 const struct device_node *np = smmu->dev->of_node;
432
433 #ifdef CONFIG_ACPI
434 if (np == NULL) {
435 /* Match platform for ACPI boot */
436 if (acpi_match_platform_list(qcom_acpi_platlist) >= 0)
437 return qcom_smmu_create(smmu, &qcom_smmu_impl);
438 }
439 #endif
440
441 /*
442 * Do not change this order of implementation, i.e., first adreno
443 * smmu impl and then apss smmu since we can have both implementing
444 * arm,mmu-500 in which case we will miss setting adreno smmu specific
445 * features if the order is changed.
446 */
447 if (of_device_is_compatible(np, "qcom,adreno-smmu"))
448 return qcom_smmu_create(smmu, &qcom_adreno_smmu_impl);
449
450 if (of_match_node(qcom_smmu_impl_of_match, np))
451 return qcom_smmu_create(smmu, &qcom_smmu_impl);
452
453 return smmu;
454 }
455