1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018-2019 HiSilicon Limited. */
3 #include <linux/acpi.h>
4 #include <linux/bitops.h>
5 #include <linux/debugfs.h>
6 #include <linux/init.h>
7 #include <linux/io.h>
8 #include <linux/kernel.h>
9 #include <linux/module.h>
10 #include <linux/pci.h>
11 #include <linux/pm_runtime.h>
12 #include <linux/topology.h>
13 #include <linux/uacce.h>
14 #include "hpre.h"
15
16 #define CAP_FILE_PERMISSION 0444
17 #define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0)
18 #define HPRE_CTRL_CNT_CLR_CE 0x301000
19 #define HPRE_FSM_MAX_CNT 0x301008
20 #define HPRE_VFG_AXQOS 0x30100c
21 #define HPRE_VFG_AXCACHE 0x301010
22 #define HPRE_RDCHN_INI_CFG 0x301014
23 #define HPRE_AWUSR_FP_CFG 0x301018
24 #define HPRE_BD_ENDIAN 0x301020
25 #define HPRE_ECC_BYPASS 0x301024
26 #define HPRE_RAS_WIDTH_CFG 0x301028
27 #define HPRE_POISON_BYPASS 0x30102c
28 #define HPRE_BD_ARUSR_CFG 0x301030
29 #define HPRE_BD_AWUSR_CFG 0x301034
30 #define HPRE_TYPES_ENB 0x301038
31 #define HPRE_RSA_ENB BIT(0)
32 #define HPRE_ECC_ENB BIT(1)
33 #define HPRE_DATA_RUSER_CFG 0x30103c
34 #define HPRE_DATA_WUSER_CFG 0x301040
35 #define HPRE_INT_MASK 0x301400
36 #define HPRE_INT_STATUS 0x301800
37 #define HPRE_HAC_INT_MSK 0x301400
38 #define HPRE_HAC_RAS_CE_ENB 0x301410
39 #define HPRE_HAC_RAS_NFE_ENB 0x301414
40 #define HPRE_HAC_RAS_FE_ENB 0x301418
41 #define HPRE_HAC_INT_SET 0x301500
42 #define HPRE_AXI_ERROR_MASK GENMASK(21, 10)
43 #define HPRE_RNG_TIMEOUT_NUM 0x301A34
44 #define HPRE_CORE_INT_ENABLE 0
45 #define HPRE_RDCHN_INI_ST 0x301a00
46 #define HPRE_CLSTR_BASE 0x302000
47 #define HPRE_CORE_EN_OFFSET 0x04
48 #define HPRE_CORE_INI_CFG_OFFSET 0x20
49 #define HPRE_CORE_INI_STATUS_OFFSET 0x80
50 #define HPRE_CORE_HTBT_WARN_OFFSET 0x8c
51 #define HPRE_CORE_IS_SCHD_OFFSET 0x90
52
53 #define HPRE_RAS_CE_ENB 0x301410
54 #define HPRE_RAS_NFE_ENB 0x301414
55 #define HPRE_RAS_FE_ENB 0x301418
56 #define HPRE_OOO_SHUTDOWN_SEL 0x301a3c
57 #define HPRE_HAC_RAS_FE_ENABLE 0
58
59 #define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET)
60 #define HPRE_CORE_INI_CFG (HPRE_CLSTR_BASE + HPRE_CORE_INI_CFG_OFFSET)
61 #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET)
62 #define HPRE_HAC_ECC1_CNT 0x301a04
63 #define HPRE_HAC_ECC2_CNT 0x301a08
64 #define HPRE_HAC_SOURCE_INT 0x301600
65 #define HPRE_CLSTR_ADDR_INTRVL 0x1000
66 #define HPRE_CLUSTER_INQURY 0x100
67 #define HPRE_CLSTR_ADDR_INQRY_RSLT 0x104
68 #define HPRE_PASID_EN_BIT 9
69 #define HPRE_REG_RD_INTVRL_US 10
70 #define HPRE_REG_RD_TMOUT_US 1000
71 #define HPRE_DBGFS_VAL_MAX_LEN 20
72 #define PCI_DEVICE_ID_HUAWEI_HPRE_PF 0xa258
73 #define HPRE_QM_USR_CFG_MASK GENMASK(31, 1)
74 #define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0)
75 #define HPRE_QM_VFG_AX_MASK GENMASK(7, 0)
76 #define HPRE_BD_USR_MASK GENMASK(1, 0)
77 #define HPRE_PREFETCH_CFG 0x301130
78 #define HPRE_SVA_PREFTCH_DFX 0x30115C
79 #define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30)))
80 #define HPRE_PREFETCH_DISABLE BIT(30)
81 #define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8))
82 #define HPRE_SVA_PREFTCH_DFX4 0x301144
83 #define HPRE_WAIT_SVA_READY 500000
84 #define HPRE_READ_SVA_STATUS_TIMES 3
85 #define HPRE_WAIT_US_MIN 10
86 #define HPRE_WAIT_US_MAX 20
87
88 /* clock gate */
89 #define HPRE_CLKGATE_CTL 0x301a10
90 #define HPRE_PEH_CFG_AUTO_GATE 0x301a2c
91 #define HPRE_CLUSTER_DYN_CTL 0x302010
92 #define HPRE_CORE_SHB_CFG 0x302088
93 #define HPRE_CLKGATE_CTL_EN BIT(0)
94 #define HPRE_PEH_CFG_AUTO_GATE_EN BIT(0)
95 #define HPRE_CLUSTER_DYN_CTL_EN BIT(0)
96 #define HPRE_CORE_GATE_EN (BIT(30) | BIT(31))
97
98 #define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044
99 #define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0)
100 #define HPRE_WR_MSI_PORT BIT(2)
101
102 #define HPRE_CORE_ECC_2BIT_ERR BIT(1)
103 #define HPRE_OOO_ECC_2BIT_ERR BIT(5)
104
105 #define HPRE_QM_BME_FLR BIT(7)
106 #define HPRE_QM_PM_FLR BIT(11)
107 #define HPRE_QM_SRIOV_FLR BIT(12)
108
109 #define HPRE_SHAPER_TYPE_RATE 640
110 #define HPRE_VIA_MSI_DSM 1
111 #define HPRE_SQE_MASK_OFFSET 8
112 #define HPRE_SQE_MASK_LEN 44
113 #define HPRE_CTX_Q_NUM_DEF 1
114
115 #define HPRE_DFX_BASE 0x301000
116 #define HPRE_DFX_COMMON1 0x301400
117 #define HPRE_DFX_COMMON2 0x301A00
118 #define HPRE_DFX_CORE 0x302000
119 #define HPRE_DFX_BASE_LEN 0x55
120 #define HPRE_DFX_COMMON1_LEN 0x41
121 #define HPRE_DFX_COMMON2_LEN 0xE
122 #define HPRE_DFX_CORE_LEN 0x43
123
124 #define HPRE_MAX_CHANNEL_NUM 2
125
126 static const char hpre_name[] = "hisi_hpre";
127 static struct dentry *hpre_debugfs_root;
128 static const struct pci_device_id hpre_dev_ids[] = {
129 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_PF) },
130 { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_HUAWEI_HPRE_VF) },
131 { 0, }
132 };
133
134 MODULE_DEVICE_TABLE(pci, hpre_dev_ids);
135
136 struct hpre_hw_error {
137 u32 int_msk;
138 const char *msg;
139 };
140
141 static const struct qm_dev_alg hpre_dev_algs[] = {
142 {
143 .alg_msk = BIT(0),
144 .alg = "rsa\n"
145 }, {
146 .alg_msk = BIT(1),
147 .alg = "dh\n"
148 }, {
149 .alg_msk = BIT(2),
150 .alg = "ecdh\n"
151 }, {
152 .alg_msk = BIT(3),
153 .alg = "ecdsa\n"
154 }, {
155 .alg_msk = BIT(4),
156 .alg = "sm2\n"
157 }, {
158 .alg_msk = BIT(5),
159 .alg = "x25519\n"
160 }, {
161 .alg_msk = BIT(6),
162 .alg = "x448\n"
163 }, {
164 /* sentinel */
165 }
166 };
167
168 static struct hisi_qm_list hpre_devices = {
169 .register_to_crypto = hpre_algs_register,
170 .unregister_from_crypto = hpre_algs_unregister,
171 };
172
173 static const char * const hpre_debug_file_name[] = {
174 [HPRE_CLEAR_ENABLE] = "rdclr_en",
175 [HPRE_CLUSTER_CTRL] = "cluster_ctrl",
176 };
177
178 enum hpre_cap_type {
179 HPRE_QM_NFE_MASK_CAP,
180 HPRE_QM_RESET_MASK_CAP,
181 HPRE_QM_OOO_SHUTDOWN_MASK_CAP,
182 HPRE_QM_CE_MASK_CAP,
183 HPRE_NFE_MASK_CAP,
184 HPRE_RESET_MASK_CAP,
185 HPRE_OOO_SHUTDOWN_MASK_CAP,
186 HPRE_CE_MASK_CAP,
187 HPRE_CLUSTER_NUM_CAP,
188 HPRE_CORE_TYPE_NUM_CAP,
189 HPRE_CORE_NUM_CAP,
190 HPRE_CLUSTER_CORE_NUM_CAP,
191 HPRE_CORE_ENABLE_BITMAP_CAP,
192 HPRE_DRV_ALG_BITMAP_CAP,
193 HPRE_DEV_ALG_BITMAP_CAP,
194 HPRE_CORE1_ALG_BITMAP_CAP,
195 HPRE_CORE2_ALG_BITMAP_CAP,
196 HPRE_CORE3_ALG_BITMAP_CAP,
197 HPRE_CORE4_ALG_BITMAP_CAP,
198 HPRE_CORE5_ALG_BITMAP_CAP,
199 HPRE_CORE6_ALG_BITMAP_CAP,
200 HPRE_CORE7_ALG_BITMAP_CAP,
201 HPRE_CORE8_ALG_BITMAP_CAP,
202 HPRE_CORE9_ALG_BITMAP_CAP,
203 HPRE_CORE10_ALG_BITMAP_CAP
204 };
205
206 static const struct hisi_qm_cap_info hpre_basic_info[] = {
207 {HPRE_QM_NFE_MASK_CAP, 0x3124, 0, GENMASK(31, 0), 0x0, 0x1C37, 0x7C37},
208 {HPRE_QM_RESET_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0xC37, 0x6C37},
209 {HPRE_QM_OOO_SHUTDOWN_MASK_CAP, 0x3128, 0, GENMASK(31, 0), 0x0, 0x4, 0x6C37},
210 {HPRE_QM_CE_MASK_CAP, 0x312C, 0, GENMASK(31, 0), 0x0, 0x8, 0x8},
211 {HPRE_NFE_MASK_CAP, 0x3130, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0x1FFFC3E},
212 {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFC3E},
213 {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFC3E},
214 {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1},
215 {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1},
216 {HPRE_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x0, 0x2, 0x2},
217 {HPRE_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x0, 0x8, 0xA},
218 {HPRE_CLUSTER_CORE_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x0, 0x2, 0xA},
219 {HPRE_CORE_ENABLE_BITMAP_CAP, 0x3140, 0, GENMASK(31, 0), 0x0, 0xF, 0x3FF},
220 {HPRE_DRV_ALG_BITMAP_CAP, 0x3144, 0, GENMASK(31, 0), 0x0, 0x03, 0x27},
221 {HPRE_DEV_ALG_BITMAP_CAP, 0x3148, 0, GENMASK(31, 0), 0x0, 0x03, 0x7F},
222 {HPRE_CORE1_ALG_BITMAP_CAP, 0x314c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
223 {HPRE_CORE2_ALG_BITMAP_CAP, 0x3150, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
224 {HPRE_CORE3_ALG_BITMAP_CAP, 0x3154, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
225 {HPRE_CORE4_ALG_BITMAP_CAP, 0x3158, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
226 {HPRE_CORE5_ALG_BITMAP_CAP, 0x315c, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
227 {HPRE_CORE6_ALG_BITMAP_CAP, 0x3160, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
228 {HPRE_CORE7_ALG_BITMAP_CAP, 0x3164, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
229 {HPRE_CORE8_ALG_BITMAP_CAP, 0x3168, 0, GENMASK(31, 0), 0x0, 0x7F, 0x7F},
230 {HPRE_CORE9_ALG_BITMAP_CAP, 0x316c, 0, GENMASK(31, 0), 0x0, 0x10, 0x10},
231 {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10}
232 };
233
234 static const struct hisi_qm_cap_query_info hpre_cap_query_info[] = {
235 {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C37, 0x7C37},
236 {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77},
237 {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8},
238 {HPRE_RAS_NFE_TYPE, "HPRE_RAS_NFE_TYPE ", 0x3130, 0x0, 0x3FFFFE, 0x1FFFC3E},
239 {HPRE_RAS_NFE_RESET, "HPRE_RAS_NFE_RESET ", 0x3134, 0x0, 0x3FFFFE, 0xBFFC3E},
240 {HPRE_RAS_CE_TYPE, "HPRE_RAS_CE_TYPE ", 0x3138, 0x0, 0x1, 0x1},
241 {HPRE_CORE_INFO, "HPRE_CORE_INFO ", 0x313c, 0x0, 0x420802, 0x120A0A},
242 {HPRE_CORE_EN, "HPRE_CORE_EN ", 0x3140, 0x0, 0xF, 0x3FF},
243 {HPRE_DRV_ALG_BITMAP, "HPRE_DRV_ALG_BITMAP ", 0x3144, 0x0, 0x03, 0x27},
244 {HPRE_ALG_BITMAP, "HPRE_ALG_BITMAP ", 0x3148, 0x0, 0x03, 0x7F},
245 {HPRE_CORE1_BITMAP_CAP, "HPRE_CORE1_BITMAP_CAP ", 0x314c, 0x0, 0x7F, 0x7F},
246 {HPRE_CORE2_BITMAP_CAP, "HPRE_CORE2_BITMAP_CAP ", 0x3150, 0x0, 0x7F, 0x7F},
247 {HPRE_CORE3_BITMAP_CAP, "HPRE_CORE3_BITMAP_CAP ", 0x3154, 0x0, 0x7F, 0x7F},
248 {HPRE_CORE4_BITMAP_CAP, "HPRE_CORE4_BITMAP_CAP ", 0x3158, 0x0, 0x7F, 0x7F},
249 {HPRE_CORE5_BITMAP_CAP, "HPRE_CORE5_BITMAP_CAP ", 0x315c, 0x0, 0x7F, 0x7F},
250 {HPRE_CORE6_BITMAP_CAP, "HPRE_CORE6_BITMAP_CAP ", 0x3160, 0x0, 0x7F, 0x7F},
251 {HPRE_CORE7_BITMAP_CAP, "HPRE_CORE7_BITMAP_CAP ", 0x3164, 0x0, 0x7F, 0x7F},
252 {HPRE_CORE8_BITMAP_CAP, "HPRE_CORE8_BITMAP_CAP ", 0x3168, 0x0, 0x7F, 0x7F},
253 {HPRE_CORE9_BITMAP_CAP, "HPRE_CORE9_BITMAP_CAP ", 0x316c, 0x0, 0x10, 0x10},
254 {HPRE_CORE10_BITMAP_CAP, "HPRE_CORE10_BITMAP_CAP ", 0x3170, 0x0, 0x10, 0x10},
255 };
256
257 static const struct hpre_hw_error hpre_hw_errors[] = {
258 {
259 .int_msk = BIT(0),
260 .msg = "core_ecc_1bit_err_int_set"
261 }, {
262 .int_msk = BIT(1),
263 .msg = "core_ecc_2bit_err_int_set"
264 }, {
265 .int_msk = BIT(2),
266 .msg = "dat_wb_poison_int_set"
267 }, {
268 .int_msk = BIT(3),
269 .msg = "dat_rd_poison_int_set"
270 }, {
271 .int_msk = BIT(4),
272 .msg = "bd_rd_poison_int_set"
273 }, {
274 .int_msk = BIT(5),
275 .msg = "ooo_ecc_2bit_err_int_set"
276 }, {
277 .int_msk = BIT(6),
278 .msg = "cluster1_shb_timeout_int_set"
279 }, {
280 .int_msk = BIT(7),
281 .msg = "cluster2_shb_timeout_int_set"
282 }, {
283 .int_msk = BIT(8),
284 .msg = "cluster3_shb_timeout_int_set"
285 }, {
286 .int_msk = BIT(9),
287 .msg = "cluster4_shb_timeout_int_set"
288 }, {
289 .int_msk = GENMASK(15, 10),
290 .msg = "ooo_rdrsp_err_int_set"
291 }, {
292 .int_msk = GENMASK(21, 16),
293 .msg = "ooo_wrrsp_err_int_set"
294 }, {
295 .int_msk = BIT(22),
296 .msg = "pt_rng_timeout_int_set"
297 }, {
298 .int_msk = BIT(23),
299 .msg = "sva_fsm_timeout_int_set"
300 }, {
301 .int_msk = BIT(24),
302 .msg = "sva_int_set"
303 }, {
304 /* sentinel */
305 }
306 };
307
308 static const u64 hpre_cluster_offsets[] = {
309 [HPRE_CLUSTER0] =
310 HPRE_CLSTR_BASE + HPRE_CLUSTER0 * HPRE_CLSTR_ADDR_INTRVL,
311 [HPRE_CLUSTER1] =
312 HPRE_CLSTR_BASE + HPRE_CLUSTER1 * HPRE_CLSTR_ADDR_INTRVL,
313 [HPRE_CLUSTER2] =
314 HPRE_CLSTR_BASE + HPRE_CLUSTER2 * HPRE_CLSTR_ADDR_INTRVL,
315 [HPRE_CLUSTER3] =
316 HPRE_CLSTR_BASE + HPRE_CLUSTER3 * HPRE_CLSTR_ADDR_INTRVL,
317 };
318
319 static const struct debugfs_reg32 hpre_cluster_dfx_regs[] = {
320 {"CORES_EN_STATUS ", HPRE_CORE_EN_OFFSET},
321 {"CORES_INI_CFG ", HPRE_CORE_INI_CFG_OFFSET},
322 {"CORES_INI_STATUS ", HPRE_CORE_INI_STATUS_OFFSET},
323 {"CORES_HTBT_WARN ", HPRE_CORE_HTBT_WARN_OFFSET},
324 {"CORES_IS_SCHD ", HPRE_CORE_IS_SCHD_OFFSET},
325 };
326
327 static const struct debugfs_reg32 hpre_com_dfx_regs[] = {
328 {"READ_CLR_EN ", HPRE_CTRL_CNT_CLR_CE},
329 {"AXQOS ", HPRE_VFG_AXQOS},
330 {"AWUSR_CFG ", HPRE_AWUSR_FP_CFG},
331 {"BD_ENDIAN ", HPRE_BD_ENDIAN},
332 {"ECC_CHECK_CTRL ", HPRE_ECC_BYPASS},
333 {"RAS_INT_WIDTH ", HPRE_RAS_WIDTH_CFG},
334 {"POISON_BYPASS ", HPRE_POISON_BYPASS},
335 {"BD_ARUSER ", HPRE_BD_ARUSR_CFG},
336 {"BD_AWUSER ", HPRE_BD_AWUSR_CFG},
337 {"DATA_ARUSER ", HPRE_DATA_RUSER_CFG},
338 {"DATA_AWUSER ", HPRE_DATA_WUSER_CFG},
339 {"INT_STATUS ", HPRE_INT_STATUS},
340 {"INT_MASK ", HPRE_HAC_INT_MSK},
341 {"RAS_CE_ENB ", HPRE_HAC_RAS_CE_ENB},
342 {"RAS_NFE_ENB ", HPRE_HAC_RAS_NFE_ENB},
343 {"RAS_FE_ENB ", HPRE_HAC_RAS_FE_ENB},
344 {"INT_SET ", HPRE_HAC_INT_SET},
345 {"RNG_TIMEOUT_NUM ", HPRE_RNG_TIMEOUT_NUM},
346 };
347
348 static const char *hpre_dfx_files[HPRE_DFX_FILE_NUM] = {
349 "send_cnt",
350 "recv_cnt",
351 "send_fail_cnt",
352 "send_busy_cnt",
353 "over_thrhld_cnt",
354 "overtime_thrhld",
355 "invalid_req_cnt"
356 };
357
358 /* define the HPRE's dfx regs region and region length */
359 static struct dfx_diff_registers hpre_diff_regs[] = {
360 {
361 .reg_offset = HPRE_DFX_BASE,
362 .reg_len = HPRE_DFX_BASE_LEN,
363 }, {
364 .reg_offset = HPRE_DFX_COMMON1,
365 .reg_len = HPRE_DFX_COMMON1_LEN,
366 }, {
367 .reg_offset = HPRE_DFX_COMMON2,
368 .reg_len = HPRE_DFX_COMMON2_LEN,
369 }, {
370 .reg_offset = HPRE_DFX_CORE,
371 .reg_len = HPRE_DFX_CORE_LEN,
372 },
373 };
374
375 static const char *hpre_channel_name[HPRE_MAX_CHANNEL_NUM] = {
376 "RSA",
377 "ECC",
378 };
379
380 static const struct hisi_qm_err_ini hpre_err_ini;
381
hpre_check_alg_support(struct hisi_qm * qm,u32 alg)382 bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg)
383 {
384 u32 cap_val;
385
386 cap_val = qm->cap_tables.dev_cap_table[HPRE_DRV_ALG_BITMAP].cap_val;
387 if (alg & cap_val)
388 return true;
389
390 return false;
391 }
392
hpre_diff_regs_show(struct seq_file * s,void * unused)393 static int hpre_diff_regs_show(struct seq_file *s, void *unused)
394 {
395 struct hisi_qm *qm = s->private;
396
397 hisi_qm_acc_diff_regs_dump(qm, s, qm->debug.acc_diff_regs,
398 ARRAY_SIZE(hpre_diff_regs));
399
400 return 0;
401 }
402
403 DEFINE_SHOW_ATTRIBUTE(hpre_diff_regs);
404
hpre_com_regs_show(struct seq_file * s,void * unused)405 static int hpre_com_regs_show(struct seq_file *s, void *unused)
406 {
407 hisi_qm_regs_dump(s, s->private);
408
409 return 0;
410 }
411
412 DEFINE_SHOW_ATTRIBUTE(hpre_com_regs);
413
hpre_cluster_regs_show(struct seq_file * s,void * unused)414 static int hpre_cluster_regs_show(struct seq_file *s, void *unused)
415 {
416 hisi_qm_regs_dump(s, s->private);
417
418 return 0;
419 }
420
421 DEFINE_SHOW_ATTRIBUTE(hpre_cluster_regs);
422
423 static const struct kernel_param_ops hpre_uacce_mode_ops = {
424 .set = uacce_mode_set,
425 .get = param_get_int,
426 };
427
428 /*
429 * uacce_mode = 0 means hpre only register to crypto,
430 * uacce_mode = 1 means hpre both register to crypto and uacce.
431 */
432 static u32 uacce_mode = UACCE_MODE_NOUACCE;
433 module_param_cb(uacce_mode, &hpre_uacce_mode_ops, &uacce_mode, 0444);
434 MODULE_PARM_DESC(uacce_mode, UACCE_MODE_DESC);
435
436 static bool pf_q_num_flag;
pf_q_num_set(const char * val,const struct kernel_param * kp)437 static int pf_q_num_set(const char *val, const struct kernel_param *kp)
438 {
439 pf_q_num_flag = true;
440
441 return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_HPRE_PF);
442 }
443
444 static const struct kernel_param_ops hpre_pf_q_num_ops = {
445 .set = pf_q_num_set,
446 .get = param_get_int,
447 };
448
449 static u32 pf_q_num = HPRE_PF_DEF_Q_NUM;
450 module_param_cb(pf_q_num, &hpre_pf_q_num_ops, &pf_q_num, 0444);
451 MODULE_PARM_DESC(pf_q_num, "Number of queues in PF of CS(2-1024)");
452
453 static const struct kernel_param_ops vfs_num_ops = {
454 .set = vfs_num_set,
455 .get = param_get_int,
456 };
457
458 static u32 vfs_num;
459 module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444);
460 MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)");
461
hpre_create_qp(u8 type)462 struct hisi_qp *hpre_create_qp(u8 type)
463 {
464 int node = cpu_to_node(raw_smp_processor_id());
465 struct hisi_qp *qp = NULL;
466 int ret;
467
468 if (type != HPRE_V2_ALG_TYPE && type != HPRE_V3_ECC_ALG_TYPE)
469 return NULL;
470
471 /*
472 * type: 0 - RSA/DH. algorithm supported in V2,
473 * 1 - ECC algorithm in V3.
474 */
475 ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp);
476 if (!ret)
477 return qp;
478
479 return NULL;
480 }
481
hpre_wait_sva_ready(struct hisi_qm * qm)482 static int hpre_wait_sva_ready(struct hisi_qm *qm)
483 {
484 u32 val, try_times = 0;
485 u8 count = 0;
486
487 /*
488 * Read the register value every 10-20us. If the value is 0 for three
489 * consecutive times, the SVA module is ready.
490 */
491 do {
492 val = readl(qm->io_base + HPRE_SVA_PREFTCH_DFX4);
493 if (val)
494 count = 0;
495 else if (++count == HPRE_READ_SVA_STATUS_TIMES)
496 break;
497
498 usleep_range(HPRE_WAIT_US_MIN, HPRE_WAIT_US_MAX);
499 } while (++try_times < HPRE_WAIT_SVA_READY);
500
501 if (try_times == HPRE_WAIT_SVA_READY) {
502 pci_err(qm->pdev, "failed to wait sva prefetch ready\n");
503 return -ETIMEDOUT;
504 }
505
506 return 0;
507 }
508
hpre_config_pasid(struct hisi_qm * qm)509 static void hpre_config_pasid(struct hisi_qm *qm)
510 {
511 u32 val1, val2;
512
513 if (qm->ver >= QM_HW_V3)
514 return;
515
516 val1 = readl_relaxed(qm->io_base + HPRE_DATA_RUSER_CFG);
517 val2 = readl_relaxed(qm->io_base + HPRE_DATA_WUSER_CFG);
518 if (qm->use_sva) {
519 val1 |= BIT(HPRE_PASID_EN_BIT);
520 val2 |= BIT(HPRE_PASID_EN_BIT);
521 } else {
522 val1 &= ~BIT(HPRE_PASID_EN_BIT);
523 val2 &= ~BIT(HPRE_PASID_EN_BIT);
524 }
525 writel_relaxed(val1, qm->io_base + HPRE_DATA_RUSER_CFG);
526 writel_relaxed(val2, qm->io_base + HPRE_DATA_WUSER_CFG);
527 }
528
hpre_cfg_by_dsm(struct hisi_qm * qm)529 static int hpre_cfg_by_dsm(struct hisi_qm *qm)
530 {
531 struct device *dev = &qm->pdev->dev;
532 union acpi_object *obj;
533 guid_t guid;
534
535 if (guid_parse("b06b81ab-0134-4a45-9b0c-483447b95fa7", &guid)) {
536 dev_err(dev, "Hpre GUID failed\n");
537 return -EINVAL;
538 }
539
540 /* Switch over to MSI handling due to non-standard PCI implementation */
541 obj = acpi_evaluate_dsm(ACPI_HANDLE(dev), &guid,
542 0, HPRE_VIA_MSI_DSM, NULL);
543 if (!obj) {
544 dev_err(dev, "ACPI handle failed!\n");
545 return -EIO;
546 }
547
548 ACPI_FREE(obj);
549
550 return 0;
551 }
552
hpre_set_cluster(struct hisi_qm * qm)553 static int hpre_set_cluster(struct hisi_qm *qm)
554 {
555 struct device *dev = &qm->pdev->dev;
556 u32 cluster_core_mask;
557 unsigned long offset;
558 u32 hpre_core_info;
559 u8 clusters_num;
560 u32 val = 0;
561 int ret, i;
562
563 cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_EN].cap_val;
564 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
565 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
566 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
567 for (i = 0; i < clusters_num; i++) {
568 offset = i * HPRE_CLSTR_ADDR_INTRVL;
569
570 /* clusters initiating */
571 writel(cluster_core_mask,
572 qm->io_base + offset + HPRE_CORE_ENB);
573 writel(0x1, qm->io_base + offset + HPRE_CORE_INI_CFG);
574 ret = readl_relaxed_poll_timeout(qm->io_base + offset +
575 HPRE_CORE_INI_STATUS, val,
576 ((val & cluster_core_mask) ==
577 cluster_core_mask),
578 HPRE_REG_RD_INTVRL_US,
579 HPRE_REG_RD_TMOUT_US);
580 if (ret) {
581 dev_err(dev,
582 "cluster %d int st status timeout!\n", i);
583 return -ETIMEDOUT;
584 }
585 }
586
587 return 0;
588 }
589
590 /*
591 * For Kunpeng 920, we should disable FLR triggered by hardware (BME/PM/SRIOV).
592 * Or it may stay in D3 state when we bind and unbind hpre quickly,
593 * as it does FLR triggered by hardware.
594 */
disable_flr_of_bme(struct hisi_qm * qm)595 static void disable_flr_of_bme(struct hisi_qm *qm)
596 {
597 u32 val;
598
599 val = readl(qm->io_base + QM_PEH_AXUSER_CFG);
600 val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR);
601 val |= HPRE_QM_PM_FLR;
602 writel(val, qm->io_base + QM_PEH_AXUSER_CFG);
603 writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE);
604 }
605
hpre_close_sva_prefetch(struct hisi_qm * qm)606 static void hpre_close_sva_prefetch(struct hisi_qm *qm)
607 {
608 u32 val;
609 int ret;
610
611 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
612 return;
613
614 val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG);
615 val |= HPRE_PREFETCH_DISABLE;
616 writel(val, qm->io_base + HPRE_PREFETCH_CFG);
617
618 ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX,
619 val, !(val & HPRE_SVA_DISABLE_READY),
620 HPRE_REG_RD_INTVRL_US,
621 HPRE_REG_RD_TMOUT_US);
622 if (ret)
623 pci_err(qm->pdev, "failed to close sva prefetch\n");
624
625 (void)hpre_wait_sva_ready(qm);
626 }
627
hpre_open_sva_prefetch(struct hisi_qm * qm)628 static void hpre_open_sva_prefetch(struct hisi_qm *qm)
629 {
630 u32 val;
631 int ret;
632
633 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps))
634 return;
635
636 /* Enable prefetch */
637 val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG);
638 val &= HPRE_PREFETCH_ENABLE;
639 writel(val, qm->io_base + HPRE_PREFETCH_CFG);
640
641 ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG,
642 val, !(val & HPRE_PREFETCH_DISABLE),
643 HPRE_REG_RD_INTVRL_US,
644 HPRE_REG_RD_TMOUT_US);
645 if (ret) {
646 pci_err(qm->pdev, "failed to open sva prefetch\n");
647 hpre_close_sva_prefetch(qm);
648 return;
649 }
650
651 ret = hpre_wait_sva_ready(qm);
652 if (ret)
653 hpre_close_sva_prefetch(qm);
654 }
655
hpre_enable_clock_gate(struct hisi_qm * qm)656 static void hpre_enable_clock_gate(struct hisi_qm *qm)
657 {
658 unsigned long offset;
659 u8 clusters_num, i;
660 u32 hpre_core_info;
661 u32 val;
662
663 if (qm->ver < QM_HW_V3)
664 return;
665
666 val = readl(qm->io_base + HPRE_CLKGATE_CTL);
667 val |= HPRE_CLKGATE_CTL_EN;
668 writel(val, qm->io_base + HPRE_CLKGATE_CTL);
669
670 val = readl(qm->io_base + HPRE_PEH_CFG_AUTO_GATE);
671 val |= HPRE_PEH_CFG_AUTO_GATE_EN;
672 writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE);
673
674 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
675 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
676 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
677 for (i = 0; i < clusters_num; i++) {
678 offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL;
679 val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL);
680 val |= HPRE_CLUSTER_DYN_CTL_EN;
681 writel(val, qm->io_base + offset + HPRE_CLUSTER_DYN_CTL);
682
683 val = readl(qm->io_base + offset + HPRE_CORE_SHB_CFG);
684 val |= HPRE_CORE_GATE_EN;
685 writel(val, qm->io_base + offset + HPRE_CORE_SHB_CFG);
686 }
687 }
688
hpre_disable_clock_gate(struct hisi_qm * qm)689 static void hpre_disable_clock_gate(struct hisi_qm *qm)
690 {
691 unsigned long offset;
692 u8 clusters_num, i;
693 u32 hpre_core_info;
694 u32 val;
695
696 if (qm->ver < QM_HW_V3)
697 return;
698
699 val = readl(qm->io_base + HPRE_CLKGATE_CTL);
700 val &= ~HPRE_CLKGATE_CTL_EN;
701 writel(val, qm->io_base + HPRE_CLKGATE_CTL);
702
703 val = readl(qm->io_base + HPRE_PEH_CFG_AUTO_GATE);
704 val &= ~HPRE_PEH_CFG_AUTO_GATE_EN;
705 writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE);
706
707 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
708 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
709 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
710 for (i = 0; i < clusters_num; i++) {
711 offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL;
712 val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL);
713 val &= ~HPRE_CLUSTER_DYN_CTL_EN;
714 writel(val, qm->io_base + offset + HPRE_CLUSTER_DYN_CTL);
715
716 val = readl(qm->io_base + offset + HPRE_CORE_SHB_CFG);
717 val &= ~HPRE_CORE_GATE_EN;
718 writel(val, qm->io_base + offset + HPRE_CORE_SHB_CFG);
719 }
720 }
721
hpre_set_user_domain_and_cache(struct hisi_qm * qm)722 static int hpre_set_user_domain_and_cache(struct hisi_qm *qm)
723 {
724 struct device *dev = &qm->pdev->dev;
725 u32 val;
726 int ret;
727
728 /* disabel dynamic clock gate before sram init */
729 hpre_disable_clock_gate(qm);
730
731 writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_ARUSER_M_CFG_ENABLE);
732 writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE);
733 writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG);
734
735 if (qm->ver >= QM_HW_V3)
736 writel(HPRE_RSA_ENB | HPRE_ECC_ENB,
737 qm->io_base + HPRE_TYPES_ENB);
738 else
739 writel(HPRE_RSA_ENB, qm->io_base + HPRE_TYPES_ENB);
740
741 writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE);
742 writel(0x0, qm->io_base + HPRE_BD_ENDIAN);
743 writel(0x0, qm->io_base + HPRE_POISON_BYPASS);
744 writel(0x0, qm->io_base + HPRE_ECC_BYPASS);
745
746 writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG);
747 writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_AWUSR_CFG);
748 writel(0x1, qm->io_base + HPRE_RDCHN_INI_CFG);
749 ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_RDCHN_INI_ST, val,
750 val & BIT(0),
751 HPRE_REG_RD_INTVRL_US,
752 HPRE_REG_RD_TMOUT_US);
753 if (ret) {
754 dev_err(dev, "read rd channel timeout fail!\n");
755 return -ETIMEDOUT;
756 }
757
758 ret = hpre_set_cluster(qm);
759 if (ret)
760 return -ETIMEDOUT;
761
762 /* This setting is only needed by Kunpeng 920. */
763 if (qm->ver == QM_HW_V2) {
764 ret = hpre_cfg_by_dsm(qm);
765 if (ret)
766 return ret;
767
768 disable_flr_of_bme(qm);
769 }
770
771 /* Config data buffer pasid needed by Kunpeng 920 */
772 hpre_config_pasid(qm);
773 hpre_open_sva_prefetch(qm);
774
775 hpre_enable_clock_gate(qm);
776
777 return ret;
778 }
779
hpre_cnt_regs_clear(struct hisi_qm * qm)780 static void hpre_cnt_regs_clear(struct hisi_qm *qm)
781 {
782 unsigned long offset;
783 u32 hpre_core_info;
784 u8 clusters_num;
785 int i;
786
787 /* clear clusterX/cluster_ctrl */
788 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
789 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
790 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
791 for (i = 0; i < clusters_num; i++) {
792 offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL;
793 writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY);
794 }
795
796 /* clear rdclr_en */
797 writel(0x0, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
798
799 hisi_qm_debug_regs_clear(qm);
800 }
801
hpre_master_ooo_ctrl(struct hisi_qm * qm,bool enable)802 static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable)
803 {
804 u32 val1, val2;
805
806 val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
807 if (enable) {
808 val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE;
809 val2 = qm->err_info.dev_err.shutdown_mask;
810 } else {
811 val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE;
812 val2 = 0x0;
813 }
814
815 if (qm->ver > QM_HW_V2)
816 writel(val2, qm->io_base + HPRE_OOO_SHUTDOWN_SEL);
817
818 writel(val1, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
819 }
820
hpre_hw_error_disable(struct hisi_qm * qm)821 static void hpre_hw_error_disable(struct hisi_qm *qm)
822 {
823 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
824 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
825
826 /* disable hpre hw error interrupts */
827 writel(err_mask, qm->io_base + HPRE_INT_MASK);
828 /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */
829 hpre_master_ooo_ctrl(qm, false);
830 }
831
hpre_hw_error_enable(struct hisi_qm * qm)832 static void hpre_hw_error_enable(struct hisi_qm *qm)
833 {
834 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
835 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
836
837 /* clear HPRE hw error source if having */
838 writel(err_mask, qm->io_base + HPRE_HAC_SOURCE_INT);
839
840 /* configure error type */
841 writel(dev_err->ce, qm->io_base + HPRE_RAS_CE_ENB);
842 writel(dev_err->nfe, qm->io_base + HPRE_RAS_NFE_ENB);
843 writel(dev_err->fe, qm->io_base + HPRE_RAS_FE_ENB);
844
845 /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */
846 hpre_master_ooo_ctrl(qm, true);
847
848 /* enable hpre hw error interrupts */
849 writel(~err_mask, qm->io_base + HPRE_INT_MASK);
850 }
851
hpre_file_to_qm(struct hpre_debugfs_file * file)852 static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file)
853 {
854 struct hpre *hpre = container_of(file->debug, struct hpre, debug);
855
856 return &hpre->qm;
857 }
858
hpre_clear_enable_read(struct hpre_debugfs_file * file)859 static u32 hpre_clear_enable_read(struct hpre_debugfs_file *file)
860 {
861 struct hisi_qm *qm = hpre_file_to_qm(file);
862
863 return readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
864 HPRE_CTRL_CNT_CLR_CE_BIT;
865 }
866
hpre_clear_enable_write(struct hpre_debugfs_file * file,u32 val)867 static int hpre_clear_enable_write(struct hpre_debugfs_file *file, u32 val)
868 {
869 struct hisi_qm *qm = hpre_file_to_qm(file);
870 u32 tmp;
871
872 if (val != 1 && val != 0)
873 return -EINVAL;
874
875 tmp = (readl(qm->io_base + HPRE_CTRL_CNT_CLR_CE) &
876 ~HPRE_CTRL_CNT_CLR_CE_BIT) | val;
877 writel(tmp, qm->io_base + HPRE_CTRL_CNT_CLR_CE);
878
879 return 0;
880 }
881
hpre_cluster_inqry_read(struct hpre_debugfs_file * file)882 static u32 hpre_cluster_inqry_read(struct hpre_debugfs_file *file)
883 {
884 struct hisi_qm *qm = hpre_file_to_qm(file);
885 int cluster_index = file->index - HPRE_CLUSTER_CTRL;
886 unsigned long offset = HPRE_CLSTR_BASE +
887 cluster_index * HPRE_CLSTR_ADDR_INTRVL;
888
889 return readl(qm->io_base + offset + HPRE_CLSTR_ADDR_INQRY_RSLT);
890 }
891
hpre_cluster_inqry_write(struct hpre_debugfs_file * file,u32 val)892 static void hpre_cluster_inqry_write(struct hpre_debugfs_file *file, u32 val)
893 {
894 struct hisi_qm *qm = hpre_file_to_qm(file);
895 int cluster_index = file->index - HPRE_CLUSTER_CTRL;
896 unsigned long offset = HPRE_CLSTR_BASE + cluster_index *
897 HPRE_CLSTR_ADDR_INTRVL;
898
899 writel(val, qm->io_base + offset + HPRE_CLUSTER_INQURY);
900 }
901
hpre_ctrl_debug_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)902 static ssize_t hpre_ctrl_debug_read(struct file *filp, char __user *buf,
903 size_t count, loff_t *pos)
904 {
905 struct hpre_debugfs_file *file = filp->private_data;
906 struct hisi_qm *qm = hpre_file_to_qm(file);
907 char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
908 u32 val;
909 int ret;
910
911 ret = hisi_qm_get_dfx_access(qm);
912 if (ret)
913 return ret;
914
915 spin_lock_irq(&file->lock);
916 switch (file->type) {
917 case HPRE_CLEAR_ENABLE:
918 val = hpre_clear_enable_read(file);
919 break;
920 case HPRE_CLUSTER_CTRL:
921 val = hpre_cluster_inqry_read(file);
922 break;
923 default:
924 goto err_input;
925 }
926 spin_unlock_irq(&file->lock);
927
928 hisi_qm_put_dfx_access(qm);
929 ret = snprintf(tbuf, HPRE_DBGFS_VAL_MAX_LEN, "%u\n", val);
930 return simple_read_from_buffer(buf, count, pos, tbuf, ret);
931
932 err_input:
933 spin_unlock_irq(&file->lock);
934 hisi_qm_put_dfx_access(qm);
935 return -EINVAL;
936 }
937
hpre_ctrl_debug_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)938 static ssize_t hpre_ctrl_debug_write(struct file *filp, const char __user *buf,
939 size_t count, loff_t *pos)
940 {
941 struct hpre_debugfs_file *file = filp->private_data;
942 struct hisi_qm *qm = hpre_file_to_qm(file);
943 char tbuf[HPRE_DBGFS_VAL_MAX_LEN];
944 unsigned long val;
945 int len, ret;
946
947 if (*pos != 0)
948 return 0;
949
950 if (count >= HPRE_DBGFS_VAL_MAX_LEN)
951 return -ENOSPC;
952
953 len = simple_write_to_buffer(tbuf, HPRE_DBGFS_VAL_MAX_LEN - 1,
954 pos, buf, count);
955 if (len < 0)
956 return len;
957
958 tbuf[len] = '\0';
959 if (kstrtoul(tbuf, 0, &val))
960 return -EFAULT;
961
962 ret = hisi_qm_get_dfx_access(qm);
963 if (ret)
964 return ret;
965
966 spin_lock_irq(&file->lock);
967 switch (file->type) {
968 case HPRE_CLEAR_ENABLE:
969 ret = hpre_clear_enable_write(file, val);
970 if (ret)
971 goto err_input;
972 break;
973 case HPRE_CLUSTER_CTRL:
974 hpre_cluster_inqry_write(file, val);
975 break;
976 default:
977 ret = -EINVAL;
978 goto err_input;
979 }
980
981 ret = count;
982
983 err_input:
984 spin_unlock_irq(&file->lock);
985 hisi_qm_put_dfx_access(qm);
986 return ret;
987 }
988
989 static const struct file_operations hpre_ctrl_debug_fops = {
990 .owner = THIS_MODULE,
991 .open = simple_open,
992 .read = hpre_ctrl_debug_read,
993 .write = hpre_ctrl_debug_write,
994 };
995
hpre_debugfs_atomic64_get(void * data,u64 * val)996 static int hpre_debugfs_atomic64_get(void *data, u64 *val)
997 {
998 struct hpre_dfx *dfx_item = data;
999
1000 *val = atomic64_read(&dfx_item->value);
1001
1002 return 0;
1003 }
1004
hpre_debugfs_atomic64_set(void * data,u64 val)1005 static int hpre_debugfs_atomic64_set(void *data, u64 val)
1006 {
1007 struct hpre_dfx *dfx_item = data;
1008 struct hpre_dfx *hpre_dfx = NULL;
1009
1010 if (dfx_item->type == HPRE_OVERTIME_THRHLD) {
1011 hpre_dfx = dfx_item - HPRE_OVERTIME_THRHLD;
1012 atomic64_set(&hpre_dfx[HPRE_OVER_THRHLD_CNT].value, 0);
1013 } else if (val) {
1014 return -EINVAL;
1015 }
1016
1017 atomic64_set(&dfx_item->value, val);
1018
1019 return 0;
1020 }
1021
1022 DEFINE_DEBUGFS_ATTRIBUTE(hpre_atomic64_ops, hpre_debugfs_atomic64_get,
1023 hpre_debugfs_atomic64_set, "%llu\n");
1024
hpre_create_debugfs_file(struct hisi_qm * qm,struct dentry * dir,enum hpre_ctrl_dbgfs_file type,int indx)1025 static int hpre_create_debugfs_file(struct hisi_qm *qm, struct dentry *dir,
1026 enum hpre_ctrl_dbgfs_file type, int indx)
1027 {
1028 struct hpre *hpre = container_of(qm, struct hpre, qm);
1029 struct hpre_debug *dbg = &hpre->debug;
1030 struct dentry *file_dir;
1031
1032 if (dir)
1033 file_dir = dir;
1034 else
1035 file_dir = qm->debug.debug_root;
1036
1037 if (type >= HPRE_DEBUG_FILE_NUM)
1038 return -EINVAL;
1039
1040 spin_lock_init(&dbg->files[indx].lock);
1041 dbg->files[indx].debug = dbg;
1042 dbg->files[indx].type = type;
1043 dbg->files[indx].index = indx;
1044 debugfs_create_file(hpre_debug_file_name[type], 0600, file_dir,
1045 dbg->files + indx, &hpre_ctrl_debug_fops);
1046
1047 return 0;
1048 }
1049
hpre_pf_comm_regs_debugfs_init(struct hisi_qm * qm)1050 static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm)
1051 {
1052 struct device *dev = &qm->pdev->dev;
1053 struct debugfs_regset32 *regset;
1054
1055 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
1056 if (!regset)
1057 return -ENOMEM;
1058
1059 regset->regs = hpre_com_dfx_regs;
1060 regset->nregs = ARRAY_SIZE(hpre_com_dfx_regs);
1061 regset->base = qm->io_base;
1062 regset->dev = dev;
1063
1064 debugfs_create_file("regs", 0444, qm->debug.debug_root,
1065 regset, &hpre_com_regs_fops);
1066
1067 return 0;
1068 }
1069
hpre_cluster_debugfs_init(struct hisi_qm * qm)1070 static int hpre_cluster_debugfs_init(struct hisi_qm *qm)
1071 {
1072 struct device *dev = &qm->pdev->dev;
1073 char buf[HPRE_DBGFS_VAL_MAX_LEN];
1074 struct debugfs_regset32 *regset;
1075 struct dentry *tmp_d;
1076 u32 hpre_core_info;
1077 u8 clusters_num;
1078 int i, ret;
1079
1080 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
1081 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
1082 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
1083 for (i = 0; i < clusters_num; i++) {
1084 ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i);
1085 if (ret >= HPRE_DBGFS_VAL_MAX_LEN)
1086 return -EINVAL;
1087 tmp_d = debugfs_create_dir(buf, qm->debug.debug_root);
1088
1089 regset = devm_kzalloc(dev, sizeof(*regset), GFP_KERNEL);
1090 if (!regset)
1091 return -ENOMEM;
1092
1093 regset->regs = hpre_cluster_dfx_regs;
1094 regset->nregs = ARRAY_SIZE(hpre_cluster_dfx_regs);
1095 regset->base = qm->io_base + hpre_cluster_offsets[i];
1096 regset->dev = dev;
1097
1098 debugfs_create_file("regs", 0444, tmp_d, regset,
1099 &hpre_cluster_regs_fops);
1100 ret = hpre_create_debugfs_file(qm, tmp_d, HPRE_CLUSTER_CTRL,
1101 i + HPRE_CLUSTER_CTRL);
1102 if (ret)
1103 return ret;
1104 }
1105
1106 return 0;
1107 }
1108
hpre_ctrl_debug_init(struct hisi_qm * qm)1109 static int hpre_ctrl_debug_init(struct hisi_qm *qm)
1110 {
1111 int ret;
1112
1113 ret = hpre_create_debugfs_file(qm, NULL, HPRE_CLEAR_ENABLE,
1114 HPRE_CLEAR_ENABLE);
1115 if (ret)
1116 return ret;
1117
1118 ret = hpre_pf_comm_regs_debugfs_init(qm);
1119 if (ret)
1120 return ret;
1121
1122 return hpre_cluster_debugfs_init(qm);
1123 }
1124
hpre_cap_regs_show(struct seq_file * s,void * unused)1125 static int hpre_cap_regs_show(struct seq_file *s, void *unused)
1126 {
1127 struct hisi_qm *qm = s->private;
1128 u32 i, size;
1129
1130 size = qm->cap_tables.qm_cap_size;
1131 for (i = 0; i < size; i++)
1132 seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name,
1133 qm->cap_tables.qm_cap_table[i].cap_val);
1134
1135 size = qm->cap_tables.dev_cap_size;
1136 for (i = 0; i < size; i++)
1137 seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name,
1138 qm->cap_tables.dev_cap_table[i].cap_val);
1139
1140 return 0;
1141 }
1142
1143 DEFINE_SHOW_ATTRIBUTE(hpre_cap_regs);
1144
hpre_dfx_debug_init(struct hisi_qm * qm)1145 static void hpre_dfx_debug_init(struct hisi_qm *qm)
1146 {
1147 struct dfx_diff_registers *hpre_regs = qm->debug.acc_diff_regs;
1148 struct hpre *hpre = container_of(qm, struct hpre, qm);
1149 struct hpre_dfx *dfx = hpre->debug.dfx;
1150 struct dentry *parent;
1151 int i;
1152
1153 parent = debugfs_create_dir("hpre_dfx", qm->debug.debug_root);
1154 for (i = 0; i < HPRE_DFX_FILE_NUM; i++) {
1155 dfx[i].type = i;
1156 debugfs_create_file(hpre_dfx_files[i], 0644, parent, &dfx[i],
1157 &hpre_atomic64_ops);
1158 }
1159
1160 if (qm->fun_type == QM_HW_PF && hpre_regs)
1161 debugfs_create_file("diff_regs", 0444, parent,
1162 qm, &hpre_diff_regs_fops);
1163
1164 debugfs_create_file("cap_regs", CAP_FILE_PERMISSION,
1165 qm->debug.debug_root, qm, &hpre_cap_regs_fops);
1166 }
1167
hpre_debugfs_init(struct hisi_qm * qm)1168 static int hpre_debugfs_init(struct hisi_qm *qm)
1169 {
1170 struct device *dev = &qm->pdev->dev;
1171 int ret;
1172
1173 ret = hisi_qm_regs_debugfs_init(qm, hpre_diff_regs, ARRAY_SIZE(hpre_diff_regs));
1174 if (ret) {
1175 dev_warn(dev, "Failed to init HPRE diff regs!\n");
1176 return ret;
1177 }
1178
1179 qm->debug.debug_root = debugfs_create_dir(dev_name(dev),
1180 hpre_debugfs_root);
1181 qm->debug.sqe_mask_offset = HPRE_SQE_MASK_OFFSET;
1182 qm->debug.sqe_mask_len = HPRE_SQE_MASK_LEN;
1183
1184 hisi_qm_debug_init(qm);
1185
1186 if (qm->pdev->device == PCI_DEVICE_ID_HUAWEI_HPRE_PF) {
1187 ret = hpre_ctrl_debug_init(qm);
1188 if (ret)
1189 goto debugfs_remove;
1190 }
1191
1192 hpre_dfx_debug_init(qm);
1193
1194 return 0;
1195
1196 debugfs_remove:
1197 debugfs_remove_recursive(qm->debug.debug_root);
1198 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs));
1199 return ret;
1200 }
1201
hpre_debugfs_exit(struct hisi_qm * qm)1202 static void hpre_debugfs_exit(struct hisi_qm *qm)
1203 {
1204 debugfs_remove_recursive(qm->debug.debug_root);
1205
1206 hisi_qm_regs_debugfs_uninit(qm, ARRAY_SIZE(hpre_diff_regs));
1207 }
1208
hpre_pre_store_cap_reg(struct hisi_qm * qm)1209 static int hpre_pre_store_cap_reg(struct hisi_qm *qm)
1210 {
1211 struct hisi_qm_cap_record *hpre_cap;
1212 struct device *dev = &qm->pdev->dev;
1213 u32 hpre_core_info;
1214 u8 clusters_num;
1215 size_t i, size;
1216
1217 size = ARRAY_SIZE(hpre_cap_query_info);
1218 hpre_cap = devm_kcalloc(dev, size, sizeof(*hpre_cap), GFP_KERNEL);
1219 if (!hpre_cap)
1220 return -ENOMEM;
1221
1222 for (i = 0; i < size; i++) {
1223 hpre_cap[i].type = hpre_cap_query_info[i].type;
1224 hpre_cap[i].name = hpre_cap_query_info[i].name;
1225 hpre_cap[i].cap_val = hisi_qm_get_cap_value(qm, hpre_cap_query_info,
1226 i, qm->cap_ver);
1227 }
1228
1229 hpre_core_info = hpre_cap[HPRE_CORE_INFO].cap_val;
1230 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
1231 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
1232 if (clusters_num > HPRE_CLUSTERS_NUM_MAX) {
1233 dev_err(dev, "Device cluster num %u is out of range for driver supports %d!\n",
1234 clusters_num, HPRE_CLUSTERS_NUM_MAX);
1235 return -EINVAL;
1236 }
1237
1238 qm->cap_tables.dev_cap_table = hpre_cap;
1239 qm->cap_tables.dev_cap_size = size;
1240
1241 return 0;
1242 }
1243
hpre_set_channels(struct hisi_qm * qm)1244 static void hpre_set_channels(struct hisi_qm *qm)
1245 {
1246 struct qm_channel *channel_data = &qm->channel_data;
1247 int i;
1248
1249 channel_data->channel_num = HPRE_MAX_CHANNEL_NUM;
1250 for (i = 0; i < HPRE_MAX_CHANNEL_NUM; i++)
1251 channel_data->channel_name[i] = hpre_channel_name[i];
1252 }
1253
hpre_qm_init(struct hisi_qm * qm,struct pci_dev * pdev)1254 static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
1255 {
1256 u64 alg_msk;
1257 int ret;
1258
1259 if (pdev->revision == QM_HW_V1) {
1260 pci_warn(pdev, "HPRE version 1 is not supported!\n");
1261 return -EINVAL;
1262 }
1263
1264 qm->mode = uacce_mode;
1265 qm->pdev = pdev;
1266 qm->sqe_size = HPRE_SQE_SIZE;
1267 qm->dev_name = hpre_name;
1268
1269 qm->fun_type = (pdev->device == PCI_DEVICE_ID_HUAWEI_HPRE_PF) ?
1270 QM_HW_PF : QM_HW_VF;
1271 if (qm->fun_type == QM_HW_PF) {
1272 qm->qp_base = HPRE_PF_DEF_Q_BASE;
1273 qm->qp_num = pf_q_num;
1274 qm->debug.curr_qm_qp_num = pf_q_num;
1275 qm->qm_list = &hpre_devices;
1276 qm->err_ini = &hpre_err_ini;
1277 if (pf_q_num_flag)
1278 set_bit(QM_MODULE_PARAM, &qm->misc_ctl);
1279 }
1280
1281 ret = hisi_qm_init(qm);
1282 if (ret) {
1283 pci_err(pdev, "Failed to init hpre qm configures!\n");
1284 return ret;
1285 }
1286
1287 hpre_set_channels(qm);
1288 /* Fetch and save the value of capability registers */
1289 ret = hpre_pre_store_cap_reg(qm);
1290 if (ret) {
1291 pci_err(pdev, "Failed to pre-store capability registers!\n");
1292 hisi_qm_uninit(qm);
1293 return ret;
1294 }
1295
1296 alg_msk = qm->cap_tables.dev_cap_table[HPRE_ALG_BITMAP].cap_val;
1297 ret = hisi_qm_set_algs(qm, alg_msk, hpre_dev_algs, ARRAY_SIZE(hpre_dev_algs));
1298 if (ret) {
1299 pci_err(pdev, "Failed to set hpre algs!\n");
1300 hisi_qm_uninit(qm);
1301 }
1302
1303 return ret;
1304 }
1305
hpre_show_last_regs_init(struct hisi_qm * qm)1306 static int hpre_show_last_regs_init(struct hisi_qm *qm)
1307 {
1308 int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs);
1309 int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs);
1310 struct qm_debug *debug = &qm->debug;
1311 void __iomem *io_base;
1312 u32 hpre_core_info;
1313 u8 clusters_num;
1314 int i, j, idx;
1315
1316 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
1317 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
1318 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
1319 debug->last_words = kcalloc(cluster_dfx_regs_num * clusters_num +
1320 com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL);
1321 if (!debug->last_words)
1322 return -ENOMEM;
1323
1324 for (i = 0; i < com_dfx_regs_num; i++)
1325 debug->last_words[i] = readl_relaxed(qm->io_base +
1326 hpre_com_dfx_regs[i].offset);
1327
1328 for (i = 0; i < clusters_num; i++) {
1329 io_base = qm->io_base + hpre_cluster_offsets[i];
1330 for (j = 0; j < cluster_dfx_regs_num; j++) {
1331 idx = com_dfx_regs_num + i * cluster_dfx_regs_num + j;
1332 debug->last_words[idx] = readl_relaxed(
1333 io_base + hpre_cluster_dfx_regs[j].offset);
1334 }
1335 }
1336
1337 return 0;
1338 }
1339
hpre_show_last_regs_uninit(struct hisi_qm * qm)1340 static void hpre_show_last_regs_uninit(struct hisi_qm *qm)
1341 {
1342 struct qm_debug *debug = &qm->debug;
1343
1344 if (qm->fun_type == QM_HW_VF || !debug->last_words)
1345 return;
1346
1347 kfree(debug->last_words);
1348 debug->last_words = NULL;
1349 }
1350
hpre_show_last_dfx_regs(struct hisi_qm * qm)1351 static void hpre_show_last_dfx_regs(struct hisi_qm *qm)
1352 {
1353 int cluster_dfx_regs_num = ARRAY_SIZE(hpre_cluster_dfx_regs);
1354 int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs);
1355 struct qm_debug *debug = &qm->debug;
1356 struct pci_dev *pdev = qm->pdev;
1357 void __iomem *io_base;
1358 u32 hpre_core_info;
1359 u8 clusters_num;
1360 int i, j, idx;
1361 u32 val;
1362
1363 if (qm->fun_type == QM_HW_VF || !debug->last_words)
1364 return;
1365
1366 /* dumps last word of the debugging registers during controller reset */
1367 for (i = 0; i < com_dfx_regs_num; i++) {
1368 val = readl_relaxed(qm->io_base + hpre_com_dfx_regs[i].offset);
1369 if (debug->last_words[i] != val)
1370 pci_info(pdev, "Common_core:%s \t= 0x%08x => 0x%08x\n",
1371 hpre_com_dfx_regs[i].name, debug->last_words[i], val);
1372 }
1373
1374 hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val;
1375 clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) &
1376 hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask;
1377 for (i = 0; i < clusters_num; i++) {
1378 io_base = qm->io_base + hpre_cluster_offsets[i];
1379 for (j = 0; j < cluster_dfx_regs_num; j++) {
1380 val = readl_relaxed(io_base +
1381 hpre_cluster_dfx_regs[j].offset);
1382 idx = com_dfx_regs_num + i * cluster_dfx_regs_num + j;
1383 if (debug->last_words[idx] != val)
1384 pci_info(pdev, "cluster-%d:%s \t= 0x%08x => 0x%08x\n",
1385 i, hpre_cluster_dfx_regs[j].name, debug->last_words[idx], val);
1386 }
1387 }
1388 }
1389
hpre_log_hw_error(struct hisi_qm * qm,u32 err_sts)1390 static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts)
1391 {
1392 const struct hpre_hw_error *err = hpre_hw_errors;
1393 struct device *dev = &qm->pdev->dev;
1394
1395 while (err->msg) {
1396 if (err->int_msk & err_sts)
1397 dev_warn(dev, "%s [error status=0x%x] found\n",
1398 err->msg, err->int_msk);
1399 err++;
1400 }
1401 }
1402
hpre_get_hw_err_status(struct hisi_qm * qm)1403 static u32 hpre_get_hw_err_status(struct hisi_qm *qm)
1404 {
1405 return readl(qm->io_base + HPRE_INT_STATUS);
1406 }
1407
hpre_clear_hw_err_status(struct hisi_qm * qm,u32 err_sts)1408 static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts)
1409 {
1410 writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT);
1411 }
1412
hpre_disable_error_report(struct hisi_qm * qm,u32 err_type)1413 static void hpre_disable_error_report(struct hisi_qm *qm, u32 err_type)
1414 {
1415 u32 nfe_mask = qm->err_info.dev_err.nfe;
1416
1417 writel(nfe_mask & (~err_type), qm->io_base + HPRE_RAS_NFE_ENB);
1418 }
1419
hpre_enable_error_report(struct hisi_qm * qm)1420 static void hpre_enable_error_report(struct hisi_qm *qm)
1421 {
1422 u32 nfe_mask = qm->err_info.dev_err.nfe;
1423 u32 ce_mask = qm->err_info.dev_err.ce;
1424
1425 writel(nfe_mask, qm->io_base + HPRE_RAS_NFE_ENB);
1426 writel(ce_mask, qm->io_base + HPRE_RAS_CE_ENB);
1427 }
1428
hpre_open_axi_master_ooo(struct hisi_qm * qm)1429 static void hpre_open_axi_master_ooo(struct hisi_qm *qm)
1430 {
1431 u32 value;
1432
1433 value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
1434 writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE,
1435 qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
1436 writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE,
1437 qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB);
1438 }
1439
hpre_get_err_result(struct hisi_qm * qm)1440 static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm)
1441 {
1442 u32 err_status;
1443
1444 err_status = hpre_get_hw_err_status(qm);
1445 if (err_status) {
1446 if (err_status & qm->err_info.dev_err.ecc_2bits_mask)
1447 qm->err_status.is_dev_ecc_mbit = true;
1448 hpre_log_hw_error(qm, err_status);
1449
1450 if (err_status & qm->err_info.dev_err.reset_mask) {
1451 /* Disable the same error reporting until device is recovered. */
1452 hpre_disable_error_report(qm, err_status);
1453 return ACC_ERR_NEED_RESET;
1454 }
1455 hpre_clear_hw_err_status(qm, err_status);
1456 /* Avoid firmware disable error report, re-enable. */
1457 hpre_enable_error_report(qm);
1458 }
1459
1460 return ACC_ERR_RECOVERED;
1461 }
1462
hpre_dev_is_abnormal(struct hisi_qm * qm)1463 static bool hpre_dev_is_abnormal(struct hisi_qm *qm)
1464 {
1465 u32 err_status;
1466
1467 err_status = hpre_get_hw_err_status(qm);
1468 if (err_status & qm->err_info.dev_err.shutdown_mask)
1469 return true;
1470
1471 return false;
1472 }
1473
hpre_disable_axi_error(struct hisi_qm * qm)1474 static void hpre_disable_axi_error(struct hisi_qm *qm)
1475 {
1476 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
1477 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
1478 u32 val;
1479
1480 val = ~(err_mask & (~HPRE_AXI_ERROR_MASK));
1481 writel(val, qm->io_base + HPRE_INT_MASK);
1482
1483 if (qm->ver > QM_HW_V2)
1484 writel(dev_err->shutdown_mask & (~HPRE_AXI_ERROR_MASK),
1485 qm->io_base + HPRE_OOO_SHUTDOWN_SEL);
1486 }
1487
hpre_enable_axi_error(struct hisi_qm * qm)1488 static void hpre_enable_axi_error(struct hisi_qm *qm)
1489 {
1490 struct hisi_qm_err_mask *dev_err = &qm->err_info.dev_err;
1491 u32 err_mask = dev_err->ce | dev_err->nfe | dev_err->fe;
1492
1493 /* clear axi error source */
1494 writel(HPRE_AXI_ERROR_MASK, qm->io_base + HPRE_HAC_SOURCE_INT);
1495
1496 writel(~err_mask, qm->io_base + HPRE_INT_MASK);
1497
1498 if (qm->ver > QM_HW_V2)
1499 writel(dev_err->shutdown_mask, qm->io_base + HPRE_OOO_SHUTDOWN_SEL);
1500 }
1501
hpre_err_info_init(struct hisi_qm * qm)1502 static void hpre_err_info_init(struct hisi_qm *qm)
1503 {
1504 struct hisi_qm_err_info *err_info = &qm->err_info;
1505 struct hisi_qm_err_mask *qm_err = &err_info->qm_err;
1506 struct hisi_qm_err_mask *dev_err = &err_info->dev_err;
1507
1508 qm_err->fe = HPRE_HAC_RAS_FE_ENABLE;
1509 qm_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_CE_MASK_CAP, qm->cap_ver);
1510 qm_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_QM_NFE_MASK_CAP, qm->cap_ver);
1511 qm_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info,
1512 HPRE_QM_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1513 qm_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info,
1514 HPRE_QM_RESET_MASK_CAP, qm->cap_ver);
1515 qm_err->ecc_2bits_mask = QM_ECC_MBIT;
1516
1517 dev_err->fe = HPRE_HAC_RAS_FE_ENABLE;
1518 dev_err->ce = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_CE_MASK_CAP, qm->cap_ver);
1519 dev_err->nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver);
1520 dev_err->shutdown_mask = hisi_qm_get_hw_info(qm, hpre_basic_info,
1521 HPRE_OOO_SHUTDOWN_MASK_CAP, qm->cap_ver);
1522 dev_err->reset_mask = hisi_qm_get_hw_info(qm, hpre_basic_info,
1523 HPRE_RESET_MASK_CAP, qm->cap_ver);
1524 dev_err->ecc_2bits_mask = HPRE_CORE_ECC_2BIT_ERR | HPRE_OOO_ECC_2BIT_ERR;
1525
1526 err_info->msi_wr_port = HPRE_WR_MSI_PORT;
1527 err_info->acpi_rst = "HRST";
1528 }
1529
1530 static const struct hisi_qm_err_ini hpre_err_ini = {
1531 .hw_init = hpre_set_user_domain_and_cache,
1532 .hw_err_enable = hpre_hw_error_enable,
1533 .hw_err_disable = hpre_hw_error_disable,
1534 .get_dev_hw_err_status = hpre_get_hw_err_status,
1535 .clear_dev_hw_err_status = hpre_clear_hw_err_status,
1536 .open_axi_master_ooo = hpre_open_axi_master_ooo,
1537 .open_sva_prefetch = hpre_open_sva_prefetch,
1538 .close_sva_prefetch = hpre_close_sva_prefetch,
1539 .show_last_dfx_regs = hpre_show_last_dfx_regs,
1540 .err_info_init = hpre_err_info_init,
1541 .get_err_result = hpre_get_err_result,
1542 .dev_is_abnormal = hpre_dev_is_abnormal,
1543 .disable_axi_error = hpre_disable_axi_error,
1544 .enable_axi_error = hpre_enable_axi_error,
1545 };
1546
hpre_pf_probe_init(struct hpre * hpre)1547 static int hpre_pf_probe_init(struct hpre *hpre)
1548 {
1549 struct hisi_qm *qm = &hpre->qm;
1550 int ret;
1551
1552 ret = hpre_set_user_domain_and_cache(qm);
1553 if (ret)
1554 return ret;
1555
1556 hisi_qm_dev_err_init(qm);
1557 ret = hpre_show_last_regs_init(qm);
1558 if (ret)
1559 pci_err(qm->pdev, "Failed to init last word regs!\n");
1560
1561 return ret;
1562 }
1563
hpre_probe_init(struct hpre * hpre)1564 static int hpre_probe_init(struct hpre *hpre)
1565 {
1566 u32 type_rate = HPRE_SHAPER_TYPE_RATE;
1567 struct hisi_qm *qm = &hpre->qm;
1568 int ret;
1569
1570 if (qm->fun_type == QM_HW_PF) {
1571 ret = hpre_pf_probe_init(hpre);
1572 if (ret)
1573 return ret;
1574 /* Enable shaper type 0 */
1575 if (qm->ver >= QM_HW_V3) {
1576 type_rate |= QM_SHAPER_ENABLE;
1577 qm->type_rate = type_rate;
1578 }
1579 }
1580
1581 return 0;
1582 }
1583
hpre_probe_uninit(struct hisi_qm * qm)1584 static void hpre_probe_uninit(struct hisi_qm *qm)
1585 {
1586 if (qm->fun_type == QM_HW_VF)
1587 return;
1588
1589 hpre_cnt_regs_clear(qm);
1590 qm->debug.curr_qm_qp_num = 0;
1591 hpre_show_last_regs_uninit(qm);
1592 hpre_close_sva_prefetch(qm);
1593 hisi_qm_dev_err_uninit(qm);
1594 }
1595
hpre_probe(struct pci_dev * pdev,const struct pci_device_id * id)1596 static int hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1597 {
1598 struct hisi_qm *qm;
1599 struct hpre *hpre;
1600 int ret;
1601
1602 hpre = devm_kzalloc(&pdev->dev, sizeof(*hpre), GFP_KERNEL);
1603 if (!hpre)
1604 return -ENOMEM;
1605
1606 qm = &hpre->qm;
1607 ret = hpre_qm_init(qm, pdev);
1608 if (ret) {
1609 pci_err(pdev, "Failed to init HPRE QM (%d)!\n", ret);
1610 return ret;
1611 }
1612
1613 ret = hpre_probe_init(hpre);
1614 if (ret) {
1615 pci_err(pdev, "Failed to probe (%d)!\n", ret);
1616 goto err_with_qm_init;
1617 }
1618
1619 ret = hisi_qm_start(qm);
1620 if (ret)
1621 goto err_with_probe_init;
1622
1623 ret = hpre_debugfs_init(qm);
1624 if (ret)
1625 dev_warn(&pdev->dev, "init debugfs fail!\n");
1626
1627 hisi_qm_add_list(qm, &hpre_devices);
1628 ret = hisi_qm_alg_register(qm, &hpre_devices, HPRE_CTX_Q_NUM_DEF);
1629 if (ret < 0) {
1630 pci_err(pdev, "fail to register algs to crypto!\n");
1631 goto err_qm_del_list;
1632 }
1633
1634 if (qm->uacce) {
1635 ret = uacce_register(qm->uacce);
1636 if (ret) {
1637 pci_err(pdev, "failed to register uacce (%d)!\n", ret);
1638 goto err_with_alg_register;
1639 }
1640 }
1641
1642 if (qm->fun_type == QM_HW_PF && vfs_num) {
1643 ret = hisi_qm_sriov_enable(pdev, vfs_num);
1644 if (ret < 0)
1645 goto err_with_alg_register;
1646 }
1647
1648 hisi_qm_pm_init(qm);
1649
1650 return 0;
1651
1652 err_with_alg_register:
1653 hisi_qm_alg_unregister(qm, &hpre_devices, HPRE_CTX_Q_NUM_DEF);
1654
1655 err_qm_del_list:
1656 hisi_qm_del_list(qm, &hpre_devices);
1657 hpre_debugfs_exit(qm);
1658 hisi_qm_stop(qm, QM_NORMAL);
1659
1660 err_with_probe_init:
1661 hpre_probe_uninit(qm);
1662
1663 err_with_qm_init:
1664 hisi_qm_uninit(qm);
1665
1666 return ret;
1667 }
1668
hpre_remove(struct pci_dev * pdev)1669 static void hpre_remove(struct pci_dev *pdev)
1670 {
1671 struct hisi_qm *qm = pci_get_drvdata(pdev);
1672
1673 hisi_qm_pm_uninit(qm);
1674 hisi_qm_wait_task_finish(qm, &hpre_devices);
1675 hisi_qm_alg_unregister(qm, &hpre_devices, HPRE_CTX_Q_NUM_DEF);
1676 hisi_qm_del_list(qm, &hpre_devices);
1677 if (qm->fun_type == QM_HW_PF && qm->vfs_num)
1678 hisi_qm_sriov_disable(pdev, true);
1679
1680 hpre_debugfs_exit(qm);
1681 hisi_qm_stop(qm, QM_NORMAL);
1682
1683 hpre_probe_uninit(qm);
1684 hisi_qm_uninit(qm);
1685 }
1686
1687 static const struct dev_pm_ops hpre_pm_ops = {
1688 SET_RUNTIME_PM_OPS(hisi_qm_suspend, hisi_qm_resume, NULL)
1689 };
1690
1691 static const struct pci_error_handlers hpre_err_handler = {
1692 .error_detected = hisi_qm_dev_err_detected,
1693 .slot_reset = hisi_qm_dev_slot_reset,
1694 .reset_prepare = hisi_qm_reset_prepare,
1695 .reset_done = hisi_qm_reset_done,
1696 };
1697
1698 static struct pci_driver hpre_pci_driver = {
1699 .name = hpre_name,
1700 .id_table = hpre_dev_ids,
1701 .probe = hpre_probe,
1702 .remove = hpre_remove,
1703 .sriov_configure = IS_ENABLED(CONFIG_PCI_IOV) ?
1704 hisi_qm_sriov_configure : NULL,
1705 .err_handler = &hpre_err_handler,
1706 .shutdown = hisi_qm_dev_shutdown,
1707 .driver.pm = &hpre_pm_ops,
1708 };
1709
hisi_hpre_get_pf_driver(void)1710 struct pci_driver *hisi_hpre_get_pf_driver(void)
1711 {
1712 return &hpre_pci_driver;
1713 }
1714 EXPORT_SYMBOL_GPL(hisi_hpre_get_pf_driver);
1715
hpre_register_debugfs(void)1716 static void hpre_register_debugfs(void)
1717 {
1718 if (!debugfs_initialized())
1719 return;
1720
1721 hpre_debugfs_root = debugfs_create_dir(hpre_name, NULL);
1722 }
1723
hpre_unregister_debugfs(void)1724 static void hpre_unregister_debugfs(void)
1725 {
1726 debugfs_remove_recursive(hpre_debugfs_root);
1727 }
1728
hpre_init(void)1729 static int __init hpre_init(void)
1730 {
1731 int ret;
1732
1733 hisi_qm_init_list(&hpre_devices);
1734 hpre_register_debugfs();
1735
1736 ret = pci_register_driver(&hpre_pci_driver);
1737 if (ret) {
1738 hpre_unregister_debugfs();
1739 pr_err("hpre: can't register hisi hpre driver.\n");
1740 }
1741
1742 return ret;
1743 }
1744
hpre_exit(void)1745 static void __exit hpre_exit(void)
1746 {
1747 pci_unregister_driver(&hpre_pci_driver);
1748 hpre_unregister_debugfs();
1749 }
1750
1751 module_init(hpre_init);
1752 module_exit(hpre_exit);
1753
1754 MODULE_LICENSE("GPL v2");
1755 MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
1756 MODULE_AUTHOR("Meng Yu <yumeng18@huawei.com>");
1757 MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator");
1758