1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef __NET_TC_WRAPPER_H
3 #define __NET_TC_WRAPPER_H
4
5 #include <net/pkt_cls.h>
6
7 #if IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)
8
9 #include <linux/cpufeature.h>
10 #include <linux/static_key.h>
11 #include <linux/indirect_call_wrapper.h>
12
13 #define TC_INDIRECT_SCOPE
14
15 extern struct static_key_false tc_skip_wrapper_act;
16 extern struct static_key_false tc_skip_wrapper_cls;
17
18 /* TC Actions */
19 #ifdef CONFIG_NET_CLS_ACT
20
21 #define TC_INDIRECT_ACTION_DECLARE(fname) \
22 INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
23 const struct tc_action *a, \
24 struct tcf_result *res))
25
26 TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);
27 TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);
28 TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);
29 TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);
30 TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);
31 TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);
32 TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);
33 TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);
34 TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);
35 TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);
36 TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);
37 TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);
38 TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);
39 TC_INDIRECT_ACTION_DECLARE(tcf_police_act);
40 TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);
41 TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);
42 TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);
43 TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);
44 TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);
45 TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
46
tc_act(struct sk_buff * skb,const struct tc_action * a,struct tcf_result * res)47 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
48 struct tcf_result *res)
49 {
50 if (static_branch_likely(&tc_skip_wrapper_act))
51 goto skip;
52
53 #if IS_BUILTIN(CONFIG_NET_ACT_GACT)
54 if (a->ops->act == tcf_gact_act)
55 return tcf_gact_act(skb, a, res);
56 #endif
57 #if IS_BUILTIN(CONFIG_NET_ACT_MIRRED)
58 if (a->ops->act == tcf_mirred_act)
59 return tcf_mirred_act(skb, a, res);
60 #endif
61 #if IS_BUILTIN(CONFIG_NET_ACT_PEDIT)
62 if (a->ops->act == tcf_pedit_act)
63 return tcf_pedit_act(skb, a, res);
64 #endif
65 #if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT)
66 if (a->ops->act == tcf_skbedit_act)
67 return tcf_skbedit_act(skb, a, res);
68 #endif
69 #if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD)
70 if (a->ops->act == tcf_skbmod_act)
71 return tcf_skbmod_act(skb, a, res);
72 #endif
73 #if IS_BUILTIN(CONFIG_NET_ACT_POLICE)
74 if (a->ops->act == tcf_police_act)
75 return tcf_police_act(skb, a, res);
76 #endif
77 #if IS_BUILTIN(CONFIG_NET_ACT_BPF)
78 if (a->ops->act == tcf_bpf_act)
79 return tcf_bpf_act(skb, a, res);
80 #endif
81 #if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK)
82 if (a->ops->act == tcf_connmark_act)
83 return tcf_connmark_act(skb, a, res);
84 #endif
85 #if IS_BUILTIN(CONFIG_NET_ACT_CSUM)
86 if (a->ops->act == tcf_csum_act)
87 return tcf_csum_act(skb, a, res);
88 #endif
89 #if IS_BUILTIN(CONFIG_NET_ACT_CT)
90 if (a->ops->act == tcf_ct_act)
91 return tcf_ct_act(skb, a, res);
92 #endif
93 #if IS_BUILTIN(CONFIG_NET_ACT_CTINFO)
94 if (a->ops->act == tcf_ctinfo_act)
95 return tcf_ctinfo_act(skb, a, res);
96 #endif
97 #if IS_BUILTIN(CONFIG_NET_ACT_GATE)
98 if (a->ops->act == tcf_gate_act)
99 return tcf_gate_act(skb, a, res);
100 #endif
101 #if IS_BUILTIN(CONFIG_NET_ACT_MPLS)
102 if (a->ops->act == tcf_mpls_act)
103 return tcf_mpls_act(skb, a, res);
104 #endif
105 #if IS_BUILTIN(CONFIG_NET_ACT_NAT)
106 if (a->ops->act == tcf_nat_act)
107 return tcf_nat_act(skb, a, res);
108 #endif
109 #if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY)
110 if (a->ops->act == tunnel_key_act)
111 return tunnel_key_act(skb, a, res);
112 #endif
113 #if IS_BUILTIN(CONFIG_NET_ACT_VLAN)
114 if (a->ops->act == tcf_vlan_act)
115 return tcf_vlan_act(skb, a, res);
116 #endif
117 #if IS_BUILTIN(CONFIG_NET_ACT_IFE)
118 if (a->ops->act == tcf_ife_act)
119 return tcf_ife_act(skb, a, res);
120 #endif
121 #if IS_BUILTIN(CONFIG_NET_ACT_SIMP)
122 if (a->ops->act == tcf_simp_act)
123 return tcf_simp_act(skb, a, res);
124 #endif
125 #if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE)
126 if (a->ops->act == tcf_sample_act)
127 return tcf_sample_act(skb, a, res);
128 #endif
129
130 skip:
131 return a->ops->act(skb, a, res);
132 }
133
134 #endif /* CONFIG_NET_CLS_ACT */
135
136 /* TC Filters */
137 #ifdef CONFIG_NET_CLS
138
139 #define TC_INDIRECT_FILTER_DECLARE(fname) \
140 INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \
141 const struct tcf_proto *tp, \
142 struct tcf_result *res))
143
144 TC_INDIRECT_FILTER_DECLARE(basic_classify);
145 TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);
146 TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);
147 TC_INDIRECT_FILTER_DECLARE(fl_classify);
148 TC_INDIRECT_FILTER_DECLARE(flow_classify);
149 TC_INDIRECT_FILTER_DECLARE(fw_classify);
150 TC_INDIRECT_FILTER_DECLARE(mall_classify);
151 TC_INDIRECT_FILTER_DECLARE(route4_classify);
152 TC_INDIRECT_FILTER_DECLARE(u32_classify);
153
tc_classify(struct sk_buff * skb,const struct tcf_proto * tp,struct tcf_result * res)154 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
155 struct tcf_result *res)
156 {
157 if (static_branch_likely(&tc_skip_wrapper_cls))
158 goto skip;
159
160 #if IS_BUILTIN(CONFIG_NET_CLS_BPF)
161 if (tp->classify == cls_bpf_classify)
162 return cls_bpf_classify(skb, tp, res);
163 #endif
164 #if IS_BUILTIN(CONFIG_NET_CLS_U32)
165 if (tp->classify == u32_classify)
166 return u32_classify(skb, tp, res);
167 #endif
168 #if IS_BUILTIN(CONFIG_NET_CLS_FLOWER)
169 if (tp->classify == fl_classify)
170 return fl_classify(skb, tp, res);
171 #endif
172 #if IS_BUILTIN(CONFIG_NET_CLS_FW)
173 if (tp->classify == fw_classify)
174 return fw_classify(skb, tp, res);
175 #endif
176 #if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL)
177 if (tp->classify == mall_classify)
178 return mall_classify(skb, tp, res);
179 #endif
180 #if IS_BUILTIN(CONFIG_NET_CLS_BASIC)
181 if (tp->classify == basic_classify)
182 return basic_classify(skb, tp, res);
183 #endif
184 #if IS_BUILTIN(CONFIG_NET_CLS_CGROUP)
185 if (tp->classify == cls_cgroup_classify)
186 return cls_cgroup_classify(skb, tp, res);
187 #endif
188 #if IS_BUILTIN(CONFIG_NET_CLS_FLOW)
189 if (tp->classify == flow_classify)
190 return flow_classify(skb, tp, res);
191 #endif
192 #if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4)
193 if (tp->classify == route4_classify)
194 return route4_classify(skb, tp, res);
195 #endif
196
197 skip:
198 return tp->classify(skb, tp, res);
199 }
200
201 #endif /* CONFIG_NET_CLS */
202
tc_wrapper_init(void)203 static inline void tc_wrapper_init(void)
204 {
205 #ifdef CONFIG_X86
206 int cnt_cls = IS_BUILTIN(CONFIG_NET_CLS_BPF) +
207 IS_BUILTIN(CONFIG_NET_CLS_U32) +
208 IS_BUILTIN(CONFIG_NET_CLS_FLOWER) +
209 IS_BUILTIN(CONFIG_NET_CLS_FW) +
210 IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) +
211 IS_BUILTIN(CONFIG_NET_CLS_BASIC) +
212 IS_BUILTIN(CONFIG_NET_CLS_CGROUP) +
213 IS_BUILTIN(CONFIG_NET_CLS_FLOW) +
214 IS_BUILTIN(CONFIG_NET_CLS_ROUTE4);
215
216 int cnt_act = IS_BUILTIN(CONFIG_NET_ACT_GACT) +
217 IS_BUILTIN(CONFIG_NET_ACT_MIRRED) +
218 IS_BUILTIN(CONFIG_NET_ACT_PEDIT) +
219 IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) +
220 IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) +
221 IS_BUILTIN(CONFIG_NET_ACT_POLICE) +
222 IS_BUILTIN(CONFIG_NET_ACT_BPF) +
223 IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) +
224 IS_BUILTIN(CONFIG_NET_ACT_CSUM) +
225 IS_BUILTIN(CONFIG_NET_ACT_CT) +
226 IS_BUILTIN(CONFIG_NET_ACT_CTINFO) +
227 IS_BUILTIN(CONFIG_NET_ACT_GATE) +
228 IS_BUILTIN(CONFIG_NET_ACT_MPLS) +
229 IS_BUILTIN(CONFIG_NET_ACT_NAT) +
230 IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) +
231 IS_BUILTIN(CONFIG_NET_ACT_VLAN) +
232 IS_BUILTIN(CONFIG_NET_ACT_IFE) +
233 IS_BUILTIN(CONFIG_NET_ACT_SIMP) +
234 IS_BUILTIN(CONFIG_NET_ACT_SAMPLE);
235
236 if (cpu_feature_enabled(X86_FEATURE_RETPOLINE))
237 return;
238
239 if (cnt_cls > 1)
240 static_branch_enable(&tc_skip_wrapper_cls);
241
242 if (cnt_act > 1)
243 static_branch_enable(&tc_skip_wrapper_act);
244 #endif
245 }
246
247 #else
248
249 #define TC_INDIRECT_SCOPE static
250
tc_act(struct sk_buff * skb,const struct tc_action * a,struct tcf_result * res)251 static inline int tc_act(struct sk_buff *skb, const struct tc_action *a,
252 struct tcf_result *res)
253 {
254 return a->ops->act(skb, a, res);
255 }
256
tc_classify(struct sk_buff * skb,const struct tcf_proto * tp,struct tcf_result * res)257 static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp,
258 struct tcf_result *res)
259 {
260 return tp->classify(skb, tp, res);
261 }
262
tc_wrapper_init(void)263 static inline void tc_wrapper_init(void)
264 {
265 }
266
267 #endif
268
269 #endif /* __NET_TC_WRAPPER_H */
270