cls_flower.c 43 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
/*
 * net/sched/cls_flower.c		Flower classifier
 *
 * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/module.h>
#include <linux/rhashtable.h>
16
#include <linux/workqueue.h>
17
18
19
20

#include <linux/if_ether.h>
#include <linux/in6.h>
#include <linux/ip.h>
21
#include <linux/mpls.h>
22
23
24
25
26
27

#include <net/sch_generic.h>
#include <net/pkt_cls.h>
#include <net/ip.h>
#include <net/flow_dissector.h>

28
29
30
#include <net/dst.h>
#include <net/dst_metadata.h>

31
32
33
struct fl_flow_key {
	int	indev_ifindex;
	struct flow_dissector_key_control control;
34
	struct flow_dissector_key_control enc_control;
35
36
	struct flow_dissector_key_basic basic;
	struct flow_dissector_key_eth_addrs eth;
37
	struct flow_dissector_key_vlan vlan;
38
39
40
41
42
	union {
		struct flow_dissector_key_ipv4_addrs ipv4;
		struct flow_dissector_key_ipv6_addrs ipv6;
	};
	struct flow_dissector_key_ports tp;
43
44
	struct flow_dissector_key_icmp icmp;
	struct flow_dissector_key_arp arp;
45
46
47
48
49
	struct flow_dissector_key_keyid enc_key_id;
	union {
		struct flow_dissector_key_ipv4_addrs enc_ipv4;
		struct flow_dissector_key_ipv6_addrs enc_ipv6;
	};
50
	struct flow_dissector_key_ports enc_tp;
51
	struct flow_dissector_key_mpls mpls;
52
53
	struct flow_dissector_key_tcp tcp;
	struct flow_dissector_key_ip ip;
54
55
56
57
58
59
60
61
62
63
} __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */

struct fl_flow_mask_range {
	unsigned short int start;
	unsigned short int end;
};

struct fl_flow_mask {
	struct fl_flow_key key;
	struct fl_flow_mask_range range;
64
65
66
67
68
69
70
	struct rhash_head ht_node;
	struct rhashtable ht;
	struct rhashtable_params filter_ht_params;
	struct flow_dissector dissector;
	struct list_head filters;
	struct rcu_work rwork;
	struct list_head list;
71
72
73
74
};

struct cls_fl_head {
	struct rhashtable ht;
75
76
	struct list_head masks;
	struct rcu_work rwork;
77
	struct idr handle_idr;
78
79
80
};

struct cls_fl_filter {
81
	struct fl_flow_mask *mask;
82
83
84
85
86
87
88
	struct rhash_head ht_node;
	struct fl_flow_key mkey;
	struct tcf_exts exts;
	struct tcf_result res;
	struct fl_flow_key key;
	struct list_head list;
	u32 handle;
89
	u32 flags;
90
	struct rcu_work rwork;
91
	struct net_device *hw_dev;
92
93
};

94
95
96
97
98
99
100
static const struct rhashtable_params mask_ht_params = {
	.key_offset = offsetof(struct fl_flow_mask, key),
	.key_len = sizeof(struct fl_flow_key),
	.head_offset = offsetof(struct fl_flow_mask, ht_node),
	.automatic_shrinking = true,
};

101
102
103
104
105
106
107
108
109
static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
{
	return mask->range.end - mask->range.start;
}

static void fl_mask_update_range(struct fl_flow_mask *mask)
{
	const u8 *bytes = (const u8 *) &mask->key;
	size_t size = sizeof(mask->key);
110
	size_t i, first = 0, last;
111

112
113
114
115
116
117
118
119
	for (i = 0; i < size; i++) {
		if (bytes[i]) {
			first = i;
			break;
		}
	}
	last = first;
	for (i = size - 1; i != first; i--) {
120
121
		if (bytes[i]) {
			last = i;
122
			break;
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
		}
	}
	mask->range.start = rounddown(first, sizeof(long));
	mask->range.end = roundup(last + 1, sizeof(long));
}

static void *fl_key_get_start(struct fl_flow_key *key,
			      const struct fl_flow_mask *mask)
{
	return (u8 *) key + mask->range.start;
}

static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
			      struct fl_flow_mask *mask)
{
	const long *lkey = fl_key_get_start(key, mask);
	const long *lmask = fl_key_get_start(&mask->key, mask);
	long *lmkey = fl_key_get_start(mkey, mask);
	int i;

	for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
		*lmkey++ = *lkey++ & *lmask++;
}

static void fl_clear_masked_range(struct fl_flow_key *key,
				  struct fl_flow_mask *mask)
{
	memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
}

153
static struct cls_fl_filter *fl_lookup(struct fl_flow_mask *mask,
154
155
				       struct fl_flow_key *mkey)
{
156
157
	return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask),
				      mask->filter_ht_params);
158
159
}

160
161
162
163
164
static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
		       struct tcf_result *res)
{
	struct cls_fl_head *head = rcu_dereference_bh(tp->root);
	struct cls_fl_filter *f;
165
	struct fl_flow_mask *mask;
166
167
168
	struct fl_flow_key skb_key;
	struct fl_flow_key skb_mkey;

169
170
	list_for_each_entry_rcu(mask, &head->masks, list) {
		fl_clear_masked_range(&skb_key, mask);
171

172
173
174
175
176
177
178
		skb_key.indev_ifindex = skb->skb_iif;
		/* skb_flow_dissect() does not set n_proto in case an unknown
		 * protocol, so do it rather here.
		 */
		skb_key.basic.n_proto = skb->protocol;
		skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key);
		skb_flow_dissect(skb, &mask->dissector, &skb_key, 0);
179

180
		fl_set_masked_key(&skb_mkey, &skb_key, mask);
181

182
183
184
185
186
		f = fl_lookup(mask, &skb_mkey);
		if (f && !tc_skip_sw(f->flags)) {
			*res = f->res;
			return tcf_exts_exec(skb, &f->exts, res);
		}
187
188
189
190
191
192
193
194
195
196
197
198
	}
	return -1;
}

static int fl_init(struct tcf_proto *tp)
{
	struct cls_fl_head *head;

	head = kzalloc(sizeof(*head), GFP_KERNEL);
	if (!head)
		return -ENOBUFS;

199
	INIT_LIST_HEAD_RCU(&head->masks);
200
	rcu_assign_pointer(tp->root, head);
201
	idr_init(&head->handle_idr);
202

203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
	return rhashtable_init(&head->ht, &mask_ht_params);
}

static void fl_mask_free(struct fl_flow_mask *mask)
{
	rhashtable_destroy(&mask->ht);
	kfree(mask);
}

static void fl_mask_free_work(struct work_struct *work)
{
	struct fl_flow_mask *mask = container_of(to_rcu_work(work),
						 struct fl_flow_mask, rwork);

	fl_mask_free(mask);
}

static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask,
			bool async)
{
	if (!list_empty(&mask->filters))
		return false;

	rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params);
	list_del_rcu(&mask->list);
	if (async)
		tcf_queue_work(&mask->rwork, fl_mask_free_work);
	else
		fl_mask_free(mask);

	return true;
234
235
}

236
237
238
239
240
241
242
243
244
static void __fl_destroy_filter(struct cls_fl_filter *f)
{
	tcf_exts_destroy(&f->exts);
	tcf_exts_put_net(&f->exts);
	kfree(f);
}

static void fl_destroy_filter_work(struct work_struct *work)
{
245
246
	struct cls_fl_filter *f = container_of(to_rcu_work(work),
					struct cls_fl_filter, rwork);
247
248
249
250
251
252

	rtnl_lock();
	__fl_destroy_filter(f);
	rtnl_unlock();
}

253
254
static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f,
				 struct netlink_ext_ack *extack)
255
{
256
	struct tc_cls_flower_offload cls_flower = {};
257
	struct tcf_block *block = tp->chain->block;
258

259
	tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
260
261
	cls_flower.command = TC_CLSFLOWER_DESTROY;
	cls_flower.cookie = (unsigned long) f;
262

263
264
	tc_setup_cb_call(block, &f->exts, TC_SETUP_CLSFLOWER,
			 &cls_flower, false);
265
	tcf_block_offload_dec(block, &f->flags);
266
267
}

268
static int fl_hw_replace_filter(struct tcf_proto *tp,
269
270
				struct cls_fl_filter *f,
				struct netlink_ext_ack *extack)
271
{
272
	struct tc_cls_flower_offload cls_flower = {};
273
274
	struct tcf_block *block = tp->chain->block;
	bool skip_sw = tc_skip_sw(f->flags);
275
	int err;
276

277
	tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
278
279
	cls_flower.command = TC_CLSFLOWER_REPLACE;
	cls_flower.cookie = (unsigned long) f;
280
281
	cls_flower.dissector = &f->mask->dissector;
	cls_flower.mask = &f->mask->key;
282
283
	cls_flower.key = &f->mkey;
	cls_flower.exts = &f->exts;
284
	cls_flower.classid = f->res.classid;
285

286
287
288
	err = tc_setup_cb_call(block, &f->exts, TC_SETUP_CLSFLOWER,
			       &cls_flower, skip_sw);
	if (err < 0) {
289
		fl_hw_destroy_filter(tp, f, NULL);
290
291
		return err;
	} else if (err > 0) {
292
		tcf_block_offload_inc(block, &f->flags);
293
294
295
296
	}

	if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW))
		return -EINVAL;
297
298

	return 0;
299
300
301
302
}

static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f)
{
303
	struct tc_cls_flower_offload cls_flower = {};
304
	struct tcf_block *block = tp->chain->block;
305

306
	tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL);
307
308
309
	cls_flower.command = TC_CLSFLOWER_STATS;
	cls_flower.cookie = (unsigned long) f;
	cls_flower.exts = &f->exts;
310
	cls_flower.classid = f->res.classid;
311

312
313
	tc_setup_cb_call(block, &f->exts, TC_SETUP_CLSFLOWER,
			 &cls_flower, false);
314
}
315

316
static bool __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f,
317
			struct netlink_ext_ack *extack)
318
{
319
	struct cls_fl_head *head = rtnl_dereference(tp->root);
320
321
	bool async = tcf_exts_get_net(&f->exts);
	bool last;
322

323
	idr_remove(&head->handle_idr, f->handle);
324
	list_del_rcu(&f->list);
325
	last = fl_mask_put(head, f->mask, async);
326
	if (!tc_skip_hw(f->flags))
327
		fl_hw_destroy_filter(tp, f, extack);
328
	tcf_unbind_filter(tp, &f->res);
329
330
	if (async)
		tcf_queue_work(&f->rwork, fl_destroy_filter_work);
331
332
	else
		__fl_destroy_filter(f);
333
334

	return last;
335
336
}

337
338
static void fl_destroy_sleepable(struct work_struct *work)
{
339
340
341
342
343
	struct cls_fl_head *head = container_of(to_rcu_work(work),
						struct cls_fl_head,
						rwork);

	rhashtable_destroy(&head->ht);
344
345
346
347
	kfree(head);
	module_put(THIS_MODULE);
}

348
static void fl_destroy(struct tcf_proto *tp, struct netlink_ext_ack *extack)
349
350
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);
351
	struct fl_flow_mask *mask, *next_mask;
352
353
	struct cls_fl_filter *f, *next;

354
355
356
357
358
359
	list_for_each_entry_safe(mask, next_mask, &head->masks, list) {
		list_for_each_entry_safe(f, next, &mask->filters, list) {
			if (__fl_delete(tp, f, extack))
				break;
		}
	}
360
	idr_destroy(&head->handle_idr);
361
362

	__module_get(THIS_MODULE);
363
	tcf_queue_work(&head->rwork, fl_destroy_sleepable);
364
365
}

366
static void *fl_get(struct tcf_proto *tp, u32 handle)
367
368
369
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);

370
	return idr_find(&head->handle_idr, handle);
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
}

static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
	[TCA_FLOWER_UNSPEC]		= { .type = NLA_UNSPEC },
	[TCA_FLOWER_CLASSID]		= { .type = NLA_U32 },
	[TCA_FLOWER_INDEV]		= { .type = NLA_STRING,
					    .len = IFNAMSIZ },
	[TCA_FLOWER_KEY_ETH_DST]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_DST_MASK]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_SRC]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_SRC_MASK]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ETH_TYPE]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_IP_PROTO]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IPV4_SRC]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV4_SRC_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV4_DST]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV4_DST_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_IPV6_SRC]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_IPV6_SRC_MASK]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_IPV6_DST]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_IPV6_DST_MASK]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_TCP_SRC]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_TCP_DST]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_SRC]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_DST]	= { .type = NLA_U16 },
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
	[TCA_FLOWER_KEY_VLAN_ID]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_VLAN_PRIO]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_VLAN_ETH_TYPE]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_KEY_ID]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_SRC]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_DST]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ENC_IPV6_SRC]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_ENC_IPV6_DST]	= { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
	[TCA_FLOWER_KEY_TCP_SRC_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_TCP_DST_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_SRC_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_UDP_DST_MASK]	= { .type = NLA_U16 },
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
	[TCA_FLOWER_KEY_SCTP_SRC_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_SCTP_DST_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_SCTP_SRC]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_SCTP_DST]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_FLAGS]		= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_FLAGS_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ICMPV4_TYPE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV4_CODE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_TYPE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_CODE]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ARP_SIP]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_SIP_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_TIP]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_TIP_MASK]	= { .type = NLA_U32 },
	[TCA_FLOWER_KEY_ARP_OP]		= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ARP_OP_MASK]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_ARP_SHA]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ARP_SHA_MASK]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ARP_THA]	= { .len = ETH_ALEN },
	[TCA_FLOWER_KEY_ARP_THA_MASK]	= { .len = ETH_ALEN },
440
441
442
443
	[TCA_FLOWER_KEY_MPLS_TTL]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_MPLS_BOS]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_MPLS_TC]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_MPLS_LABEL]	= { .type = NLA_U32 },
444
445
446
447
448
449
	[TCA_FLOWER_KEY_TCP_FLAGS]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_TCP_FLAGS_MASK]	= { .type = NLA_U16 },
	[TCA_FLOWER_KEY_IP_TOS]		= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IP_TOS_MASK]	= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IP_TTL]		= { .type = NLA_U8 },
	[TCA_FLOWER_KEY_IP_TTL_MASK]	= { .type = NLA_U8 },
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
};

static void fl_set_key_val(struct nlattr **tb,
			   void *val, int val_type,
			   void *mask, int mask_type, int len)
{
	if (!tb[val_type])
		return;
	memcpy(val, nla_data(tb[val_type]), len);
	if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
		memset(mask, 0xff, len);
	else
		memcpy(mask, nla_data(tb[mask_type]), len);
}

465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
static int fl_set_key_mpls(struct nlattr **tb,
			   struct flow_dissector_key_mpls *key_val,
			   struct flow_dissector_key_mpls *key_mask)
{
	if (tb[TCA_FLOWER_KEY_MPLS_TTL]) {
		key_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]);
		key_mask->mpls_ttl = MPLS_TTL_MASK;
	}
	if (tb[TCA_FLOWER_KEY_MPLS_BOS]) {
		u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]);

		if (bos & ~MPLS_BOS_MASK)
			return -EINVAL;
		key_val->mpls_bos = bos;
		key_mask->mpls_bos = MPLS_BOS_MASK;
	}
	if (tb[TCA_FLOWER_KEY_MPLS_TC]) {
		u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]);

		if (tc & ~MPLS_TC_MASK)
			return -EINVAL;
		key_val->mpls_tc = tc;
		key_mask->mpls_tc = MPLS_TC_MASK;
	}
	if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
		u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]);

		if (label & ~MPLS_LABEL_MASK)
			return -EINVAL;
		key_val->mpls_label = label;
		key_mask->mpls_label = MPLS_LABEL_MASK;
	}
	return 0;
}

500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
static void fl_set_key_vlan(struct nlattr **tb,
			    struct flow_dissector_key_vlan *key_val,
			    struct flow_dissector_key_vlan *key_mask)
{
#define VLAN_PRIORITY_MASK	0x7

	if (tb[TCA_FLOWER_KEY_VLAN_ID]) {
		key_val->vlan_id =
			nla_get_u16(tb[TCA_FLOWER_KEY_VLAN_ID]) & VLAN_VID_MASK;
		key_mask->vlan_id = VLAN_VID_MASK;
	}
	if (tb[TCA_FLOWER_KEY_VLAN_PRIO]) {
		key_val->vlan_priority =
			nla_get_u8(tb[TCA_FLOWER_KEY_VLAN_PRIO]) &
			VLAN_PRIORITY_MASK;
		key_mask->vlan_priority = VLAN_PRIORITY_MASK;
	}
}

519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
			    u32 *dissector_key, u32 *dissector_mask,
			    u32 flower_flag_bit, u32 dissector_flag_bit)
{
	if (flower_mask & flower_flag_bit) {
		*dissector_mask |= dissector_flag_bit;
		if (flower_key & flower_flag_bit)
			*dissector_key |= dissector_flag_bit;
	}
}

static int fl_set_key_flags(struct nlattr **tb,
			    u32 *flags_key, u32 *flags_mask)
{
	u32 key, mask;

	/* mask is mandatory for flags */
	if (!tb[TCA_FLOWER_KEY_FLAGS_MASK])
		return -EINVAL;

	key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
	mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));

	*flags_key  = 0;
	*flags_mask = 0;

	fl_set_key_flag(key, mask, flags_key, flags_mask,
			TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
547
548
549
	fl_set_key_flag(key, mask, flags_key, flags_mask,
			TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
			FLOW_DIS_FIRST_FRAG);
550
551
552
553

	return 0;
}

554
555
556
557
558
559
560
561
562
563
564
565
566
static void fl_set_key_ip(struct nlattr **tb,
			  struct flow_dissector_key_ip *key,
			  struct flow_dissector_key_ip *mask)
{
		fl_set_key_val(tb, &key->tos, TCA_FLOWER_KEY_IP_TOS,
			       &mask->tos, TCA_FLOWER_KEY_IP_TOS_MASK,
			       sizeof(key->tos));

		fl_set_key_val(tb, &key->ttl, TCA_FLOWER_KEY_IP_TTL,
			       &mask->ttl, TCA_FLOWER_KEY_IP_TTL_MASK,
			       sizeof(key->ttl));
}

567
static int fl_set_key(struct net *net, struct nlattr **tb,
568
569
		      struct fl_flow_key *key, struct fl_flow_key *mask,
		      struct netlink_ext_ack *extack)
570
{
571
	__be16 ethertype;
572
	int ret = 0;
573
574
#ifdef CONFIG_NET_CLS_IND
	if (tb[TCA_FLOWER_INDEV]) {
575
		int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack);
576
577
578
579
580
581
582
583
584
585
586
587
588
589
		if (err < 0)
			return err;
		key->indev_ifindex = err;
		mask->indev_ifindex = 0xffffffff;
	}
#endif

	fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
		       mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
		       sizeof(key->eth.dst));
	fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
		       mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
		       sizeof(key->eth.src));

590
591
592
593
594
595
596
597
598
599
600
601
602
603
	if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
		ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);

		if (ethertype == htons(ETH_P_8021Q)) {
			fl_set_key_vlan(tb, &key->vlan, &mask->vlan);
			fl_set_key_val(tb, &key->basic.n_proto,
				       TCA_FLOWER_KEY_VLAN_ETH_TYPE,
				       &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
				       sizeof(key->basic.n_proto));
		} else {
			key->basic.n_proto = ethertype;
			mask->basic.n_proto = cpu_to_be16(~0);
		}
	}
604
605
606
607
608
609

	if (key->basic.n_proto == htons(ETH_P_IP) ||
	    key->basic.n_proto == htons(ETH_P_IPV6)) {
		fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
			       &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
			       sizeof(key->basic.ip_proto));
610
		fl_set_key_ip(tb, &key->ip, &mask->ip);
611
612
613
614
	}

	if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
615
		mask->control.addr_type = ~0;
616
617
618
619
620
621
622
623
		fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
			       &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
			       sizeof(key->ipv4.src));
		fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
			       &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
			       sizeof(key->ipv4.dst));
	} else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
		key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
624
		mask->control.addr_type = ~0;
625
626
627
628
629
630
631
632
633
634
		fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
			       &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
			       sizeof(key->ipv6.src));
		fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
			       &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
			       sizeof(key->ipv6.dst));
	}

	if (key->basic.ip_proto == IPPROTO_TCP) {
		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
635
			       &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
636
637
			       sizeof(key->tp.src));
		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
638
			       &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
639
			       sizeof(key->tp.dst));
640
641
642
		fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
			       &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
			       sizeof(key->tcp.flags));
643
644
	} else if (key->basic.ip_proto == IPPROTO_UDP) {
		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
645
			       &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
646
647
			       sizeof(key->tp.src));
		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
648
			       &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
649
			       sizeof(key->tp.dst));
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
	} else if (key->basic.ip_proto == IPPROTO_SCTP) {
		fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
			       &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
			       sizeof(key->tp.src));
		fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
			       &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
			       sizeof(key->tp.dst));
	} else if (key->basic.n_proto == htons(ETH_P_IP) &&
		   key->basic.ip_proto == IPPROTO_ICMP) {
		fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE,
			       &mask->icmp.type,
			       TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
			       sizeof(key->icmp.type));
		fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
			       &mask->icmp.code,
			       TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
			       sizeof(key->icmp.code));
	} else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
		   key->basic.ip_proto == IPPROTO_ICMPV6) {
		fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE,
			       &mask->icmp.type,
			       TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
			       sizeof(key->icmp.type));
		fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE,
			       &mask->icmp.code,
			       TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
			       sizeof(key->icmp.code));
677
678
679
680
681
	} else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) ||
		   key->basic.n_proto == htons(ETH_P_MPLS_MC)) {
		ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls);
		if (ret)
			return ret;
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
	} else if (key->basic.n_proto == htons(ETH_P_ARP) ||
		   key->basic.n_proto == htons(ETH_P_RARP)) {
		fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP,
			       &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK,
			       sizeof(key->arp.sip));
		fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP,
			       &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK,
			       sizeof(key->arp.tip));
		fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP,
			       &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK,
			       sizeof(key->arp.op));
		fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
			       mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
			       sizeof(key->arp.sha));
		fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
			       mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
			       sizeof(key->arp.tha));
699
700
	}

701
702
703
	if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
	    tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
704
		mask->enc_control.addr_type = ~0;
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
		fl_set_key_val(tb, &key->enc_ipv4.src,
			       TCA_FLOWER_KEY_ENC_IPV4_SRC,
			       &mask->enc_ipv4.src,
			       TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
			       sizeof(key->enc_ipv4.src));
		fl_set_key_val(tb, &key->enc_ipv4.dst,
			       TCA_FLOWER_KEY_ENC_IPV4_DST,
			       &mask->enc_ipv4.dst,
			       TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
			       sizeof(key->enc_ipv4.dst));
	}

	if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
	    tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
		key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
720
		mask->enc_control.addr_type = ~0;
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
		fl_set_key_val(tb, &key->enc_ipv6.src,
			       TCA_FLOWER_KEY_ENC_IPV6_SRC,
			       &mask->enc_ipv6.src,
			       TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
			       sizeof(key->enc_ipv6.src));
		fl_set_key_val(tb, &key->enc_ipv6.dst,
			       TCA_FLOWER_KEY_ENC_IPV6_DST,
			       &mask->enc_ipv6.dst,
			       TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
			       sizeof(key->enc_ipv6.dst));
	}

	fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
		       &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
		       sizeof(key->enc_key_id.keyid));

737
738
739
740
741
742
743
744
745
746
747
748
	fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
		       &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
		       sizeof(key->enc_tp.src));

	fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
		       &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
		       sizeof(key->enc_tp.dst));

	if (tb[TCA_FLOWER_KEY_FLAGS])
		ret = fl_set_key_flags(tb, &key->control.flags, &mask->control.flags);

	return ret;
749
750
}

751
752
static void fl_mask_copy(struct fl_flow_mask *dst,
			 struct fl_flow_mask *src)
753
{
754
755
	const void *psrc = fl_key_get_start(&src->key, src);
	void *pdst = fl_key_get_start(&dst->key, src);
756

757
758
	memcpy(pdst, psrc, fl_mask_range(src));
	dst->range = src->range;
759
760
761
762
763
764
765
766
}

static const struct rhashtable_params fl_ht_params = {
	.key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
	.head_offset = offsetof(struct cls_fl_filter, ht_node),
	.automatic_shrinking = true,
};

767
static int fl_init_mask_hashtable(struct fl_flow_mask *mask)
768
{
769
770
771
	mask->filter_ht_params = fl_ht_params;
	mask->filter_ht_params.key_len = fl_mask_range(mask);
	mask->filter_ht_params.key_offset += mask->range.start;
772

773
	return rhashtable_init(&mask->ht, &mask->filter_ht_params);
774
775
776
777
778
}

#define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
#define FL_KEY_MEMBER_SIZE(member) (sizeof(((struct fl_flow_key *) 0)->member))

779
780
781
#define FL_KEY_IS_MASKED(mask, member)						\
	memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member),		\
		   0, FL_KEY_MEMBER_SIZE(member))				\
782
783
784
785
786
787
788
789

#define FL_KEY_SET(keys, cnt, id, member)					\
	do {									\
		keys[cnt].key_id = id;						\
		keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member);		\
		cnt++;								\
	} while(0);

790
#define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member)			\
791
	do {									\
792
		if (FL_KEY_IS_MASKED(mask, member))				\
793
794
795
			FL_KEY_SET(keys, cnt, id, member);			\
	} while(0);

796
static void fl_init_dissector(struct fl_flow_mask *mask)
797
798
799
800
801
802
{
	struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
	size_t cnt = 0;

	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
	FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
803
804
805
806
807
808
809
810
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_PORTS, tp);
811
812
813
814
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_IP, ip);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_TCP, tcp);
815
816
817
818
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ICMP, icmp);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ARP, arp);
819
820
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_MPLS, mpls);
821
822
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_VLAN, vlan);
823
824
825
826
827
828
829
830
831
832
833
834
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
	if (FL_KEY_IS_MASKED(&mask->key, enc_ipv4) ||
	    FL_KEY_IS_MASKED(&mask->key, enc_ipv6))
		FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
			   enc_control);
	FL_KEY_SET_IF_MASKED(&mask->key, keys, cnt,
			     FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
835

836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
	skb_flow_dissector_init(&mask->dissector, keys, cnt);
}

static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head,
					       struct fl_flow_mask *mask)
{
	struct fl_flow_mask *newmask;
	int err;

	newmask = kzalloc(sizeof(*newmask), GFP_KERNEL);
	if (!newmask)
		return ERR_PTR(-ENOMEM);

	fl_mask_copy(newmask, mask);

	err = fl_init_mask_hashtable(newmask);
	if (err)
		goto errout_free;

	fl_init_dissector(newmask);

	INIT_LIST_HEAD_RCU(&newmask->filters);

	err = rhashtable_insert_fast(&head->ht, &newmask->ht_node,
				     mask_ht_params);
	if (err)
		goto errout_destroy;

	list_add_tail_rcu(&newmask->list, &head->masks);

	return newmask;

errout_destroy:
	rhashtable_destroy(&newmask->ht);
errout_free:
	kfree(newmask);

	return ERR_PTR(err);
874
875
876
}

static int fl_check_assign_mask(struct cls_fl_head *head,
877
878
				struct cls_fl_filter *fnew,
				struct cls_fl_filter *fold,
879
880
				struct fl_flow_mask *mask)
{
881
	struct fl_flow_mask *newmask;
882

883
884
885
	fnew->mask = rhashtable_lookup_fast(&head->ht, mask, mask_ht_params);
	if (!fnew->mask) {
		if (fold)
886
887
			return -EINVAL;

888
889
890
		newmask = fl_create_new_mask(head, mask);
		if (IS_ERR(newmask))
			return PTR_ERR(newmask);
891

892
893
894
895
		fnew->mask = newmask;
	} else if (fold && fold->mask != fnew->mask) {
		return -EINVAL;
	}
896
897
898
899
900
901
902

	return 0;
}

static int fl_set_parms(struct net *net, struct tcf_proto *tp,
			struct cls_fl_filter *f, struct fl_flow_mask *mask,
			unsigned long base, struct nlattr **tb,
903
904
			struct nlattr *est, bool ovr,
			struct netlink_ext_ack *extack)
905
906
907
{
	int err;

908
	err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, extack);
909
910
911
912
913
914
915
916
	if (err < 0)
		return err;

	if (tb[TCA_FLOWER_CLASSID]) {
		f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
		tcf_bind_filter(tp, &f->res, base);
	}

917
	err = fl_set_key(net, tb, &f->key, &mask->key, extack);
918
	if (err)
919
		return err;
920
921
922
923
924
925
926
927
928
929

	fl_mask_update_range(mask);
	fl_set_masked_key(&f->mkey, &f->key, mask);

	return 0;
}

static int fl_change(struct net *net, struct sk_buff *in_skb,
		     struct tcf_proto *tp, unsigned long base,
		     u32 handle, struct nlattr **tca,
930
		     void **arg, bool ovr, struct netlink_ext_ack *extack)
931
932
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);
933
	struct cls_fl_filter *fold = *arg;
934
	struct cls_fl_filter *fnew;
935
	struct nlattr **tb;
936
937
938
939
940
941
	struct fl_flow_mask mask = {};
	int err;

	if (!tca[TCA_OPTIONS])
		return -EINVAL;

942
943
944
945
	tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
	if (!tb)
		return -ENOBUFS;

946
947
	err = nla_parse_nested(tb, TCA_FLOWER_MAX, tca[TCA_OPTIONS],
			       fl_policy, NULL);
948
	if (err < 0)
949
		goto errout_tb;
950

951
952
953
954
	if (fold && handle && fold->handle != handle) {
		err = -EINVAL;
		goto errout_tb;
	}
955
956

	fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
957
958
959
960
	if (!fnew) {
		err = -ENOBUFS;
		goto errout_tb;
	}
961

962
963
964
	err = tcf_exts_init(&fnew->exts, TCA_FLOWER_ACT, 0);
	if (err < 0)
		goto errout;
965
966

	if (!handle) {
967
968
969
970
971
972
973
		handle = 1;
		err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
				    INT_MAX, GFP_KERNEL);
	} else if (!fold) {
		/* user specifies a handle and it doesn't exist */
		err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
				    handle, GFP_KERNEL);
974
	}
975
976
977
	if (err)
		goto errout;
	fnew->handle = handle;
978

979
980
981
982
983
	if (tb[TCA_FLOWER_FLAGS]) {
		fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);

		if (!tc_flags_valid(fnew->flags)) {
			err = -EINVAL;
984
			goto errout_idr;
985
986
		}
	}
987

988
989
	err = fl_set_parms(net, tp, fnew, &mask, base, tb, tca[TCA_RATE], ovr,
			   extack);
990
	if (err)
991
		goto errout_idr;
992

993
	err = fl_check_assign_mask(head, fnew, fold, &mask);
994
	if (err)
995
		goto errout_idr;
996

997
	if (!tc_skip_sw(fnew->flags)) {
998
		if (!fold && fl_lookup(fnew->mask, &fnew->mkey)) {
999
			err = -EEXIST;
1000
			goto errout_mask;
1001
1002
		}

1003
1004
		err = rhashtable_insert_fast(&fnew->mask->ht, &fnew->ht_node,
					     fnew->mask->filter_ht_params);
1005
		if (err)
1006
			goto errout_mask;
1007
1008
	}

1009
	if (!tc_skip_hw(fnew->flags)) {
1010
		err = fl_hw_replace_filter(tp, fnew, extack);
1011
		if (err)
1012
			goto errout_mask;
1013
1014
1015
1016
	}

	if (!tc_in_hw(fnew->flags))
		fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
1017
1018

	if (fold) {
1019
		if (!tc_skip_sw(fold->flags))
1020
1021
1022
			rhashtable_remove_fast(&fold->mask->ht,
					       &fold->ht_node,
					       fold->mask->filter_ht_params);
1023
		if (!tc_skip_hw(fold->flags))
1024
			fl_hw_destroy_filter(tp, fold, NULL);
1025
1026
	}

1027
	*arg = fnew;
1028
1029

	if (fold) {
1030
		idr_replace(&head->handle_idr, fnew, fnew->handle);
1031
1032
		list_replace_rcu(&fold->list, &fnew->list);
		tcf_unbind_filter(tp, &fold->res);
1033
		tcf_exts_get_net(&fold->exts);
1034
		tcf_queue_work(&fold->rwork, fl_destroy_filter_work);
1035
	} else {
1036
		list_add_tail_rcu(&fnew->list, &fnew->mask->filters);
1037
1038
	}

1039
	kfree(tb);
1040
1041
	return 0;

1042
1043
1044
errout_mask:
	fl_mask_put(head, fnew->mask, false);

1045
errout_idr:
1046
	if (!fold)
1047
		idr_remove(&head->handle_idr, fnew->handle);
1048
errout:
1049
	tcf_exts_destroy(&fnew->exts);
1050
	kfree(fnew);
1051
1052
errout_tb:
	kfree(tb);
1053
1054
1055
	return err;
}

1056
1057
static int fl_delete(struct tcf_proto *tp, void *arg, bool *last,
		     struct netlink_ext_ack *extack)
1058
1059
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);
1060
	struct cls_fl_filter *f = arg;
1061

1062
	if (!tc_skip_sw(f->flags))
1063
1064
		rhashtable_remove_fast(&f->mask->ht, &f->ht_node,
				       f->mask->filter_ht_params);
1065
	__fl_delete(tp, f, extack);
1066
	*last = list_empty(&head->masks);
1067
1068
1069
1070
1071
1072
1073
	return 0;
}

static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg)
{
	struct cls_fl_head *head = rtnl_dereference(tp->root);
	struct cls_fl_filter *f;
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
	struct fl_flow_mask *mask;

	list_for_each_entry_rcu(mask, &head->masks, list) {
		list_for_each_entry_rcu(f, &mask->filters, list) {
			if (arg->count < arg->skip)
				goto skip;
			if (arg->fn(tp, f, arg) < 0) {
				arg->stop = 1;
				break;
			}
1084
skip:
1085
1086
			arg->count++;
		}
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
	}
}

static int fl_dump_key_val(struct sk_buff *skb,
			   void *val, int val_type,
			   void *mask, int mask_type, int len)
{
	int err;

	if (!memchr_inv(mask, 0, len))
		return 0;
	err = nla_put(skb, val_type, len, val);
	if (err)
		return err;
	if (mask_type != TCA_FLOWER_UNSPEC) {
		err = nla_put(skb, mask_type, len, mask);
		if (err)
			return err;
	}
	return 0;
}

1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
static int fl_dump_key_mpls(struct sk_buff *skb,
			    struct flow_dissector_key_mpls *mpls_key,
			    struct flow_dissector_key_mpls *mpls_mask)
{
	int err;

	if (!memchr_inv(mpls_mask, 0, sizeof(*mpls_mask)))
		return 0;
	if (mpls_mask->mpls_ttl) {
		err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL,
				 mpls_key->mpls_ttl);
		if (err)
			return err;
	}
	if (mpls_mask->mpls_tc) {
		err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC,
				 mpls_key->mpls_tc);
		if (err)
			return err;
	}
	if (mpls_mask->mpls_label) {
		err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL,
				  mpls_key->mpls_label);
		if (err)
			return err;
	}
	if (mpls_mask->mpls_bos) {
		err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS,
				 mpls_key->mpls_bos);
		if (err)
			return err;
	}
	return 0;
}

1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
static int fl_dump_key_ip(struct sk_buff *skb,
			  struct flow_dissector_key_ip *key,
			  struct flow_dissector_key_ip *mask)
{
	if (fl_dump_key_val(skb, &key->tos, TCA_FLOWER_KEY_IP_TOS, &mask->tos,
			    TCA_FLOWER_KEY_IP_TOS_MASK, sizeof(key->tos)) ||
	    fl_dump_key_val(skb, &key->ttl, TCA_FLOWER_KEY_IP_TTL, &mask->ttl,
			    TCA_FLOWER_KEY_IP_TTL_MASK, sizeof(key->ttl)))
		return -1;

	return 0;
}

1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
static int fl_dump_key_vlan(struct sk_buff *skb,
			    struct flow_dissector_key_vlan *vlan_key,
			    struct flow_dissector_key_vlan *vlan_mask)
{
	int err;

	if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
		return 0;
	if (vlan_mask->vlan_id) {
		err = nla_put_u16(skb, TCA_FLOWER_KEY_VLAN_ID,
				  vlan_key->vlan_id);
		if (err)
			return err;
	}
	if (vlan_mask->vlan_priority) {
		err = nla_put_u8(skb, TCA_FLOWER_KEY_VLAN_PRIO,
				 vlan_key->vlan_priority);
		if (err)
			return err;
	}
	return 0;
}

1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
			    u32 *flower_key, u32 *flower_mask,
			    u32 flower_flag_bit, u32 dissector_flag_bit)
{
	if (dissector_mask & dissector_flag_bit) {
		*flower_mask |= flower_flag_bit;
		if (dissector_key & dissector_flag_bit)
			*flower_key |= flower_flag_bit;
	}
}

static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
{
	u32 key, mask;
	__be32 _key, _mask;
	int err;

	if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask)))
		return 0;

	key = 0;
	mask = 0;

	fl_get_key_flag(flags_key, flags_mask, &key, &mask,
			TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
1205
1206
1207
	fl_get_key_flag(flags_key, flags_mask, &key, &mask,
			TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
			FLOW_DIS_FIRST_FRAG);
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218

	_key = cpu_to_be32(key);
	_mask = cpu_to_be32(mask);

	err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key);
	if (err)
		return err;

	return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask);
}

1219
static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh,
1220
1221
		   struct sk_buff *skb, struct tcmsg *t)
{
1222
	struct cls_fl_filter *f = fh;
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
	struct nlattr *nest;
	struct fl_flow_key *key, *mask;

	if (!f)
		return skb->len;

	t->tcm_handle = f->handle;

	nest = nla_nest_start(skb, TCA_OPTIONS);
	if (!nest)
		goto nla_put_failure;

	if (f->res.classid &&
	    nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
		goto nla_put_failure;

	key = &f->key;
1240
	mask = &f->mask->key;
1241
1242
1243
1244
1245
1246
1247
1248
1249

	if (mask->indev_ifindex) {
		struct net_device *dev;

		dev = __dev_get_by_index(net, key->indev_ifindex);
		if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
			goto nla_put_failure;
	}

1250
1251
	if (!tc_skip_hw(f->flags))
		fl_hw_update_stats(tp, f);
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262

	if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
			    mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
			    sizeof(key->eth.dst)) ||
	    fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
			    mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
			    sizeof(key->eth.src)) ||
	    fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
			    &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
			    sizeof(key->basic.n_proto)))
		goto nla_put_failure;
1263

1264
1265
1266
	if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls))
		goto nla_put_failure;

1267
1268
1269
	if (fl_dump_key_vlan(skb, &key->vlan, &mask->vlan))
		goto nla_put_failure;

1270
1271
	if ((key->basic.n_proto == htons(ETH_P_IP) ||
	     key->basic.n_proto == htons(ETH_P_IPV6)) &&
1272
	    (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
1273
			    &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
1274
1275
			    sizeof(key->basic.ip_proto)) ||
	    fl_dump_key_ip(skb, &key->ip, &mask->ip)))
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
		goto nla_put_failure;

	if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
	    (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
			     &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
			     sizeof(key->ipv4.src)) ||
	     fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
			     &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
			     sizeof(key->ipv4.dst))))
		goto nla_put_failure;
	else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
		 (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
				  &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
				  sizeof(key->ipv6.src)) ||
		  fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
				  &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
				  sizeof(key->ipv6.dst))))
		goto nla_put_failure;

	if (key->basic.ip_proto == IPPROTO_TCP &&
	    (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
1297
			     &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
1298
1299
			     sizeof(key->tp.src)) ||
	     fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
1300
			     &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
1301
1302
1303
1304
			     sizeof(key->tp.dst)) ||
	     fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
			     &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
			     sizeof(key->tcp.flags))))
1305
1306
1307
		goto nla_put_failure;
	else if (key->basic.ip_proto == IPPROTO_UDP &&
		 (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
1308
				  &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
1309
1310
				  sizeof(key->tp.src)) ||
		  fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
1311
				  &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
1312
1313
				  sizeof(key->tp.dst))))
		goto nla_put_failure;
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337