qca-nss-clients/nss_qdisc/nss_qdisc.h

/*
 **************************************************************************
 * Copyright (c) 2014-2018, 2020 The Linux Foundation. All rights reserved.
 * Permission to use, copy, modify, and/or distribute this software for
 * any purpose with or without fee is hereby granted, provided that the
 * above copyright notice and this permission notice appear in all copies.
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
 * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 **************************************************************************
 */

#include <linux/module.h>
#include <linux/of.h>
#include <linux/types.h>
#include <linux/kernel.h>
#include <linux/skbuff.h>
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/inet_ecn.h>
#include <net/netfilter/nf_conntrack.h>
#include <linux/if_bridge.h>
#include <linux/list.h>
#include <linux/version.h>
#include <br_private.h>
#include <nss_api_if.h>
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
#include <linux/netlink.h>
#endif

#if defined(NSS_QDISC_PPE_SUPPORT)
#include "nss_ppe.h"
#endif

#define NSS_QDISC_DEBUG_LEVEL_ERROR 1
#define NSS_QDISC_DEBUG_LEVEL_WARN 2
#define NSS_QDISC_DEBUG_LEVEL_INFO 3
#define NSS_QDISC_DEBUG_LEVEL_TRACE 4

/*
 * Debug message for module init and exit
 */
#define nss_qdisc_info_always(s, ...) printk(KERN_INFO"%s[%d]:" s, __func__, __LINE__, ##__VA_ARGS__)

/*
 * Error and warn message will be enabled by default in Makefile
 */
#if (NSS_QDISC_DEBUG_LEVEL < NSS_QDISC_DEBUG_LEVEL_ERROR)
#define nss_qdisc_assert(s, ...)
#define nss_qdisc_error(s, ...)
#else
#define nss_qdisc_assert(c, s, ...) { if (!(c)) { pr_emerg("ASSERT: %s:%d:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__); BUG(); } }
#define nss_qdisc_error(s, ...) pr_err("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif

#if (NSS_QDISC_DEBUG_LEVEL < NSS_QDISC_DEBUG_LEVEL_WARN)
#define nss_qdisc_warning(s, ...)
#else
#define nss_qdisc_warning(s, ...) pr_warn("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif

#if defined(CONFIG_DYNAMIC_DEBUG)
/*
 * Compile messages for dynamic enable/disable
 */
#define nss_qdisc_info(s, ...) pr_debug("%s[%d]:" s, __func__, __LINE__, ##__VA_ARGS__)
#define nss_qdisc_trace(s, ...) pr_debug("%s[%d]:" s, __func__, __LINE__, ##__VA_ARGS__)

#else
/*
 * Statically compile messages at different levels
 */
#if (NSS_QDISC_DEBUG_LEVEL < NSS_QDISC_DEBUG_LEVEL_INFO)
#define nss_qdisc_info(s, ...)
#else
#define nss_qdisc_info(s, ...) pr_notice("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif

#if (NSS_QDISC_DEBUG_LEVEL < NSS_QDISC_DEBUG_LEVEL_TRACE)
#define nss_qdisc_trace(s, ...)
#else
#define nss_qdisc_trace(s, ...) pr_info("%s[%d]:" s, __FUNCTION__, __LINE__, ##__VA_ARGS__)
#endif
#endif

/*
 * State values
 */
#define NSS_QDISC_STATE_IDLE 0
#define NSS_QDISC_STATE_READY 1

#define NSS_QDISC_STATE_INIT_FAILED -1
#define NSS_QDISC_STATE_ASSIGN_SHAPER_SEND_FAIL -2
#define NSS_QDISC_STATE_SHAPER_ASSIGN_FAILED -3
#define NSS_QDISC_STATE_NODE_ALLOC_SEND_FAIL -4
#define NSS_QDISC_STATE_NODE_ALLOC_FAIL -5
#define NSS_QDISC_STATE_ROOT_SET_SEND_FAIL -6
#define NSS_QDISC_STATE_ROOT_SET_FAIL -7
#define NSS_QDISC_STATE_DEFAULT_SET_SEND_FAIL -8
#define NSS_QDISC_STATE_DEFAULT_SET_FAIL -9
#define NSS_QDISC_STATE_CHILD_ALLOC_SEND_FAIL -10
#define NSS_QDISC_STATE_NODE_ALLOC_FAIL_CHILD -11
#define NSS_QDISC_STATE_FAILED_RESPONSE -12
#define NSS_QDISC_STATE_UNASSIGN_SHAPER_SEND_FAIL -13
#define NSS_QDISC_STATE_UNASSIGN_SHAPER_FAIL -14
#define NSS_QDISC_STATE_NODE_FREE_FAIL -15
#define NSS_QDISC_STATE_NODE_FREE_SEND_FAIL -16

#define NSS_QDISC_BRIDGE_PORT_MAX 100

#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3,8,0))
#define nss_qdisc_hlist_for_each_entry(tpos, pos, head, member) hlist_for_each_entry(tpos, pos, head, member)
#define nss_qdisc_hlist_for_each_entry_safe(tpos, pos, n, head, member) hlist_for_each_entry_safe(tpos, pos, n, head, member)
#define nss_qdisc_get_dev_master(dev) (dev->master)
#define nss_qdisc_get_dev(ptr) (struct net_device *)ptr
#else
#define nss_qdisc_hlist_for_each_entry(tpos, pos, head, member) hlist_for_each_entry(tpos, head, member)
#define nss_qdisc_hlist_for_each_entry_safe(tpos, pos, n, head, member) hlist_for_each_entry_safe(tpos, n, head, member)
#define nss_qdisc_get_dev_master(dev) netdev_master_upper_dev_get(dev)
#define nss_qdisc_get_dev(ptr) netdev_notifier_info_to_dev(ptr)
#endif

/*
 * Mode of Qdisc/class
 * These are defined as Magic numbers to avoid false positives
 */
#define NSS_QDISC_MODE_NSS 0x6243
#define NSS_QDISC_MODE_PPE 0x6245

struct nss_qdisc;

typedef void (*nss_qdisc_stats_callback_t)(struct nss_qdisc *nq, struct nss_shaper_node_stats_response *response);
typedef void (*nss_qdisc_configure_callback_t)(struct nss_qdisc *nq, struct nss_shaper_configure *response);

struct nss_qdisc {
	struct Qdisc *qdisc;			/* Handy pointer back to containing qdisc */
	struct nss_qdisc *parent;		/* Pointer to parent nss qdisc */
	void *nss_shaping_ctx;			/* NSS context for general operations */
	int32_t nss_interface_number;		/* NSS Interface number we are shaping on */
	nss_shaper_node_type_t type;		/* Type of shaper node */
	bool is_class;				/* True if this represents a class and not a qdisc */
	bool is_root;				/* True if root qdisc on a net device */
	bool is_bridge;				/* True when qdisc is a bridge */
	bool is_virtual;			/* True when the device is represented as a virtual in
						 * the NSS e.g. perhaps operating on a wifi interface
						 * or bridge.
						 */
	bool needs_ppe_loopback;		/* True when qdisc is on bridge or igs */
	bool destroy_virtual_interface;		/* Set if the interface is first registered in NSS by
						 * us. This means it needs to be un-regisreted when the
						 * module goes down.
						 */
	atomic_t state;				/* < 0: Signal that qdisc has 'failed'. 0
						 * indicates 'pending' setup.  > 0 is READY.
						 * NOTE: volatile AND atomic - this is polled
						 * AND is used for syncronisation.
						 */
	uint32_t shaper_id;			/* Used when is_root. Child qdiscs use this
						 * information to know what shaper under
						 * which to create shaper nodes
						 */
	uint32_t qos_tag;			/* QoS tag of this node */
	volatile int32_t pending_final_state;	/* Used to let the callback cycle know what
						 * state to set the qdisc in on successful
						 * completion.
						 */
	struct nss_virt_if_handle *virt_if_ctx;	/* Context provided by the NSS driver for
						 * new interfaces that are registered.
						 */
	void *bounce_context;			/* Context for bounce registration. Bounce
						 * enables packets to be sent to NSS for
						 * shaping purposes, and is returned to
						 * Linux for transmit.
						 */
	spinlock_t bounce_protection_lock;	/* Lock to protect the enqueue and dequeue
						 * operation on skb lists triggeret by bounce
						 * callbacks.
						 */
	nss_qdisc_stats_callback_t stats_cb;
						/* Stats update callback function for qdisc specific
						 * stats update.
						 */
	nss_qdisc_configure_callback_t config_cb;
						/* Shaper configure callback for reading shaper specific
						 * responses (e.g. memory size).
						 */
	struct gnet_stats_basic_packed bstats;	/* Basic class statistics */
	struct gnet_stats_queue qstats;		/* Qstats for use by classes */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
	atomic_t refcnt;			/* Reference count for class use */
#else
	refcount_t refcnt;			/* Reference count for class use */
#endif
	struct timer_list stats_get_timer;	/* Timer used to poll for stats */
	atomic_t pending_stat_requests;		/* Number of pending stats responses */
	wait_queue_head_t wait_queue;		/* Wait queue used to wait on responses from the NSS */
	spinlock_t lock;			/* Lock to protect the nss qdisc structure */
	uint16_t mode;				/* Mode of Qdisc/class */

#if defined(NSS_QDISC_PPE_SUPPORT)
	struct nss_ppe_qdisc npq;		/* PPE Qdisc */
	bool ppe_init_failed;			/* Flag is set if PPE initialization fails */
	bool hybrid_configured;			/* Flag is set only in root qdisc when first NSS Qdisc
						 * is attached to PPE qdisc in the tree.
						 */
#endif
	struct tcf_proto __rcu *filter_list;	/* Filter list */
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 4, 0))
	struct tcf_block *block;
#endif
};

/*
 * nss_qdisc bridge update structure
 */
struct nss_qdisc_bridge_update {
	int port_list[NSS_QDISC_BRIDGE_PORT_MAX];
	int port_list_count;
	int unassign_count;
};

/*
 * Task types for bridge scanner.
 */
enum nss_qdisc_bshaper_tasks {
	NSS_QDISC_SCAN_AND_ASSIGN_BSHAPER,
	NSS_QDISC_SCAN_AND_UNASSIGN_BSHAPER,
};

/*
 * Types of messages sent down to NSS interfaces
 */
enum nss_qdisc_interface_msgs {
	NSS_QDISC_IF_SHAPER_ASSIGN,
	NSS_QDISC_IF_SHAPER_UNASSIGN,
	NSS_QDISC_IF_SHAPER_CONFIG,
};

/*
 * Types of mode for hybrid configuration.
 */
enum nss_qdisc_hybrid_mode {
	NSS_QDISC_HYBRID_MODE_DISABLE,
	NSS_QDISC_HYBRID_MODE_ENABLE,
};

/*
 * nss_qdisc_nla_nest_start()
 *	Returns the container attribute
 */
static inline struct nlattr * nss_qdisc_nla_nest_start(struct sk_buff *skb, int attrtype)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
	return nla_nest_start(skb, TCA_OPTIONS);
#else
	return nla_nest_start_noflag(skb, TCA_OPTIONS);
#endif
}

/*
 * nss_qdisc_atomic_sub()
 *	Atomically decrements the ref count by 1
 */
static inline void nss_qdisc_atomic_sub(struct nss_qdisc *nq)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
	atomic_sub(1, &nq->refcnt);
#else
	atomic_sub(1, &nq->refcnt.refs);
#endif
}

/*
 * nss_qdisc_atomic_sub_return()
 *	Atomically decrements the ref count by 1 and return ref count
 */
static inline int nss_qdisc_atomic_sub_return(struct nss_qdisc *nq)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
	return atomic_sub_return(1, &nq->refcnt);
#else
	return atomic_sub_return(1, &nq->refcnt.refs);
#endif
}

/*
 * nss_qdisc_atomic_set()
 *	Atomically sets the ref count by 1
 */
static inline void nss_qdisc_atomic_set(struct nss_qdisc *nq)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 13, 0))
	atomic_set(&nq->refcnt, 1);
#else
	refcount_set(&nq->refcnt, 1);
#endif
}

/*
 * nss_qdisc_put()
 *	Destroy the qdisc
 */
static inline void nss_qdisc_put(struct Qdisc *sch)
{
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 20, 0))
	qdisc_destroy(sch);
#else
	qdisc_put(sch);
#endif
}

/*
 * nss_qdisc_qopt_get()
 *	Extracts qopt from opt.
 */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 16, 0))
extern void *nss_qdisc_qopt_get(struct nlattr *opt, struct nla_policy *policy,
				struct nlattr *tb[], uint32_t tca_max, uint32_t tca_params);
#else
extern void *nss_qdisc_qopt_get(struct nlattr *opt, struct nla_policy *policy,
				struct nlattr *tb[], uint32_t tca_max, uint32_t tca_params, struct netlink_ext_ack *extack);
#endif

/*
 * nss_qdisc_mode_get()
 *	Returns the operating mode of nss_qdisc, 0 = nss-fw, 1 = ppe.
 */
extern uint8_t nss_qdisc_accel_mode_get(struct nss_qdisc *nq);

/*
 * nss_qdisc_peek()
 *	Called to peek at the head of an nss qdisc
 */
extern struct sk_buff *nss_qdisc_peek(struct Qdisc *sch);

#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
/*
 * nss_qdisc_drop()
 *	Called to drop the packet at the head of queue
 */
extern unsigned int nss_qdisc_drop(struct Qdisc *sch);
#endif

/*
 * nss_qdisc_reset()
 *	Called when a qdisc is reset
 */
extern void nss_qdisc_reset(struct Qdisc *sch);

/*
 * nss_qdisc_enqueue()
 *	Generic enqueue call for enqueuing packets into NSS for shaping
 */
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 8, 0))
extern int nss_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch);
#else
extern int nss_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, struct sk_buff **to_free);
#endif

/*
 * nss_qdisc_dequeue()
 *	Generic dequeue call for dequeuing bounced packets.
 */
extern struct sk_buff *nss_qdisc_dequeue(struct Qdisc *sch);

/*
 * nss_qdisc_set_hybrid_mode()
 *	Configuration function that enables/disables hybrid mode
 */
extern int nss_qdisc_set_hybrid_mode(struct nss_qdisc *nq, enum nss_qdisc_hybrid_mode mode, uint32_t offset);

/*
 * nss_qdisc_node_set_default()
 *	Configuration function that sets shaper node as default for packet enqueue
 */
extern int nss_qdisc_set_default(struct nss_qdisc *nq);

/*
 * nss_qdisc_node_attach()
 *	Configuration function that helps attach a child shaper node to a parent.
 */
extern int nss_qdisc_node_attach(struct nss_qdisc *nq, struct nss_qdisc *nq_child,
					struct nss_if_msg *nim, int32_t attach_type);

/*
 * nss_qdisc_node_detach()
 *	Configuration function that helps detach a child shaper node to a parent.
 */
extern int nss_qdisc_node_detach(struct nss_qdisc *nq, struct nss_qdisc *nq_child,
					struct nss_if_msg *nim, int32_t detach_type);

/*
 * nss_qdisc_configure()
 *	Configuration function that aids in tuning of queuing parameters.
 */
extern int nss_qdisc_configure(struct nss_qdisc *nq,
	struct nss_if_msg *nim, int32_t config_type);


/*
 * nss_qdisc_register_configure_callback()
 *	Register shaper configure callback, which gets invoked on receiving a response.
 */
extern void nss_qdisc_register_configure_callback(struct nss_qdisc *nq, nss_qdisc_configure_callback_t cb);

/*
 * nss_qdisc_register_stats_callback()
 *	Register shaper stats callback, which gets invoked on receiving a stats response.
 */
extern void nss_qdisc_register_stats_callback(struct nss_qdisc *nq, nss_qdisc_stats_callback_t cb);

/*
 * nss_qdisc_destroy()
 *	Destroys a shaper in NSS, and the sequence is based on the position of
 *	this qdisc (child or root) and the interface to which it is attached to.
 */
extern void nss_qdisc_destroy(struct nss_qdisc *nq);

/*
 * nss_qdisc_init()
 *	Initializes a shaper in NSS, based on the position of this qdisc (child or root)
 *	and if its a normal interface or a bridge interface.
 */
extern int nss_qdisc_init(struct Qdisc *sch, struct nss_qdisc *nq, nss_shaper_node_type_t type, uint32_t classid, uint32_t accel_mode,
		void *extack);

/*
 * nss_qdisc_start_basic_stats_polling()
 *	Call to initiate the stats polling timer
 */
extern void nss_qdisc_start_basic_stats_polling(struct nss_qdisc *nq);

/*
 * nss_qdisc_stop_basic_stats_polling()
 *	Call to stop polling of basic stats
 */
extern void nss_qdisc_stop_basic_stats_polling(struct nss_qdisc *nq);

/*
 * nss_qdisc_gnet_stats_copy_basic()
 *  Wrapper around gnet_stats_copy_basic()
 */
extern int nss_qdisc_gnet_stats_copy_basic(struct Qdisc *sch,
				struct gnet_dump *d, struct gnet_stats_basic_packed *b);

/*
 * nss_qdisc_gnet_stats_copy_queue()
 *  Wrapper around gnet_stats_copy_queue()
 */
extern int nss_qdisc_gnet_stats_copy_queue(struct gnet_dump *d,
				struct gnet_stats_queue *q);

/*
 * nss_qdisc_replace()
 *	Used to replace old qdisc with a new one
 */
extern struct Qdisc *nss_qdisc_replace(struct Qdisc *sch, struct Qdisc *new,
					struct Qdisc **pold);

#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 4, 0))
/*
 * nss_qdisc_tcf_chain()
 *	Return the filter list of qdisc.
 */
extern struct tcf_proto __rcu **nss_qdisc_tcf_chain(struct Qdisc *sch, unsigned long arg);
#else
/*
 * nss_qdisc_tcf_block()
 *	Return the block containing chain of qdisc.
 */
extern struct tcf_block *nss_qdisc_tcf_block(struct Qdisc *sch, unsigned long cl, struct netlink_ext_ack *extack);
#endif

/*
 * nss_qdisc_tcf_bind()
 *	Bind the filter to the qdisc.
 */
extern unsigned long nss_qdisc_tcf_bind(struct Qdisc *sch, unsigned long parent, u32 classid);

/*
 * nss_qdisc_tcf_unbind()
 *	Unbind the filter from the qdisc.
 */
extern void nss_qdisc_tcf_unbind(struct Qdisc *sch, unsigned long arg);