Project import generated by Copybara.
GitOrigin-RevId: 7c20c0fc9d25f1105432955e3f8e88efad9b63f4
diff --git a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp.h b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp.h
index 54c9ca3..a8bc27c 100644
--- a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp.h
+++ b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp.h
@@ -43,6 +43,9 @@
extern int tx_requeue_stop;
extern int tx_desc_threshold_size;
+extern int tx_completion_load_balance;
+extern int tx_completion_load_balance_threshold;
+extern int tx_completion_load_balance_cpu;
/*
* syn_dp_info
diff --git a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_cfg_tx.c b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_cfg_tx.c
index bf5e19a..83238be 100644
--- a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_cfg_tx.c
+++ b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_cfg_tx.c
@@ -48,6 +48,11 @@
tx_info->tx_idx = 0;
tx_info->busy_tx_desc_cnt = 0;
+ tx_info->tx_free_batch_size = SYN_DP_NAPI_BUDGET_TX;
+ tx_info->csd.func = syn_dp_free_tx_done_skb;
+ tx_info->csd.info = &tx_info->tx_free_batch_size;
+ tx_info->csd.flags = 0;
+
return NSS_DP_SUCCESS;
}
@@ -82,7 +87,9 @@
int i;
struct sk_buff *skb;
uint32_t busy_tx_desc_cnt = atomic_read((atomic_t *)&tx_info->busy_tx_desc_cnt);
+ int budget = SYN_DP_TX_DESC_SIZE;
+ syn_dp_free_tx_done_skb(&budget);
/*
* Tx Ring cleaning
*/
diff --git a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.c b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.c
index 70c395f..29e6a4c 100644
--- a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.c
+++ b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.c
@@ -409,6 +409,11 @@
tx_info->tx_idx = syn_dp_tx_inc_index(tx_idx, 1);
}
+void syn_dp_schedule_free_tx_skb(call_single_data_t *csd)
+{
+ smp_call_function_single_async(tx_completion_load_balance_cpu, csd);
+}
+
/*
* syn_dp_tx_complete()
* Xmit complete, clear descriptor and free the skb
@@ -502,13 +507,34 @@
* in performance.
* All the completed skb's shinfo are prefetched and skb's are freed in batch.
*/
- for (free_idx = 0; free_idx < count; free_idx++) {
- if (likely((free_idx + 1) < count)) {
- prefetch((void *)tx_info->shinfo_addr_virt[free_idx+1]);
+
+ if (unlikely(tx_completion_load_balance &&
+ count > tx_completion_load_balance_threshold)) {
+ unsigned long flags;
+ struct list_head skb_list;
+ struct sk_buff *skb;
+
+ INIT_LIST_HEAD(&skb_list);
+ for (free_idx = 0; free_idx < count; free_idx++) {
+ skb = tx_info->skb_free_list[free_idx];
+ tx_info->skb_free_list[free_idx] = NULL;
+ tx_info->shinfo_addr_virt[free_idx] = 0;
+ list_add_tail(&skb->list, &skb_list);
}
- dev_kfree_skb_any(tx_info->skb_free_list[free_idx]);
- tx_info->skb_free_list[free_idx] = NULL;
- tx_info->shinfo_addr_virt[free_idx] = 0;
+ atomic_add(count, &syn_dp_tx_done_skb_count);
+ spin_lock_irqsave(&syn_dp_tx_done_skb_list_lock, flags);
+ list_splice_tail_init(&skb_list, &syn_dp_tx_done_skb_list);
+ spin_unlock_irqrestore(&syn_dp_tx_done_skb_list_lock, flags);
+ syn_dp_schedule_free_tx_skb(&tx_info->csd);
+ } else {
+ for (free_idx = 0; free_idx < count; free_idx++) {
+ if (likely((free_idx + 1) < count)) {
+ prefetch((void *)tx_info->shinfo_addr_virt[free_idx+1]);
+ }
+ dev_kfree_skb_any(tx_info->skb_free_list[free_idx]);
+ tx_info->skb_free_list[free_idx] = NULL;
+ tx_info->shinfo_addr_virt[free_idx] = 0;
+ }
}
atomic64_add(tx_packets, (atomic64_t *)&tx_info->tx_stats.tx_packets);
@@ -598,3 +624,42 @@
atomic_inc((atomic_t *)&tx_info->busy_tx_desc_cnt);
return 0;
}
+
+void syn_dp_free_tx_done_skb(void *data)
+{
+ int max = *(int *)data;
+ int count = 0;
+ unsigned long flags;
+ struct sk_buff *skb, *next, *skb_prefetch = NULL;
+ struct list_head work_list;
+
+ INIT_LIST_HEAD(&work_list);
+ spin_lock_irqsave(&syn_dp_tx_done_skb_list_lock, flags);
+ list_splice_init(&syn_dp_tx_done_skb_list, &work_list);
+ spin_unlock_irqrestore(&syn_dp_tx_done_skb_list_lock, flags);
+
+ if (list_empty(&work_list))
+ return;
+
+ skb_prefetch = list_first_entry(&work_list, struct sk_buff, list);
+ if (skb_prefetch) prefetchw(skb_prefetch);
+
+ list_for_each_entry_safe(skb, next, &work_list, list) {
+ prefetch(skb_shinfo(skb));
+ skb_prefetch = list_next_entry(skb, list);
+ if (skb_prefetch) prefetchw(skb_prefetch);
+ count++;
+ skb_list_del_init(skb);
+ dev_kfree_skb_any(skb);
+ if (count >= max)
+ break;
+ }
+ atomic_sub(count, &syn_dp_tx_done_skb_count);
+ if (list_empty(&work_list))
+ return;
+
+ spin_lock_irqsave(&syn_dp_tx_done_skb_list_lock, flags);
+ list_splice_init(&work_list, &syn_dp_tx_done_skb_list);
+ spin_unlock_irqrestore(&syn_dp_tx_done_skb_list_lock, flags);
+ return;
+}
diff --git a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.h b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.h
index 8ffbaa8..3f74b7f 100644
--- a/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.h
+++ b/qca-nss-dp/hal/dp_ops/syn_gmac_dp/syn_dp_tx.h
@@ -23,6 +23,8 @@
#define SYN_DP_TX_DESC_MAX_INDEX (SYN_DP_TX_DESC_SIZE - 1)
#define SYN_DP_TX_INVALID_DESC_INDEX SYN_DP_TX_DESC_SIZE
#define NSS_DP_TX_MAX_DESC_SIZE SYN_DP_TX_DESC_SIZE
+
+
/*
* syn_dp_tx_buf
*/
@@ -51,6 +53,8 @@
/* GMAC driver Tx statistics */
struct net_device *netdev; /* Net-device corresponding to the GMAC */
struct device *dev; /* Platform device corresponding to the GMAC */
+ call_single_data_t csd;
+ int tx_free_batch_size;
struct sk_buff *skb_free_list[SYN_DP_NAPI_BUDGET_TX];
/* Array to hold SKBs before free during Tx completion */
size_t shinfo_addr_virt[SYN_DP_NAPI_BUDGET_TX];
@@ -85,4 +89,8 @@
return tx_info->tx_comp_idx;
}
+extern void syn_dp_free_tx_done_skb(void *data);
+extern spinlock_t syn_dp_tx_done_skb_list_lock;
+extern struct list_head syn_dp_tx_done_skb_list;
+extern atomic_t syn_dp_tx_done_skb_count;
#endif /* __NSS_DP_SYN_DP_TX__ */
diff --git a/qca-nss-dp/nss_dp_main.c b/qca-nss-dp/nss_dp_main.c
index 5ae217f..f9607de 100644
--- a/qca-nss-dp/nss_dp_main.c
+++ b/qca-nss-dp/nss_dp_main.c
@@ -107,6 +107,10 @@
MODULE_PARM_DESC(nss_dp_rx_mitigation_pkt_cnt, "Rx mitigation packet count value");
#endif
+spinlock_t syn_dp_tx_done_skb_list_lock;
+struct list_head syn_dp_tx_done_skb_list;
+atomic_t syn_dp_tx_done_skb_count = ATOMIC_INIT(0);
+
/*
* Sysctl table
*/
@@ -114,6 +118,9 @@
int tx_requeue_stop;
int tx_desc_threshold_size = 0;
+int tx_completion_load_balance = false;
+int tx_completion_load_balance_threshold = 6;
+int tx_completion_load_balance_cpu = 1;
/*
* nss_dp_tx_requeue_stop()
@@ -181,6 +188,108 @@
return ret;
}
+/*
+ * nss_dp_tx_completion_load_balance()
+ * Tx completion load balance sysctl handler
+ */
+static int nss_dp_tx_completion_load_balance(struct ctl_table *ctl, int write,
+ void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ int ret;
+
+ int current_value = tx_completion_load_balance;
+
+ ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
+
+ if (!write) {
+ return ret;
+ }
+
+ if (ret) {
+ pr_err("Errno: -%d.\n", ret);
+ return ret;
+ }
+
+ /*
+ * Check if tx_completion_load_balance is holding a valid value
+ */
+ if ((tx_completion_load_balance != 1) &&
+ (tx_completion_load_balance != 0)) {
+ pr_err(" Invalid input. Valid values are 0/1\n");
+ tx_completion_load_balance = current_value;
+ return -EINVAL;
+ }
+ return ret;
+}
+
+/*
+ * nss_dp_tx_completion_load_balance_threshold()
+ * tx_completion_load_balance_threshold sysctl handler
+ */
+static int nss_dp_tx_completion_load_balance_threshold(struct ctl_table *ctl,
+ int write, void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ int ret;
+
+ int current_value = tx_completion_load_balance_threshold;
+
+ ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
+
+ if (!write) {
+ return ret;
+ }
+
+ if (ret) {
+ pr_err("Errno: -%d.\n", ret);
+ return ret;
+ }
+
+ /*
+ * Check if tx_completion_load_balance is holding a valid value
+ */
+ if (tx_completion_load_balance_threshold < 0) {
+ pr_err(" Invalid input\n");
+ tx_completion_load_balance_threshold = current_value;
+ return -EINVAL;
+ }
+ return ret;
+}
+
+/*
+ * nss_dp_tx_completion_load_balance_cput()
+ * tx_completion_load_balance_cpu sysctl handler
+ */
+static int nss_dp_tx_completion_load_balance_cpu(struct ctl_table *ctl,
+ int write, void __user *buffer, size_t *lenp, loff_t *ppos)
+{
+ int ret;
+
+ int current_value = tx_completion_load_balance_cpu;
+
+ ret = proc_dointvec(ctl, write, buffer, lenp, ppos);
+
+ if (!write) {
+ return ret;
+ }
+
+ if (ret) {
+ pr_err("Errno: -%d.\n", ret);
+ return ret;
+ }
+
+ /*
+ * Check if tx_completion_load_balance is holding a valid value
+ */
+ if (tx_completion_load_balance_cpu < 0 ||
+ tx_completion_load_balance_cpu > 1) {
+ pr_err(" Invalid input\n");
+ tx_completion_load_balance_cpu = current_value;
+ return -EINVAL;
+ }
+ return ret;
+}
+
+
/* nss_dp_table
* Sysctl entries which are part of nss dp
*/
@@ -199,6 +308,27 @@
.mode = 0666,
.proc_handler = &nss_dp_tx_desc_threshold_set,
},
+ {
+ .procname = "tx_completion_load_balance",
+ .data = &tx_completion_load_balance,
+ .maxlen = sizeof(int),
+ .mode = 0666,
+ .proc_handler = &nss_dp_tx_completion_load_balance,
+ },
+ {
+ .procname = "tx_completion_load_balance_threshold",
+ .data = &tx_completion_load_balance_threshold,
+ .maxlen = sizeof(int),
+ .mode = 0666,
+ .proc_handler = &nss_dp_tx_completion_load_balance_threshold,
+ },
+ {
+ .procname = "tx_completion_load_balance_cpu",
+ .data = &tx_completion_load_balance_cpu,
+ .maxlen = sizeof(int),
+ .mode = 0666,
+ .proc_handler = &nss_dp_tx_completion_load_balance_cpu,
+ },
{ }
};
@@ -992,6 +1122,9 @@
}
#endif
+ spin_lock_init(&syn_dp_tx_done_skb_list_lock);
+ INIT_LIST_HEAD(&syn_dp_tx_done_skb_list);
+
/* TODO: Features: CSUM, tx/rx offload... configure */
/* Register the network interface */
diff --git a/qca-nss-ecm/ecm_classifier_mark.c b/qca-nss-ecm/ecm_classifier_mark.c
index d054d8e..cd112fa 100644
--- a/qca-nss-ecm/ecm_classifier_mark.c
+++ b/qca-nss-ecm/ecm_classifier_mark.c
@@ -523,8 +523,8 @@
int dst_port;
ip_addr_t src_ip;
ip_addr_t dst_ip;
- struct in6_addr src_ip6;
- struct in6_addr dest_ip6;
+ struct in6_addr src_ip6 = IN6ADDR_ANY_INIT;
+ struct in6_addr dest_ip6 = IN6ADDR_ANY_INIT;
struct ecm_db_connection_instance *ci;
ecm_classifier_mark_sync_to_ipv6_callback_t cb = NULL;
ecm_classifier_mark_type_t type = ECM_CLASSIFIER_MARK_TYPE_L2_ENCAP;
diff --git a/qca-nss-ecm/ecm_classifier_pcc.c b/qca-nss-ecm/ecm_classifier_pcc.c
index 6c6fcd4..71f5c2a 100644
--- a/qca-nss-ecm/ecm_classifier_pcc.c
+++ b/qca-nss-ecm/ecm_classifier_pcc.c
@@ -828,8 +828,8 @@
}
#ifdef ECM_IPV6_ENABLE
if (ip_version == 6) {
- struct in6_addr src_ip6;
- struct in6_addr dest_ip6;
+ struct in6_addr src_ip6 = IN6ADDR_ANY_INIT;
+ struct in6_addr dest_ip6 = IN6ADDR_ANY_INIT;
ECM_IP_ADDR_TO_NIN6_ADDR(src_ip6, src_ip);
ECM_IP_ADDR_TO_NIN6_ADDR(dest_ip6, dst_ip);
diff --git a/qca-nss-ecm/ecm_db/ecm_db.c b/qca-nss-ecm/ecm_db/ecm_db.c
index 722d6d5..3b899e9 100644
--- a/qca-nss-ecm/ecm_db/ecm_db.c
+++ b/qca-nss-ecm/ecm_db/ecm_db.c
@@ -275,7 +275,7 @@
while (ci) {
struct ecm_db_connection_instance *cin;
struct in6_addr prefix_addr;
- struct in6_addr ecm_in6;
+ struct in6_addr ecm_in6 = IN6ADDR_ANY_INIT;
ip_addr_t ecm_addr;
struct ecm_db_iface_instance *interfaces[ECM_DB_IFACE_HEIRARCHY_MAX];
int32_t if_first;
diff --git a/qca-nss-ecm/ecm_db/ecm_db_connection.c b/qca-nss-ecm/ecm_db/ecm_db_connection.c
index fd68f23..717aa3a 100644
--- a/qca-nss-ecm/ecm_db/ecm_db_connection.c
+++ b/qca-nss-ecm/ecm_db/ecm_db_connection.c
@@ -1239,7 +1239,7 @@
/*
* Remove from database if inserted
*/
- if (!ci->flags & ECM_DB_CONNECTION_FLAGS_INSERTED) {
+ if (!(ci->flags & ECM_DB_CONNECTION_FLAGS_INSERTED)) {
spin_unlock_bh(&ecm_db_lock);
} else {
struct ecm_db_listener_instance *li;
diff --git a/qca-nss-ecm/ecm_db/ecm_db_host.c b/qca-nss-ecm/ecm_db/ecm_db_host.c
index 8820576..1bc3a94 100644
--- a/qca-nss-ecm/ecm_db/ecm_db_host.c
+++ b/qca-nss-ecm/ecm_db/ecm_db_host.c
@@ -249,7 +249,7 @@
/*
* Remove from database if inserted
*/
- if (!hi->flags & ECM_DB_HOST_FLAGS_INSERTED) {
+ if (!(hi->flags & ECM_DB_HOST_FLAGS_INSERTED)) {
spin_unlock_bh(&ecm_db_lock);
} else {
struct ecm_db_listener_instance *li;
diff --git a/qca-nss-ecm/ecm_db/ecm_db_iface.c b/qca-nss-ecm/ecm_db/ecm_db_iface.c
index 53c254c..108540c 100644
--- a/qca-nss-ecm/ecm_db/ecm_db_iface.c
+++ b/qca-nss-ecm/ecm_db/ecm_db_iface.c
@@ -1032,7 +1032,7 @@
/*
* Remove from database if inserted
*/
- if (!ii->flags & ECM_DB_IFACE_FLAGS_INSERTED) {
+ if (!(ii->flags & ECM_DB_IFACE_FLAGS_INSERTED)) {
spin_unlock_bh(&ecm_db_lock);
} else {
struct ecm_db_listener_instance *li;
@@ -1510,7 +1510,7 @@
/*
* Remove from database if inserted
*/
- if (!ii->flags & ECM_DB_IFACE_FLAGS_INSERTED) {
+ if (!(ii->flags & ECM_DB_IFACE_FLAGS_INSERTED)) {
return;
}
diff --git a/qca-nss-ecm/ecm_db/ecm_db_mapping.c b/qca-nss-ecm/ecm_db/ecm_db_mapping.c
index 1d06ef4..329c7cf 100644
--- a/qca-nss-ecm/ecm_db/ecm_db_mapping.c
+++ b/qca-nss-ecm/ecm_db/ecm_db_mapping.c
@@ -390,7 +390,7 @@
/*
* Remove from database if inserted
*/
- if (!mi->flags & ECM_DB_MAPPING_FLAGS_INSERTED) {
+ if (!(mi->flags & ECM_DB_MAPPING_FLAGS_INSERTED)) {
spin_unlock_bh(&ecm_db_lock);
} else {
struct ecm_db_listener_instance *li;
diff --git a/qca-nss-ecm/ecm_db/ecm_db_node.c b/qca-nss-ecm/ecm_db/ecm_db_node.c
index 18d6aba..8221e77 100644
--- a/qca-nss-ecm/ecm_db/ecm_db_node.c
+++ b/qca-nss-ecm/ecm_db/ecm_db_node.c
@@ -254,7 +254,7 @@
/*
* Remove from database if inserted
*/
- if (!ni->flags & ECM_DB_NODE_FLAGS_INSERTED) {
+ if (!(ni->flags & ECM_DB_NODE_FLAGS_INSERTED)) {
spin_unlock_bh(&ecm_db_lock);
} else {
struct ecm_db_listener_instance *li;
diff --git a/qca-nss-ecm/ecm_interface.c b/qca-nss-ecm/ecm_interface.c
index 52bd11b..7058f15 100644
--- a/qca-nss-ecm/ecm_interface.c
+++ b/qca-nss-ecm/ecm_interface.c
@@ -330,7 +330,7 @@
*/
static struct net_device *ecm_interface_dev_find_by_local_addr_ipv6(ip_addr_t addr)
{
- struct in6_addr addr6;
+ struct in6_addr addr6 = IN6ADDR_ANY_INIT;
struct net_device *dev;
ECM_IP_ADDR_TO_NIN6_ADDR(addr6, addr);
@@ -421,7 +421,7 @@
*/
static bool ecm_interface_mac_addr_get_ipv6(ip_addr_t addr, uint8_t *mac_addr, bool *on_link, ip_addr_t gw_addr)
{
- struct in6_addr daddr;
+ struct in6_addr daddr = IN6ADDR_ANY_INIT;
struct ecm_interface_route ecm_rt;
struct neighbour *neigh;
struct rt6_info *rt;
@@ -726,7 +726,7 @@
*/
static bool ecm_interface_mac_addr_get_ipv6_no_route(struct net_device *dev, ip_addr_t addr, uint8_t *mac_addr)
{
- struct in6_addr daddr;
+ struct in6_addr daddr = IN6ADDR_ANY_INIT;
struct neighbour *neigh;
struct net_device *local_dev;
@@ -1129,7 +1129,7 @@
*/
static bool ecm_interface_find_route_by_addr_ipv6(ip_addr_t addr, struct ecm_interface_route *ecm_rt)
{
- struct in6_addr naddr;
+ struct in6_addr naddr = IN6ADDR_ANY_INIT;
ECM_IP_ADDR_TO_NIN6_ADDR(naddr, addr);
@@ -1193,7 +1193,8 @@
*/
void ecm_interface_send_neighbour_solicitation(struct net_device *dev, ip_addr_t addr)
{
- struct in6_addr dst_addr, src_addr;
+ struct in6_addr dst_addr = IN6ADDR_ANY_INIT;
+ struct in6_addr src_addr;
struct in6_addr mc_dst_addr;
struct rt6_info *rt6i;
struct neighbour *neigh;
@@ -1321,7 +1322,7 @@
struct neighbour *neigh;
struct rt6_info *rt;
struct dst_entry *dst;
- struct in6_addr ipv6_addr;
+ struct in6_addr ipv6_addr = IN6ADDR_ANY_INIT;
ECM_IP_ADDR_TO_NIN6_ADDR(ipv6_addr, addr);
@@ -4156,7 +4157,10 @@
#endif
) {
br_dev_src = ecm_interface_get_and_hold_dev_master(in_dev);
- DEBUG_ASSERT(br_dev_src, "Expected a master\n");
+ if (!br_dev_src) {
+ DEBUG_WARN("Expected a master\n");
+ return 0;
+ }
/*
* Source netdev is part of a bridge. First make sure that this bridge
diff --git a/qca-nss-ecm/frontends/cmn/ecm_ipv4.c b/qca-nss-ecm/frontends/cmn/ecm_ipv4.c
index e5ceae4..f542566 100644
--- a/qca-nss-ecm/frontends/cmn/ecm_ipv4.c
+++ b/qca-nss-ecm/frontends/cmn/ecm_ipv4.c
@@ -1,7 +1,7 @@
/*
**************************************************************************
* Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
*
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
@@ -475,7 +475,11 @@
} else {
struct net_device *master;
master = ecm_interface_get_and_hold_dev_master(dev);
- DEBUG_ASSERT(master, "%px: Expected a master\n", feci);
+ if (!master) {
+ DEBUG_WARN("%px: Expected a master\n", feci);
+ return NULL;
+ }
+
ecm_interface_send_arp_request(master, addr, on_link, gw_addr);
dev_put(master);
}
@@ -1508,50 +1512,6 @@
DEBUG_TRACE("%px: IPv4 CMN Routing: %s\n", out, out->name);
/*
- * If operations have stopped then do not process packets
- */
- spin_lock_bh(&ecm_ipv4_lock);
- if (unlikely(ecm_front_end_ipv4_stopped)) {
- spin_unlock_bh(&ecm_ipv4_lock);
- DEBUG_TRACE("Front end stopped\n");
- return NF_ACCEPT;
- }
- spin_unlock_bh(&ecm_ipv4_lock);
-
- /*
- * Don't process broadcast or multicast
- */
- if (skb->pkt_type == PACKET_BROADCAST) {
- DEBUG_TRACE("Broadcast, ignoring: %px\n", skb);
- return NF_ACCEPT;
- }
-
-#ifndef ECM_INTERFACE_PPTP_ENABLE
- /*
- * skip pptp because we don't accelerate them
- */
- if (ecm_interface_is_pptp(skb, out)) {
- return NF_ACCEPT;
- }
-#endif
-
-#ifndef ECM_INTERFACE_L2TPV2_ENABLE
- /*
- * skip l2tpv2 because we don't accelerate them
- */
- if (ecm_interface_is_l2tp_packet_by_version(skb, out, 2)) {
- return NF_ACCEPT;
- }
-#endif
-
- /*
- * skip l2tpv3 because we don't accelerate them
- */
- if (ecm_interface_is_l2tp_packet_by_version(skb, out, 3)) {
- return NF_ACCEPT;
- }
-
- /*
* Identify interface from where this packet came
*/
in = dev_get_by_index(&init_net, skb->skb_iif);
@@ -1562,13 +1522,56 @@
return NF_ACCEPT;
}
+ /*
+ * If operations have stopped then do not process packets
+ */
+ spin_lock_bh(&ecm_ipv4_lock);
+ if (unlikely(ecm_front_end_ipv4_stopped)) {
+ spin_unlock_bh(&ecm_ipv4_lock);
+ DEBUG_TRACE("Front end stopped\n");
+ goto skip_ipv4_route_flow;
+ }
+ spin_unlock_bh(&ecm_ipv4_lock);
+
+ /*
+ * Don't process broadcast or multicast
+ */
+ if (skb->pkt_type == PACKET_BROADCAST) {
+ DEBUG_TRACE("Broadcast, ignoring: %px\n", skb);
+ goto skip_ipv4_route_flow;
+ }
+
+#ifndef ECM_INTERFACE_PPTP_ENABLE
+ /*
+ * skip pptp because we don't accelerate them
+ */
+ if (ecm_interface_is_pptp(skb, out)) {
+ goto skip_ipv4_route_flow;
+ }
+#endif
+
+#ifndef ECM_INTERFACE_L2TPV2_ENABLE
+ /*
+ * skip l2tpv2 because we don't accelerate them
+ */
+ if (ecm_interface_is_l2tp_packet_by_version(skb, out, 2)) {
+ goto skip_ipv4_route_flow;
+ }
+#endif
+
+ /*
+ * skip l2tpv3 because we don't accelerate them
+ */
+ if (ecm_interface_is_l2tp_packet_by_version(skb, out, 3)) {
+ goto skip_ipv4_route_flow;
+ }
+
#ifndef ECM_INTERFACE_OVS_BRIDGE_ENABLE
/*
* skip OpenVSwitch flows because we don't accelerate them
*/
if (netif_is_ovs_master(out) || netif_is_ovs_master(in)) {
- dev_put(in);
- return NF_ACCEPT;
+ goto skip_ipv4_route_flow;
}
#endif
@@ -1577,6 +1580,10 @@
can_accel, true, false, skb, 0);
dev_put(in);
return result;
+
+skip_ipv4_route_flow:
+ dev_put(in);
+ return NF_ACCEPT;
}
/*
@@ -1650,6 +1657,36 @@
DEBUG_TRACE("%px: IPv4 CMN Bridge: %s\n", out, out->name);
/*
+ * Identify interface from where this packet came.
+ * There are three scenarios to consider here:
+ * 1. Packet came from a local source.
+ * Ignore - local is not handled.
+ * 2. Packet came from a routed path.
+ * Ignore - it was handled in INET post routing.
+ * 3. Packet is bridged from another port.
+ * Process.
+ *
+ * Begin by identifying case 1.
+ * NOTE: We are given 'out' (which we implicitly know is a bridge port) so out interface's master is the 'bridge'.
+ */
+ in = dev_get_by_index(&init_net, skb->skb_iif);
+ if (!in) {
+ /*
+ * Case 1.
+ */
+ bridge = ecm_interface_get_and_hold_dev_master((struct net_device *)out);
+
+ if (!bridge) {
+ DEBUG_WARN("Expected bridge\n");
+ return NF_ACCEPT;
+ }
+ DEBUG_TRACE("Local traffic: %px, ignoring traffic to bridge: %px (%s) \n", skb, bridge, bridge->name);
+ dev_put(bridge);
+ return NF_ACCEPT;
+ }
+ dev_put(in);
+
+ /*
* If operations have stopped then do not process packets
*/
spin_lock_bh(&ecm_ipv4_lock);
@@ -1689,31 +1726,11 @@
return NF_ACCEPT;
}
- /*
- * Identify interface from where this packet came.
- * There are three scenarios to consider here:
- * 1. Packet came from a local source.
- * Ignore - local is not handled.
- * 2. Packet came from a routed path.
- * Ignore - it was handled in INET post routing.
- * 3. Packet is bridged from another port.
- * Process.
- *
- * Begin by identifying case 1.
- * NOTE: We are given 'out' (which we implicitly know is a bridge port) so out interface's master is the 'bridge'.
- */
bridge = ecm_interface_get_and_hold_dev_master((struct net_device *)out);
- DEBUG_ASSERT(bridge, "Expected bridge\n");
- in = dev_get_by_index(&init_net, skb->skb_iif);
- if (!in) {
- /*
- * Case 1.
- */
- DEBUG_TRACE("Local traffic: %px, ignoring traffic to bridge: %px (%s) \n", skb, bridge, bridge->name);
- dev_put(bridge);
+ if (!bridge) {
+ DEBUG_WARN("Expected bridge\n");
return NF_ACCEPT;
}
- dev_put(in);
/*
* Case 2:
diff --git a/qca-nss-ecm/frontends/cmn/ecm_ipv6.c b/qca-nss-ecm/frontends/cmn/ecm_ipv6.c
index fc37eb8..dac809c 100644
--- a/qca-nss-ecm/frontends/cmn/ecm_ipv6.c
+++ b/qca-nss-ecm/frontends/cmn/ecm_ipv6.c
@@ -1,7 +1,7 @@
/*
**************************************************************************
* Copyright (c) 2014-2021 The Linux Foundation. All rights reserved.
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
* Permission to use, copy, modify, and/or distribute this software for
* any purpose with or without fee is hereby granted, provided that the
* above copyright notice and this permission notice appear in all copies.
@@ -519,7 +519,10 @@
if (ecm_front_end_is_bridge_port(dev)) {
struct net_device *master;
master = ecm_interface_get_and_hold_dev_master(dev);
- DEBUG_ASSERT(master, "%px: Expected a master\n", feci);
+ if (!master) {
+ DEBUG_WARN("%px: Expected a master for bridge port %s\n", feci, dev->name);
+ return NULL;
+ }
ecm_interface_send_neighbour_solicitation(master, gw_addr);
dev_put(master);
} else {
@@ -1264,50 +1267,6 @@
DEBUG_TRACE("%px: Routing: %s\n", out, out->name);
/*
- * If operations have stopped then do not process packets
- */
- spin_lock_bh(&ecm_ipv6_lock);
- if (unlikely(ecm_front_end_ipv6_stopped)) {
- spin_unlock_bh(&ecm_ipv6_lock);
- DEBUG_TRACE("Front end stopped\n");
- return NF_ACCEPT;
- }
- spin_unlock_bh(&ecm_ipv6_lock);
-
- /*
- * Don't process broadcast or multicast
- */
- if (skb->pkt_type == PACKET_BROADCAST) {
- DEBUG_TRACE("Broadcast, ignoring: %px\n", skb);
- return NF_ACCEPT;
- }
-
-#ifndef ECM_INTERFACE_PPTP_ENABLE
- /*
- * skip pptp because we don't accelerate them
- */
- if (ecm_interface_is_pptp(skb, out)) {
- return NF_ACCEPT;
- }
-#endif
-
-#ifndef ECM_INTERFACE_L2TPV2_ENABLE
- /*
- * skip l2tpv2 because we don't accelerate them
- */
- if (ecm_interface_is_l2tp_packet_by_version(skb, out, 2)) {
- return NF_ACCEPT;
- }
-#endif
-
- /*
- * skip l2tpv3 because we don't accelerate them
- */
- if (ecm_interface_is_l2tp_packet_by_version(skb, out, 3)) {
- return NF_ACCEPT;
- }
-
- /*
* Identify interface from where this packet came
*/
in = dev_get_by_index(&init_net, skb->skb_iif);
@@ -1318,13 +1277,56 @@
return NF_ACCEPT;
}
+ /*
+ * If operations have stopped then do not process packets
+ */
+ spin_lock_bh(&ecm_ipv6_lock);
+ if (unlikely(ecm_front_end_ipv6_stopped)) {
+ spin_unlock_bh(&ecm_ipv6_lock);
+ DEBUG_TRACE("Front end stopped\n");
+ goto skip_ipv6_route_flow;
+ }
+ spin_unlock_bh(&ecm_ipv6_lock);
+
+ /*
+ * Don't process broadcast or multicast
+ */
+ if (skb->pkt_type == PACKET_BROADCAST) {
+ DEBUG_TRACE("Broadcast, ignoring: %px\n", skb);
+ goto skip_ipv6_route_flow;
+ }
+
+#ifndef ECM_INTERFACE_PPTP_ENABLE
+ /*
+ * skip pptp because we don't accelerate them
+ */
+ if (ecm_interface_is_pptp(skb, out)) {
+ goto skip_ipv6_route_flow;
+ }
+#endif
+
+#ifndef ECM_INTERFACE_L2TPV2_ENABLE
+ /*
+ * skip l2tpv2 because we don't accelerate them
+ */
+ if (ecm_interface_is_l2tp_packet_by_version(skb, out, 2)) {
+ goto skip_ipv6_route_flow;
+ }
+#endif
+
+ /*
+ * skip l2tpv3 because we don't accelerate them
+ */
+ if (ecm_interface_is_l2tp_packet_by_version(skb, out, 3)) {
+ goto skip_ipv6_route_flow;
+ }
+
#ifndef ECM_INTERFACE_OVS_BRIDGE_ENABLE
/*
* skip OpenVSwitch flows because we don't accelerate them
*/
if (netif_is_ovs_master(out) || netif_is_ovs_master(in)) {
- dev_put(in);
- return NF_ACCEPT;
+ goto skip_ipv6_route_flow;
}
#endif
@@ -1332,6 +1334,10 @@
result = ecm_ipv6_ip_process((struct net_device *)out, in, NULL, NULL, can_accel, true, false, skb, 0);
dev_put(in);
return result;
+
+skip_ipv6_route_flow:
+ dev_put(in);
+ return NF_ACCEPT;
}
/*
@@ -1405,6 +1411,36 @@
DEBUG_TRACE("%px: IPv6 CMN Bridge: %s\n", out, out->name);
/*
+ * Identify interface from where this packet came.
+ * There are three scenarios to consider here:
+ * 1. Packet came from a local source.
+ * Ignore - local is not handled.
+ * 2. Packet came from a routed path.
+ * Ignore - it was handled in INET post routing.
+ * 3. Packet is bridged from another port.
+ * Process.
+ *
+ * Begin by identifying case 1.
+ * NOTE: We are given 'out' (which we implicitly know is a bridge port) so out interface's master is the 'bridge'.
+ */
+ in = dev_get_by_index(&init_net, skb->skb_iif);
+ if (!in) {
+ /*
+ * Case 1.
+ */
+ bridge = ecm_interface_get_and_hold_dev_master((struct net_device *)out);
+
+ if (!bridge) {
+ DEBUG_WARN("Expected bridge\n");
+ return NF_ACCEPT;
+ }
+ DEBUG_TRACE("Local traffic: %px, ignoring traffic to bridge: %px (%s) \n", skb, bridge, bridge->name);
+ dev_put(bridge);
+ return NF_ACCEPT;
+ }
+ dev_put(in);
+
+ /*
* If operations have stopped then do not process packets
*/
spin_lock_bh(&ecm_ipv6_lock);
@@ -1444,31 +1480,11 @@
return NF_ACCEPT;
}
- /*
- * Identify interface from where this packet came.
- * There are three scenarios to consider here:
- * 1. Packet came from a local source.
- * Ignore - local is not handled.
- * 2. Packet came from a routed path.
- * Ignore - it was handled in INET post routing.
- * 3. Packet is bridged from another port.
- * Process.
- *
- * Begin by identifying case 1.
- * NOTE: We are given 'out' (which we implicitly know is a bridge port) so out interface's master is the 'bridge'.
- */
bridge = ecm_interface_get_and_hold_dev_master((struct net_device *)out);
- DEBUG_ASSERT(bridge, "Expected bridge\n");
- in = dev_get_by_index(&init_net, skb->skb_iif);
- if (!in) {
- /*
- * Case 1.
- */
- DEBUG_TRACE("Local traffic: %px, ignoring traffic to bridge: %px (%s) \n", skb, bridge, bridge->name);
- dev_put(bridge);
+ if (!bridge) {
+ DEBUG_WARN("Expected bridge\n");
return NF_ACCEPT;
}
- dev_put(in);
/*
* Case 2:
diff --git a/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv4.c b/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv4.c
index ea91eda..e61c1c7 100644
--- a/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv4.c
+++ b/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv4.c
@@ -682,7 +682,11 @@
* interface list
*/
out_dev_master = ecm_interface_get_and_hold_dev_master(out_dev);
- DEBUG_ASSERT(out_dev_master, "Expected a master\n");
+ if (!out_dev_master) {
+ DEBUG_WARN("Expected a master\n");
+ goto done;
+ }
+
if_cnt = mc_bridge_ipv4_get_if(out_dev_master, ip_src, ip_grp, ECM_DB_MULTICAST_IF_MAX, dst_dev);
if (if_cnt <= 0) {
DEBUG_WARN("Not found a valid MCS if count %d\n", if_cnt);
diff --git a/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv6.c b/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv6.c
index e25318c..21046a5 100644
--- a/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv6.c
+++ b/qca-nss-ecm/frontends/cmn/ecm_multicast_ipv6.c
@@ -366,7 +366,11 @@
if (ecm_front_end_is_bridge_port(dev)) {
struct net_device *master;
master = ecm_interface_get_and_hold_dev_master(dev);
- DEBUG_ASSERT(master, "Expected a master\n");
+ if (!master) {
+ DEBUG_WARN("Expected a master\n");
+ return NULL;
+ }
+
ecm_interface_send_neighbour_solicitation(master, addr);
dev_put(master);
} else {
diff --git a/qca-nss-ecm/frontends/include/ecm_front_end_common.h b/qca-nss-ecm/frontends/include/ecm_front_end_common.h
index 0c70a1d..1c0a6d0 100644
--- a/qca-nss-ecm/frontends/include/ecm_front_end_common.h
+++ b/qca-nss-ecm/frontends/include/ecm_front_end_common.h
@@ -119,8 +119,8 @@
ip_addr_t ct_src_ip;
if (ip_version == 4) {
- uint32_t flow_ip_32;
- uint32_t ct_src_ip_32;
+ uint32_t flow_ip_32 = 0;
+ uint32_t ct_src_ip_32 = 0;
ECM_NIN4_ADDR_TO_IP_ADDR(ct_src_ip, ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple.src.u3.ip);
/*
diff --git a/qca-nss-sfe/sfe.c b/qca-nss-sfe/sfe.c
index 0195d41..70f1c2c 100644
--- a/qca-nss-sfe/sfe.c
+++ b/qca-nss-sfe/sfe.c
@@ -132,6 +132,9 @@
static struct sfe_ctx_instance_internal __sfe_ctx;
+// Temporarily pause sfe_ipvX_periodic_sync() if enabled to reduce CPU overhead
+bool sfe_pause_stats_sync = false;
+
/*
* Convert public SFE context to internal context
*/
@@ -1551,6 +1554,32 @@
__ATTR(bypass_mark, S_IWUSR | S_IRUGO, sfe_get_bypass_mark,
sfe_set_bypass_mark);
+static ssize_t
+sfe_get_pause_stats_sync(struct device *dev, struct device_attribute *attr,
+ char *buf)
+{
+ return snprintf(buf, (ssize_t)PAGE_SIZE, "%d\n", sfe_pause_stats_sync);
+}
+
+static ssize_t
+sfe_set_pause_stats_sync(struct device *dev, struct device_attribute *attr,
+ const char *buf, size_t count)
+{
+ int ret;
+ bool enable;
+
+ ret = kstrtobool(buf, &enable);
+ if (ret) {
+ return ret;
+ }
+ sfe_pause_stats_sync = enable;
+ return count;
+}
+
+static const struct device_attribute sfe_pause_stats_sync_attr =
+ __ATTR(pause_stats_sync, S_IWUSR | S_IRUGO, sfe_get_pause_stats_sync,
+ sfe_set_pause_stats_sync);
+
/*
* sfe_init_if()
*/
@@ -1603,6 +1632,13 @@
goto exit2;
}
+ result = sysfs_create_file(sfe_ctx->sys_sfe,
+ &sfe_pause_stats_sync_attr.attr);
+ if (result) {
+ DEBUG_ERROR("failed to register Pause Stats sync sysfs file: %d\n",
+ result);
+ goto exit2;
+ }
spin_lock_init(&sfe_ctx->lock);
INIT_LIST_HEAD(&sfe_ctx->msg_queue);
diff --git a/qca-nss-sfe/sfe.h b/qca-nss-sfe/sfe.h
index 59203ad..c613c7f 100644
--- a/qca-nss-sfe/sfe.h
+++ b/qca-nss-sfe/sfe.h
@@ -327,4 +327,5 @@
int sfe_init_if(void);
void sfe_exit_if(void);
+extern bool sfe_pause_stats_sync;
#endif /* __SFE_H */
diff --git a/qca-nss-sfe/sfe_ipv4.c b/qca-nss-sfe/sfe_ipv4.c
index 1fd2883..3650022 100644
--- a/qca-nss-sfe/sfe_ipv4.c
+++ b/qca-nss-sfe/sfe_ipv4.c
@@ -1859,6 +1859,9 @@
sfe_sync_rule_callback_t sync_rule_callback;
struct sfe_ipv4_connection *c;
+ if (sfe_pause_stats_sync)
+ goto done;
+
now_jiffies = get_jiffies_64();
rcu_read_lock();
diff --git a/qca-nss-sfe/sfe_ipv4_tcp.c b/qca-nss-sfe/sfe_ipv4_tcp.c
index b2d5ec9..5761564 100644
--- a/qca-nss-sfe/sfe_ipv4_tcp.c
+++ b/qca-nss-sfe/sfe_ipv4_tcp.c
@@ -133,6 +133,7 @@
bool bridge_flow;
bool fast_xmit;
netdev_features_t features;
+ bool segmentation_needed;
/*
* Is our packet too short to contain a valid TCP header?
@@ -268,7 +269,8 @@
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
- if (unlikely((len > cm->xmit_dev_mtu) && !skb_is_gso(skb))) {
+ segmentation_needed = len > cm->xmit_dev_mtu;
+ if (unlikely(segmentation_needed && !skb_is_gso(skb))) {
sfe_ipv4_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
@@ -555,6 +557,12 @@
}
/*
+ * We're going to check for GSO flags when we transmit the packet so
+ * start fetching the necessary cache line now.
+ */
+ if (segmentation_needed) prefetch(skb_shinfo(skb));
+
+ /*
* From this point on we're good to modify the packet.
*/
@@ -724,18 +732,19 @@
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
- * We're going to check for GSO flags when we transmit the packet so
- * start fetching the necessary cache line now.
- */
- prefetch(skb_shinfo(skb));
-
- /*
* We do per packet condition check before we could fast xmit the
* packet.
*/
- if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
- this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
- return 1;
+ if (likely(fast_xmit)) {
+ if (likely(!segmentation_needed || !skb_is_gso(skb))) {
+ if (likely(dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+ } else {
+ cm->flags &= ~SFE_IPV4_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ DEBUG_TRACE("%px: fast xmit disabled for xmit dev %s", skb, xmit_dev->name);
+ }
}
/*
diff --git a/qca-nss-sfe/sfe_ipv6.c b/qca-nss-sfe/sfe_ipv6.c
index 488f3a4..9670c7e 100644
--- a/qca-nss-sfe/sfe_ipv6.c
+++ b/qca-nss-sfe/sfe_ipv6.c
@@ -1839,6 +1839,8 @@
sfe_sync_rule_callback_t sync_rule_callback;
struct sfe_ipv6_connection *c;
+ if (sfe_pause_stats_sync)
+ goto done;
now_jiffies = get_jiffies_64();
rcu_read_lock();
diff --git a/qca-nss-sfe/sfe_ipv6_tcp.c b/qca-nss-sfe/sfe_ipv6_tcp.c
index 6ba30b3..672289c 100644
--- a/qca-nss-sfe/sfe_ipv6_tcp.c
+++ b/qca-nss-sfe/sfe_ipv6_tcp.c
@@ -132,6 +132,7 @@
bool bridge_flow;
bool fast_xmit;
netdev_features_t features;
+ bool segmentation_needed;
/*
* Is our packet too short to contain a valid TCP header?
@@ -269,7 +270,8 @@
* If our packet is larger than the MTU of the transmit interface then
* we can't forward it easily.
*/
- if (unlikely((len > cm->xmit_dev_mtu) && !skb_is_gso(skb))) {
+ segmentation_needed = len > cm->xmit_dev_mtu;
+ if (unlikely(segmentation_needed && !skb_is_gso(skb))) {
sfe_ipv6_sync_status(si, cm->connection, SFE_SYNC_REASON_STATS);
rcu_read_unlock();
@@ -566,6 +568,12 @@
}
/*
+ * We're going to check for GSO flags when we transmit the packet so
+ * start fetching the necessary cache line now.
+ */
+ if (len > cm->xmit_dev_mtu) prefetch(skb_shinfo(skb));
+
+ /*
* From this point on we're good to modify the packet.
*/
@@ -729,18 +737,19 @@
this_cpu_inc(si->stats_pcpu->packets_forwarded64);
/*
- * We're going to check for GSO flags when we transmit the packet so
- * start fetching the necessary cache line now.
- */
- prefetch(skb_shinfo(skb));
-
- /*
* We do per packet condition check before we could fast xmit the
* packet.
*/
- if (likely(fast_xmit && dev_fast_xmit(skb, xmit_dev, features))) {
- this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
- return 1;
+ if (likely(fast_xmit)) {
+ if (likely(!segmentation_needed || !skb_is_gso(skb))) {
+ if (likely(dev_fast_xmit(skb, xmit_dev, features))) {
+ this_cpu_inc(si->stats_pcpu->packets_fast_xmited64);
+ return 1;
+ }
+ } else {
+ cm->flags &= ~SFE_IPV6_CONNECTION_MATCH_FLAG_FAST_XMIT;
+ DEBUG_TRACE("%px: fast xmit disabled for xmit dev %s", skb, xmit_dev->name);
+ }
}
/*
