eth: fbnic: Make Rx completion coalescing configurable

The Rx completion queue writeback coalescing window controls how long
the queue manager may defer writing completions to host memory.

Expose this window through the standard rx-cqe-nsecs ethtool
parameter. Preserve the existing 2000 ns default and accept values
from 1000 ns through the device maximum (~109 us).

Hardware expresses COAL_WAIT in 600 MHz core clock cycles, or 1.667 ns
per cycle. Programming rounds the configured value to the nearest clock
cycle, resulting in an error of less than 1 ns. Retain the requested
value for get_coalesce() because this error is below the one-nanosecond
resolution of the ethtool interface.

Signed-off-by: Mohsin Bashir <hmohsin@meta.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Link: https://patch.msgid.link/20260909211111.1795726-1-mohsin.bashr@gmail.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

drivers/net/ethernet/meta/fbnic/fbnic.h

@@ -57,6 +57,7 @@ struct fbnic_dev {
 	u64 dsn;
 	u32 mps;
 	u32 readrq;
+	u32 rx_cqe_nsecs;
 	u8 relaxed_ord;
 
 	/* Local copy of the devices TCAM */
@@ -250,6 +251,7 @@ int fbnic_csr_regs_len(struct fbnic_dev *fbd);
 
 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm);
 void fbnic_config_rx_frames(struct fbnic_napi_vector *nv);
+void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd);
 
 enum fbnic_boards {
 	fbnic_board_asic

drivers/net/ethernet/meta/fbnic/fbnic_devlink.c

@@ -10,6 +10,7 @@
 #include "fbnic.h"
 #include "fbnic_fw.h"
 #include "fbnic_tlv.h"
+#include "fbnic_txrx.h"
 
 #define FBNIC_SN_STR_LEN	24
 
@@ -648,6 +649,7 @@ struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev)
 	fbd->mps = pcie_get_mps(pdev);
 	fbd->readrq = pcie_get_readrq(pdev);
 	fbd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev);
+	fbd->rx_cqe_nsecs = FBNIC_RX_CQE_NSECS_DEFAULT;
 
 	fbd->mac_addr_boundary = FBNIC_RPC_TCAM_MACDA_DEFAULT_BOUNDARY;
 

drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c

@@ -246,6 +246,7 @@ static int fbnic_get_coalesce(struct net_device *netdev,
 	ec->tx_coalesce_usecs = fbn->tx_usecs;
 	ec->rx_coalesce_usecs = fbn->rx_usecs;
 	ec->rx_max_coalesced_frames = fbn->rx_max_frames;
+	kernel_coal->rx_cqe_nsecs = fbn->fbd->rx_cqe_nsecs;
 
 	return 0;
 }
@@ -256,6 +257,7 @@ static int fbnic_set_coalesce(struct net_device *netdev,
 			      struct netlink_ext_ack *extack)
 {
 	struct fbnic_net *fbn = netdev_priv(netdev);
+	struct fbnic_dev *fbd = fbn->fbd;
 
 	/* Verify against hardware limits */
 	if (ec->rx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT)) {
@@ -272,10 +274,20 @@ static int fbnic_set_coalesce(struct net_device *netdev,
 		NL_SET_ERR_MSG_MOD(extack, "rx_frames is above device max");
 		return -EINVAL;
 	}
+	if (kernel_coal->rx_cqe_nsecs < FBNIC_RX_CQE_NSECS_MIN ||
+	    kernel_coal->rx_cqe_nsecs > FBNIC_RX_CQE_NSECS_MAX) {
+		NL_SET_ERR_MSG_FMT_MOD(extack,
+				       "rx-cqe-nsecs must be between %u and %u",
+				       FBNIC_RX_CQE_NSECS_MIN,
+				       FBNIC_RX_CQE_NSECS_MAX);
+		return -EINVAL;
+	}
 
 	fbn->tx_usecs = ec->tx_coalesce_usecs;
 	fbn->rx_usecs = ec->rx_coalesce_usecs;
 	fbn->rx_max_frames = ec->rx_max_coalesced_frames;
+	fbd->rx_cqe_nsecs = kernel_coal->rx_cqe_nsecs;
+	fbnic_config_rx_cqe_nsecs(fbd);
 
 	if (netif_running(netdev)) {
 		int i;
@@ -2016,7 +2028,8 @@ static void fbnic_get_link_ext_stats(struct net_device *netdev,
 static const struct ethtool_ops fbnic_ethtool_ops = {
 	.cap_link_lanes_supported	= true,
 	.supported_coalesce_params	= ETHTOOL_COALESCE_USECS |
-					  ETHTOOL_COALESCE_RX_MAX_FRAMES,
+					  ETHTOOL_COALESCE_RX_MAX_FRAMES |
+					  ETHTOOL_COALESCE_RX_CQE_NSECS,
 	.supported_ring_params		= ETHTOOL_RING_USE_TCP_DATA_SPLIT |
 					  ETHTOOL_RING_USE_HDS_THRS,
 	.rxfh_max_num_contexts		= FBNIC_RPC_RSS_TBL_COUNT,

drivers/net/ethernet/meta/fbnic/fbnic_mac.c

@@ -125,14 +125,7 @@ static void fbnic_mac_init_qm(struct fbnic_dev *fbd)
 	     FIELD_PREP(FBNIC_QM_TCQ_CTL0_COAL_WAIT,
 			clock_freq / 12500));
 
-	/* We will have the interrupt threshold timer tick once every
-	 * 1 usec and coalesce writes for up to 2 usecs.
-	 */
-	wr32(fbd, FBNIC_QM_RCQ_CTL0,
-	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES,
-			clock_freq / 1000000) |
-	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT,
-			clock_freq / 500000));
+	fbnic_config_rx_cqe_nsecs(fbd);
 
 	/* Configure spacer control to 64 beats. */
 	wr32(fbd, FBNIC_FAB_AXI4_AR_SPACER_2_CFG,

drivers/net/ethernet/meta/fbnic/fbnic_txrx.c

@@ -6,6 +6,7 @@
 #include <linux/bpf_trace.h>
 #include <linux/iopoll.h>
 #include <linux/pci.h>
+#include <linux/time64.h>
 #include <net/netdev_queues.h>
 #include <net/page_pool/helpers.h>
 #include <net/tcp.h>
@@ -2643,6 +2644,22 @@ static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 r
 	fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, threshold);
 }
 
+void fbnic_config_rx_cqe_nsecs(struct fbnic_dev *fbd)
+{
+	u32 coal_wait;
+
+	coal_wait = DIV_ROUND_CLOSEST_ULL((u64)fbd->rx_cqe_nsecs *
+					  FBNIC_CLOCK_FREQ, NSEC_PER_SEC);
+
+	/* TICK_CYCLES controls the interrupt threshold timer. COAL_WAIT is
+	 * independent and measured in core clock cycles.
+	 */
+	wr32(fbd, FBNIC_QM_RCQ_CTL0,
+	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES,
+			FBNIC_CLOCK_FREQ / USEC_PER_SEC) |
+	     FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT, coal_wait));
+}
+
 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm)
 {
 	struct fbnic_net *fbn = netdev_priv(nv->napi.dev);

drivers/net/ethernet/meta/fbnic/fbnic_txrx.h

@@ -4,12 +4,16 @@
 #ifndef _FBNIC_TXRX_H_
 #define _FBNIC_TXRX_H_
 
+#include <linux/bitfield.h>
 #include <linux/netdevice.h>
 #include <linux/skbuff.h>
+#include <linux/time64.h>
 #include <linux/types.h>
 #include <linux/u64_stats_sync.h>
 #include <net/xdp.h>
 
+#include "fbnic_csr.h"
+
 struct fbnic_net;
 
 /* Guarantee we have space needed for storing the buffer
@@ -49,6 +53,12 @@ struct fbnic_net;
 #define FBNIC_RX_USECS_DEFAULT		30
 #define FBNIC_RX_FRAMES_DEFAULT		0
 
+#define FBNIC_RX_CQE_NSECS_MIN		1000
+#define FBNIC_RX_CQE_NSECS_DEFAULT	2000
+#define FBNIC_RX_CQE_NSECS_MAX \
+	((u32)(((u64)FIELD_MAX(FBNIC_QM_RCQ_CTL0_COAL_WAIT) * NSEC_PER_SEC) / \
+	       FBNIC_CLOCK_FREQ))
+
 #define FBNIC_RX_TROOM \
 	SKB_DATA_ALIGN(sizeof(struct skb_shared_info))
 #define FBNIC_RX_HROOM_PAD		128