diff options
Diffstat (limited to 'drivers/net/ethernet/intel')
101 files changed, 8106 insertions, 4536 deletions
diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index b05cc0d7a15d..a563a94e2780 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -146,6 +146,7 @@ config IXGBE tristate "Intel(R) 10GbE PCI Express adapters support" depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + select LIBIE_FWLOG select MDIO select NET_DEVLINK select PLDMFW @@ -297,6 +298,7 @@ config ICE select DIMLIB select LIBIE select LIBIE_ADMINQ + select LIBIE_FWLOG select NET_DEVLINK select PACKING select PLDMFW diff --git a/drivers/net/ethernet/intel/Makefile b/drivers/net/ethernet/intel/Makefile index 04c844ef4964..9a37dc76aef0 100644 --- a/drivers/net/ethernet/intel/Makefile +++ b/drivers/net/ethernet/intel/Makefile @@ -4,7 +4,7 @@ # obj-$(CONFIG_LIBETH) += libeth/ -obj-$(CONFIG_LIBIE) += libie/ +obj-y += libie/ obj-$(CONFIG_E100) += e100.o obj-$(CONFIG_E1000) += e1000/ diff --git a/drivers/net/ethernet/intel/e1000/e1000.h b/drivers/net/ethernet/intel/e1000/e1000.h index 75f3fd1d8d6e..ea6ccf4b728b 100644 --- a/drivers/net/ethernet/intel/e1000/e1000.h +++ b/drivers/net/ethernet/intel/e1000/e1000.h @@ -116,7 +116,7 @@ struct e1000_adapter; #define E1000_MASTER_SLAVE e1000_ms_hw_default #endif -#define E1000_MNG_VLAN_NONE (-1) +#define E1000_MNG_VLAN_NONE 0xFFFF /* wrapper around a pointer to a socket buffer, * so a DMA handle can be stored along with the buffer diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index d06d29c6c037..726365c567ef 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -806,7 +806,7 @@ static int e1000_eeprom_test(struct e1000_adapter *adapter, u64 *data) } /* If Checksum is not Correct return error else test passed */ - if ((checksum != (u16)EEPROM_SUM) && !(*data)) + if (checksum != EEPROM_SUM && !(*data)) *data = 2; return *data; diff --git a/drivers/net/ethernet/intel/e1000/e1000_hw.c b/drivers/net/ethernet/intel/e1000/e1000_hw.c index f9328f2e669f..0e5de52b1067 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_hw.c +++ b/drivers/net/ethernet/intel/e1000/e1000_hw.c @@ -3970,7 +3970,7 @@ s32 e1000_validate_eeprom_checksum(struct e1000_hw *hw) return E1000_SUCCESS; #endif - if (checksum == (u16)EEPROM_SUM) + if (checksum == EEPROM_SUM) return E1000_SUCCESS; else { e_dbg("EEPROM Checksum Invalid\n"); @@ -3997,7 +3997,7 @@ s32 e1000_update_eeprom_checksum(struct e1000_hw *hw) } checksum += eeprom_data; } - checksum = (u16)EEPROM_SUM - checksum; + checksum = EEPROM_SUM - checksum; if (e1000_write_eeprom(hw, EEPROM_CHECKSUM_REG, 1, &checksum) < 0) { e_dbg("EEPROM Write Error\n"); return -E1000_ERR_EEPROM; diff --git a/drivers/net/ethernet/intel/e1000/e1000_main.c b/drivers/net/ethernet/intel/e1000/e1000_main.c index d8595e84326d..292389aceb2d 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_main.c +++ b/drivers/net/ethernet/intel/e1000/e1000_main.c @@ -313,8 +313,7 @@ static void e1000_update_mng_vlan(struct e1000_adapter *adapter) } else { adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; } - if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && - (vid != old_vid) && + if (old_vid != E1000_MNG_VLAN_NONE && vid != old_vid && !test_bit(old_vid, adapter->active_vlans)) e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); diff --git a/drivers/net/ethernet/intel/e1000e/e1000.h b/drivers/net/ethernet/intel/e1000e/e1000.h index 952898151565..018e61aea787 100644 --- a/drivers/net/ethernet/intel/e1000e/e1000.h +++ b/drivers/net/ethernet/intel/e1000e/e1000.h @@ -64,7 +64,7 @@ struct e1000_info; #define AUTO_ALL_MODES 0 #define E1000_EEPROM_APME 0x0400 -#define E1000_MNG_VLAN_NONE (-1) +#define E1000_MNG_VLAN_NONE 0xFFFF #define DEFAULT_JUMBO 9234 diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index cf01a108a5bb..8e40bb50a01e 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -963,7 +963,7 @@ static int e1000_eeprom_test(struct e1000_adapter *adapter, u64 *data) } /* If Checksum is not Correct return error else test passed */ - if ((checksum != (u16)NVM_SUM) && !(*data)) + if (checksum != NVM_SUM && !(*data)) *data = 2; return *data; diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index b27a61fab371..201322dac233 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -2761,7 +2761,7 @@ static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter) rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN); ew32(RCTL, rctl); - if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) { + if (adapter->mng_vlan_id != E1000_MNG_VLAN_NONE) { e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), adapter->mng_vlan_id); adapter->mng_vlan_id = E1000_MNG_VLAN_NONE; @@ -2828,7 +2828,7 @@ static void e1000_update_mng_vlan(struct e1000_adapter *adapter) adapter->mng_vlan_id = vid; } - if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid)) + if (old_vid != E1000_MNG_VLAN_NONE && vid != old_vid) e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid); } diff --git a/drivers/net/ethernet/intel/e1000e/nvm.c b/drivers/net/ethernet/intel/e1000e/nvm.c index 16369e6d245a..4bde1c9de1b9 100644 --- a/drivers/net/ethernet/intel/e1000e/nvm.c +++ b/drivers/net/ethernet/intel/e1000e/nvm.c @@ -564,7 +564,7 @@ s32 e1000e_validate_nvm_checksum_generic(struct e1000_hw *hw) return 0; } - if (checksum != (u16)NVM_SUM) { + if (checksum != NVM_SUM) { e_dbg("NVM Checksum Invalid\n"); return -E1000_ERR_NVM; } @@ -594,7 +594,7 @@ s32 e1000e_update_nvm_checksum_generic(struct e1000_hw *hw) } checksum += nvm_data; } - checksum = (u16)NVM_SUM - checksum; + checksum = NVM_SUM - checksum; ret_val = e1000_write_nvm(hw, NVM_CHECKSUM_REG, 1, &checksum); if (ret_val) e_dbg("NVM Write Error while updating checksum.\n"); diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.c b/drivers/net/ethernet/intel/fm10k/fm10k_common.c index f51a63fca513..1f919a50c765 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_common.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.c @@ -447,17 +447,16 @@ void fm10k_update_hw_stats_q(struct fm10k_hw *hw, struct fm10k_hw_stats_q *q, /** * fm10k_unbind_hw_stats_q - Unbind the queue counters from their queues * @q: pointer to the ring of hardware statistics queue - * @idx: index pointing to the start of the ring iteration * @count: number of queues to iterate over * * Function invalidates the index values for the queues so any updates that * may have happened are ignored and the base for the queue stats is reset. **/ -void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 idx, u32 count) +void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 count) { u32 i; - for (i = 0; i < count; i++, idx++, q++) { + for (i = 0; i < count; i++, q++) { q->rx_stats_idx = 0; q->tx_stats_idx = 0; } diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_common.h b/drivers/net/ethernet/intel/fm10k/fm10k_common.h index 4c48fb73b3e7..13fca6a91a01 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_common.h +++ b/drivers/net/ethernet/intel/fm10k/fm10k_common.h @@ -43,6 +43,6 @@ u32 fm10k_read_hw_stats_32b(struct fm10k_hw *hw, u32 addr, void fm10k_update_hw_stats_q(struct fm10k_hw *hw, struct fm10k_hw_stats_q *q, u32 idx, u32 count); #define fm10k_unbind_hw_stats_32b(s) ((s)->base_h = 0) -void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 idx, u32 count); +void fm10k_unbind_hw_stats_q(struct fm10k_hw_stats_q *q, u32 count); s32 fm10k_get_host_state_generic(struct fm10k_hw *hw, bool *host_ready); #endif /* _FM10K_COMMON_H_ */ diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c index 1954a04460d1..bf2029144c1d 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c @@ -560,7 +560,7 @@ static int fm10k_set_ringparam(struct net_device *netdev, /* allocate temporary buffer to store rings in */ i = max_t(int, interface->num_tx_queues, interface->num_rx_queues); - temp_ring = vmalloc(array_size(i, sizeof(struct fm10k_ring))); + temp_ring = vmalloc_array(i, sizeof(struct fm10k_ring)); if (!temp_ring) { err = -ENOMEM; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_main.c b/drivers/net/ethernet/intel/fm10k/fm10k_main.c index 142f07ca8bc0..b8c15b837fda 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_main.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_main.c @@ -37,7 +37,7 @@ static int __init fm10k_init_module(void) pr_info("%s\n", fm10k_copyright); /* create driver workqueue */ - fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, + fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM | WQ_PERCPU, 0, fm10k_driver_name); if (!fm10k_workqueue) return -ENOMEM; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c index b9dd7b719832..3394645a18fe 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_pf.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_pf.c @@ -1389,7 +1389,7 @@ static void fm10k_rebind_hw_stats_pf(struct fm10k_hw *hw, fm10k_unbind_hw_stats_32b(&stats->nodesc_drop); /* Unbind Queue Statistics */ - fm10k_unbind_hw_stats_q(stats->q, 0, hw->mac.max_queues); + fm10k_unbind_hw_stats_q(stats->q, hw->mac.max_queues); /* Reinitialize bases for all stats */ fm10k_update_hw_stats_pf(hw, stats); diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c index 7fb1961f2921..6861a0bdc14e 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_vf.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_vf.c @@ -465,7 +465,7 @@ static void fm10k_rebind_hw_stats_vf(struct fm10k_hw *hw, struct fm10k_hw_stats *stats) { /* Unbind Queue Statistics */ - fm10k_unbind_hw_stats_q(stats->q, 0, hw->mac.max_queues); + fm10k_unbind_hw_stats_q(stats->q, hw->mac.max_queues); /* Reinitialize bases for all stats */ fm10k_update_hw_stats_vf(hw, stats); diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 529d5501baac..50be0a60ae13 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -16635,7 +16635,7 @@ static int __init i40e_init_module(void) * since we need to be able to guarantee forward progress even under * memory pressure. */ - i40e_wq = alloc_workqueue("%s", 0, 0, i40e_driver_name); + i40e_wq = alloc_workqueue("%s", WQ_PERCPU, 0, i40e_driver_name); if (!i40e_wq) { pr_err("%s: Failed to create workqueue\n", i40e_driver_name); return -ENOMEM; diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index b194eae03208..cc0b9efc2637 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -2148,10 +2148,10 @@ static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring, memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0], sizeof(skb_frag_t) * nr_frags); - xdp_update_skb_shared_info(skb, skinfo->nr_frags + nr_frags, - sinfo->xdp_frags_size, - nr_frags * xdp->frame_sz, - xdp_buff_is_frag_pfmemalloc(xdp)); + xdp_update_skb_frags_info(skb, skinfo->nr_frags + nr_frags, + sinfo->xdp_frags_size, + nr_frags * xdp->frame_sz, + xdp_buff_get_skb_flags(xdp)); /* First buffer has already been processed, so bump ntc */ if (++rx_ring->next_to_clean == rx_ring->count) @@ -2203,10 +2203,9 @@ static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring, skb_metadata_set(skb, metasize); if (unlikely(xdp_buff_has_frags(xdp))) { - xdp_update_skb_shared_info(skb, nr_frags, - sinfo->xdp_frags_size, - nr_frags * xdp->frame_sz, - xdp_buff_is_frag_pfmemalloc(xdp)); + xdp_update_skb_frags_info(skb, nr_frags, sinfo->xdp_frags_size, + nr_frags * xdp->frame_sz, + xdp_buff_get_skb_flags(xdp)); i40e_process_rx_buffs(rx_ring, I40E_XDP_PASS, xdp); } else { diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index 69054af4689a..c2fbe443ef85 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -5491,7 +5491,7 @@ static int iavf_resume(struct device *dev_d) { struct pci_dev *pdev = to_pci_dev(dev_d); struct iavf_adapter *adapter; - u32 err; + int err; adapter = iavf_pdev_to_adapter(pdev); diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index d0f9c9492363..5b2c666496e7 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -42,14 +42,15 @@ ice-y := ice_main.o \ ice_ethtool.o \ ice_repr.o \ ice_tc_lib.o \ - ice_fwlog.o \ ice_debugfs.o \ ice_adapter.o ice-$(CONFIG_PCI_IOV) += \ ice_sriov.o \ - ice_virtchnl.o \ - ice_virtchnl_allowlist.o \ - ice_virtchnl_fdir.o \ + virt/allowlist.o \ + virt/fdir.o \ + virt/queues.o \ + virt/virtchnl.o \ + virt/rss.o \ ice_vf_mbx.o \ ice_vf_vsi_vlan_ops.o \ ice_vf_lib.o diff --git a/drivers/net/ethernet/intel/ice/devlink/health.c b/drivers/net/ethernet/intel/ice/devlink/health.c index ab519c0f28bf..8e9a8a8178d4 100644 --- a/drivers/net/ethernet/intel/ice/devlink/health.c +++ b/drivers/net/ethernet/intel/ice/devlink/health.c @@ -450,9 +450,8 @@ ice_init_devlink_rep(struct ice_pf *pf, { struct devlink *devlink = priv_to_devlink(pf); struct devlink_health_reporter *rep; - const u64 graceful_period = 0; - rep = devl_health_reporter_create(devlink, ops, graceful_period, pf); + rep = devl_health_reporter_create(devlink, ops, pf); if (IS_ERR(rep)) { struct device *dev = ice_pf_to_dev(pf); diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index 8a8a01a4bb40..22b8323ff0d0 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -84,7 +84,11 @@ #define ICE_BAR0 0 #define ICE_REQ_DESC_MULTIPLE 32 #define ICE_MIN_NUM_DESC 64 -#define ICE_MAX_NUM_DESC 8160 +#define ICE_MAX_NUM_DESC_E810 8160 +#define ICE_MAX_NUM_DESC_E830 8096 +#define ICE_MAX_NUM_DESC_BY_MAC(hw) ((hw)->mac_type == ICE_MAC_E830 ? \ + ICE_MAX_NUM_DESC_E830 : \ + ICE_MAX_NUM_DESC_E810) #define ICE_DFLT_MIN_RX_DESC 512 #define ICE_DFLT_NUM_TX_DESC 256 #define ICE_DFLT_NUM_RX_DESC 2048 @@ -200,9 +204,11 @@ enum ice_feature { ICE_F_SMA_CTRL, ICE_F_CGU, ICE_F_GNSS, + ICE_F_TXTIME, ICE_F_GCS, ICE_F_ROCE_LAG, ICE_F_SRIOV_LAG, + ICE_F_SRIOV_AA_LAG, ICE_F_MBX_LIMIT, ICE_F_MAX }; @@ -567,9 +573,6 @@ struct ice_pf { struct ice_sw *first_sw; /* first switch created by firmware */ u16 eswitch_mode; /* current mode of eswitch */ struct dentry *ice_debugfs_pf; - struct dentry *ice_debugfs_pf_fwlog; - /* keep track of all the dentrys for FW log modules */ - struct dentry **ice_debugfs_pf_fwlog_modules; struct ice_vfs vfs; DECLARE_BITMAP(features, ICE_F_MAX); DECLARE_BITMAP(state, ICE_STATE_NBITS); @@ -577,6 +580,7 @@ struct ice_pf { DECLARE_BITMAP(misc_thread, ICE_MISC_THREAD_NBITS); unsigned long *avail_txqs; /* bitmap to track PF Tx queue usage */ unsigned long *avail_rxqs; /* bitmap to track PF Rx queue usage */ + unsigned long *txtime_txqs; /* bitmap to track PF Tx Time queue */ unsigned long serv_tmr_period; unsigned long serv_tmr_prev; struct timer_list serv_tmr; @@ -751,6 +755,31 @@ static inline void ice_set_ring_xdp(struct ice_tx_ring *ring) } /** + * ice_is_txtime_ena - check if Tx Time is enabled on the Tx ring + * @ring: pointer to Tx ring + * + * Return: true if the Tx ring has Tx Time enabled, false otherwise. + */ +static inline bool ice_is_txtime_ena(const struct ice_tx_ring *ring) +{ + struct ice_vsi *vsi = ring->vsi; + struct ice_pf *pf = vsi->back; + + return test_bit(ring->q_index, pf->txtime_txqs); +} + +/** + * ice_is_txtime_cfg - check if Tx Time is configured on the Tx ring + * @ring: pointer to Tx ring + * + * Return: true if the Tx ring is configured for Tx ring, false otherwise. + */ +static inline bool ice_is_txtime_cfg(const struct ice_tx_ring *ring) +{ + return !!(ring->flags & ICE_TX_FLAGS_TXTIME); +} + +/** * ice_get_xp_from_qid - get ZC XSK buffer pool bound to a queue ID * @vsi: pointer to VSI * @qid: index of a queue to look at XSK buff pool presence @@ -907,11 +936,10 @@ static inline bool ice_is_adq_active(struct ice_pf *pf) return false; } -void ice_debugfs_fwlog_init(struct ice_pf *pf); +int ice_debugfs_pf_init(struct ice_pf *pf); void ice_debugfs_pf_deinit(struct ice_pf *pf); void ice_debugfs_init(void); void ice_debugfs_exit(void); -void ice_pf_fwlog_update_module(struct ice_pf *pf, int log_level, int module); bool netif_is_ice(const struct net_device *dev); int ice_vsi_setup_tx_rings(struct ice_vsi *vsi); diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 3bd3ea3af888..859e9c66f3e7 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -33,6 +33,10 @@ typedef struct __packed { u8 buf[ICE_TXQ_CTX_SZ]; } ice_txq_ctx_buf_t; typedef struct __packed { u8 buf[ICE_TXQ_CTX_FULL_SZ]; } ice_txq_ctx_buf_full_t; +#define ICE_TXTIME_CTX_SZ 25 + +typedef struct __packed { u8 buf[ICE_TXTIME_CTX_SZ]; } ice_txtime_ctx_buf_t; + /* Queue Shutdown (direct 0x0003) */ struct ice_aqc_q_shutdown { u8 driver_unloading; @@ -2060,6 +2064,10 @@ struct ice_aqc_cfg_txqs { #define ICE_AQC_Q_CFG_SRC_PRT_M 0x7 #define ICE_AQC_Q_CFG_DST_PRT_S 3 #define ICE_AQC_Q_CFG_DST_PRT_M (0x7 << ICE_AQC_Q_CFG_DST_PRT_S) +#define ICE_AQC_Q_CFG_MODE_M GENMASK(7, 6) +#define ICE_AQC_Q_CFG_MODE_SAME_PF 0x0 +#define ICE_AQC_Q_CFG_MODE_GIVE_OWN 0x1 +#define ICE_AQC_Q_CFG_MODE_KEEP_OWN 0x2 u8 time_out; #define ICE_AQC_Q_CFG_TIMEOUT_S 2 #define ICE_AQC_Q_CFG_TIMEOUT_M (0x1F << ICE_AQC_Q_CFG_TIMEOUT_S) @@ -2113,6 +2121,34 @@ struct ice_aqc_add_rdma_qset_data { struct ice_aqc_add_tx_rdma_qset_entry rdma_qsets[]; }; +/* Set Tx Time LAN Queue (indirect 0x0C35) */ +struct ice_aqc_set_txtimeqs { + __le16 q_id; + __le16 q_amount; + u8 reserved[4]; + __le32 addr_high; + __le32 addr_low; +}; + +/* This is the descriptor of each queue entry for the Set Tx Time Queue + * command (0x0C35). Only used within struct ice_aqc_set_txtime_qgrp. + */ +struct ice_aqc_set_txtimeqs_perq { + u8 reserved[4]; + ice_txtime_ctx_buf_t txtime_ctx; + u8 reserved1[3]; +}; + +/* The format of the command buffer for Set Tx Time Queue (0x0C35) + * is an array of the following structs. Please note that the length of + * each struct ice_aqc_set_txtime_qgrp is variable due to the variable + * number of queues in each group! + */ +struct ice_aqc_set_txtime_qgrp { + u8 reserved[8]; + struct ice_aqc_set_txtimeqs_perq txtimeqs[]; +}; + /* Download Package (indirect 0x0C40) */ /* Also used for Update Package (indirect 0x0C41 and 0x0C42) */ struct ice_aqc_download_pkg { @@ -2395,42 +2431,6 @@ struct ice_aqc_event_lan_overflow { u8 reserved[8]; }; -enum ice_aqc_fw_logging_mod { - ICE_AQC_FW_LOG_ID_GENERAL = 0, - ICE_AQC_FW_LOG_ID_CTRL, - ICE_AQC_FW_LOG_ID_LINK, - ICE_AQC_FW_LOG_ID_LINK_TOPO, - ICE_AQC_FW_LOG_ID_DNL, - ICE_AQC_FW_LOG_ID_I2C, - ICE_AQC_FW_LOG_ID_SDP, - ICE_AQC_FW_LOG_ID_MDIO, - ICE_AQC_FW_LOG_ID_ADMINQ, - ICE_AQC_FW_LOG_ID_HDMA, - ICE_AQC_FW_LOG_ID_LLDP, - ICE_AQC_FW_LOG_ID_DCBX, - ICE_AQC_FW_LOG_ID_DCB, - ICE_AQC_FW_LOG_ID_XLR, - ICE_AQC_FW_LOG_ID_NVM, - ICE_AQC_FW_LOG_ID_AUTH, - ICE_AQC_FW_LOG_ID_VPD, - ICE_AQC_FW_LOG_ID_IOSF, - ICE_AQC_FW_LOG_ID_PARSER, - ICE_AQC_FW_LOG_ID_SW, - ICE_AQC_FW_LOG_ID_SCHEDULER, - ICE_AQC_FW_LOG_ID_TXQ, - ICE_AQC_FW_LOG_ID_RSVD, - ICE_AQC_FW_LOG_ID_POST, - ICE_AQC_FW_LOG_ID_WATCHDOG, - ICE_AQC_FW_LOG_ID_TASK_DISPATCH, - ICE_AQC_FW_LOG_ID_MNG, - ICE_AQC_FW_LOG_ID_SYNCE, - ICE_AQC_FW_LOG_ID_HEALTH, - ICE_AQC_FW_LOG_ID_TSDRV, - ICE_AQC_FW_LOG_ID_PFREG, - ICE_AQC_FW_LOG_ID_MDLVER, - ICE_AQC_FW_LOG_ID_MAX, -}; - enum ice_aqc_health_status_mask { ICE_AQC_HEALTH_STATUS_SET_PF_SPECIFIC_MASK = BIT(0), ICE_AQC_HEALTH_STATUS_SET_ALL_PF_MASK = BIT(1), @@ -2512,48 +2512,6 @@ struct ice_aqc_health_status_elem { __le32 internal_data2; }; -/* Set FW Logging configuration (indirect 0xFF30) - * Register for FW Logging (indirect 0xFF31) - * Query FW Logging (indirect 0xFF32) - * FW Log Event (indirect 0xFF33) - */ -struct ice_aqc_fw_log { - u8 cmd_flags; -#define ICE_AQC_FW_LOG_CONF_UART_EN BIT(0) -#define ICE_AQC_FW_LOG_CONF_AQ_EN BIT(1) -#define ICE_AQC_FW_LOG_QUERY_REGISTERED BIT(2) -#define ICE_AQC_FW_LOG_CONF_SET_VALID BIT(3) -#define ICE_AQC_FW_LOG_AQ_REGISTER BIT(0) -#define ICE_AQC_FW_LOG_AQ_QUERY BIT(2) - - u8 rsp_flag; - __le16 fw_rt_msb; - union { - struct { - __le32 fw_rt_lsb; - } sync; - struct { - __le16 log_resolution; -#define ICE_AQC_FW_LOG_MIN_RESOLUTION (1) -#define ICE_AQC_FW_LOG_MAX_RESOLUTION (128) - - __le16 mdl_cnt; - } cfg; - } ops; - __le32 addr_high; - __le32 addr_low; -}; - -/* Response Buffer for: - * Set Firmware Logging Configuration (0xFF30) - * Query FW Logging (0xFF32) - */ -struct ice_aqc_fw_log_cfg_resp { - __le16 module_identifier; - u8 log_level; - u8 rsvd0; -}; - /* Admin Queue command opcodes */ enum ice_adminq_opc { /* AQ commands */ @@ -2688,6 +2646,9 @@ enum ice_adminq_opc { ice_aqc_opc_cfg_txqs = 0x0C32, ice_aqc_opc_add_rdma_qset = 0x0C33, + /* Tx Time queue commands */ + ice_aqc_opc_set_txtimeqs = 0x0C35, + /* package commands */ ice_aqc_opc_download_pkg = 0x0C40, ice_aqc_opc_upload_section = 0x0C41, diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index c5da8e9cc0a0..2d35a278c555 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -242,7 +242,8 @@ static void ice_cfg_itr_gran(struct ice_hw *hw) * @ring: ring to get the absolute queue index * @tc: traffic class number */ -static u16 ice_calc_txq_handle(struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 tc) +static u16 +ice_calc_txq_handle(const struct ice_vsi *vsi, struct ice_tx_ring *ring, u8 tc) { WARN_ONCE(ice_ring_is_xdp(ring) && tc, "XDP ring can't belong to TC other than 0\n"); @@ -278,30 +279,20 @@ static void ice_cfg_xps_tx_ring(struct ice_tx_ring *ring) } /** - * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance - * @ring: The Tx ring to configure - * @tlan_ctx: Pointer to the Tx LAN queue context structure to be initialized - * @pf_q: queue index in the PF space + * ice_set_txq_ctx_vmvf - set queue context VM/VF type and number by VSI type + * @ring: the Tx ring to configure + * @vmvf_type: VM/VF type + * @vmvf_num: VM/VF number * - * Configure the Tx descriptor ring in TLAN context. + * Return: 0 on success and a negative value on error. */ -static void -ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) +static int +ice_set_txq_ctx_vmvf(struct ice_tx_ring *ring, u8 *vmvf_type, u16 *vmvf_num) { struct ice_vsi *vsi = ring->vsi; - struct ice_hw *hw = &vsi->back->hw; - - tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S; - - tlan_ctx->port_num = vsi->port_info->lport; - - /* Transmit Queue Length */ - tlan_ctx->qlen = ring->count; - - ice_set_cgd_num(tlan_ctx, ring->dcb_tc); + struct ice_hw *hw; - /* PF number */ - tlan_ctx->pf_num = hw->pf_id; + hw = &vsi->back->hw; /* queue belongs to a specific VSI type * VF / VM index should be programmed per vmvf_type setting: @@ -314,21 +305,60 @@ ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf case ICE_VSI_CTRL: case ICE_VSI_PF: if (ring->ch) - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; else - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF; break; case ICE_VSI_VF: /* Firmware expects vmvf_num to be absolute VF ID */ - tlan_ctx->vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; + *vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; break; case ICE_VSI_SF: - tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; + *vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; break; default: - return; + dev_info(ice_pf_to_dev(vsi->back), + "Unable to set VMVF type for VSI type %d\n", + vsi->type); + return -EINVAL; } + return 0; +} + +/** + * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance + * @ring: the Tx ring to configure + * @tlan_ctx: pointer to the Tx LAN queue context structure to be initialized + * @pf_q: queue index in the PF space + * + * Configure the Tx descriptor ring in TLAN context. + * + * Return: 0 on success and a negative value on error. + */ +static int +ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q) +{ + struct ice_vsi *vsi = ring->vsi; + struct ice_hw *hw; + int err; + + hw = &vsi->back->hw; + tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S; + tlan_ctx->port_num = vsi->port_info->lport; + + /* Transmit Queue Length */ + tlan_ctx->qlen = ring->count; + + ice_set_cgd_num(tlan_ctx, ring->dcb_tc); + + /* PF number */ + tlan_ctx->pf_num = hw->pf_id; + + err = ice_set_txq_ctx_vmvf(ring, &tlan_ctx->vmvf_type, + &tlan_ctx->vmvf_num); + if (err) + return err; /* make sure the context is associated with the right VSI */ if (ring->ch) @@ -355,6 +385,80 @@ ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf * 1: Legacy Host Interface */ tlan_ctx->legacy_int = ICE_TX_LEGACY; + + return 0; +} + +/** + * ice_setup_txtime_ctx - setup a struct ice_txtime_ctx instance + * @ring: the tstamp ring to configure + * @txtime_ctx: pointer to the Tx time queue context structure to be initialized + * + * Return: 0 on success and a negative value on error. + */ +static int +ice_setup_txtime_ctx(const struct ice_tstamp_ring *ring, + struct ice_txtime_ctx *txtime_ctx) +{ + struct ice_tx_ring *tx_ring = ring->tx_ring; + struct ice_vsi *vsi = tx_ring->vsi; + struct ice_hw *hw = &vsi->back->hw; + int err; + + txtime_ctx->base = ring->dma >> ICE_TXTIME_CTX_BASE_S; + + /* Tx time Queue Length */ + txtime_ctx->qlen = ring->count; + txtime_ctx->txtime_ena_q = 1; + + /* PF number */ + txtime_ctx->pf_num = hw->pf_id; + + err = ice_set_txq_ctx_vmvf(tx_ring, &txtime_ctx->vmvf_type, + &txtime_ctx->vmvf_num); + if (err) + return err; + + /* make sure the context is associated with the right VSI */ + if (tx_ring->ch) + txtime_ctx->src_vsi = tx_ring->ch->vsi_num; + else + txtime_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx); + + txtime_ctx->ts_res = ICE_TXTIME_CTX_RESOLUTION_128NS; + txtime_ctx->drbell_mode_32 = ICE_TXTIME_CTX_DRBELL_MODE_32; + txtime_ctx->ts_fetch_prof_id = ICE_TXTIME_CTX_FETCH_PROF_ID_0; + + return 0; +} + +/** + * ice_calc_ts_ring_count - calculate the number of Tx time stamp descriptors + * @tx_ring: Tx ring to calculate the count for + * + * Return: the number of Tx time stamp descriptors. + */ +u16 ice_calc_ts_ring_count(struct ice_tx_ring *tx_ring) +{ + u16 prof = ICE_TXTIME_CTX_FETCH_PROF_ID_0; + struct ice_vsi *vsi = tx_ring->vsi; + struct ice_hw *hw = &vsi->back->hw; + u16 max_fetch_desc = 0, fetch, i; + u32 reg; + + for (i = 0; i < ICE_TXTIME_FETCH_PROFILE_CNT; i++) { + reg = rd32(hw, E830_GLTXTIME_FETCH_PROFILE(prof, 0)); + fetch = FIELD_GET(E830_GLTXTIME_FETCH_PROFILE_FETCH_TS_DESC_M, + reg); + max_fetch_desc = max(fetch, max_fetch_desc); + } + + if (!max_fetch_desc) + max_fetch_desc = ICE_TXTIME_FETCH_TS_DESC_DFLT; + + max_fetch_desc = ALIGN(max_fetch_desc, ICE_REQ_DESC_MULTIPLE); + + return tx_ring->count + max_fetch_desc; } /** @@ -882,13 +986,49 @@ void ice_vsi_free_q_vectors(struct ice_vsi *vsi) } /** + * ice_cfg_tstamp - Configure Tx time stamp queue + * @tx_ring: Tx ring to be configured with timestamping + * + * Return: 0 on success and a negative value on error. + */ +static int +ice_cfg_tstamp(struct ice_tx_ring *tx_ring) +{ + DEFINE_RAW_FLEX(struct ice_aqc_set_txtime_qgrp, txtime_qg_buf, + txtimeqs, 1); + u8 txtime_buf_len = struct_size(txtime_qg_buf, txtimeqs, 1); + struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; + struct ice_txtime_ctx txtime_ctx = {}; + struct ice_vsi *vsi = tx_ring->vsi; + struct ice_pf *pf = vsi->back; + struct ice_hw *hw = &pf->hw; + u16 pf_q = tx_ring->reg_idx; + int err; + + err = ice_setup_txtime_ctx(tstamp_ring, &txtime_ctx); + if (err) { + dev_err(ice_pf_to_dev(pf), "Failed to setup Tx time queue context for queue %d, error: %d\n", + pf_q, err); + return err; + } + ice_pack_txtime_ctx(&txtime_ctx, + &txtime_qg_buf->txtimeqs[0].txtime_ctx); + + tstamp_ring->tail = hw->hw_addr + E830_GLQTX_TXTIME_DBELL_LSB(pf_q); + return ice_aq_set_txtimeq(hw, pf_q, 1, txtime_qg_buf, + txtime_buf_len, NULL); +} + +/** * ice_vsi_cfg_txq - Configure single Tx queue * @vsi: the VSI that queue belongs to * @ring: Tx ring to be configured * @qg_buf: queue group buffer + * + * Return: 0 on success and a negative value on error. */ static int -ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, +ice_vsi_cfg_txq(const struct ice_vsi *vsi, struct ice_tx_ring *ring, struct ice_aqc_add_tx_qgrp *qg_buf) { u8 buf_len = struct_size(qg_buf, txqs, 1); @@ -897,15 +1037,20 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, struct ice_channel *ch = ring->ch; struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; + u32 pf_q, vsi_idx; int status; - u16 pf_q; u8 tc; /* Configure XPS */ ice_cfg_xps_tx_ring(ring); pf_q = ring->reg_idx; - ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); + status = ice_setup_tx_ctx(ring, &tlan_ctx, pf_q); + if (status) { + dev_err(ice_pf_to_dev(pf), "Failed to setup Tx context for queue %d, error: %d\n", + pf_q, status); + return status; + } /* copy context contents into the qg_buf */ qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q); ice_pack_txq_ctx(&tlan_ctx, &qg_buf->txqs[0].txq_ctx); @@ -925,14 +1070,15 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, */ ring->q_handle = ice_calc_txq_handle(vsi, ring, tc); - if (ch) - status = ice_ena_vsi_txq(vsi->port_info, ch->ch_vsi->idx, 0, - ring->q_handle, 1, qg_buf, buf_len, - NULL); - else - status = ice_ena_vsi_txq(vsi->port_info, vsi->idx, tc, - ring->q_handle, 1, qg_buf, buf_len, - NULL); + if (ch) { + tc = 0; + vsi_idx = ch->ch_vsi->idx; + } else { + vsi_idx = vsi->idx; + } + + status = ice_ena_vsi_txq(vsi->port_info, vsi_idx, tc, ring->q_handle, + 1, qg_buf, buf_len, NULL); if (status) { dev_err(ice_pf_to_dev(pf), "Failed to set LAN Tx queue context, error: %d\n", status); @@ -947,7 +1093,32 @@ ice_vsi_cfg_txq(struct ice_vsi *vsi, struct ice_tx_ring *ring, if (pf_q == le16_to_cpu(txq->txq_id)) ring->txq_teid = le32_to_cpu(txq->q_teid); + if (ice_is_txtime_ena(ring)) { + status = ice_alloc_setup_tstamp_ring(ring); + if (status) { + dev_err(ice_pf_to_dev(pf), + "Failed to allocate Tx timestamp ring, error: %d\n", + status); + goto err_setup_tstamp; + } + + status = ice_cfg_tstamp(ring); + if (status) { + dev_err(ice_pf_to_dev(pf), "Failed to set Tx Time queue context, error: %d\n", + status); + goto err_cfg_tstamp; + } + } return 0; + +err_cfg_tstamp: + ice_free_tx_tstamp_ring(ring); +err_setup_tstamp: + ice_dis_vsi_txq(vsi->port_info, vsi_idx, tc, 1, &ring->q_handle, + &ring->reg_idx, &ring->txq_teid, ICE_NO_RESET, + tlan_ctx.vmvf_num, NULL); + + return status; } int ice_vsi_cfg_single_txq(struct ice_vsi *vsi, struct ice_tx_ring **tx_rings, @@ -1206,3 +1377,148 @@ ice_fill_txq_meta(const struct ice_vsi *vsi, struct ice_tx_ring *ring, txq_meta->tc = tc; } } + +/** + * ice_qp_reset_stats - Resets all stats for rings of given index + * @vsi: VSI that contains rings of interest + * @q_idx: ring index in array + */ +static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) +{ + struct ice_vsi_stats *vsi_stat; + struct ice_pf *pf; + + pf = vsi->back; + if (!pf->vsi_stats) + return; + + vsi_stat = pf->vsi_stats[vsi->idx]; + if (!vsi_stat) + return; + + memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, + sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); + memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, + sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); + if (vsi->xdp_rings) + memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, + sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); +} + +/** + * ice_qp_clean_rings - Cleans all the rings of a given index + * @vsi: VSI that contains rings of interest + * @q_idx: ring index in array + */ +static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) +{ + ice_clean_tx_ring(vsi->tx_rings[q_idx]); + if (vsi->xdp_rings) + ice_clean_tx_ring(vsi->xdp_rings[q_idx]); + ice_clean_rx_ring(vsi->rx_rings[q_idx]); +} + +/** + * ice_qp_dis - Disables a queue pair + * @vsi: VSI of interest + * @q_idx: ring index in array + * + * Returns 0 on success, negative on failure. + */ +int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) +{ + struct ice_txq_meta txq_meta = { }; + struct ice_q_vector *q_vector; + struct ice_tx_ring *tx_ring; + struct ice_rx_ring *rx_ring; + int fail = 0; + int err; + + if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) + return -EINVAL; + + tx_ring = vsi->tx_rings[q_idx]; + rx_ring = vsi->rx_rings[q_idx]; + q_vector = rx_ring->q_vector; + + synchronize_net(); + netif_carrier_off(vsi->netdev); + netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); + + ice_qvec_dis_irq(vsi, rx_ring, q_vector); + ice_qvec_toggle_napi(vsi, q_vector, false); + + ice_fill_txq_meta(vsi, tx_ring, &txq_meta); + err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); + if (!fail) + fail = err; + if (vsi->xdp_rings) { + struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; + + memset(&txq_meta, 0, sizeof(txq_meta)); + ice_fill_txq_meta(vsi, xdp_ring, &txq_meta); + err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring, + &txq_meta); + if (!fail) + fail = err; + } + + ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, false); + ice_qp_clean_rings(vsi, q_idx); + ice_qp_reset_stats(vsi, q_idx); + + return fail; +} + +/** + * ice_qp_ena - Enables a queue pair + * @vsi: VSI of interest + * @q_idx: ring index in array + * + * Returns 0 on success, negative on failure. + */ +int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) +{ + struct ice_q_vector *q_vector; + int fail = 0; + bool link_up; + int err; + + err = ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx); + if (!fail) + fail = err; + + if (ice_is_xdp_ena_vsi(vsi)) { + struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; + + err = ice_vsi_cfg_single_txq(vsi, vsi->xdp_rings, q_idx); + if (!fail) + fail = err; + ice_set_ring_xdp(xdp_ring); + ice_tx_xsk_pool(vsi, q_idx); + } + + err = ice_vsi_cfg_single_rxq(vsi, q_idx); + if (!fail) + fail = err; + + q_vector = vsi->rx_rings[q_idx]->q_vector; + ice_qvec_cfg_msix(vsi, q_vector, q_idx); + + err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true); + if (!fail) + fail = err; + + ice_qvec_toggle_napi(vsi, q_vector, true); + ice_qvec_ena_irq(vsi, q_vector); + + /* make sure NAPI sees updated ice_{t,x}_ring::xsk_pool */ + synchronize_net(); + ice_get_link_status(vsi->port_info, &link_up); + if (link_up) { + netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); + netif_carrier_on(vsi->netdev); + } + + return fail; +} diff --git a/drivers/net/ethernet/intel/ice/ice_base.h b/drivers/net/ethernet/intel/ice/ice_base.h index b711bc921928..d28294247599 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.h +++ b/drivers/net/ethernet/intel/ice/ice_base.h @@ -32,4 +32,7 @@ ice_vsi_stop_tx_ring(struct ice_vsi *vsi, enum ice_disq_rst_src rst_src, void ice_fill_txq_meta(const struct ice_vsi *vsi, struct ice_tx_ring *ring, struct ice_txq_meta *txq_meta); +int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx); +int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx); +u16 ice_calc_ts_ring_count(struct ice_tx_ring *tx_ring); #endif /* _ICE_BASE_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_common.c b/drivers/net/ethernet/intel/ice/ice_common.c index 003d60a4db21..2250426ec91b 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.c +++ b/drivers/net/ethernet/intel/ice/ice_common.c @@ -984,6 +984,37 @@ static int ice_wait_for_fw(struct ice_hw *hw, u32 timeout) return -ETIMEDOUT; } +static int __fwlog_send_cmd(void *priv, struct libie_aq_desc *desc, void *buf, + u16 size) +{ + struct ice_hw *hw = priv; + + return ice_aq_send_cmd(hw, desc, buf, size, NULL); +} + +static int __fwlog_init(struct ice_hw *hw) +{ + struct ice_pf *pf = hw->back; + struct libie_fwlog_api api = { + .pdev = pf->pdev, + .send_cmd = __fwlog_send_cmd, + .priv = hw, + }; + int err; + + /* only support fw log commands on PF 0 */ + if (hw->bus.func) + return -EINVAL; + + err = ice_debugfs_pf_init(pf); + if (err) + return err; + + api.debugfs_root = pf->ice_debugfs_pf; + + return libie_fwlog_init(&hw->fwlog, &api); +} + /** * ice_init_hw - main hardware initialization routine * @hw: pointer to the hardware structure @@ -1012,7 +1043,7 @@ int ice_init_hw(struct ice_hw *hw) if (status) goto err_unroll_cqinit; - status = ice_fwlog_init(hw); + status = __fwlog_init(hw); if (status) ice_debug(hw, ICE_DBG_FW_LOG, "Error initializing FW logging: %d\n", status); @@ -1159,6 +1190,16 @@ err_unroll_cqinit: return status; } +static void __fwlog_deinit(struct ice_hw *hw) +{ + /* only support fw log commands on PF 0 */ + if (hw->bus.func) + return; + + ice_debugfs_pf_deinit(hw->back); + libie_fwlog_deinit(&hw->fwlog); +} + /** * ice_deinit_hw - unroll initialization operations done by ice_init_hw * @hw: pointer to the hardware structure @@ -1177,8 +1218,7 @@ void ice_deinit_hw(struct ice_hw *hw) ice_free_seg(hw); ice_free_hw_tbls(hw); mutex_destroy(&hw->tnl_lock); - - ice_fwlog_deinit(hw); + __fwlog_deinit(hw); ice_destroy_all_ctrlq(hw); /* Clear VSI contexts if not already cleared */ @@ -1693,6 +1733,44 @@ int ice_write_txq_ctx(struct ice_hw *hw, struct ice_tlan_ctx *tlan_ctx, return 0; } +/* Tx time Queue Context */ +static const struct packed_field_u8 ice_txtime_ctx_fields[] = { + /* Field Width LSB */ + ICE_CTX_STORE(ice_txtime_ctx, base, 57, 0), + ICE_CTX_STORE(ice_txtime_ctx, pf_num, 3, 57), + ICE_CTX_STORE(ice_txtime_ctx, vmvf_num, 10, 60), + ICE_CTX_STORE(ice_txtime_ctx, vmvf_type, 2, 70), + ICE_CTX_STORE(ice_txtime_ctx, src_vsi, 10, 72), + ICE_CTX_STORE(ice_txtime_ctx, cpuid, 8, 82), + ICE_CTX_STORE(ice_txtime_ctx, tphrd_desc, 1, 90), + ICE_CTX_STORE(ice_txtime_ctx, qlen, 13, 91), + ICE_CTX_STORE(ice_txtime_ctx, timer_num, 1, 104), + ICE_CTX_STORE(ice_txtime_ctx, txtime_ena_q, 1, 105), + ICE_CTX_STORE(ice_txtime_ctx, drbell_mode_32, 1, 106), + ICE_CTX_STORE(ice_txtime_ctx, ts_res, 4, 107), + ICE_CTX_STORE(ice_txtime_ctx, ts_round_type, 2, 111), + ICE_CTX_STORE(ice_txtime_ctx, ts_pacing_slot, 3, 113), + ICE_CTX_STORE(ice_txtime_ctx, merging_ena, 1, 116), + ICE_CTX_STORE(ice_txtime_ctx, ts_fetch_prof_id, 4, 117), + ICE_CTX_STORE(ice_txtime_ctx, ts_fetch_cache_line_aln_thld, 4, 121), + ICE_CTX_STORE(ice_txtime_ctx, tx_pipe_delay_mode, 1, 125), +}; + +/** + * ice_pack_txtime_ctx - pack Tx time queue context into a HW buffer + * @ctx: the Tx time queue context to pack + * @buf: the HW buffer to pack into + * + * Pack the Tx time queue context from the CPU-friendly unpacked buffer into + * its bit-packed HW layout. + */ +void ice_pack_txtime_ctx(const struct ice_txtime_ctx *ctx, + ice_txtime_ctx_buf_t *buf) +{ + pack_fields(buf, sizeof(*buf), ctx, ice_txtime_ctx_fields, + QUIRK_LITTLE_ENDIAN | QUIRK_LSW32_IS_FIRST); +} + /* Sideband Queue command wrappers */ /** @@ -2418,12 +2496,15 @@ ice_parse_common_caps(struct ice_hw *hw, struct ice_hw_common_caps *caps, caps->reset_restrict_support); break; case LIBIE_AQC_CAPS_FW_LAG_SUPPORT: - caps->roce_lag = !!(number & LIBIE_AQC_BIT_ROCEV2_LAG); + caps->roce_lag = number & LIBIE_AQC_BIT_ROCEV2_LAG; ice_debug(hw, ICE_DBG_INIT, "%s: roce_lag = %u\n", prefix, caps->roce_lag); - caps->sriov_lag = !!(number & LIBIE_AQC_BIT_SRIOV_LAG); + caps->sriov_lag = number & LIBIE_AQC_BIT_SRIOV_LAG; ice_debug(hw, ICE_DBG_INIT, "%s: sriov_lag = %u\n", prefix, caps->sriov_lag); + caps->sriov_aa_lag = number & LIBIE_AQC_BIT_SRIOV_AA_LAG; + ice_debug(hw, ICE_DBG_INIT, "%s: sriov_aa_lag = %u\n", + prefix, caps->sriov_aa_lag); break; case LIBIE_AQC_CAPS_TX_SCHED_TOPO_COMP_MODE: caps->tx_sched_topo_comp_mode_en = (number == 1); @@ -4712,24 +4793,24 @@ do_aq: } /** - * ice_aq_cfg_lan_txq + * ice_aq_cfg_lan_txq - send AQ command 0x0C32 to FW * @hw: pointer to the hardware structure * @buf: buffer for command * @buf_size: size of buffer in bytes * @num_qs: number of queues being configured * @oldport: origination lport * @newport: destination lport + * @mode: cmd_type for move to use * @cd: pointer to command details structure or NULL * * Move/Configure LAN Tx queue (0x0C32) * - * There is a better AQ command to use for moving nodes, so only coding - * this one for configuring the node. + * Return: Zero on success, associated error code on failure. */ int ice_aq_cfg_lan_txq(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *buf, u16 buf_size, u16 num_qs, u8 oldport, u8 newport, - struct ice_sq_cd *cd) + u8 mode, struct ice_sq_cd *cd) { struct ice_aqc_cfg_txqs *cmd; struct libie_aq_desc desc; @@ -4742,10 +4823,12 @@ ice_aq_cfg_lan_txq(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *buf, if (!buf) return -EINVAL; - cmd->cmd_type = ICE_AQC_Q_CFG_TC_CHNG; + cmd->cmd_type = mode; cmd->num_qs = num_qs; cmd->port_num_chng = (oldport & ICE_AQC_Q_CFG_SRC_PRT_M); cmd->port_num_chng |= FIELD_PREP(ICE_AQC_Q_CFG_DST_PRT_M, newport); + cmd->port_num_chng |= FIELD_PREP(ICE_AQC_Q_CFG_MODE_M, + ICE_AQC_Q_CFG_MODE_KEEP_OWN); cmd->time_out = FIELD_PREP(ICE_AQC_Q_CFG_TIMEOUT_M, 5); cmd->blocked_cgds = 0; @@ -4801,6 +4884,46 @@ ice_aq_add_rdma_qsets(struct ice_hw *hw, u8 num_qset_grps, return ice_aq_send_cmd(hw, &desc, qset_list, buf_size, cd); } +/** + * ice_aq_set_txtimeq - set Tx time queues + * @hw: pointer to the hardware structure + * @txtimeq: first Tx time queue id to configure + * @q_count: number of queues to configure + * @txtime_qg: queue group to be set + * @buf_size: size of buffer for indirect command + * @cd: pointer to command details structure or NULL + * + * Set Tx Time queue (0x0C35) + * Return: 0 on success or negative value on failure. + */ +int +ice_aq_set_txtimeq(struct ice_hw *hw, u16 txtimeq, u8 q_count, + struct ice_aqc_set_txtime_qgrp *txtime_qg, u16 buf_size, + struct ice_sq_cd *cd) +{ + struct ice_aqc_set_txtimeqs *cmd; + struct libie_aq_desc desc; + u16 size; + + if (!txtime_qg || txtimeq > ICE_TXTIME_MAX_QUEUE || + q_count < 1 || q_count > ICE_SET_TXTIME_MAX_Q_AMOUNT) + return -EINVAL; + + size = struct_size(txtime_qg, txtimeqs, q_count); + if (buf_size != size) + return -EINVAL; + + cmd = libie_aq_raw(&desc); + + ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_set_txtimeqs); + + desc.flags |= cpu_to_le16(LIBIE_AQ_FLAG_RD); + + cmd->q_id = cpu_to_le16(txtimeq); + cmd->q_amount = cpu_to_le16(q_count); + return ice_aq_send_cmd(hw, &desc, txtime_qg, buf_size, cd); +} + /* End of FW Admin Queue command wrappers */ /** diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 60320cdf7804..e700ac0dc347 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -270,11 +270,17 @@ ice_ena_vsi_txq(struct ice_port_info *pi, u16 vsi_handle, u8 tc, u16 q_handle, int ice_aq_cfg_lan_txq(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *buf, u16 buf_size, u16 num_qs, u8 oldport, u8 newport, - struct ice_sq_cd *cd); + u8 mode, struct ice_sq_cd *cd); int ice_replay_vsi(struct ice_hw *hw, u16 vsi_handle); void ice_replay_post(struct ice_hw *hw); struct ice_q_ctx * ice_get_lan_q_ctx(struct ice_hw *hw, u16 vsi_handle, u8 tc, u16 q_handle); +int +ice_aq_set_txtimeq(struct ice_hw *hw, u16 txtimeq, u8 q_count, + struct ice_aqc_set_txtime_qgrp *txtime_qg, + u16 buf_size, struct ice_sq_cd *cd); +void ice_pack_txtime_ctx(const struct ice_txtime_ctx *ctx, + ice_txtime_ctx_buf_t *buf); int ice_sbq_rw_reg(struct ice_hw *hw, struct ice_sbq_msg_input *in, u16 flag); int ice_aq_get_cgu_input_pin_measure(struct ice_hw *hw, u8 dpll_idx, struct ice_cgu_input_measure *meas, diff --git a/drivers/net/ethernet/intel/ice/ice_debugfs.c b/drivers/net/ethernet/intel/ice/ice_debugfs.c index cb71eca6a85b..f450250fc827 100644 --- a/drivers/net/ethernet/intel/ice/ice_debugfs.c +++ b/drivers/net/ethernet/intel/ice/ice_debugfs.c @@ -1,647 +1,20 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2022, Intel Corporation. */ -#include <linux/fs.h> #include <linux/debugfs.h> -#include <linux/random.h> -#include <linux/vmalloc.h> #include "ice.h" static struct dentry *ice_debugfs_root; -/* create a define that has an extra module that doesn't really exist. this - * is so we can add a module 'all' to easily enable/disable all the modules - */ -#define ICE_NR_FW_LOG_MODULES (ICE_AQC_FW_LOG_ID_MAX + 1) - -/* the ordering in this array is important. it matches the ordering of the - * values in the FW so the index is the same value as in ice_aqc_fw_logging_mod - */ -static const char * const ice_fwlog_module_string[] = { - "general", - "ctrl", - "link", - "link_topo", - "dnl", - "i2c", - "sdp", - "mdio", - "adminq", - "hdma", - "lldp", - "dcbx", - "dcb", - "xlr", - "nvm", - "auth", - "vpd", - "iosf", - "parser", - "sw", - "scheduler", - "txq", - "rsvd", - "post", - "watchdog", - "task_dispatch", - "mng", - "synce", - "health", - "tsdrv", - "pfreg", - "mdlver", - "all", -}; - -/* the ordering in this array is important. it matches the ordering of the - * values in the FW so the index is the same value as in ice_fwlog_level - */ -static const char * const ice_fwlog_level_string[] = { - "none", - "error", - "warning", - "normal", - "verbose", -}; - -static const char * const ice_fwlog_log_size[] = { - "128K", - "256K", - "512K", - "1M", - "2M", -}; - -/** - * ice_fwlog_print_module_cfg - print current FW logging module configuration - * @hw: pointer to the HW structure - * @module: module to print - * @s: the seq file to put data into - */ -static void -ice_fwlog_print_module_cfg(struct ice_hw *hw, int module, struct seq_file *s) -{ - struct ice_fwlog_cfg *cfg = &hw->fwlog_cfg; - struct ice_fwlog_module_entry *entry; - - if (module != ICE_AQC_FW_LOG_ID_MAX) { - entry = &cfg->module_entries[module]; - - seq_printf(s, "\tModule: %s, Log Level: %s\n", - ice_fwlog_module_string[entry->module_id], - ice_fwlog_level_string[entry->log_level]); - } else { - int i; - - for (i = 0; i < ICE_AQC_FW_LOG_ID_MAX; i++) { - entry = &cfg->module_entries[i]; - - seq_printf(s, "\tModule: %s, Log Level: %s\n", - ice_fwlog_module_string[entry->module_id], - ice_fwlog_level_string[entry->log_level]); - } - } -} - -static int ice_find_module_by_dentry(struct ice_pf *pf, struct dentry *d) -{ - int i, module; - - module = -1; - /* find the module based on the dentry */ - for (i = 0; i < ICE_NR_FW_LOG_MODULES; i++) { - if (d == pf->ice_debugfs_pf_fwlog_modules[i]) { - module = i; - break; - } - } - - return module; -} - -/** - * ice_debugfs_module_show - read from 'module' file - * @s: the opened file - * @v: pointer to the offset - */ -static int ice_debugfs_module_show(struct seq_file *s, void *v) -{ - const struct file *filp = s->file; - struct dentry *dentry; - struct ice_pf *pf; - int module; - - dentry = file_dentry(filp); - pf = s->private; - - module = ice_find_module_by_dentry(pf, dentry); - if (module < 0) { - dev_info(ice_pf_to_dev(pf), "unknown module\n"); - return -EINVAL; - } - - ice_fwlog_print_module_cfg(&pf->hw, module, s); - - return 0; -} - -static int ice_debugfs_module_open(struct inode *inode, struct file *filp) -{ - return single_open(filp, ice_debugfs_module_show, inode->i_private); -} - -/** - * ice_debugfs_module_write - write into 'module' file - * @filp: the opened file - * @buf: where to find the user's data - * @count: the length of the user's data - * @ppos: file position offset - */ -static ssize_t -ice_debugfs_module_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct ice_pf *pf = file_inode(filp)->i_private; - struct dentry *dentry = file_dentry(filp); - struct device *dev = ice_pf_to_dev(pf); - char user_val[16], *cmd_buf; - int module, log_level, cnt; - - /* don't allow partial writes or invalid input */ - if (*ppos != 0 || count > 8) - return -EINVAL; - - cmd_buf = memdup_user_nul(buf, count); - if (IS_ERR(cmd_buf)) - return PTR_ERR(cmd_buf); - - module = ice_find_module_by_dentry(pf, dentry); - if (module < 0) { - dev_info(dev, "unknown module\n"); - return -EINVAL; - } - - cnt = sscanf(cmd_buf, "%s", user_val); - if (cnt != 1) - return -EINVAL; - - log_level = sysfs_match_string(ice_fwlog_level_string, user_val); - if (log_level < 0) { - dev_info(dev, "unknown log level '%s'\n", user_val); - return -EINVAL; - } - - if (module != ICE_AQC_FW_LOG_ID_MAX) { - ice_pf_fwlog_update_module(pf, log_level, module); - } else { - /* the module 'all' is a shortcut so that we can set - * all of the modules to the same level quickly - */ - int i; - - for (i = 0; i < ICE_AQC_FW_LOG_ID_MAX; i++) - ice_pf_fwlog_update_module(pf, log_level, i); - } - - return count; -} - -static const struct file_operations ice_debugfs_module_fops = { - .owner = THIS_MODULE, - .open = ice_debugfs_module_open, - .read = seq_read, - .release = single_release, - .write = ice_debugfs_module_write, -}; - -/** - * ice_debugfs_nr_messages_read - read from 'nr_messages' file - * @filp: the opened file - * @buffer: where to write the data for the user to read - * @count: the size of the user's buffer - * @ppos: file position offset - */ -static ssize_t ice_debugfs_nr_messages_read(struct file *filp, - char __user *buffer, size_t count, - loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct ice_hw *hw = &pf->hw; - char buff[32] = {}; - - snprintf(buff, sizeof(buff), "%d\n", - hw->fwlog_cfg.log_resolution); - - return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); -} - -/** - * ice_debugfs_nr_messages_write - write into 'nr_messages' file - * @filp: the opened file - * @buf: where to find the user's data - * @count: the length of the user's data - * @ppos: file position offset - */ -static ssize_t -ice_debugfs_nr_messages_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct device *dev = ice_pf_to_dev(pf); - struct ice_hw *hw = &pf->hw; - char user_val[8], *cmd_buf; - s16 nr_messages; - ssize_t ret; - - /* don't allow partial writes or invalid input */ - if (*ppos != 0 || count > 4) - return -EINVAL; - - cmd_buf = memdup_user_nul(buf, count); - if (IS_ERR(cmd_buf)) - return PTR_ERR(cmd_buf); - - ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; - - ret = kstrtos16(user_val, 0, &nr_messages); - if (ret) - return ret; - - if (nr_messages < ICE_AQC_FW_LOG_MIN_RESOLUTION || - nr_messages > ICE_AQC_FW_LOG_MAX_RESOLUTION) { - dev_err(dev, "Invalid FW log number of messages %d, value must be between %d - %d\n", - nr_messages, ICE_AQC_FW_LOG_MIN_RESOLUTION, - ICE_AQC_FW_LOG_MAX_RESOLUTION); - return -EINVAL; - } - - hw->fwlog_cfg.log_resolution = nr_messages; - - return count; -} - -static const struct file_operations ice_debugfs_nr_messages_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = ice_debugfs_nr_messages_read, - .write = ice_debugfs_nr_messages_write, -}; - -/** - * ice_debugfs_enable_read - read from 'enable' file - * @filp: the opened file - * @buffer: where to write the data for the user to read - * @count: the size of the user's buffer - * @ppos: file position offset - */ -static ssize_t ice_debugfs_enable_read(struct file *filp, - char __user *buffer, size_t count, - loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct ice_hw *hw = &pf->hw; - char buff[32] = {}; - - snprintf(buff, sizeof(buff), "%u\n", - (u16)(hw->fwlog_cfg.options & - ICE_FWLOG_OPTION_IS_REGISTERED) >> 3); - - return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); -} - -/** - * ice_debugfs_enable_write - write into 'enable' file - * @filp: the opened file - * @buf: where to find the user's data - * @count: the length of the user's data - * @ppos: file position offset - */ -static ssize_t -ice_debugfs_enable_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct ice_hw *hw = &pf->hw; - char user_val[8], *cmd_buf; - bool enable; - ssize_t ret; - - /* don't allow partial writes or invalid input */ - if (*ppos != 0 || count > 2) - return -EINVAL; - - cmd_buf = memdup_user_nul(buf, count); - if (IS_ERR(cmd_buf)) - return PTR_ERR(cmd_buf); - - ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; - - ret = kstrtobool(user_val, &enable); - if (ret) - goto enable_write_error; - - if (enable) - hw->fwlog_cfg.options |= ICE_FWLOG_OPTION_ARQ_ENA; - else - hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_ARQ_ENA; - - ret = ice_fwlog_set(hw, &hw->fwlog_cfg); - if (ret) - goto enable_write_error; - - if (enable) - ret = ice_fwlog_register(hw); - else - ret = ice_fwlog_unregister(hw); - - if (ret) - goto enable_write_error; - - /* if we get here, nothing went wrong; return count since we didn't - * really write anything - */ - ret = (ssize_t)count; - -enable_write_error: - /* This function always consumes all of the written input, or produces - * an error. Check and enforce this. Otherwise, the write operation - * won't complete properly. - */ - if (WARN_ON(ret != (ssize_t)count && ret >= 0)) - ret = -EIO; - - return ret; -} - -static const struct file_operations ice_debugfs_enable_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = ice_debugfs_enable_read, - .write = ice_debugfs_enable_write, -}; - -/** - * ice_debugfs_log_size_read - read from 'log_size' file - * @filp: the opened file - * @buffer: where to write the data for the user to read - * @count: the size of the user's buffer - * @ppos: file position offset - */ -static ssize_t ice_debugfs_log_size_read(struct file *filp, - char __user *buffer, size_t count, - loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct ice_hw *hw = &pf->hw; - char buff[32] = {}; - int index; - - index = hw->fwlog_ring.index; - snprintf(buff, sizeof(buff), "%s\n", ice_fwlog_log_size[index]); - - return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); -} - -/** - * ice_debugfs_log_size_write - write into 'log_size' file - * @filp: the opened file - * @buf: where to find the user's data - * @count: the length of the user's data - * @ppos: file position offset - */ -static ssize_t -ice_debugfs_log_size_write(struct file *filp, const char __user *buf, - size_t count, loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct device *dev = ice_pf_to_dev(pf); - struct ice_hw *hw = &pf->hw; - char user_val[8], *cmd_buf; - ssize_t ret; - int index; - - /* don't allow partial writes or invalid input */ - if (*ppos != 0 || count > 5) - return -EINVAL; - - cmd_buf = memdup_user_nul(buf, count); - if (IS_ERR(cmd_buf)) - return PTR_ERR(cmd_buf); - - ret = sscanf(cmd_buf, "%s", user_val); - if (ret != 1) - return -EINVAL; - - index = sysfs_match_string(ice_fwlog_log_size, user_val); - if (index < 0) { - dev_info(dev, "Invalid log size '%s'. The value must be one of 128K, 256K, 512K, 1M, 2M\n", - user_val); - ret = -EINVAL; - goto log_size_write_error; - } else if (hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED) { - dev_info(dev, "FW logging is currently running. Please disable FW logging to change log_size\n"); - ret = -EINVAL; - goto log_size_write_error; - } - - /* free all the buffers and the tracking info and resize */ - ice_fwlog_realloc_rings(hw, index); - - /* if we get here, nothing went wrong; return count since we didn't - * really write anything - */ - ret = (ssize_t)count; - -log_size_write_error: - /* This function always consumes all of the written input, or produces - * an error. Check and enforce this. Otherwise, the write operation - * won't complete properly. - */ - if (WARN_ON(ret != (ssize_t)count && ret >= 0)) - ret = -EIO; - - return ret; -} - -static const struct file_operations ice_debugfs_log_size_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = ice_debugfs_log_size_read, - .write = ice_debugfs_log_size_write, -}; - -/** - * ice_debugfs_data_read - read from 'data' file - * @filp: the opened file - * @buffer: where to write the data for the user to read - * @count: the size of the user's buffer - * @ppos: file position offset - */ -static ssize_t ice_debugfs_data_read(struct file *filp, char __user *buffer, - size_t count, loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct ice_hw *hw = &pf->hw; - int data_copied = 0; - bool done = false; - - if (ice_fwlog_ring_empty(&hw->fwlog_ring)) - return 0; - - while (!ice_fwlog_ring_empty(&hw->fwlog_ring) && !done) { - struct ice_fwlog_data *log; - u16 cur_buf_len; - - log = &hw->fwlog_ring.rings[hw->fwlog_ring.head]; - cur_buf_len = log->data_size; - if (cur_buf_len >= count) { - done = true; - continue; - } - - if (copy_to_user(buffer, log->data, cur_buf_len)) { - /* if there is an error then bail and return whatever - * the driver has copied so far - */ - done = true; - continue; - } - - data_copied += cur_buf_len; - buffer += cur_buf_len; - count -= cur_buf_len; - *ppos += cur_buf_len; - ice_fwlog_ring_increment(&hw->fwlog_ring.head, - hw->fwlog_ring.size); - } - - return data_copied; -} - -/** - * ice_debugfs_data_write - write into 'data' file - * @filp: the opened file - * @buf: where to find the user's data - * @count: the length of the user's data - * @ppos: file position offset - */ -static ssize_t -ice_debugfs_data_write(struct file *filp, const char __user *buf, size_t count, - loff_t *ppos) -{ - struct ice_pf *pf = filp->private_data; - struct device *dev = ice_pf_to_dev(pf); - struct ice_hw *hw = &pf->hw; - ssize_t ret; - - /* don't allow partial writes */ - if (*ppos != 0) - return 0; - - /* any value is allowed to clear the buffer so no need to even look at - * what the value is - */ - if (!(hw->fwlog_cfg.options & ICE_FWLOG_OPTION_IS_REGISTERED)) { - hw->fwlog_ring.head = 0; - hw->fwlog_ring.tail = 0; - } else { - dev_info(dev, "Can't clear FW log data while FW log running\n"); - ret = -EINVAL; - goto nr_buffs_write_error; - } - - /* if we get here, nothing went wrong; return count since we didn't - * really write anything - */ - ret = (ssize_t)count; - -nr_buffs_write_error: - /* This function always consumes all of the written input, or produces - * an error. Check and enforce this. Otherwise, the write operation - * won't complete properly. - */ - if (WARN_ON(ret != (ssize_t)count && ret >= 0)) - ret = -EIO; - - return ret; -} - -static const struct file_operations ice_debugfs_data_fops = { - .owner = THIS_MODULE, - .open = simple_open, - .read = ice_debugfs_data_read, - .write = ice_debugfs_data_write, -}; - -/** - * ice_debugfs_fwlog_init - setup the debugfs directory - * @pf: the ice that is starting up - */ -void ice_debugfs_fwlog_init(struct ice_pf *pf) +int ice_debugfs_pf_init(struct ice_pf *pf) { const char *name = pci_name(pf->pdev); - struct dentry *fw_modules_dir; - struct dentry **fw_modules; - int i; - - /* only support fw log commands on PF 0 */ - if (pf->hw.bus.func) - return; - - /* allocate space for this first because if it fails then we don't - * need to unwind - */ - fw_modules = kcalloc(ICE_NR_FW_LOG_MODULES, sizeof(*fw_modules), - GFP_KERNEL); - if (!fw_modules) - return; pf->ice_debugfs_pf = debugfs_create_dir(name, ice_debugfs_root); if (IS_ERR(pf->ice_debugfs_pf)) - goto err_create_module_files; - - pf->ice_debugfs_pf_fwlog = debugfs_create_dir("fwlog", - pf->ice_debugfs_pf); - if (IS_ERR(pf->ice_debugfs_pf_fwlog)) - goto err_create_module_files; + return PTR_ERR(pf->ice_debugfs_pf); - fw_modules_dir = debugfs_create_dir("modules", - pf->ice_debugfs_pf_fwlog); - if (IS_ERR(fw_modules_dir)) - goto err_create_module_files; - - for (i = 0; i < ICE_NR_FW_LOG_MODULES; i++) { - fw_modules[i] = debugfs_create_file(ice_fwlog_module_string[i], - 0600, fw_modules_dir, pf, - &ice_debugfs_module_fops); - if (IS_ERR(fw_modules[i])) - goto err_create_module_files; - } - - debugfs_create_file("nr_messages", 0600, - pf->ice_debugfs_pf_fwlog, pf, - &ice_debugfs_nr_messages_fops); - - pf->ice_debugfs_pf_fwlog_modules = fw_modules; - - debugfs_create_file("enable", 0600, pf->ice_debugfs_pf_fwlog, - pf, &ice_debugfs_enable_fops); - - debugfs_create_file("log_size", 0600, pf->ice_debugfs_pf_fwlog, - pf, &ice_debugfs_log_size_fops); - - debugfs_create_file("data", 0600, pf->ice_debugfs_pf_fwlog, - pf, &ice_debugfs_data_fops); - - return; - -err_create_module_files: - debugfs_remove_recursive(pf->ice_debugfs_pf_fwlog); - kfree(fw_modules); + return 0; } /** diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 55e0f2c6af9e..dc131779d426 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3147,9 +3147,11 @@ ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; + struct ice_hw *hw; - ring->rx_max_pending = ICE_MAX_NUM_DESC; - ring->tx_max_pending = ICE_MAX_NUM_DESC; + hw = &vsi->back->hw; + ring->rx_max_pending = ICE_MAX_NUM_DESC_BY_MAC(hw); + ring->tx_max_pending = ICE_MAX_NUM_DESC_BY_MAC(hw); if (vsi->tx_rings && vsi->rx_rings) { ring->rx_pending = vsi->rx_rings[0]->count; ring->tx_pending = vsi->tx_rings[0]->count; @@ -3177,15 +3179,16 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct ice_vsi *vsi = np->vsi; struct ice_pf *pf = vsi->back; int i, timeout = 50, err = 0; + struct ice_hw *hw = &pf->hw; u16 new_rx_cnt, new_tx_cnt; - if (ring->tx_pending > ICE_MAX_NUM_DESC || + if (ring->tx_pending > ICE_MAX_NUM_DESC_BY_MAC(hw) || ring->tx_pending < ICE_MIN_NUM_DESC || - ring->rx_pending > ICE_MAX_NUM_DESC || + ring->rx_pending > ICE_MAX_NUM_DESC_BY_MAC(hw) || ring->rx_pending < ICE_MIN_NUM_DESC) { netdev_err(netdev, "Descriptors requested (Tx: %d / Rx: %d) out of range [%d-%d] (increment %d)\n", ring->tx_pending, ring->rx_pending, - ICE_MIN_NUM_DESC, ICE_MAX_NUM_DESC, + ICE_MIN_NUM_DESC, ICE_MAX_NUM_DESC_BY_MAC(hw), ICE_REQ_DESC_MULTIPLE); return -EINVAL; } @@ -3258,6 +3261,7 @@ ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, tx_rings[i].count = new_tx_cnt; tx_rings[i].desc = NULL; tx_rings[i].tx_buf = NULL; + tx_rings[i].tstamp_ring = NULL; tx_rings[i].tx_tstamps = &pf->ptp.port.tx; err = ice_setup_tx_ring(&tx_rings[i]); if (err) { @@ -4620,10 +4624,12 @@ static int ice_get_port_fec_stats(struct ice_hw *hw, u16 pcs_quad, u16 pcs_port, * ice_get_fec_stats - returns FEC correctable, uncorrectable stats per netdev * @netdev: network interface device structure * @fec_stats: buffer to hold FEC statistics for given port + * @hist: buffer to put FEC histogram statistics for given port * */ static void ice_get_fec_stats(struct net_device *netdev, - struct ethtool_fec_stats *fec_stats) + struct ethtool_fec_stats *fec_stats, + struct ethtool_fec_hist *hist) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_port_topology port_topology; diff --git a/drivers/net/ethernet/intel/ice/ice_fwlog.c b/drivers/net/ethernet/intel/ice/ice_fwlog.c deleted file mode 100644 index a31bb026ad34..000000000000 --- a/drivers/net/ethernet/intel/ice/ice_fwlog.c +++ /dev/null @@ -1,474 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* Copyright (c) 2022, Intel Corporation. */ - -#include <linux/vmalloc.h> -#include "ice.h" -#include "ice_common.h" -#include "ice_fwlog.h" - -bool ice_fwlog_ring_full(struct ice_fwlog_ring *rings) -{ - u16 head, tail; - - head = rings->head; - tail = rings->tail; - - if (head < tail && (tail - head == (rings->size - 1))) - return true; - else if (head > tail && (tail == (head - 1))) - return true; - - return false; -} - -bool ice_fwlog_ring_empty(struct ice_fwlog_ring *rings) -{ - return rings->head == rings->tail; -} - -void ice_fwlog_ring_increment(u16 *item, u16 size) -{ - *item = (*item + 1) & (size - 1); -} - -static int ice_fwlog_alloc_ring_buffs(struct ice_fwlog_ring *rings) -{ - int i, nr_bytes; - u8 *mem; - - nr_bytes = rings->size * ICE_AQ_MAX_BUF_LEN; - mem = vzalloc(nr_bytes); - if (!mem) - return -ENOMEM; - - for (i = 0; i < rings->size; i++) { - struct ice_fwlog_data *ring = &rings->rings[i]; - - ring->data_size = ICE_AQ_MAX_BUF_LEN; - ring->data = mem; - mem += ICE_AQ_MAX_BUF_LEN; - } - - return 0; -} - -static void ice_fwlog_free_ring_buffs(struct ice_fwlog_ring *rings) -{ - int i; - - for (i = 0; i < rings->size; i++) { - struct ice_fwlog_data *ring = &rings->rings[i]; - - /* the first ring is the base memory for the whole range so - * free it - */ - if (!i) - vfree(ring->data); - - ring->data = NULL; - ring->data_size = 0; - } -} - -#define ICE_FWLOG_INDEX_TO_BYTES(n) ((128 * 1024) << (n)) -/** - * ice_fwlog_realloc_rings - reallocate the FW log rings - * @hw: pointer to the HW structure - * @index: the new index to use to allocate memory for the log data - * - */ -void ice_fwlog_realloc_rings(struct ice_hw *hw, int index) -{ - struct ice_fwlog_ring ring; - int status, ring_size; - - /* convert the number of bytes into a number of 4K buffers. externally - * the driver presents the interface to the FW log data as a number of - * bytes because that's easy for users to understand. internally the - * driver uses a ring of buffers because the driver doesn't know where - * the beginning and end of any line of log data is so the driver has - * to overwrite data as complete blocks. when the data is returned to - * the user the driver knows that the data is correct and the FW log - * can be correctly parsed by the tools - */ - ring_size = ICE_FWLOG_INDEX_TO_BYTES(index) / ICE_AQ_MAX_BUF_LEN; - if (ring_size == hw->fwlog_ring.size) - return; - - /* allocate space for the new rings and buffers then release the - * old rings and buffers. that way if we don't have enough - * memory then we at least have what we had before - */ - ring.rings = kcalloc(ring_size, sizeof(*ring.rings), GFP_KERNEL); - if (!ring.rings) - return; - - ring.size = ring_size; - - status = ice_fwlog_alloc_ring_buffs(&ring); - if (status) { - dev_warn(ice_hw_to_dev(hw), "Unable to allocate memory for FW log ring data buffers\n"); - ice_fwlog_free_ring_buffs(&ring); - kfree(ring.rings); - return; - } - - ice_fwlog_free_ring_buffs(&hw->fwlog_ring); - kfree(hw->fwlog_ring.rings); - - hw->fwlog_ring.rings = ring.rings; - hw->fwlog_ring.size = ring.size; - hw->fwlog_ring.index = index; - hw->fwlog_ring.head = 0; - hw->fwlog_ring.tail = 0; -} - -/** - * ice_fwlog_init - Initialize FW logging configuration - * @hw: pointer to the HW structure - * - * This function should be called on driver initialization during - * ice_init_hw(). - */ -int ice_fwlog_init(struct ice_hw *hw) -{ - /* only support fw log commands on PF 0 */ - if (hw->bus.func) - return -EINVAL; - - ice_fwlog_set_supported(hw); - - if (ice_fwlog_supported(hw)) { - int status; - - /* read the current config from the FW and store it */ - status = ice_fwlog_get(hw, &hw->fwlog_cfg); - if (status) - return status; - - hw->fwlog_ring.rings = kcalloc(ICE_FWLOG_RING_SIZE_DFLT, - sizeof(*hw->fwlog_ring.rings), - GFP_KERNEL); - if (!hw->fwlog_ring.rings) { - dev_warn(ice_hw_to_dev(hw), "Unable to allocate memory for FW log rings\n"); - return -ENOMEM; - } - - hw->fwlog_ring.size = ICE_FWLOG_RING_SIZE_DFLT; - hw->fwlog_ring.index = ICE_FWLOG_RING_SIZE_INDEX_DFLT; - - status = ice_fwlog_alloc_ring_buffs(&hw->fwlog_ring); - if (status) { - dev_warn(ice_hw_to_dev(hw), "Unable to allocate memory for FW log ring data buffers\n"); - ice_fwlog_free_ring_buffs(&hw->fwlog_ring); - kfree(hw->fwlog_ring.rings); - return status; - } - - ice_debugfs_fwlog_init(hw->back); - } else { - dev_warn(ice_hw_to_dev(hw), "FW logging is not supported in this NVM image. Please update the NVM to get FW log support\n"); - } - - return 0; -} - -/** - * ice_fwlog_deinit - unroll FW logging configuration - * @hw: pointer to the HW structure - * - * This function should be called in ice_deinit_hw(). - */ -void ice_fwlog_deinit(struct ice_hw *hw) -{ - struct ice_pf *pf = hw->back; - int status; - - /* only support fw log commands on PF 0 */ - if (hw->bus.func) - return; - - ice_debugfs_pf_deinit(hw->back); - - /* make sure FW logging is disabled to not put the FW in a weird state - * for the next driver load - */ - hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_ARQ_ENA; - status = ice_fwlog_set(hw, &hw->fwlog_cfg); - if (status) - dev_warn(ice_hw_to_dev(hw), "Unable to turn off FW logging, status: %d\n", - status); - - kfree(pf->ice_debugfs_pf_fwlog_modules); - - pf->ice_debugfs_pf_fwlog_modules = NULL; - - status = ice_fwlog_unregister(hw); - if (status) - dev_warn(ice_hw_to_dev(hw), "Unable to unregister FW logging, status: %d\n", - status); - - if (hw->fwlog_ring.rings) { - ice_fwlog_free_ring_buffs(&hw->fwlog_ring); - kfree(hw->fwlog_ring.rings); - } -} - -/** - * ice_fwlog_supported - Cached for whether FW supports FW logging or not - * @hw: pointer to the HW structure - * - * This will always return false if called before ice_init_hw(), so it must be - * called after ice_init_hw(). - */ -bool ice_fwlog_supported(struct ice_hw *hw) -{ - return hw->fwlog_supported; -} - -/** - * ice_aq_fwlog_set - Set FW logging configuration AQ command (0xFF30) - * @hw: pointer to the HW structure - * @entries: entries to configure - * @num_entries: number of @entries - * @options: options from ice_fwlog_cfg->options structure - * @log_resolution: logging resolution - */ -static int -ice_aq_fwlog_set(struct ice_hw *hw, struct ice_fwlog_module_entry *entries, - u16 num_entries, u16 options, u16 log_resolution) -{ - struct ice_aqc_fw_log_cfg_resp *fw_modules; - struct ice_aqc_fw_log *cmd; - struct libie_aq_desc desc; - int status; - int i; - - fw_modules = kcalloc(num_entries, sizeof(*fw_modules), GFP_KERNEL); - if (!fw_modules) - return -ENOMEM; - - for (i = 0; i < num_entries; i++) { - fw_modules[i].module_identifier = - cpu_to_le16(entries[i].module_id); - fw_modules[i].log_level = entries[i].log_level; - } - - ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_config); - desc.flags |= cpu_to_le16(LIBIE_AQ_FLAG_RD); - - cmd = libie_aq_raw(&desc); - - cmd->cmd_flags = ICE_AQC_FW_LOG_CONF_SET_VALID; - cmd->ops.cfg.log_resolution = cpu_to_le16(log_resolution); - cmd->ops.cfg.mdl_cnt = cpu_to_le16(num_entries); - - if (options & ICE_FWLOG_OPTION_ARQ_ENA) - cmd->cmd_flags |= ICE_AQC_FW_LOG_CONF_AQ_EN; - if (options & ICE_FWLOG_OPTION_UART_ENA) - cmd->cmd_flags |= ICE_AQC_FW_LOG_CONF_UART_EN; - - status = ice_aq_send_cmd(hw, &desc, fw_modules, - sizeof(*fw_modules) * num_entries, - NULL); - - kfree(fw_modules); - - return status; -} - -/** - * ice_fwlog_set - Set the firmware logging settings - * @hw: pointer to the HW structure - * @cfg: config used to set firmware logging - * - * This function should be called whenever the driver needs to set the firmware - * logging configuration. It can be called on initialization, reset, or during - * runtime. - * - * If the PF wishes to receive FW logging then it must register via - * ice_fwlog_register. Note, that ice_fwlog_register does not need to be called - * for init. - */ -int ice_fwlog_set(struct ice_hw *hw, struct ice_fwlog_cfg *cfg) -{ - if (!ice_fwlog_supported(hw)) - return -EOPNOTSUPP; - - return ice_aq_fwlog_set(hw, cfg->module_entries, - ICE_AQC_FW_LOG_ID_MAX, cfg->options, - cfg->log_resolution); -} - -/** - * ice_aq_fwlog_get - Get the current firmware logging configuration (0xFF32) - * @hw: pointer to the HW structure - * @cfg: firmware logging configuration to populate - */ -static int ice_aq_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg) -{ - struct ice_aqc_fw_log_cfg_resp *fw_modules; - struct ice_aqc_fw_log *cmd; - struct libie_aq_desc desc; - u16 module_id_cnt; - int status; - void *buf; - int i; - - memset(cfg, 0, sizeof(*cfg)); - - buf = kzalloc(ICE_AQ_MAX_BUF_LEN, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_query); - cmd = libie_aq_raw(&desc); - - cmd->cmd_flags = ICE_AQC_FW_LOG_AQ_QUERY; - - status = ice_aq_send_cmd(hw, &desc, buf, ICE_AQ_MAX_BUF_LEN, NULL); - if (status) { - ice_debug(hw, ICE_DBG_FW_LOG, "Failed to get FW log configuration\n"); - goto status_out; - } - - module_id_cnt = le16_to_cpu(cmd->ops.cfg.mdl_cnt); - if (module_id_cnt < ICE_AQC_FW_LOG_ID_MAX) { - ice_debug(hw, ICE_DBG_FW_LOG, "FW returned less than the expected number of FW log module IDs\n"); - } else if (module_id_cnt > ICE_AQC_FW_LOG_ID_MAX) { - ice_debug(hw, ICE_DBG_FW_LOG, "FW returned more than expected number of FW log module IDs, setting module_id_cnt to software expected max %u\n", - ICE_AQC_FW_LOG_ID_MAX); - module_id_cnt = ICE_AQC_FW_LOG_ID_MAX; - } - - cfg->log_resolution = le16_to_cpu(cmd->ops.cfg.log_resolution); - if (cmd->cmd_flags & ICE_AQC_FW_LOG_CONF_AQ_EN) - cfg->options |= ICE_FWLOG_OPTION_ARQ_ENA; - if (cmd->cmd_flags & ICE_AQC_FW_LOG_CONF_UART_EN) - cfg->options |= ICE_FWLOG_OPTION_UART_ENA; - if (cmd->cmd_flags & ICE_AQC_FW_LOG_QUERY_REGISTERED) - cfg->options |= ICE_FWLOG_OPTION_IS_REGISTERED; - - fw_modules = (struct ice_aqc_fw_log_cfg_resp *)buf; - - for (i = 0; i < module_id_cnt; i++) { - struct ice_aqc_fw_log_cfg_resp *fw_module = &fw_modules[i]; - - cfg->module_entries[i].module_id = - le16_to_cpu(fw_module->module_identifier); - cfg->module_entries[i].log_level = fw_module->log_level; - } - -status_out: - kfree(buf); - return status; -} - -/** - * ice_fwlog_get - Get the firmware logging settings - * @hw: pointer to the HW structure - * @cfg: config to populate based on current firmware logging settings - */ -int ice_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg) -{ - if (!ice_fwlog_supported(hw)) - return -EOPNOTSUPP; - - return ice_aq_fwlog_get(hw, cfg); -} - -/** - * ice_aq_fwlog_register - Register PF for firmware logging events (0xFF31) - * @hw: pointer to the HW structure - * @reg: true to register and false to unregister - */ -static int ice_aq_fwlog_register(struct ice_hw *hw, bool reg) -{ - struct ice_aqc_fw_log *cmd; - struct libie_aq_desc desc; - - ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_fw_logs_register); - cmd = libie_aq_raw(&desc); - - if (reg) - cmd->cmd_flags = ICE_AQC_FW_LOG_AQ_REGISTER; - - return ice_aq_send_cmd(hw, &desc, NULL, 0, NULL); -} - -/** - * ice_fwlog_register - Register the PF for firmware logging - * @hw: pointer to the HW structure - * - * After this call the PF will start to receive firmware logging based on the - * configuration set in ice_fwlog_set. - */ -int ice_fwlog_register(struct ice_hw *hw) -{ - int status; - - if (!ice_fwlog_supported(hw)) - return -EOPNOTSUPP; - - status = ice_aq_fwlog_register(hw, true); - if (status) - ice_debug(hw, ICE_DBG_FW_LOG, "Failed to register for firmware logging events over ARQ\n"); - else - hw->fwlog_cfg.options |= ICE_FWLOG_OPTION_IS_REGISTERED; - - return status; -} - -/** - * ice_fwlog_unregister - Unregister the PF from firmware logging - * @hw: pointer to the HW structure - */ -int ice_fwlog_unregister(struct ice_hw *hw) -{ - int status; - - if (!ice_fwlog_supported(hw)) - return -EOPNOTSUPP; - - status = ice_aq_fwlog_register(hw, false); - if (status) - ice_debug(hw, ICE_DBG_FW_LOG, "Failed to unregister from firmware logging events over ARQ\n"); - else - hw->fwlog_cfg.options &= ~ICE_FWLOG_OPTION_IS_REGISTERED; - - return status; -} - -/** - * ice_fwlog_set_supported - Set if FW logging is supported by FW - * @hw: pointer to the HW struct - * - * If FW returns success to the ice_aq_fwlog_get call then it supports FW - * logging, else it doesn't. Set the fwlog_supported flag accordingly. - * - * This function is only meant to be called during driver init to determine if - * the FW support FW logging. - */ -void ice_fwlog_set_supported(struct ice_hw *hw) -{ - struct ice_fwlog_cfg *cfg; - int status; - - hw->fwlog_supported = false; - - cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); - if (!cfg) - return; - - /* don't call ice_fwlog_get() because that would check to see if FW - * logging is supported which is what the driver is determining now - */ - status = ice_aq_fwlog_get(hw, cfg); - if (status) - ice_debug(hw, ICE_DBG_FW_LOG, "ice_aq_fwlog_get failed, FW logging is not supported on this version of FW, status %d\n", - status); - else - hw->fwlog_supported = true; - - kfree(cfg); -} diff --git a/drivers/net/ethernet/intel/ice/ice_fwlog.h b/drivers/net/ethernet/intel/ice/ice_fwlog.h deleted file mode 100644 index 287e71fa4b86..000000000000 --- a/drivers/net/ethernet/intel/ice/ice_fwlog.h +++ /dev/null @@ -1,79 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* Copyright (C) 2022, Intel Corporation. */ - -#ifndef _ICE_FWLOG_H_ -#define _ICE_FWLOG_H_ -#include "ice_adminq_cmd.h" - -struct ice_hw; - -/* Only a single log level should be set and all log levels under the set value - * are enabled, e.g. if log level is set to ICE_FW_LOG_LEVEL_VERBOSE, then all - * other log levels are included (except ICE_FW_LOG_LEVEL_NONE) - */ -enum ice_fwlog_level { - ICE_FWLOG_LEVEL_NONE = 0, - ICE_FWLOG_LEVEL_ERROR = 1, - ICE_FWLOG_LEVEL_WARNING = 2, - ICE_FWLOG_LEVEL_NORMAL = 3, - ICE_FWLOG_LEVEL_VERBOSE = 4, - ICE_FWLOG_LEVEL_INVALID, /* all values >= this entry are invalid */ -}; - -struct ice_fwlog_module_entry { - /* module ID for the corresponding firmware logging event */ - u16 module_id; - /* verbosity level for the module_id */ - u8 log_level; -}; - -struct ice_fwlog_cfg { - /* list of modules for configuring log level */ - struct ice_fwlog_module_entry module_entries[ICE_AQC_FW_LOG_ID_MAX]; - /* options used to configure firmware logging */ - u16 options; -#define ICE_FWLOG_OPTION_ARQ_ENA BIT(0) -#define ICE_FWLOG_OPTION_UART_ENA BIT(1) - /* set before calling ice_fwlog_init() so the PF registers for firmware - * logging on initialization - */ -#define ICE_FWLOG_OPTION_REGISTER_ON_INIT BIT(2) - /* set in the ice_fwlog_get() response if the PF is registered for FW - * logging events over ARQ - */ -#define ICE_FWLOG_OPTION_IS_REGISTERED BIT(3) - - /* minimum number of log events sent per Admin Receive Queue event */ - u16 log_resolution; -}; - -struct ice_fwlog_data { - u16 data_size; - u8 *data; -}; - -struct ice_fwlog_ring { - struct ice_fwlog_data *rings; - u16 index; - u16 size; - u16 head; - u16 tail; -}; - -#define ICE_FWLOG_RING_SIZE_INDEX_DFLT 3 -#define ICE_FWLOG_RING_SIZE_DFLT 256 -#define ICE_FWLOG_RING_SIZE_MAX 512 - -bool ice_fwlog_ring_full(struct ice_fwlog_ring *rings); -bool ice_fwlog_ring_empty(struct ice_fwlog_ring *rings); -void ice_fwlog_ring_increment(u16 *item, u16 size); -void ice_fwlog_set_supported(struct ice_hw *hw); -bool ice_fwlog_supported(struct ice_hw *hw); -int ice_fwlog_init(struct ice_hw *hw); -void ice_fwlog_deinit(struct ice_hw *hw); -int ice_fwlog_set(struct ice_hw *hw, struct ice_fwlog_cfg *cfg); -int ice_fwlog_get(struct ice_hw *hw, struct ice_fwlog_cfg *cfg); -int ice_fwlog_register(struct ice_hw *hw); -int ice_fwlog_unregister(struct ice_hw *hw); -void ice_fwlog_realloc_rings(struct ice_hw *hw, int index); -#endif /* _ICE_FWLOG_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_hw_autogen.h b/drivers/net/ethernet/intel/ice/ice_hw_autogen.h index dd520aa4d1d6..082ad33c53dc 100644 --- a/drivers/net/ethernet/intel/ice/ice_hw_autogen.h +++ b/drivers/net/ethernet/intel/ice/ice_hw_autogen.h @@ -19,6 +19,7 @@ #define QTX_COMM_HEAD_MAX_INDEX 16383 #define QTX_COMM_HEAD_HEAD_S 0 #define QTX_COMM_HEAD_HEAD_M ICE_M(0x1FFF, 0) +#define E830_GLQTX_TXTIME_DBELL_LSB(_DBQM) (0x002E0000 + ((_DBQM) * 8)) #define PF_FW_ARQBAH 0x00080180 #define PF_FW_ARQBAL 0x00080080 #define PF_FW_ARQH 0x00080380 @@ -571,6 +572,8 @@ #define E830_PFPTM_SEM_BUSY_M BIT(0) #define VFINT_DYN_CTLN(_i) (0x00003800 + ((_i) * 4)) #define VFINT_DYN_CTLN_CLEARPBA_M BIT(1) +#define E830_GLTXTIME_FETCH_PROFILE(_i, _j) (0x002D3500 + ((_i) * 4 + (_j) * 64)) +#define E830_GLTXTIME_FETCH_PROFILE_FETCH_TS_DESC_M ICE_M(0x1FF, 0) #define E830_MBX_PF_IN_FLIGHT_VF_MSGS_THRESH 0x00234000 #define E830_MBX_VF_DEC_TRIG(_VF) (0x00233800 + (_VF) * 4) #define E830_MBX_VF_IN_FLIGHT_MSGS_AT_PF_CNT(_VF) (0x00233000 + (_VF) * 4) diff --git a/drivers/net/ethernet/intel/ice/ice_lag.c b/drivers/net/ethernet/intel/ice/ice_lag.c index b1129da72139..aebf8e08a297 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.c +++ b/drivers/net/ethernet/intel/ice/ice_lag.c @@ -10,12 +10,17 @@ #define ICE_LAG_RES_SHARED BIT(14) #define ICE_LAG_RES_VALID BIT(15) -#define LACP_TRAIN_PKT_LEN 16 -static const u8 lacp_train_pkt[LACP_TRAIN_PKT_LEN] = { 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - 0x88, 0x09, 0, 0 }; +#define ICE_TRAIN_PKT_LEN 16 +static const u8 lacp_train_pkt[ICE_TRAIN_PKT_LEN] = { 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0x88, 0x09, 0, 0 }; +static const u8 act_act_train_pkt[ICE_TRAIN_PKT_LEN] = { 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0 }; #define ICE_RECIPE_LEN 64 +#define ICE_LAG_SRIOV_CP_RECIPE 10 + static const u8 ice_dflt_vsi_rcp[ICE_RECIPE_LEN] = { 0x05, 0, 0, 0, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x85, 0, 0x01, 0, 0, 0, 0xff, 0xff, 0x08, 0, 0, 0, 0, 0, 0, 0, @@ -46,10 +51,10 @@ static void ice_lag_set_primary(struct ice_lag *lag) } /** - * ice_lag_set_backup - set PF LAG state to Backup + * ice_lag_set_bkup - set PF LAG state to Backup * @lag: LAG info struct */ -static void ice_lag_set_backup(struct ice_lag *lag) +static void ice_lag_set_bkup(struct ice_lag *lag) { struct ice_pf *pf = lag->pf; @@ -99,6 +104,28 @@ static bool netif_is_same_ice(struct ice_pf *pf, struct net_device *netdev) } /** + * ice_lag_config_eswitch - configure eswitch to work with LAG + * @lag: lag info struct + * @netdev: active network interface device struct + * + * Updates all port representors in eswitch to use @netdev for Tx. + * + * Configures the netdev to keep dst metadata (also used in representor Tx). + * This is required for an uplink without switchdev mode configured. + */ +static void ice_lag_config_eswitch(struct ice_lag *lag, + struct net_device *netdev) +{ + struct ice_repr *repr; + unsigned long id; + + xa_for_each(&lag->pf->eswitch.reprs, id, repr) + repr->dst->u.port_info.lower_dev = netdev; + + netif_keep_dst(netdev); +} + +/** * ice_netdev_to_lag - return pointer to associated lag struct from netdev * @netdev: pointer to net_device struct to query */ @@ -210,13 +237,12 @@ ice_lag_cfg_fltr(struct ice_lag *lag, u32 act, u16 recipe_id, u16 *rule_idx, u8 direction, bool add) { struct ice_sw_rule_lkup_rx_tx *s_rule; + struct ice_hw *hw = &lag->pf->hw; u16 s_rule_sz, vsi_num; - struct ice_hw *hw; u8 *eth_hdr; u32 opc; int err; - hw = &lag->pf->hw; vsi_num = ice_get_hw_vsi_num(hw, 0); s_rule_sz = ICE_SW_RULE_RX_TX_ETH_HDR_SIZE(s_rule); @@ -314,26 +340,15 @@ ice_lag_cfg_drop_fltr(struct ice_lag *lag, bool add) } /** - * ice_lag_cfg_pf_fltrs - set filters up for new active port + * ice_lag_cfg_pf_fltrs_act_bkup - set filters up for new active port * @lag: local interfaces lag struct - * @ptr: opaque data containing notifier event + * @bonding_info: netdev event bonding info */ static void -ice_lag_cfg_pf_fltrs(struct ice_lag *lag, void *ptr) +ice_lag_cfg_pf_fltrs_act_bkup(struct ice_lag *lag, + struct netdev_bonding_info *bonding_info) { - struct netdev_notifier_bonding_info *info; - struct netdev_bonding_info *bonding_info; - struct net_device *event_netdev; - struct device *dev; - - event_netdev = netdev_notifier_info_to_dev(ptr); - /* not for this netdev */ - if (event_netdev != lag->netdev) - return; - - info = (struct netdev_notifier_bonding_info *)ptr; - bonding_info = &info->bonding_info; - dev = ice_pf_to_dev(lag->pf); + struct device *dev = ice_pf_to_dev(lag->pf); /* interface not active - remove old default VSI rule */ if (bonding_info->slave.state && lag->pf_rx_rule_id) { @@ -354,6 +369,105 @@ ice_lag_cfg_pf_fltrs(struct ice_lag *lag, void *ptr) } /** + * ice_lag_cfg_lp_fltr - configure lport filters + * @lag: local interface's lag struct + * @add: add or remove rule + * @cp: control packet only or general PF lport rule + */ +static void +ice_lag_cfg_lp_fltr(struct ice_lag *lag, bool add, bool cp) +{ + struct ice_sw_rule_lkup_rx_tx *s_rule; + struct ice_vsi *vsi = lag->pf->vsi[0]; + u16 buf_len, opc; + + buf_len = ICE_SW_RULE_RX_TX_HDR_SIZE(s_rule, ICE_TRAIN_PKT_LEN); + s_rule = kzalloc(buf_len, GFP_KERNEL); + if (!s_rule) { + netdev_warn(lag->netdev, "-ENOMEM error configuring CP filter\n"); + return; + } + + if (add) { + if (cp) { + s_rule->recipe_id = + cpu_to_le16(ICE_LAG_SRIOV_CP_RECIPE); + memcpy(s_rule->hdr_data, lacp_train_pkt, + ICE_TRAIN_PKT_LEN); + } else { + s_rule->recipe_id = cpu_to_le16(lag->act_act_recipe); + memcpy(s_rule->hdr_data, act_act_train_pkt, + ICE_TRAIN_PKT_LEN); + } + + s_rule->src = cpu_to_le16(vsi->port_info->lport); + s_rule->act = cpu_to_le32(ICE_FWD_TO_VSI | + ICE_SINGLE_ACT_LAN_ENABLE | + ICE_SINGLE_ACT_VALID_BIT | + FIELD_PREP(ICE_SINGLE_ACT_VSI_ID_M, + vsi->vsi_num)); + s_rule->hdr_len = cpu_to_le16(ICE_TRAIN_PKT_LEN); + s_rule->hdr.type = cpu_to_le16(ICE_AQC_SW_RULES_T_LKUP_RX); + opc = ice_aqc_opc_add_sw_rules; + } else { + opc = ice_aqc_opc_remove_sw_rules; + if (cp) + s_rule->index = cpu_to_le16(lag->cp_rule_idx); + else + s_rule->index = cpu_to_le16(lag->act_act_rule_idx); + } + if (ice_aq_sw_rules(&lag->pf->hw, s_rule, buf_len, 1, opc, NULL)) { + netdev_warn(lag->netdev, "Error %s %s rule for aggregate\n", + add ? "ADDING" : "REMOVING", + cp ? "CONTROL PACKET" : "LPORT"); + goto err_cp_free; + } + + if (add) { + if (cp) + lag->cp_rule_idx = le16_to_cpu(s_rule->index); + else + lag->act_act_rule_idx = le16_to_cpu(s_rule->index); + } else { + if (cp) + lag->cp_rule_idx = 0; + else + lag->act_act_rule_idx = 0; + } + +err_cp_free: + kfree(s_rule); +} + +/** + * ice_lag_cfg_pf_fltrs - set filters up for PF traffic + * @lag: local interfaces lag struct + * @ptr: opaque data containing notifier event + */ +static void +ice_lag_cfg_pf_fltrs(struct ice_lag *lag, void *ptr) +{ + struct netdev_notifier_bonding_info *info = ptr; + struct netdev_bonding_info *bonding_info; + struct net_device *event_netdev; + + event_netdev = netdev_notifier_info_to_dev(ptr); + if (event_netdev != lag->netdev) + return; + + bonding_info = &info->bonding_info; + + if (lag->bond_aa) { + if (lag->need_fltr_cfg) { + ice_lag_cfg_lp_fltr(lag, true, false); + lag->need_fltr_cfg = false; + } + } else { + ice_lag_cfg_pf_fltrs_act_bkup(lag, bonding_info); + } +} + +/** * ice_display_lag_info - print LAG info * @lag: LAG info struct */ @@ -402,12 +516,11 @@ static u16 ice_lag_qbuf_recfg(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *qbuf, u16 vsi_num, u16 numq, u8 tc) { + struct ice_pf *pf = hw->back; struct ice_q_ctx *q_ctx; u16 qid, count = 0; - struct ice_pf *pf; int i; - pf = hw->back; for (i = 0; i < numq; i++) { q_ctx = ice_get_lan_q_ctx(hw, vsi_num, tc, i); if (!q_ctx) { @@ -577,7 +690,7 @@ ice_lag_move_vf_node_tc(struct ice_lag *lag, u8 oldport, u8 newport, } if (ice_aq_cfg_lan_txq(&lag->pf->hw, qbuf, qbuf_size, valq, oldport, - newport, NULL)) { + newport, ICE_AQC_Q_CFG_TC_CHNG, NULL)) { dev_warn(dev, "Failure to configure queues for LAG failover\n"); goto qbuf_err; } @@ -677,54 +790,6 @@ ice_lag_move_single_vf_nodes(struct ice_lag *lag, u8 oldport, u8 newport, } /** - * ice_lag_move_new_vf_nodes - Move Tx scheduling nodes for a VF if required - * @vf: the VF to move Tx nodes for - * - * Called just after configuring new VF queues. Check whether the VF Tx - * scheduling nodes need to be updated to fail over to the active port. If so, - * move them now. - */ -void ice_lag_move_new_vf_nodes(struct ice_vf *vf) -{ - struct ice_lag_netdev_list ndlist; - u8 pri_port, act_port; - struct ice_lag *lag; - struct ice_vsi *vsi; - struct ice_pf *pf; - - vsi = ice_get_vf_vsi(vf); - - if (WARN_ON(!vsi)) - return; - - if (WARN_ON(vsi->type != ICE_VSI_VF)) - return; - - pf = vf->pf; - lag = pf->lag; - - mutex_lock(&pf->lag_mutex); - if (!lag->bonded) - goto new_vf_unlock; - - pri_port = pf->hw.port_info->lport; - act_port = lag->active_port; - - if (lag->upper_netdev) - ice_lag_build_netdev_list(lag, &ndlist); - - if (ice_is_feature_supported(pf, ICE_F_SRIOV_LAG) && - lag->bonded && lag->primary && pri_port != act_port && - !list_empty(lag->netdev_head)) - ice_lag_move_single_vf_nodes(lag, pri_port, act_port, vsi->idx); - - ice_lag_destroy_netdev_list(lag, &ndlist); - -new_vf_unlock: - mutex_unlock(&pf->lag_mutex); -} - -/** * ice_lag_move_vf_nodes - move Tx scheduling nodes for all VFs to new port * @lag: lag info struct * @oldport: lport of previous interface @@ -767,61 +832,6 @@ void ice_lag_move_vf_nodes_cfg(struct ice_lag *lag, u8 src_prt, u8 dst_prt) ice_lag_destroy_netdev_list(lag, &ndlist); } -#define ICE_LAG_SRIOV_CP_RECIPE 10 -#define ICE_LAG_SRIOV_TRAIN_PKT_LEN 16 - -/** - * ice_lag_cfg_cp_fltr - configure filter for control packets - * @lag: local interface's lag struct - * @add: add or remove rule - */ -static void -ice_lag_cfg_cp_fltr(struct ice_lag *lag, bool add) -{ - struct ice_sw_rule_lkup_rx_tx *s_rule = NULL; - struct ice_vsi *vsi; - u16 buf_len, opc; - - vsi = lag->pf->vsi[0]; - - buf_len = ICE_SW_RULE_RX_TX_HDR_SIZE(s_rule, - ICE_LAG_SRIOV_TRAIN_PKT_LEN); - s_rule = kzalloc(buf_len, GFP_KERNEL); - if (!s_rule) { - netdev_warn(lag->netdev, "-ENOMEM error configuring CP filter\n"); - return; - } - - if (add) { - s_rule->hdr.type = cpu_to_le16(ICE_AQC_SW_RULES_T_LKUP_RX); - s_rule->recipe_id = cpu_to_le16(ICE_LAG_SRIOV_CP_RECIPE); - s_rule->src = cpu_to_le16(vsi->port_info->lport); - s_rule->act = cpu_to_le32(ICE_FWD_TO_VSI | - ICE_SINGLE_ACT_LAN_ENABLE | - ICE_SINGLE_ACT_VALID_BIT | - FIELD_PREP(ICE_SINGLE_ACT_VSI_ID_M, vsi->vsi_num)); - s_rule->hdr_len = cpu_to_le16(ICE_LAG_SRIOV_TRAIN_PKT_LEN); - memcpy(s_rule->hdr_data, lacp_train_pkt, LACP_TRAIN_PKT_LEN); - opc = ice_aqc_opc_add_sw_rules; - } else { - opc = ice_aqc_opc_remove_sw_rules; - s_rule->index = cpu_to_le16(lag->cp_rule_idx); - } - if (ice_aq_sw_rules(&lag->pf->hw, s_rule, buf_len, 1, opc, NULL)) { - netdev_warn(lag->netdev, "Error %s CP rule for fail-over\n", - add ? "ADDING" : "REMOVING"); - goto cp_free; - } - - if (add) - lag->cp_rule_idx = le16_to_cpu(s_rule->index); - else - lag->cp_rule_idx = 0; - -cp_free: - kfree(s_rule); -} - /** * ice_lag_prepare_vf_reset - helper to adjust vf lag for reset * @lag: lag struct for interface that owns VF @@ -835,11 +845,20 @@ u8 ice_lag_prepare_vf_reset(struct ice_lag *lag) u8 pri_prt, act_prt; if (lag && lag->bonded && lag->primary && lag->upper_netdev) { - pri_prt = lag->pf->hw.port_info->lport; - act_prt = lag->active_port; - if (act_prt != pri_prt && act_prt != ICE_LAG_INVALID_PORT) { - ice_lag_move_vf_nodes_cfg(lag, act_prt, pri_prt); - return act_prt; + if (!lag->bond_aa) { + pri_prt = lag->pf->hw.port_info->lport; + act_prt = lag->active_port; + if (act_prt != pri_prt && + act_prt != ICE_LAG_INVALID_PORT) { + ice_lag_move_vf_nodes_cfg(lag, act_prt, pri_prt); + return act_prt; + } + } else { + if (lag->port_bitmap & ICE_LAGS_M) { + lag->port_bitmap &= ~ICE_LAGS_M; + ice_lag_aa_failover(lag, ICE_LAGP_IDX, NULL); + lag->port_bitmap |= ICE_LAGS_M; + } } } @@ -857,10 +876,15 @@ void ice_lag_complete_vf_reset(struct ice_lag *lag, u8 act_prt) { u8 pri_prt; - if (lag && lag->bonded && lag->primary && - act_prt != ICE_LAG_INVALID_PORT) { - pri_prt = lag->pf->hw.port_info->lport; - ice_lag_move_vf_nodes_cfg(lag, pri_prt, act_prt); + if (lag && lag->bonded && lag->primary) { + if (!lag->bond_aa) { + pri_prt = lag->pf->hw.port_info->lport; + if (act_prt != ICE_LAG_INVALID_PORT) + ice_lag_move_vf_nodes_cfg(lag, pri_prt, + act_prt); + } else { + ice_lag_aa_failover(lag, ICE_LAGS_IDX, NULL); + } } } @@ -873,13 +897,12 @@ void ice_lag_complete_vf_reset(struct ice_lag *lag, u8 act_prt) */ static void ice_lag_info_event(struct ice_lag *lag, void *ptr) { - struct netdev_notifier_bonding_info *info; + struct netdev_notifier_bonding_info *info = ptr; struct netdev_bonding_info *bonding_info; struct net_device *event_netdev; const char *lag_netdev_name; event_netdev = netdev_notifier_info_to_dev(ptr); - info = ptr; lag_netdev_name = netdev_name(lag->netdev); bonding_info = &info->bonding_info; @@ -897,7 +920,7 @@ static void ice_lag_info_event(struct ice_lag *lag, void *ptr) } if (bonding_info->slave.state) - ice_lag_set_backup(lag); + ice_lag_set_bkup(lag); else ice_lag_set_primary(lag); @@ -906,6 +929,295 @@ lag_out: } /** + * ice_lag_aa_qbuf_recfg - fill a single queue buffer for recfg cmd + * @hw: HW struct that contains the queue context + * @qbuf: pointer to single queue buffer + * @vsi_num: index of the VF VSI in PF space + * @qnum: queue index + * + * Return: Zero on success, error code on failure. + */ +static int +ice_lag_aa_qbuf_recfg(struct ice_hw *hw, struct ice_aqc_cfg_txqs_buf *qbuf, + u16 vsi_num, int qnum) +{ + struct ice_pf *pf = hw->back; + struct ice_q_ctx *q_ctx; + u16 q_id; + + q_ctx = ice_get_lan_q_ctx(hw, vsi_num, 0, qnum); + if (!q_ctx) { + dev_dbg(ice_hw_to_dev(hw), "LAG queue %d no Q context\n", qnum); + return -ENOENT; + } + + if (q_ctx->q_teid == ICE_INVAL_TEID) { + dev_dbg(ice_hw_to_dev(hw), "LAG queue %d INVAL TEID\n", qnum); + return -EINVAL; + } + + if (q_ctx->q_handle == ICE_INVAL_Q_HANDLE) { + dev_dbg(ice_hw_to_dev(hw), "LAG queue %d INVAL Q HANDLE\n", qnum); + return -EINVAL; + } + + q_id = pf->vsi[vsi_num]->txq_map[q_ctx->q_handle]; + qbuf->queue_info[0].q_handle = cpu_to_le16(q_id); + qbuf->queue_info[0].tc = 0; + qbuf->queue_info[0].q_teid = cpu_to_le32(q_ctx->q_teid); + + return 0; +} + +/** + * ice_lag_aa_move_vf_qs - Move some/all VF queues to destination + * @lag: primary interface's lag struct + * @dest: index of destination port + * @vsi_num: index of VF VSI in PF space + * @all: if true move all queues to destination + * @odd: VF wide q indicator for odd/even + * @e_pf: PF struct for the event interface + * + * the parameter "all" is to control whether we are splitting the queues + * between two interfaces or moving them all to the destination interface + */ +static void ice_lag_aa_move_vf_qs(struct ice_lag *lag, u8 dest, u16 vsi_num, + bool all, bool *odd, struct ice_pf *e_pf) +{ + DEFINE_RAW_FLEX(struct ice_aqc_cfg_txqs_buf, qbuf, queue_info, 1); + struct ice_hw *old_hw, *new_hw, *pri_hw, *sec_hw; + struct device *dev = ice_pf_to_dev(lag->pf); + struct ice_vsi_ctx *pv_ctx, *sv_ctx; + struct ice_lag_netdev_list ndlist; + u16 num_q, qbuf_size, sec_vsi_num; + u8 pri_lport, sec_lport; + u32 pvf_teid, svf_teid; + u16 vf_id; + + vf_id = lag->pf->vsi[vsi_num]->vf->vf_id; + /* If sec_vf[] not defined, then no second interface to share with */ + if (lag->sec_vf[vf_id]) + sec_vsi_num = lag->sec_vf[vf_id]->idx; + else + return; + + pri_lport = lag->bond_lport_pri; + sec_lport = lag->bond_lport_sec; + + if (pri_lport == ICE_LAG_INVALID_PORT || + sec_lport == ICE_LAG_INVALID_PORT) + return; + + if (!e_pf) + ice_lag_build_netdev_list(lag, &ndlist); + + pri_hw = &lag->pf->hw; + if (e_pf && lag->pf != e_pf) + sec_hw = &e_pf->hw; + else + sec_hw = ice_lag_find_hw_by_lport(lag, sec_lport); + + if (!pri_hw || !sec_hw) + return; + + if (dest == ICE_LAGP_IDX) { + struct ice_vsi *vsi; + + vsi = ice_get_main_vsi(lag->pf); + if (!vsi) + return; + + old_hw = sec_hw; + new_hw = pri_hw; + ice_lag_config_eswitch(lag, vsi->netdev); + } else { + struct ice_pf *sec_pf = sec_hw->back; + struct ice_vsi *vsi; + + vsi = ice_get_main_vsi(sec_pf); + if (!vsi) + return; + + old_hw = pri_hw; + new_hw = sec_hw; + ice_lag_config_eswitch(lag, vsi->netdev); + } + + pv_ctx = ice_get_vsi_ctx(pri_hw, vsi_num); + if (!pv_ctx) { + dev_warn(dev, "Unable to locate primary VSI %d context for LAG failover\n", + vsi_num); + return; + } + + sv_ctx = ice_get_vsi_ctx(sec_hw, sec_vsi_num); + if (!sv_ctx) { + dev_warn(dev, "Unable to locate secondary VSI %d context for LAG failover\n", + vsi_num); + return; + } + + num_q = pv_ctx->num_lan_q_entries[0]; + qbuf_size = __struct_size(qbuf); + + /* Suspend traffic for primary VSI VF */ + pvf_teid = le32_to_cpu(pv_ctx->sched.vsi_node[0]->info.node_teid); + ice_sched_suspend_resume_elems(pri_hw, 1, &pvf_teid, true); + + /* Suspend traffic for secondary VSI VF */ + svf_teid = le32_to_cpu(sv_ctx->sched.vsi_node[0]->info.node_teid); + ice_sched_suspend_resume_elems(sec_hw, 1, &svf_teid, true); + + for (int i = 0; i < num_q; i++) { + struct ice_sched_node *n_prt, *q_node, *parent; + struct ice_port_info *pi, *new_pi; + struct ice_vsi_ctx *src_ctx; + struct ice_sched_node *p; + struct ice_q_ctx *q_ctx; + u16 dst_vsi_num; + + pi = old_hw->port_info; + new_pi = new_hw->port_info; + + *odd = !(*odd); + if ((dest == ICE_LAGP_IDX && *odd && !all) || + (dest == ICE_LAGS_IDX && !(*odd) && !all) || + lag->q_home[vf_id][i] == dest) + continue; + + if (dest == ICE_LAGP_IDX) + dst_vsi_num = vsi_num; + else + dst_vsi_num = sec_vsi_num; + + n_prt = ice_sched_get_free_qparent(new_hw->port_info, + dst_vsi_num, 0, + ICE_SCHED_NODE_OWNER_LAN); + if (!n_prt) + continue; + + q_ctx = ice_get_lan_q_ctx(pri_hw, vsi_num, 0, i); + if (!q_ctx) + continue; + + if (dest == ICE_LAGP_IDX) + src_ctx = sv_ctx; + else + src_ctx = pv_ctx; + + q_node = ice_sched_find_node_by_teid(src_ctx->sched.vsi_node[0], + q_ctx->q_teid); + if (!q_node) + continue; + + qbuf->src_parent_teid = q_node->info.parent_teid; + qbuf->dst_parent_teid = n_prt->info.node_teid; + + /* Move the node in the HW/FW */ + if (ice_lag_aa_qbuf_recfg(pri_hw, qbuf, vsi_num, i)) + continue; + + if (dest == ICE_LAGP_IDX) + ice_aq_cfg_lan_txq(pri_hw, qbuf, qbuf_size, 1, + sec_lport, pri_lport, + ICE_AQC_Q_CFG_MOVE_TC_CHNG, + NULL); + else + ice_aq_cfg_lan_txq(pri_hw, qbuf, qbuf_size, 1, + pri_lport, sec_lport, + ICE_AQC_Q_CFG_MOVE_TC_CHNG, + NULL); + + /* Move the node in the SW */ + parent = q_node->parent; + if (!parent) + continue; + + for (int n = 0; n < parent->num_children; n++) { + int j; + + if (parent->children[n] != q_node) + continue; + + for (j = n + 1; j < parent->num_children; + j++) { + parent->children[j - 1] = + parent->children[j]; + } + parent->children[j] = NULL; + parent->num_children--; + break; + } + + p = pi->sib_head[0][q_node->tx_sched_layer]; + while (p) { + if (p->sibling == q_node) { + p->sibling = q_node->sibling; + break; + } + p = p->sibling; + } + + if (pi->sib_head[0][q_node->tx_sched_layer] == q_node) + pi->sib_head[0][q_node->tx_sched_layer] = + q_node->sibling; + + q_node->parent = n_prt; + q_node->info.parent_teid = n_prt->info.node_teid; + q_node->sibling = NULL; + p = new_pi->sib_head[0][q_node->tx_sched_layer]; + if (p) { + while (p) { + if (!p->sibling) { + p->sibling = q_node; + break; + } + p = p->sibling; + } + } else { + new_pi->sib_head[0][q_node->tx_sched_layer] = + q_node; + } + + n_prt->children[n_prt->num_children++] = q_node; + lag->q_home[vf_id][i] = dest; + } + + ice_sched_suspend_resume_elems(pri_hw, 1, &pvf_teid, false); + ice_sched_suspend_resume_elems(sec_hw, 1, &svf_teid, false); + + if (!e_pf) + ice_lag_destroy_netdev_list(lag, &ndlist); +} + +/** + * ice_lag_aa_failover - move VF queues in A/A mode + * @lag: primary lag struct + * @dest: index of destination port + * @e_pf: PF struct for event port + */ +void ice_lag_aa_failover(struct ice_lag *lag, u8 dest, struct ice_pf *e_pf) +{ + bool odd = true, all = false; + int i; + + /* Primary can be a target if down (cleanup), but secondary can't */ + if (dest == ICE_LAGS_IDX && !(lag->port_bitmap & ICE_LAGS_M)) + return; + + /* Move all queues to a destination if only one port is active, + * or no ports are active and dest is primary. + */ + if ((lag->port_bitmap ^ (ICE_LAGP_M | ICE_LAGS_M)) || + (!lag->port_bitmap && dest == ICE_LAGP_IDX)) + all = true; + + ice_for_each_vsi(lag->pf, i) + if (lag->pf->vsi[i] && lag->pf->vsi[i]->type == ICE_VSI_VF) + ice_lag_aa_move_vf_qs(lag, dest, i, all, &odd, e_pf); +} + +/** * ice_lag_reclaim_vf_tc - move scheduling nodes back to primary interface * @lag: primary interface lag struct * @src_hw: HW struct current node location @@ -921,13 +1233,12 @@ ice_lag_reclaim_vf_tc(struct ice_lag *lag, struct ice_hw *src_hw, u16 vsi_num, u16 numq, valq, num_moved, qbuf_size; u16 buf_size = __struct_size(buf); struct ice_aqc_cfg_txqs_buf *qbuf; + struct ice_hw *hw = &lag->pf->hw; struct ice_sched_node *n_prt; __le32 teid, parent_teid; struct ice_vsi_ctx *ctx; - struct ice_hw *hw; u32 tmp_teid; - hw = &lag->pf->hw; ctx = ice_get_vsi_ctx(hw, vsi_num); if (!ctx) { dev_warn(dev, "Unable to locate VSI context for LAG reclaim\n"); @@ -968,7 +1279,7 @@ ice_lag_reclaim_vf_tc(struct ice_lag *lag, struct ice_hw *src_hw, u16 vsi_num, if (ice_aq_cfg_lan_txq(hw, qbuf, qbuf_size, numq, src_hw->port_info->lport, hw->port_info->lport, - NULL)) { + ICE_AQC_Q_CFG_TC_CHNG, NULL)) { dev_warn(dev, "Failure to configure queues for LAG failover\n"); goto reclaim_qerr; } @@ -1039,36 +1350,15 @@ static void ice_lag_link(struct ice_lag *lag) lag->bonded = true; lag->role = ICE_LAG_UNSET; + lag->need_fltr_cfg = true; netdev_info(lag->netdev, "Shared SR-IOV resources in bond are active\n"); } /** - * ice_lag_config_eswitch - configure eswitch to work with LAG - * @lag: lag info struct - * @netdev: active network interface device struct - * - * Updates all port representors in eswitch to use @netdev for Tx. - * - * Configures the netdev to keep dst metadata (also used in representor Tx). - * This is required for an uplink without switchdev mode configured. - */ -static void ice_lag_config_eswitch(struct ice_lag *lag, - struct net_device *netdev) -{ - struct ice_repr *repr; - unsigned long id; - - xa_for_each(&lag->pf->eswitch.reprs, id, repr) - repr->dst->u.port_info.lower_dev = netdev; - - netif_keep_dst(netdev); -} - -/** - * ice_lag_unlink - handle unlink event + * ice_lag_act_bkup_unlink - handle unlink event for A/B bond * @lag: LAG info struct */ -static void ice_lag_unlink(struct ice_lag *lag) +static void ice_lag_act_bkup_unlink(struct ice_lag *lag) { u8 pri_port, act_port, loc_port; struct ice_pf *pf = lag->pf; @@ -1104,10 +1394,32 @@ static void ice_lag_unlink(struct ice_lag *lag) } } } +} - lag->bonded = false; - lag->role = ICE_LAG_NONE; - lag->upper_netdev = NULL; +/** + * ice_lag_aa_unlink - handle unlink event for Active-Active bond + * @lag: LAG info struct + */ +static void ice_lag_aa_unlink(struct ice_lag *lag) +{ + struct ice_lag *pri_lag; + + if (lag->primary) { + pri_lag = lag; + lag->port_bitmap &= ~ICE_LAGP_M; + } else { + pri_lag = ice_lag_find_primary(lag); + if (pri_lag) + pri_lag->port_bitmap &= ICE_LAGS_M; + } + + if (pri_lag) { + ice_lag_aa_failover(pri_lag, ICE_LAGP_IDX, lag->pf); + if (lag->primary) + pri_lag->bond_lport_pri = ICE_LAG_INVALID_PORT; + else + pri_lag->bond_lport_sec = ICE_LAG_INVALID_PORT; + } } /** @@ -1123,10 +1435,20 @@ static void ice_lag_link_unlink(struct ice_lag *lag, void *ptr) if (netdev != lag->netdev) return; - if (info->linking) + if (info->linking) { ice_lag_link(lag); - else - ice_lag_unlink(lag); + } else { + if (lag->bond_aa) + ice_lag_aa_unlink(lag); + else + ice_lag_act_bkup_unlink(lag); + + lag->bonded = false; + lag->role = ICE_LAG_NONE; + lag->upper_netdev = NULL; + lag->bond_aa = false; + lag->need_fltr_cfg = false; + } } /** @@ -1224,11 +1546,8 @@ ice_lag_set_swid(u16 primary_swid, struct ice_lag *local_lag, */ static void ice_lag_primary_swid(struct ice_lag *lag, bool link) { - struct ice_hw *hw; - u16 swid; - - hw = &lag->pf->hw; - swid = hw->port_info->sw_id; + struct ice_hw *hw = &lag->pf->hw; + u16 swid = hw->port_info->sw_id; if (ice_share_res(hw, ICE_AQC_RES_TYPE_SWID, link, swid)) dev_warn(ice_pf_to_dev(lag->pf), "Failure to set primary interface shared status\n"); @@ -1241,12 +1560,10 @@ static void ice_lag_primary_swid(struct ice_lag *lag, bool link) */ static void ice_lag_add_prune_list(struct ice_lag *lag, struct ice_pf *event_pf) { - u16 num_vsi, rule_buf_sz, vsi_list_id, event_vsi_num, prim_vsi_idx; - struct ice_sw_rule_vsi_list *s_rule = NULL; + u16 rule_buf_sz, vsi_list_id, event_vsi_num, prim_vsi_idx, num_vsi = 1; + struct ice_sw_rule_vsi_list *s_rule; struct device *dev; - num_vsi = 1; - dev = ice_pf_to_dev(lag->pf); event_vsi_num = event_pf->vsi[0]->vsi_num; prim_vsi_idx = lag->pf->vsi[0]->idx; @@ -1282,12 +1599,10 @@ static void ice_lag_add_prune_list(struct ice_lag *lag, struct ice_pf *event_pf) */ static void ice_lag_del_prune_list(struct ice_lag *lag, struct ice_pf *event_pf) { - u16 num_vsi, vsi_num, vsi_idx, rule_buf_sz, vsi_list_id; - struct ice_sw_rule_vsi_list *s_rule = NULL; + u16 vsi_num, vsi_idx, rule_buf_sz, vsi_list_id, num_vsi = 1; + struct ice_sw_rule_vsi_list *s_rule; struct device *dev; - num_vsi = 1; - dev = ice_pf_to_dev(lag->pf); vsi_num = event_pf->vsi[0]->vsi_num; vsi_idx = lag->pf->vsi[0]->idx; @@ -1335,6 +1650,11 @@ static void ice_lag_init_feature_support_flag(struct ice_pf *pf) ice_set_feature_support(pf, ICE_F_SRIOV_LAG); else ice_clear_feature_support(pf, ICE_F_SRIOV_LAG); + + if (caps->sriov_aa_lag && ice_pkg_has_lport_extract(&pf->hw)) + ice_set_feature_support(pf, ICE_F_SRIOV_AA_LAG); + else + ice_clear_feature_support(pf, ICE_F_SRIOV_AA_LAG); } /** @@ -1344,11 +1664,10 @@ static void ice_lag_init_feature_support_flag(struct ice_pf *pf) */ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) { - struct netdev_notifier_changeupper_info *info; + struct netdev_notifier_changeupper_info *info = ptr; struct ice_lag *primary_lag; struct net_device *netdev; - info = ptr; netdev = netdev_notifier_info_to_dev(ptr); /* not for this netdev */ @@ -1369,6 +1688,9 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) /* Configure primary's SWID to be shared */ ice_lag_primary_swid(lag, true); primary_lag = lag; + lag->bond_lport_pri = lag->pf->hw.port_info->lport; + lag->bond_lport_sec = ICE_LAG_INVALID_PORT; + lag->port_bitmap = 0; } else { u16 swid; @@ -1378,16 +1700,29 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) swid = primary_lag->pf->hw.port_info->sw_id; ice_lag_set_swid(swid, lag, true); ice_lag_add_prune_list(primary_lag, lag->pf); - ice_lag_cfg_drop_fltr(lag, true); + primary_lag->bond_lport_sec = + lag->pf->hw.port_info->lport; } /* add filter for primary control packets */ - ice_lag_cfg_cp_fltr(lag, true); + ice_lag_cfg_lp_fltr(lag, true, true); } else { if (!primary_lag && lag->primary) primary_lag = lag; + if (primary_lag) { + for (int i = 0; i < ICE_MAX_SRIOV_VFS; i++) { + if (primary_lag->sec_vf[i]) { + ice_vsi_release(primary_lag->sec_vf[i]); + primary_lag->sec_vf[i] = NULL; + } + } + } + if (!lag->primary) { ice_lag_set_swid(0, lag, false); + if (primary_lag) + primary_lag->bond_lport_sec = + ICE_LAG_INVALID_PORT; } else { if (primary_lag && lag->primary) { ice_lag_primary_swid(lag, false); @@ -1395,7 +1730,7 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) } } /* remove filter for control packets */ - ice_lag_cfg_cp_fltr(lag, false); + ice_lag_cfg_lp_fltr(lag, false, !lag->bond_aa); } } @@ -1408,7 +1743,7 @@ static void ice_lag_changeupper_event(struct ice_lag *lag, void *ptr) */ static void ice_lag_monitor_link(struct ice_lag *lag, void *ptr) { - struct netdev_notifier_changeupper_info *info; + struct netdev_notifier_changeupper_info *info = ptr; struct ice_hw *prim_hw, *active_hw; struct net_device *event_netdev; struct ice_pf *pf; @@ -1421,19 +1756,34 @@ static void ice_lag_monitor_link(struct ice_lag *lag, void *ptr) if (!netif_is_same_ice(lag->pf, event_netdev)) return; + if (info->upper_dev != lag->upper_netdev) + return; + + if (info->linking) + return; + pf = lag->pf; prim_hw = &pf->hw; prim_port = prim_hw->port_info->lport; - info = (struct netdev_notifier_changeupper_info *)ptr; - if (info->upper_dev != lag->upper_netdev) - return; - - if (!info->linking) { - /* Since there are only two interfaces allowed in SRIOV+LAG, if - * one port is leaving, then nodes need to be on primary - * interface. - */ + /* Since there are only two interfaces allowed in SRIOV+LAG, if + * one port is leaving, then nodes need to be on primary + * interface. + */ + if (lag->bond_aa) { + struct ice_netdev_priv *e_ndp; + struct ice_pf *e_pf; + + e_ndp = netdev_priv(event_netdev); + e_pf = e_ndp->vsi->back; + + if (lag->bond_lport_pri != ICE_LAG_INVALID_PORT && + lag->port_bitmap & ICE_LAGS_M) { + lag->port_bitmap &= ~ICE_LAGS_M; + ice_lag_aa_failover(lag, ICE_LAGP_IDX, e_pf); + lag->bond_lport_sec = ICE_LAG_INVALID_PORT; + } + } else { if (prim_port != lag->active_port && lag->active_port != ICE_LAG_INVALID_PORT) { active_hw = ice_lag_find_hw_by_lport(lag, @@ -1445,45 +1795,32 @@ static void ice_lag_monitor_link(struct ice_lag *lag, void *ptr) } /** - * ice_lag_monitor_active - main PF keep track of which port is active + * ice_lag_monitor_act_bkup - keep track of which port is active in A/B LAG * @lag: lag info struct - * @ptr: opaque data containing notifier event + * @b_info: bonding info + * @event_netdev: net_device got target netdev * * This function is for the primary PF to monitor changes in which port is * active and handle changes for SRIOV VF functionality */ -static void ice_lag_monitor_active(struct ice_lag *lag, void *ptr) +static void ice_lag_monitor_act_bkup(struct ice_lag *lag, + struct netdev_bonding_info *b_info, + struct net_device *event_netdev) { - struct net_device *event_netdev, *event_upper; - struct netdev_notifier_bonding_info *info; - struct netdev_bonding_info *bonding_info; struct ice_netdev_priv *event_np; struct ice_pf *pf, *event_pf; u8 prim_port, event_port; - if (!lag->primary) - return; - pf = lag->pf; if (!pf) return; - event_netdev = netdev_notifier_info_to_dev(ptr); - rcu_read_lock(); - event_upper = netdev_master_upper_dev_get_rcu(event_netdev); - rcu_read_unlock(); - if (!netif_is_ice(event_netdev) || event_upper != lag->upper_netdev) - return; - event_np = netdev_priv(event_netdev); event_pf = event_np->vsi->back; event_port = event_pf->hw.port_info->lport; prim_port = pf->hw.port_info->lport; - info = (struct netdev_notifier_bonding_info *)ptr; - bonding_info = &info->bonding_info; - - if (!bonding_info->slave.state) { + if (!b_info->slave.state) { /* if no port is currently active, then nodes and filters exist * on primary port, check if we need to move them */ @@ -1520,6 +1857,128 @@ static void ice_lag_monitor_active(struct ice_lag *lag, void *ptr) } /** + * ice_lag_aa_clear_spoof - adjust the placeholder VSI spoofing for A/A LAG + * @vsi: placeholder VSI to adjust + */ +static void ice_lag_aa_clear_spoof(struct ice_vsi *vsi) +{ + ice_vsi_update_security(vsi, ice_vsi_ctx_clear_antispoof); +} + +/** + * ice_lag_monitor_act_act - Keep track of active ports in A/A LAG + * @lag: lag struct for primary interface + * @b_info: bonding_info for event + * @event_netdev: net_device for target netdev + */ +static void ice_lag_monitor_act_act(struct ice_lag *lag, + struct netdev_bonding_info *b_info, + struct net_device *event_netdev) +{ + struct ice_netdev_priv *event_np; + u8 prim_port, event_port; + struct ice_pf *event_pf; + + event_np = netdev_priv(event_netdev); + event_pf = event_np->vsi->back; + event_port = event_pf->hw.port_info->lport; + prim_port = lag->pf->hw.port_info->lport; + + if (b_info->slave.link == BOND_LINK_UP) { + /* Port is coming up */ + if (prim_port == event_port) { + /* Processing event for primary interface */ + if (lag->bond_lport_pri == ICE_LAG_INVALID_PORT) + return; + + if (!(lag->port_bitmap & ICE_LAGP_M)) { + /* Primary port was not marked up before, move + * some|all VF queues to it and mark as up + */ + lag->port_bitmap |= ICE_LAGP_M; + ice_lag_aa_failover(lag, ICE_LAGP_IDX, event_pf); + } + } else { + if (lag->bond_lport_sec == ICE_LAG_INVALID_PORT) + return; + + /* Create placeholder VSIs on secondary PF. + * The placeholder is necessary so that we have + * an element that represents the VF on the secondary + * interface's scheduling tree. This will be a tree + * root for scheduling nodes when they are moved to + * the secondary interface. + */ + if (!lag->sec_vf[0]) { + struct ice_vsi_cfg_params params = {}; + struct ice_vsi *nvsi; + struct ice_vf *vf; + unsigned int bkt; + + params.type = ICE_VSI_VF; + params.port_info = event_pf->hw.port_info; + params.flags = ICE_VSI_FLAG_INIT; + + ice_for_each_vf(lag->pf, bkt, vf) { + params.vf = vf; + nvsi = ice_vsi_setup(event_pf, + ¶ms); + ice_lag_aa_clear_spoof(nvsi); + lag->sec_vf[vf->vf_id] = nvsi; + } + } + + if (!(lag->port_bitmap & ICE_LAGS_M)) { + /* Secondary port was not marked up before, + * move some|all VF queues to it and mark as up + */ + lag->port_bitmap |= ICE_LAGS_M; + ice_lag_aa_failover(lag, ICE_LAGS_IDX, event_pf); + } + } + } else { + /* Port is going down */ + if (prim_port == event_port) { + lag->port_bitmap &= ~ICE_LAGP_M; + ice_lag_aa_failover(lag, ICE_LAGS_IDX, event_pf); + } else { + lag->port_bitmap &= ~ICE_LAGS_M; + ice_lag_aa_failover(lag, ICE_LAGP_IDX, event_pf); + } + } +} + +/** + * ice_lag_monitor_info - Calls relevant A/A or A/B monitoring function + * @lag: lag info struct + * @ptr: opaque data containing notifier event + * + * This function is for the primary PF to monitor changes in which port is + * active and handle changes for SRIOV VF functionality + */ +static void ice_lag_monitor_info(struct ice_lag *lag, void *ptr) +{ + struct netdev_notifier_bonding_info *info = ptr; + struct net_device *event_netdev, *event_upper; + struct netdev_bonding_info *bonding_info; + + if (!lag->primary) + return; + + event_netdev = netdev_notifier_info_to_dev(ptr); + bonding_info = &info->bonding_info; + rcu_read_lock(); + event_upper = netdev_master_upper_dev_get_rcu(event_netdev); + rcu_read_unlock(); + if (!netif_is_ice(event_netdev) || event_upper != lag->upper_netdev) + return; + + if (lag->bond_aa) + ice_lag_monitor_act_act(lag, bonding_info, event_netdev); + else + ice_lag_monitor_act_bkup(lag, bonding_info, event_netdev); +} +/** * ice_lag_chk_comp - evaluate bonded interface for feature support * @lag: lag info struct * @ptr: opaque data for netdev event info @@ -1527,13 +1986,21 @@ static void ice_lag_monitor_active(struct ice_lag *lag, void *ptr) static bool ice_lag_chk_comp(struct ice_lag *lag, void *ptr) { + struct netdev_notifier_bonding_info *info = ptr; struct net_device *event_netdev, *event_upper; - struct netdev_notifier_bonding_info *info; struct netdev_bonding_info *bonding_info; struct list_head *tmp; struct device *dev; int count = 0; + /* All members need to know if bond A/A or A/B */ + bonding_info = &info->bonding_info; + lag->bond_mode = bonding_info->master.bond_mode; + if (lag->bond_mode != BOND_MODE_ACTIVEBACKUP) + lag->bond_aa = true; + else + lag->bond_aa = false; + if (!lag->primary) return true; @@ -1554,13 +2021,9 @@ ice_lag_chk_comp(struct ice_lag *lag, void *ptr) return false; } - info = (struct netdev_notifier_bonding_info *)ptr; - bonding_info = &info->bonding_info; - lag->bond_mode = bonding_info->master.bond_mode; - if (lag->bond_mode != BOND_MODE_ACTIVEBACKUP) { - dev_info(dev, "Bond Mode not ACTIVE-BACKUP - VF LAG disabled\n"); + if (lag->bond_aa && !ice_is_feature_supported(lag->pf, + ICE_F_SRIOV_AA_LAG)) return false; - } list_for_each(tmp, lag->netdev_head) { struct ice_dcbx_cfg *dcb_cfg, *peer_dcb_cfg; @@ -1664,10 +2127,9 @@ ice_lag_unregister(struct ice_lag *lag, struct net_device *event_netdev) static void ice_lag_monitor_rdma(struct ice_lag *lag, void *ptr) { - struct netdev_notifier_changeupper_info *info; + struct netdev_notifier_changeupper_info *info = ptr; struct net_device *netdev; - info = ptr; netdev = netdev_notifier_info_to_dev(ptr); if (netdev != lag->netdev) @@ -1715,12 +2177,30 @@ static void ice_lag_chk_disabled_bond(struct ice_lag *lag, void *ptr) */ static void ice_lag_disable_sriov_bond(struct ice_lag *lag) { - struct ice_netdev_priv *np; - struct ice_pf *pf; + struct ice_netdev_priv *np = netdev_priv(lag->netdev); + struct ice_pf *pf = np->vsi->back; - np = netdev_priv(lag->netdev); - pf = np->vsi->back; ice_clear_feature_support(pf, ICE_F_SRIOV_LAG); + ice_clear_feature_support(pf, ICE_F_SRIOV_AA_LAG); +} + +/** + * ice_lag_preset_drop_fltr - preset drop filter for A/B bonds + * @lag: local lag struct + * @ptr: opaque data containing event + * + * Sets the initial drop filter for secondary interface in an + * active-backup bond + */ +static void ice_lag_preset_drop_fltr(struct ice_lag *lag, void *ptr) +{ + struct net_device *netdev = netdev_notifier_info_to_dev(ptr); + + if (netdev != lag->netdev || lag->primary || !lag->need_fltr_cfg) + return; + + ice_lag_cfg_drop_fltr(lag, true); + lag->need_fltr_cfg = false; } /** @@ -1761,10 +2241,12 @@ static void ice_lag_process_event(struct work_struct *work) ice_lag_unregister(lag_work->lag, netdev); goto lag_cleanup; } - ice_lag_monitor_active(lag_work->lag, - &lag_work->info.bonding_info); ice_lag_cfg_pf_fltrs(lag_work->lag, &lag_work->info.bonding_info); + ice_lag_preset_drop_fltr(lag_work->lag, + &lag_work->info.bonding_info); + ice_lag_monitor_info(lag_work->lag, + &lag_work->info.bonding_info); } ice_lag_info_event(lag_work->lag, &lag_work->info.bonding_info); break; @@ -1837,9 +2319,8 @@ ice_lag_event_handler(struct notifier_block *notif_blk, unsigned long event, lag_work->lag = lag; lag_work->event = event; if (event == NETDEV_CHANGEUPPER) { - struct netdev_notifier_changeupper_info *info; + struct netdev_notifier_changeupper_info *info = ptr; - info = ptr; upper_netdev = info->upper_dev; } else { upper_netdev = netdev_master_upper_dev_get(netdev); @@ -1889,10 +2370,8 @@ ice_lag_event_handler(struct notifier_block *notif_blk, unsigned long event, */ static int ice_register_lag_handler(struct ice_lag *lag) { + struct notifier_block *notif_blk = &lag->notif_block; struct device *dev = ice_pf_to_dev(lag->pf); - struct notifier_block *notif_blk; - - notif_blk = &lag->notif_block; if (!notif_blk->notifier_call) { notif_blk->notifier_call = ice_lag_event_handler; @@ -1912,10 +2391,9 @@ static int ice_register_lag_handler(struct ice_lag *lag) */ static void ice_unregister_lag_handler(struct ice_lag *lag) { + struct notifier_block *notif_blk = &lag->notif_block; struct device *dev = ice_pf_to_dev(lag->pf); - struct notifier_block *notif_blk; - notif_blk = &lag->notif_block; if (notif_blk->notifier_call) { unregister_netdevice_notifier(notif_blk); dev_dbg(dev, "LAG event handler unregistered\n"); @@ -1977,13 +2455,12 @@ ice_lag_move_vf_nodes_tc_sync(struct ice_lag *lag, struct ice_hw *dest_hw, u16 numq, valq, num_moved, qbuf_size; u16 buf_size = __struct_size(buf); struct ice_aqc_cfg_txqs_buf *qbuf; + struct ice_hw *hw = &lag->pf->hw; struct ice_sched_node *n_prt; __le32 teid, parent_teid; struct ice_vsi_ctx *ctx; - struct ice_hw *hw; u32 tmp_teid; - hw = &lag->pf->hw; ctx = ice_get_vsi_ctx(hw, vsi_num); if (!ctx) { dev_warn(dev, "LAG rebuild failed after reset due to VSI Context failure\n"); @@ -2020,7 +2497,8 @@ ice_lag_move_vf_nodes_tc_sync(struct ice_lag *lag, struct ice_hw *dest_hw, } if (ice_aq_cfg_lan_txq(hw, qbuf, qbuf_size, numq, hw->port_info->lport, - dest_hw->port_info->lport, NULL)) { + dest_hw->port_info->lport, + ICE_AQC_Q_CFG_TC_CHNG, NULL)) { dev_warn(dev, "Failure to configure queues for LAG reset rebuild\n"); goto sync_qerr; } @@ -2116,9 +2594,13 @@ int ice_init_lag(struct ice_pf *pf) lag->netdev = vsi->netdev; lag->role = ICE_LAG_NONE; lag->active_port = ICE_LAG_INVALID_PORT; + lag->port_bitmap = 0x0; lag->bonded = false; + lag->bond_aa = false; + lag->need_fltr_cfg = false; lag->upper_netdev = NULL; lag->notif_block.notifier_call = NULL; + memset(lag->sec_vf, 0, sizeof(lag->sec_vf)); err = ice_register_lag_handler(lag); if (err) { @@ -2136,6 +2618,11 @@ int ice_init_lag(struct ice_pf *pf) if (err) goto free_rcp_res; + err = ice_create_lag_recipe(&pf->hw, &lag->act_act_recipe, + ice_lport_rcp, 1); + if (err) + goto free_lport_res; + /* associate recipes to profiles */ for (n = 0; n < ICE_PROFID_IPV6_GTPU_IPV6_TCP_INNER; n++) { err = ice_aq_get_recipe_to_profile(&pf->hw, n, @@ -2145,7 +2632,8 @@ int ice_init_lag(struct ice_pf *pf) if (recipe_bits & BIT(ICE_SW_LKUP_DFLT)) { recipe_bits |= BIT(lag->pf_recipe) | - BIT(lag->lport_recipe); + BIT(lag->lport_recipe) | + BIT(lag->act_act_recipe); ice_aq_map_recipe_to_profile(&pf->hw, n, recipe_bits, NULL); } @@ -2156,9 +2644,13 @@ int ice_init_lag(struct ice_pf *pf) dev_dbg(dev, "INIT LAG complete\n"); return 0; +free_lport_res: + ice_free_hw_res(&pf->hw, ICE_AQC_RES_TYPE_RECIPE, 1, + &lag->lport_recipe); + free_rcp_res: ice_free_hw_res(&pf->hw, ICE_AQC_RES_TYPE_RECIPE, 1, - &pf->lag->pf_recipe); + &lag->pf_recipe); lag_error: kfree(lag); pf->lag = NULL; @@ -2174,9 +2666,7 @@ lag_error: */ void ice_deinit_lag(struct ice_pf *pf) { - struct ice_lag *lag; - - lag = pf->lag; + struct ice_lag *lag = pf->lag; if (!lag) return; @@ -2245,11 +2735,15 @@ void ice_lag_rebuild(struct ice_pf *pf) ice_lag_move_vf_nodes_sync(prim_lag, &pf->hw); } - ice_lag_cfg_cp_fltr(lag, true); + if (!lag->bond_aa) { + ice_lag_cfg_lp_fltr(lag, true, true); + if (lag->pf_rx_rule_id) + if (ice_lag_cfg_dflt_fltr(lag, true)) + dev_err(ice_pf_to_dev(pf), "Error adding default VSI rule in rebuild\n"); + } else { + ice_lag_cfg_lp_fltr(lag, true, false); + } - if (lag->pf_rx_rule_id) - if (ice_lag_cfg_dflt_fltr(lag, true)) - dev_err(ice_pf_to_dev(pf), "Error adding default VSI rule in rebuild\n"); ice_clear_rdma_cap(pf); lag_rebuild_out: diff --git a/drivers/net/ethernet/intel/ice/ice_lag.h b/drivers/net/ethernet/intel/ice/ice_lag.h index 69347d9f986b..f77ebcd61042 100644 --- a/drivers/net/ethernet/intel/ice/ice_lag.h +++ b/drivers/net/ethernet/intel/ice/ice_lag.h @@ -14,7 +14,11 @@ enum ice_lag_role { ICE_LAG_UNSET }; -#define ICE_LAG_INVALID_PORT 0xFF +#define ICE_LAG_INVALID_PORT 0xFF +#define ICE_LAGP_IDX 0 +#define ICE_LAGS_IDX 1 +#define ICE_LAGP_M 0x1 +#define ICE_LAGS_M 0x2 #define ICE_LAG_RESET_RETRIES 5 #define ICE_SW_DEFAULT_PROFILE 0 @@ -41,12 +45,26 @@ struct ice_lag { u8 active_port; /* lport value for the current active port */ u8 bonded:1; /* currently bonded */ u8 primary:1; /* this is primary */ + u8 bond_aa:1; /* is this bond active-active */ + u8 need_fltr_cfg:1; /* fltrs for A/A bond still need to be make */ + u8 port_bitmap:2; /* bitmap of active ports */ + u8 bond_lport_pri; /* lport values for primary PF */ + u8 bond_lport_sec; /* lport values for secondary PF */ + + /* q_home keeps track of which interface the q is currently on */ + u8 q_home[ICE_MAX_SRIOV_VFS][ICE_MAX_RSS_QS_PER_VF]; + + /* placeholder VSI for hanging VF queues from on secondary interface */ + struct ice_vsi *sec_vf[ICE_MAX_SRIOV_VFS]; + u16 pf_recipe; u16 lport_recipe; + u16 act_act_recipe; u16 pf_rx_rule_id; u16 pf_tx_rule_id; u16 cp_rule_idx; u16 lport_rule_idx; + u16 act_act_rule_idx; u8 role; }; @@ -64,7 +82,7 @@ struct ice_lag_work { } info; }; -void ice_lag_move_new_vf_nodes(struct ice_vf *vf); +void ice_lag_aa_failover(struct ice_lag *lag, u8 dest, struct ice_pf *e_pf); int ice_init_lag(struct ice_pf *pf); void ice_deinit_lag(struct ice_pf *pf); void ice_lag_rebuild(struct ice_pf *pf); diff --git a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h index 77ba26538b07..10c312d49e05 100644 --- a/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h +++ b/drivers/net/ethernet/intel/ice/ice_lan_tx_rx.h @@ -569,4 +569,45 @@ struct ice_tlan_ctx { u8 pkt_shaper_prof_idx; }; +#define ICE_TXTIME_TX_DESC_IDX_M GENMASK(12, 0) +#define ICE_TXTIME_STAMP_M GENMASK(31, 13) + +/* Tx time stamp descriptor */ +struct ice_ts_desc { + __le32 tx_desc_idx_tstamp; +}; + +#define ICE_TS_DESC(R, i) (&(((struct ice_ts_desc *)((R)->desc))[i])) + +#define ICE_TXTIME_MAX_QUEUE 2047 +#define ICE_SET_TXTIME_MAX_Q_AMOUNT 127 +#define ICE_TXTIME_FETCH_TS_DESC_DFLT 8 +#define ICE_TXTIME_FETCH_PROFILE_CNT 16 + +/* Tx Time queue context data */ +struct ice_txtime_ctx { +#define ICE_TXTIME_CTX_BASE_S 7 + u64 base; /* base is defined in 128-byte units */ + u8 pf_num; + u16 vmvf_num; + u8 vmvf_type; + u16 src_vsi; + u8 cpuid; + u8 tphrd_desc; + u16 qlen; + u8 timer_num; + u8 txtime_ena_q; + u8 drbell_mode_32; +#define ICE_TXTIME_CTX_DRBELL_MODE_32 1 + u8 ts_res; +#define ICE_TXTIME_CTX_RESOLUTION_128NS 7 + u8 ts_round_type; + u8 ts_pacing_slot; +#define ICE_TXTIME_CTX_FETCH_PROF_ID_0 0 + u8 merging_ena; + u8 ts_fetch_prof_id; + u8 ts_fetch_cache_line_aln_thld; + u8 tx_pipe_delay_mode; +}; + #endif /* _ICE_LAN_TX_RX_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index a439b5a61a56..4479c824561e 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -3950,6 +3950,7 @@ void ice_init_feature_support(struct ice_pf *pf) if (pf->hw.mac_type == ICE_MAC_E830) { ice_set_feature_support(pf, ICE_F_MBX_LIMIT); ice_set_feature_support(pf, ICE_F_GCS); + ice_set_feature_support(pf, ICE_F_TXTIME); } } diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 77781277aa8e..86f5859e88ef 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -39,6 +39,7 @@ static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; MODULE_DESCRIPTION(DRV_SUMMARY); MODULE_IMPORT_NS("LIBIE"); MODULE_IMPORT_NS("LIBIE_ADMINQ"); +MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_LICENSE("GPL v2"); MODULE_FIRMWARE(ICE_DDP_PKG_FILE); @@ -1251,32 +1252,6 @@ ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) } /** - * ice_get_fwlog_data - copy the FW log data from ARQ event - * @pf: PF that the FW log event is associated with - * @event: event structure containing FW log data - */ -static void -ice_get_fwlog_data(struct ice_pf *pf, struct ice_rq_event_info *event) -{ - struct ice_fwlog_data *fwlog; - struct ice_hw *hw = &pf->hw; - - fwlog = &hw->fwlog_ring.rings[hw->fwlog_ring.tail]; - - memset(fwlog->data, 0, PAGE_SIZE); - fwlog->data_size = le16_to_cpu(event->desc.datalen); - - memcpy(fwlog->data, event->msg_buf, fwlog->data_size); - ice_fwlog_ring_increment(&hw->fwlog_ring.tail, hw->fwlog_ring.size); - - if (ice_fwlog_ring_full(&hw->fwlog_ring)) { - /* the rings are full so bump the head to create room */ - ice_fwlog_ring_increment(&hw->fwlog_ring.head, - hw->fwlog_ring.size); - } -} - -/** * ice_aq_prep_for_event - Prepare to wait for an AdminQ event from firmware * @pf: pointer to the PF private structure * @task: intermediate helper storage and identifier for waiting @@ -1566,7 +1541,8 @@ static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) } break; case ice_aqc_opc_fw_logs_event: - ice_get_fwlog_data(pf, &event); + libie_get_fwlog_data(&hw->fwlog, event.msg_buf, + le16_to_cpu(event.desc.datalen)); break; case ice_aqc_opc_lldp_set_mib_change: ice_dcb_process_lldp_set_mib_change(pf, &event); @@ -3993,6 +3969,11 @@ static void ice_deinit_pf(struct ice_pf *pf) pf->avail_rxqs = NULL; } + if (pf->txtime_txqs) { + bitmap_free(pf->txtime_txqs); + pf->txtime_txqs = NULL; + } + if (pf->ptp.clock) ptp_clock_unregister(pf->ptp.clock); @@ -4086,6 +4067,15 @@ static int ice_init_pf(struct ice_pf *pf) return -ENOMEM; } + pf->txtime_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); + if (!pf->txtime_txqs) { + bitmap_free(pf->avail_txqs); + pf->avail_txqs = NULL; + bitmap_free(pf->avail_rxqs); + pf->avail_rxqs = NULL; + return -ENOMEM; + } + mutex_init(&pf->vfs.table_lock); hash_init(pf->vfs.table); if (ice_is_feature_supported(pf, ICE_F_MBX_LIMIT)) @@ -4654,19 +4644,6 @@ static void ice_print_wake_reason(struct ice_pf *pf) } /** - * ice_pf_fwlog_update_module - update 1 module - * @pf: pointer to the PF struct - * @log_level: log_level to use for the @module - * @module: module to update - */ -void ice_pf_fwlog_update_module(struct ice_pf *pf, int log_level, int module) -{ - struct ice_hw *hw = &pf->hw; - - hw->fwlog_cfg.module_entries[module].log_level = log_level; -} - -/** * ice_register_netdev - register netdev * @vsi: pointer to the VSI struct */ @@ -7521,7 +7498,8 @@ int ice_vsi_open(struct ice_vsi *vsi) if (err) goto err_setup_rx; - ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); + if (bitmap_empty(pf->txtime_txqs, pf->max_pf_txqs)) + ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); if (vsi->type == ICE_VSI_PF || vsi->type == ICE_VSI_SF) { /* Notify the stack of the actual queue counts. */ @@ -9125,7 +9103,7 @@ static int ice_create_q_channels(struct ice_vsi *vsi) list_add_tail(&ch->list, &vsi->ch_list); vsi->tc_map_vsi[i] = ch->ch_vsi; dev_dbg(ice_pf_to_dev(pf), - "successfully created channel: VSI %pK\n", ch->ch_vsi); + "successfully created channel: VSI %p\n", ch->ch_vsi); } return 0; @@ -9310,6 +9288,96 @@ exit: return ret; } +/** + * ice_cfg_txtime - configure Tx Time for the Tx ring + * @tx_ring: pointer to the Tx ring structure + * + * Return: 0 on success, negative value on failure. + */ +static int ice_cfg_txtime(struct ice_tx_ring *tx_ring) +{ + int err, timeout = 50; + struct ice_vsi *vsi; + struct device *dev; + struct ice_pf *pf; + u32 queue; + + if (!tx_ring) + return -EINVAL; + + vsi = tx_ring->vsi; + pf = vsi->back; + while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) { + timeout--; + if (!timeout) + return -EBUSY; + usleep_range(1000, 2000); + } + + queue = tx_ring->q_index; + dev = ice_pf_to_dev(pf); + + /* Ignore return value, and always attempt to enable queue. */ + ice_qp_dis(vsi, queue); + + err = ice_qp_ena(vsi, queue); + if (err) + dev_err(dev, "Failed to enable Tx queue %d for TxTime configuration\n", + queue); + + clear_bit(ICE_CFG_BUSY, pf->state); + return err; +} + +/** + * ice_offload_txtime - set earliest TxTime first + * @netdev: network interface device structure + * @qopt_off: etf queue option offload from the skb to set + * + * Return: 0 on success, negative value on failure. + */ +static int ice_offload_txtime(struct net_device *netdev, + void *qopt_off) +{ + struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_pf *pf = np->vsi->back; + struct tc_etf_qopt_offload *qopt; + struct ice_vsi *vsi = np->vsi; + struct ice_tx_ring *tx_ring; + int ret = 0; + + if (!ice_is_feature_supported(pf, ICE_F_TXTIME)) + return -EOPNOTSUPP; + + qopt = qopt_off; + if (!qopt_off || qopt->queue < 0 || qopt->queue >= vsi->num_txq) + return -EINVAL; + + if (qopt->enable) + set_bit(qopt->queue, pf->txtime_txqs); + else + clear_bit(qopt->queue, pf->txtime_txqs); + + if (netif_running(vsi->netdev)) { + tx_ring = vsi->tx_rings[qopt->queue]; + ret = ice_cfg_txtime(tx_ring); + if (ret) + goto err; + } + + netdev_info(netdev, "%s TxTime on queue: %i\n", + str_enable_disable(qopt->enable), qopt->queue); + return 0; + +err: + netdev_err(netdev, "Failed to %s TxTime on queue: %i\n", + str_enable_disable(qopt->enable), qopt->queue); + + if (qopt->enable) + clear_bit(qopt->queue, pf->txtime_txqs); + return ret; +} + static LIST_HEAD(ice_block_cb_list); static int @@ -9373,6 +9441,8 @@ adev_unlock: mutex_unlock(&pf->adev_mutex); } return err; + case TC_SETUP_QDISC_ETF: + return ice_offload_txtime(netdev, type_data); default: return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c index 9ce4c4db400e..843e82fd3bf9 100644 --- a/drivers/net/ethernet/intel/ice/ice_sriov.c +++ b/drivers/net/ethernet/intel/ice/ice_sriov.c @@ -9,7 +9,7 @@ #include "ice_dcb_lib.h" #include "ice_flow.h" #include "ice_eswitch.h" -#include "ice_virtchnl_allowlist.h" +#include "virt/allowlist.h" #include "ice_flex_pipe.h" #include "ice_vf_vsi_vlan_ops.h" #include "ice_vlan.h" diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.h b/drivers/net/ethernet/intel/ice/ice_sriov.h index d1a998a4bef6..6c4fad09a527 100644 --- a/drivers/net/ethernet/intel/ice/ice_sriov.h +++ b/drivers/net/ethernet/intel/ice/ice_sriov.h @@ -3,9 +3,9 @@ #ifndef _ICE_SRIOV_H_ #define _ICE_SRIOV_H_ -#include "ice_virtchnl_fdir.h" +#include "virt/fdir.h" #include "ice_vf_lib.h" -#include "ice_virtchnl.h" +#include "virt/virtchnl.h" /* Static VF transaction/status register def */ #define VF_DEVICE_STATUS 0xAA diff --git a/drivers/net/ethernet/intel/ice/ice_trace.h b/drivers/net/ethernet/intel/ice/ice_trace.h index 07aab6e130cd..4f35ef8d6b29 100644 --- a/drivers/net/ethernet/intel/ice/ice_trace.h +++ b/drivers/net/ethernet/intel/ice/ice_trace.h @@ -130,7 +130,7 @@ DECLARE_EVENT_CLASS(ice_tx_template, __entry->buf = buf; __assign_str(devname);), - TP_printk("netdev: %s ring: %pK desc: %pK buf %pK", __get_str(devname), + TP_printk("netdev: %s ring: %p desc: %p buf %p", __get_str(devname), __entry->ring, __entry->desc, __entry->buf) ); @@ -158,7 +158,7 @@ DECLARE_EVENT_CLASS(ice_rx_template, __entry->desc = desc; __assign_str(devname);), - TP_printk("netdev: %s ring: %pK desc: %pK", __get_str(devname), + TP_printk("netdev: %s ring: %p desc: %p", __get_str(devname), __entry->ring, __entry->desc) ); DEFINE_EVENT(ice_rx_template, ice_clean_rx_irq, @@ -182,7 +182,7 @@ DECLARE_EVENT_CLASS(ice_rx_indicate_template, __entry->skb = skb; __assign_str(devname);), - TP_printk("netdev: %s ring: %pK desc: %pK skb %pK", __get_str(devname), + TP_printk("netdev: %s ring: %p desc: %p skb %p", __get_str(devname), __entry->ring, __entry->desc, __entry->skb) ); @@ -205,7 +205,7 @@ DECLARE_EVENT_CLASS(ice_xmit_template, __entry->skb = skb; __assign_str(devname);), - TP_printk("netdev: %s skb: %pK ring: %pK", __get_str(devname), + TP_printk("netdev: %s skb: %p ring: %p", __get_str(devname), __entry->skb, __entry->ring) ); @@ -228,7 +228,7 @@ DECLARE_EVENT_CLASS(ice_tx_tstamp_template, TP_fast_assign(__entry->skb = skb; __entry->idx = idx;), - TP_printk("skb %pK idx %d", + TP_printk("skb %p idx %d", __entry->skb, __entry->idx) ); #define DEFINE_TX_TSTAMP_OP_EVENT(name) \ diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index 41e7e29879a3..73f08d02f9c7 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -144,6 +144,56 @@ static struct netdev_queue *txring_txq(const struct ice_tx_ring *ring) } /** + * ice_clean_tstamp_ring - clean time stamp ring + * @tx_ring: Tx ring to clean the Time Stamp ring for + */ +static void ice_clean_tstamp_ring(struct ice_tx_ring *tx_ring) +{ + struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; + u32 size; + + if (!tstamp_ring->desc) + return; + + size = ALIGN(tstamp_ring->count * sizeof(struct ice_ts_desc), + PAGE_SIZE); + memset(tstamp_ring->desc, 0, size); + tstamp_ring->next_to_use = 0; +} + +/** + * ice_free_tstamp_ring - free time stamp resources per queue + * @tx_ring: Tx ring to free the Time Stamp ring for + */ +void ice_free_tstamp_ring(struct ice_tx_ring *tx_ring) +{ + struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; + u32 size; + + if (!tstamp_ring->desc) + return; + + ice_clean_tstamp_ring(tx_ring); + size = ALIGN(tstamp_ring->count * sizeof(struct ice_ts_desc), + PAGE_SIZE); + dmam_free_coherent(tx_ring->dev, size, tstamp_ring->desc, + tstamp_ring->dma); + tstamp_ring->desc = NULL; +} + +/** + * ice_free_tx_tstamp_ring - free time stamp resources per Tx ring + * @tx_ring: Tx ring to free the Time Stamp ring for + */ +void ice_free_tx_tstamp_ring(struct ice_tx_ring *tx_ring) +{ + ice_free_tstamp_ring(tx_ring); + kfree_rcu(tx_ring->tstamp_ring, rcu); + tx_ring->tstamp_ring = NULL; + tx_ring->flags &= ~ICE_TX_FLAGS_TXTIME; +} + +/** * ice_clean_tx_ring - Free any empty Tx buffers * @tx_ring: ring to be cleaned */ @@ -181,6 +231,9 @@ tx_skip_free: /* cleanup Tx queue statistics */ netdev_tx_reset_queue(txring_txq(tx_ring)); + + if (ice_is_txtime_cfg(tx_ring)) + ice_free_tx_tstamp_ring(tx_ring); } /** @@ -332,6 +385,84 @@ static bool ice_clean_tx_irq(struct ice_tx_ring *tx_ring, int napi_budget) } /** + * ice_alloc_tstamp_ring - allocate the Time Stamp ring + * @tx_ring: Tx ring to allocate the Time Stamp ring for + * + * Return: 0 on success, negative on error + */ +static int ice_alloc_tstamp_ring(struct ice_tx_ring *tx_ring) +{ + struct ice_tstamp_ring *tstamp_ring; + + /* allocate with kzalloc(), free with kfree_rcu() */ + tstamp_ring = kzalloc(sizeof(*tstamp_ring), GFP_KERNEL); + if (!tstamp_ring) + return -ENOMEM; + + tstamp_ring->tx_ring = tx_ring; + tx_ring->tstamp_ring = tstamp_ring; + tstamp_ring->desc = NULL; + tstamp_ring->count = ice_calc_ts_ring_count(tx_ring); + tx_ring->flags |= ICE_TX_FLAGS_TXTIME; + return 0; +} + +/** + * ice_setup_tstamp_ring - allocate the Time Stamp ring + * @tx_ring: Tx ring to set up the Time Stamp ring for + * + * Return: 0 on success, negative on error + */ +static int ice_setup_tstamp_ring(struct ice_tx_ring *tx_ring) +{ + struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; + struct device *dev = tx_ring->dev; + u32 size; + + /* round up to nearest page */ + size = ALIGN(tstamp_ring->count * sizeof(struct ice_ts_desc), + PAGE_SIZE); + tstamp_ring->desc = dmam_alloc_coherent(dev, size, &tstamp_ring->dma, + GFP_KERNEL); + if (!tstamp_ring->desc) { + dev_err(dev, "Unable to allocate memory for Time stamp Ring, size=%d\n", + size); + return -ENOMEM; + } + + tstamp_ring->next_to_use = 0; + return 0; +} + +/** + * ice_alloc_setup_tstamp_ring - Allocate and setup the Time Stamp ring + * @tx_ring: Tx ring to allocate and setup the Time Stamp ring for + * + * Return: 0 on success, negative on error + */ +int ice_alloc_setup_tstamp_ring(struct ice_tx_ring *tx_ring) +{ + struct device *dev = tx_ring->dev; + int err; + + err = ice_alloc_tstamp_ring(tx_ring); + if (err) { + dev_err(dev, "Unable to allocate Time stamp ring for Tx ring %d\n", + tx_ring->q_index); + return err; + } + + err = ice_setup_tstamp_ring(tx_ring); + if (err) { + dev_err(dev, "Unable to setup Time stamp ring for Tx ring %d\n", + tx_ring->q_index); + ice_free_tx_tstamp_ring(tx_ring); + return err; + } + return 0; +} + +/** * ice_setup_tx_ring - Allocate the Tx descriptors * @tx_ring: the Tx ring to set up * @@ -1031,10 +1162,9 @@ ice_build_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) skb_metadata_set(skb, metasize); if (unlikely(xdp_buff_has_frags(xdp))) - xdp_update_skb_shared_info(skb, nr_frags, - sinfo->xdp_frags_size, - nr_frags * xdp->frame_sz, - xdp_buff_is_frag_pfmemalloc(xdp)); + xdp_update_skb_frags_info(skb, nr_frags, sinfo->xdp_frags_size, + nr_frags * xdp->frame_sz, + xdp_buff_get_skb_flags(xdp)); return skb; } @@ -1111,10 +1241,10 @@ ice_construct_skb(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp) memcpy(&skinfo->frags[skinfo->nr_frags], &sinfo->frags[0], sizeof(skb_frag_t) * nr_frags); - xdp_update_skb_shared_info(skb, skinfo->nr_frags + nr_frags, - sinfo->xdp_frags_size, - nr_frags * xdp->frame_sz, - xdp_buff_is_frag_pfmemalloc(xdp)); + xdp_update_skb_frags_info(skb, skinfo->nr_frags + nr_frags, + sinfo->xdp_frags_size, + nr_frags * xdp->frame_sz, + xdp_buff_get_skb_flags(xdp)); } return skb; @@ -1823,10 +1953,46 @@ ice_tx_map(struct ice_tx_ring *tx_ring, struct ice_tx_buf *first, /* notify HW of packet */ kick = __netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount, netdev_xmit_more()); - if (kick) - /* notify HW of packet */ - writel(i, tx_ring->tail); + if (!kick) + return; + if (ice_is_txtime_cfg(tx_ring)) { + struct ice_tstamp_ring *tstamp_ring = tx_ring->tstamp_ring; + u32 tstamp_count = tstamp_ring->count; + u32 j = tstamp_ring->next_to_use; + struct ice_ts_desc *ts_desc; + struct timespec64 ts; + u32 tstamp; + + ts = ktime_to_timespec64(first->skb->tstamp); + tstamp = ts.tv_nsec >> ICE_TXTIME_CTX_RESOLUTION_128NS; + + ts_desc = ICE_TS_DESC(tstamp_ring, j); + ts_desc->tx_desc_idx_tstamp = ice_build_tstamp_desc(i, tstamp); + + j++; + if (j == tstamp_count) { + u32 fetch = tstamp_count - tx_ring->count; + + j = 0; + + /* To prevent an MDD, when wrapping the tstamp ring + * create additional TS descriptors equal to the number + * of the fetch TS descriptors value. HW will merge the + * TS descriptors with the same timestamp value into a + * single descriptor. + */ + for (; j < fetch; j++) { + ts_desc = ICE_TS_DESC(tstamp_ring, j); + ts_desc->tx_desc_idx_tstamp = + ice_build_tstamp_desc(i, tstamp); + } + } + tstamp_ring->next_to_use = j; + writel_relaxed(j, tstamp_ring->tail); + } else { + writel_relaxed(i, tx_ring->tail); + } return; dma_error: diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.h b/drivers/net/ethernet/intel/ice/ice_txrx.h index 2fd8e78178a2..841a07bfba54 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx.h @@ -310,6 +310,16 @@ enum ice_dynamic_itr { #define ICE_TX_LEGACY 1 /* descriptor ring, associated with a VSI */ +struct ice_tstamp_ring { + struct ice_tx_ring *tx_ring; /* Backreference to associated Tx ring */ + dma_addr_t dma; /* physical address of ring */ + struct rcu_head rcu; /* to avoid race on free */ + u8 __iomem *tail; + void *desc; + u16 next_to_use; + u16 count; +} ____cacheline_internodealigned_in_smp; + struct ice_rx_ring { /* CL1 - 1st cacheline starts here */ void *desc; /* Descriptor ring memory */ @@ -402,9 +412,11 @@ struct ice_tx_ring { spinlock_t tx_lock; u32 txq_teid; /* Added Tx queue TEID */ /* CL4 - 4th cacheline starts here */ + struct ice_tstamp_ring *tstamp_ring; #define ICE_TX_FLAGS_RING_XDP BIT(0) #define ICE_TX_FLAGS_RING_VLAN_L2TAG1 BIT(1) #define ICE_TX_FLAGS_RING_VLAN_L2TAG2 BIT(2) +#define ICE_TX_FLAGS_TXTIME BIT(3) u8 flags; u8 dcb_tc; /* Traffic class of ring */ u16 quanta_prof_id; @@ -500,6 +512,7 @@ void ice_clean_tx_ring(struct ice_tx_ring *tx_ring); void ice_clean_rx_ring(struct ice_rx_ring *rx_ring); int ice_setup_tx_ring(struct ice_tx_ring *tx_ring); int ice_setup_rx_ring(struct ice_rx_ring *rx_ring); +int ice_alloc_setup_tstamp_ring(struct ice_tx_ring *tx_ring); void ice_free_tx_ring(struct ice_tx_ring *tx_ring); void ice_free_rx_ring(struct ice_rx_ring *rx_ring); int ice_napi_poll(struct napi_struct *napi, int budget); @@ -508,4 +521,6 @@ ice_prgm_fdir_fltr(struct ice_vsi *vsi, struct ice_fltr_desc *fdir_desc, u8 *raw_packet); void ice_clean_ctrl_tx_irq(struct ice_tx_ring *tx_ring); void ice_clean_ctrl_rx_irq(struct ice_rx_ring *rx_ring); +void ice_free_tx_tstamp_ring(struct ice_tx_ring *tx_ring); +void ice_free_tstamp_ring(struct ice_tx_ring *tx_ring); #endif /* _ICE_TXRX_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h index 6cf32b404127..99717730f21a 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.h @@ -54,6 +54,20 @@ ice_build_ctob(u64 td_cmd, u64 td_offset, unsigned int size, u64 td_tag) } /** + * ice_build_tstamp_desc - build Tx time stamp descriptor + * @tx_desc: Tx LAN descriptor index + * @tstamp: time stamp + * + * Return: Tx time stamp descriptor + */ +static inline __le32 +ice_build_tstamp_desc(u16 tx_desc, u32 tstamp) +{ + return cpu_to_le32(FIELD_PREP(ICE_TXTIME_TX_DESC_IDX_M, tx_desc) | + FIELD_PREP(ICE_TXTIME_STAMP_M, tstamp)); +} + +/** * ice_get_vlan_tci - get VLAN TCI from Rx flex descriptor * @rx_desc: Rx 32b flex descriptor with RXDID=2 * diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 03c6c271865d..b0a1b67071c5 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -17,7 +17,7 @@ #include "ice_protocol_type.h" #include "ice_sbq_cmd.h" #include "ice_vlan_mode.h" -#include "ice_fwlog.h" +#include <linux/net/intel/libie/fwlog.h> #include <linux/wait.h> #include <net/dscp.h> @@ -293,8 +293,10 @@ struct ice_hw_common_caps { u8 dcb; u8 ieee_1588; u8 rdma; - u8 roce_lag; - u8 sriov_lag; + + bool roce_lag; + bool sriov_lag; + bool sriov_aa_lag; bool nvm_update_pending_nvm; bool nvm_update_pending_orom; @@ -946,9 +948,7 @@ struct ice_hw { u8 fw_patch; /* firmware patch version */ u32 fw_build; /* firmware build number */ - struct ice_fwlog_cfg fwlog_cfg; - bool fwlog_supported; /* does hardware support FW logging? */ - struct ice_fwlog_ring fwlog_ring; + struct libie_fwlog fwlog; /* Device max aggregate bandwidths corresponding to the GL_PWR_MODE_CTL * register. Used for determining the ITR/INTRL granularity during diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c index 5ee74f3e82dc..de9e81ccee66 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c @@ -5,7 +5,7 @@ #include "ice.h" #include "ice_lib.h" #include "ice_fltr.h" -#include "ice_virtchnl_allowlist.h" +#include "virt/allowlist.h" /* Public functions which may be accessed by all driver files */ diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h index ffe1f9f830ea..b00708907176 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h @@ -13,7 +13,7 @@ #include <linux/avf/virtchnl.h> #include "ice_type.h" #include "ice_flow.h" -#include "ice_virtchnl_fdir.h" +#include "virt/fdir.h" #include "ice_vsi_vlan_ops.h" #define ICE_MAX_SRIOV_VFS 256 diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index a3a4eaa17739..575fd48f485f 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -19,52 +19,12 @@ static struct xdp_buff **ice_xdp_buf(struct ice_rx_ring *rx_ring, u32 idx) } /** - * ice_qp_reset_stats - Resets all stats for rings of given index - * @vsi: VSI that contains rings of interest - * @q_idx: ring index in array - */ -static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) -{ - struct ice_vsi_stats *vsi_stat; - struct ice_pf *pf; - - pf = vsi->back; - if (!pf->vsi_stats) - return; - - vsi_stat = pf->vsi_stats[vsi->idx]; - if (!vsi_stat) - return; - - memset(&vsi_stat->rx_ring_stats[q_idx]->rx_stats, 0, - sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); - memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, - sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); - if (vsi->xdp_rings) - memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, - sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); -} - -/** - * ice_qp_clean_rings - Cleans all the rings of a given index - * @vsi: VSI that contains rings of interest - * @q_idx: ring index in array - */ -static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) -{ - ice_clean_tx_ring(vsi->tx_rings[q_idx]); - if (vsi->xdp_rings) - ice_clean_tx_ring(vsi->xdp_rings[q_idx]); - ice_clean_rx_ring(vsi->rx_rings[q_idx]); -} - -/** * ice_qvec_toggle_napi - Enables/disables NAPI for a given q_vector * @vsi: VSI that has netdev * @q_vector: q_vector that has NAPI context * @enable: true for enable, false for disable */ -static void +void ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, bool enable) { @@ -83,7 +43,7 @@ ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, * @rx_ring: Rx ring that will have its IRQ disabled * @q_vector: queue vector */ -static void +void ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, struct ice_q_vector *q_vector) { @@ -113,7 +73,7 @@ ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, * @q_vector: queue vector * @qid: queue index */ -static void +void ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector, u16 qid) { u16 reg_idx = q_vector->reg_idx; @@ -143,7 +103,7 @@ ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector, u16 qid) * @vsi: the VSI that contains queue vector * @q_vector: queue vector */ -static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) +void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) { struct ice_pf *pf = vsi->back; struct ice_hw *hw = &pf->hw; @@ -154,111 +114,6 @@ static void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) } /** - * ice_qp_dis - Disables a queue pair - * @vsi: VSI of interest - * @q_idx: ring index in array - * - * Returns 0 on success, negative on failure. - */ -static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) -{ - struct ice_txq_meta txq_meta = { }; - struct ice_q_vector *q_vector; - struct ice_tx_ring *tx_ring; - struct ice_rx_ring *rx_ring; - int fail = 0; - int err; - - if (q_idx >= vsi->num_rxq || q_idx >= vsi->num_txq) - return -EINVAL; - - tx_ring = vsi->tx_rings[q_idx]; - rx_ring = vsi->rx_rings[q_idx]; - q_vector = rx_ring->q_vector; - - synchronize_net(); - netif_carrier_off(vsi->netdev); - netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); - - ice_qvec_dis_irq(vsi, rx_ring, q_vector); - ice_qvec_toggle_napi(vsi, q_vector, false); - - ice_fill_txq_meta(vsi, tx_ring, &txq_meta); - err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); - if (!fail) - fail = err; - if (vsi->xdp_rings) { - struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; - - memset(&txq_meta, 0, sizeof(txq_meta)); - ice_fill_txq_meta(vsi, xdp_ring, &txq_meta); - err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, xdp_ring, - &txq_meta); - if (!fail) - fail = err; - } - - ice_vsi_ctrl_one_rx_ring(vsi, false, q_idx, false); - ice_qp_clean_rings(vsi, q_idx); - ice_qp_reset_stats(vsi, q_idx); - - return fail; -} - -/** - * ice_qp_ena - Enables a queue pair - * @vsi: VSI of interest - * @q_idx: ring index in array - * - * Returns 0 on success, negative on failure. - */ -static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) -{ - struct ice_q_vector *q_vector; - int fail = 0; - bool link_up; - int err; - - err = ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx); - if (!fail) - fail = err; - - if (ice_is_xdp_ena_vsi(vsi)) { - struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; - - err = ice_vsi_cfg_single_txq(vsi, vsi->xdp_rings, q_idx); - if (!fail) - fail = err; - ice_set_ring_xdp(xdp_ring); - ice_tx_xsk_pool(vsi, q_idx); - } - - err = ice_vsi_cfg_single_rxq(vsi, q_idx); - if (!fail) - fail = err; - - q_vector = vsi->rx_rings[q_idx]->q_vector; - ice_qvec_cfg_msix(vsi, q_vector, q_idx); - - err = ice_vsi_ctrl_one_rx_ring(vsi, true, q_idx, true); - if (!fail) - fail = err; - - ice_qvec_toggle_napi(vsi, q_vector, true); - ice_qvec_ena_irq(vsi, q_vector); - - /* make sure NAPI sees updated ice_{t,x}_ring::xsk_pool */ - synchronize_net(); - ice_get_link_status(vsi->port_info, &link_up); - if (link_up) { - netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); - netif_carrier_on(vsi->netdev); - } - - return fail; -} - -/** * ice_xsk_pool_disable - disable a buffer pool region * @vsi: Current VSI * @qid: queue ID diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.h b/drivers/net/ethernet/intel/ice/ice_xsk.h index 8dc5d55e26c5..600cbeeaa203 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.h +++ b/drivers/net/ethernet/intel/ice/ice_xsk.h @@ -23,6 +23,13 @@ void ice_xsk_clean_rx_ring(struct ice_rx_ring *rx_ring); void ice_xsk_clean_xdp_ring(struct ice_tx_ring *xdp_ring); bool ice_xmit_zc(struct ice_tx_ring *xdp_ring, struct xsk_buff_pool *xsk_pool); int ice_realloc_zc_buf(struct ice_vsi *vsi, bool zc); +void ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector, + u16 qid); +void ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, + bool enable); +void ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector); +void ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, + struct ice_q_vector *q_vector); #else static inline bool ice_xmit_zc(struct ice_tx_ring __always_unused *xdp_ring, struct xsk_buff_pool __always_unused *xsk_pool) @@ -75,5 +82,20 @@ ice_realloc_zc_buf(struct ice_vsi __always_unused *vsi, { return 0; } + +static inline void +ice_qvec_cfg_msix(struct ice_vsi *vsi, struct ice_q_vector *q_vector, + u16 qid) { } + +static inline void +ice_qvec_toggle_napi(struct ice_vsi *vsi, struct ice_q_vector *q_vector, + bool enable) { } + +static inline void +ice_qvec_ena_irq(struct ice_vsi *vsi, struct ice_q_vector *q_vector) { } + +static inline void +ice_qvec_dis_irq(struct ice_vsi *vsi, struct ice_rx_ring *rx_ring, + struct ice_q_vector *q_vector) { } #endif /* CONFIG_XDP_SOCKETS */ #endif /* !_ICE_XSK_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.c b/drivers/net/ethernet/intel/ice/virt/allowlist.c index 4c2ec2337b38..a07efec19c45 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.c +++ b/drivers/net/ethernet/intel/ice/virt/allowlist.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2021, Intel Corporation. */ -#include "ice_virtchnl_allowlist.h" +#include "allowlist.h" /* Purpose of this file is to share functionality to allowlist or denylist * opcodes used in PF <-> VF communication. Group of opcodes: diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.h b/drivers/net/ethernet/intel/ice/virt/allowlist.h index d3ae86ded219..d3ae86ded219 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_allowlist.h +++ b/drivers/net/ethernet/intel/ice/virt/allowlist.h diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c b/drivers/net/ethernet/intel/ice/virt/fdir.c index ae83c3914e29..ae83c3914e29 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c +++ b/drivers/net/ethernet/intel/ice/virt/fdir.c diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.h b/drivers/net/ethernet/intel/ice/virt/fdir.h index ac6dcab454b4..ac6dcab454b4 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.h +++ b/drivers/net/ethernet/intel/ice/virt/fdir.h diff --git a/drivers/net/ethernet/intel/ice/virt/queues.c b/drivers/net/ethernet/intel/ice/virt/queues.c new file mode 100644 index 000000000000..370f6ec2a374 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/virt/queues.c @@ -0,0 +1,973 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022, Intel Corporation. */ + +#include "virtchnl.h" +#include "queues.h" +#include "ice_vf_lib_private.h" +#include "ice.h" +#include "ice_base.h" +#include "ice_lib.h" + +/** + * ice_vc_get_max_frame_size - get max frame size allowed for VF + * @vf: VF used to determine max frame size + * + * Max frame size is determined based on the current port's max frame size and + * whether a port VLAN is configured on this VF. The VF is not aware whether + * it's in a port VLAN so the PF needs to account for this in max frame size + * checks and sending the max frame size to the VF. + */ +u16 ice_vc_get_max_frame_size(struct ice_vf *vf) +{ + struct ice_port_info *pi = ice_vf_get_port_info(vf); + u16 max_frame_size; + + max_frame_size = pi->phy.link_info.max_frame_size; + + if (ice_vf_is_port_vlan_ena(vf)) + max_frame_size -= VLAN_HLEN; + + return max_frame_size; +} + +/** + * ice_vc_isvalid_q_id + * @vsi: VSI to check queue ID against + * @qid: VSI relative queue ID + * + * check for the valid queue ID + */ +static bool ice_vc_isvalid_q_id(struct ice_vsi *vsi, u16 qid) +{ + /* allocated Tx and Rx queues should be always equal for VF VSI */ + return qid < vsi->alloc_txq; +} + +/** + * ice_vc_isvalid_ring_len + * @ring_len: length of ring + * + * check for the valid ring count, should be multiple of ICE_REQ_DESC_MULTIPLE + * or zero + */ +static bool ice_vc_isvalid_ring_len(u16 ring_len) +{ + return ring_len == 0 || + (ring_len >= ICE_MIN_NUM_DESC && + ring_len <= ICE_MAX_NUM_DESC_E810 && + !(ring_len % ICE_REQ_DESC_MULTIPLE)); +} + +/** + * ice_vf_cfg_qs_bw - Configure per queue bandwidth + * @vf: pointer to the VF info + * @num_queues: number of queues to be configured + * + * Configure per queue bandwidth. + * + * Return: 0 on success or negative error value. + */ +static int ice_vf_cfg_qs_bw(struct ice_vf *vf, u16 num_queues) +{ + struct ice_hw *hw = &vf->pf->hw; + struct ice_vsi *vsi; + int ret; + u16 i; + + vsi = ice_get_vf_vsi(vf); + if (!vsi) + return -EINVAL; + + for (i = 0; i < num_queues; i++) { + u32 p_rate, min_rate; + u8 tc; + + p_rate = vf->qs_bw[i].peak; + min_rate = vf->qs_bw[i].committed; + tc = vf->qs_bw[i].tc; + if (p_rate) + ret = ice_cfg_q_bw_lmt(hw->port_info, vsi->idx, tc, + vf->qs_bw[i].queue_id, + ICE_MAX_BW, p_rate); + else + ret = ice_cfg_q_bw_dflt_lmt(hw->port_info, vsi->idx, tc, + vf->qs_bw[i].queue_id, + ICE_MAX_BW); + if (ret) + return ret; + + if (min_rate) + ret = ice_cfg_q_bw_lmt(hw->port_info, vsi->idx, tc, + vf->qs_bw[i].queue_id, + ICE_MIN_BW, min_rate); + else + ret = ice_cfg_q_bw_dflt_lmt(hw->port_info, vsi->idx, tc, + vf->qs_bw[i].queue_id, + ICE_MIN_BW); + + if (ret) + return ret; + } + + return 0; +} + +/** + * ice_vf_cfg_q_quanta_profile - Configure quanta profile + * @vf: pointer to the VF info + * @quanta_prof_idx: pointer to the quanta profile index + * @quanta_size: quanta size to be set + * + * This function chooses available quanta profile and configures the register. + * The quanta profile is evenly divided by the number of device ports, and then + * available to the specific PF and VFs. The first profile for each PF is a + * reserved default profile. Only quanta size of the rest unused profile can be + * modified. + * + * Return: 0 on success or negative error value. + */ +static int ice_vf_cfg_q_quanta_profile(struct ice_vf *vf, u16 quanta_size, + u16 *quanta_prof_idx) +{ + const u16 n_desc = calc_quanta_desc(quanta_size); + struct ice_hw *hw = &vf->pf->hw; + const u16 n_cmd = 2 * n_desc; + struct ice_pf *pf = vf->pf; + u16 per_pf, begin_id; + u8 n_used; + u32 reg; + + begin_id = (GLCOMM_QUANTA_PROF_MAX_INDEX + 1) / hw->dev_caps.num_funcs * + hw->logical_pf_id; + + if (quanta_size == ICE_DFLT_QUANTA) { + *quanta_prof_idx = begin_id; + } else { + per_pf = (GLCOMM_QUANTA_PROF_MAX_INDEX + 1) / + hw->dev_caps.num_funcs; + n_used = pf->num_quanta_prof_used; + if (n_used < per_pf) { + *quanta_prof_idx = begin_id + 1 + n_used; + pf->num_quanta_prof_used++; + } else { + return -EINVAL; + } + } + + reg = FIELD_PREP(GLCOMM_QUANTA_PROF_QUANTA_SIZE_M, quanta_size) | + FIELD_PREP(GLCOMM_QUANTA_PROF_MAX_CMD_M, n_cmd) | + FIELD_PREP(GLCOMM_QUANTA_PROF_MAX_DESC_M, n_desc); + wr32(hw, GLCOMM_QUANTA_PROF(*quanta_prof_idx), reg); + + return 0; +} + +/** + * ice_vc_validate_vqs_bitmaps - validate Rx/Tx queue bitmaps from VIRTCHNL + * @vqs: virtchnl_queue_select structure containing bitmaps to validate + * + * Return true on successful validation, else false + */ +static bool ice_vc_validate_vqs_bitmaps(struct virtchnl_queue_select *vqs) +{ + if ((!vqs->rx_queues && !vqs->tx_queues) || + vqs->rx_queues >= BIT(ICE_MAX_RSS_QS_PER_VF) || + vqs->tx_queues >= BIT(ICE_MAX_RSS_QS_PER_VF)) + return false; + + return true; +} + +/** + * ice_vf_ena_txq_interrupt - enable Tx queue interrupt via QINT_TQCTL + * @vsi: VSI of the VF to configure + * @q_idx: VF queue index used to determine the queue in the PF's space + */ +void ice_vf_ena_txq_interrupt(struct ice_vsi *vsi, u32 q_idx) +{ + struct ice_hw *hw = &vsi->back->hw; + u32 pfq = vsi->txq_map[q_idx]; + u32 reg; + + reg = rd32(hw, QINT_TQCTL(pfq)); + + /* MSI-X index 0 in the VF's space is always for the OICR, which means + * this is most likely a poll mode VF driver, so don't enable an + * interrupt that was never configured via VIRTCHNL_OP_CONFIG_IRQ_MAP + */ + if (!(reg & QINT_TQCTL_MSIX_INDX_M)) + return; + + wr32(hw, QINT_TQCTL(pfq), reg | QINT_TQCTL_CAUSE_ENA_M); +} + +/** + * ice_vf_ena_rxq_interrupt - enable Tx queue interrupt via QINT_RQCTL + * @vsi: VSI of the VF to configure + * @q_idx: VF queue index used to determine the queue in the PF's space + */ +void ice_vf_ena_rxq_interrupt(struct ice_vsi *vsi, u32 q_idx) +{ + struct ice_hw *hw = &vsi->back->hw; + u32 pfq = vsi->rxq_map[q_idx]; + u32 reg; + + reg = rd32(hw, QINT_RQCTL(pfq)); + + /* MSI-X index 0 in the VF's space is always for the OICR, which means + * this is most likely a poll mode VF driver, so don't enable an + * interrupt that was never configured via VIRTCHNL_OP_CONFIG_IRQ_MAP + */ + if (!(reg & QINT_RQCTL_MSIX_INDX_M)) + return; + + wr32(hw, QINT_RQCTL(pfq), reg | QINT_RQCTL_CAUSE_ENA_M); +} + +/** + * ice_vc_ena_qs_msg + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * called from the VF to enable all or specific queue(s) + */ +int ice_vc_ena_qs_msg(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_queue_select *vqs = + (struct virtchnl_queue_select *)msg; + struct ice_vsi *vsi; + unsigned long q_map; + u16 vf_q_id; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_validate_vqs_bitmaps(vqs)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* Enable only Rx rings, Tx rings were enabled by the FW when the + * Tx queue group list was configured and the context bits were + * programmed using ice_vsi_cfg_txqs + */ + q_map = vqs->rx_queues; + for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { + if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* Skip queue if enabled */ + if (test_bit(vf_q_id, vf->rxq_ena)) + continue; + + if (ice_vsi_ctrl_one_rx_ring(vsi, true, vf_q_id, true)) { + dev_err(ice_pf_to_dev(vsi->back), "Failed to enable Rx ring %d on VSI %d\n", + vf_q_id, vsi->vsi_num); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + ice_vf_ena_rxq_interrupt(vsi, vf_q_id); + set_bit(vf_q_id, vf->rxq_ena); + } + + q_map = vqs->tx_queues; + for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { + if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* Skip queue if enabled */ + if (test_bit(vf_q_id, vf->txq_ena)) + continue; + + ice_vf_ena_txq_interrupt(vsi, vf_q_id); + set_bit(vf_q_id, vf->txq_ena); + } + + /* Set flag to indicate that queues are enabled */ + if (v_ret == VIRTCHNL_STATUS_SUCCESS) + set_bit(ICE_VF_STATE_QS_ENA, vf->vf_states); + +error_param: + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_ENABLE_QUEUES, v_ret, + NULL, 0); +} + +/** + * ice_vf_vsi_dis_single_txq - disable a single Tx queue + * @vf: VF to disable queue for + * @vsi: VSI for the VF + * @q_id: VF relative (0-based) queue ID + * + * Attempt to disable the Tx queue passed in. If the Tx queue was successfully + * disabled then clear q_id bit in the enabled queues bitmap and return + * success. Otherwise return error. + */ +int ice_vf_vsi_dis_single_txq(struct ice_vf *vf, struct ice_vsi *vsi, u16 q_id) +{ + struct ice_txq_meta txq_meta = { 0 }; + struct ice_tx_ring *ring; + int err; + + if (!test_bit(q_id, vf->txq_ena)) + dev_dbg(ice_pf_to_dev(vsi->back), "Queue %u on VSI %u is not enabled, but stopping it anyway\n", + q_id, vsi->vsi_num); + + ring = vsi->tx_rings[q_id]; + if (!ring) + return -EINVAL; + + ice_fill_txq_meta(vsi, ring, &txq_meta); + + err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, vf->vf_id, ring, &txq_meta); + if (err) { + dev_err(ice_pf_to_dev(vsi->back), "Failed to stop Tx ring %d on VSI %d\n", + q_id, vsi->vsi_num); + return err; + } + + /* Clear enabled queues flag */ + clear_bit(q_id, vf->txq_ena); + + return 0; +} + +/** + * ice_vc_dis_qs_msg + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * called from the VF to disable all or specific queue(s) + */ +int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_queue_select *vqs = + (struct virtchnl_queue_select *)msg; + struct ice_vsi *vsi; + unsigned long q_map; + u16 vf_q_id; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states) && + !test_bit(ICE_VF_STATE_QS_ENA, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_validate_vqs_bitmaps(vqs)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (vqs->tx_queues) { + q_map = vqs->tx_queues; + + for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { + if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (ice_vf_vsi_dis_single_txq(vf, vsi, vf_q_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + } + } + + q_map = vqs->rx_queues; + /* speed up Rx queue disable by batching them if possible */ + if (q_map && + bitmap_equal(&q_map, vf->rxq_ena, ICE_MAX_RSS_QS_PER_VF)) { + if (ice_vsi_stop_all_rx_rings(vsi)) { + dev_err(ice_pf_to_dev(vsi->back), "Failed to stop all Rx rings on VSI %d\n", + vsi->vsi_num); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + bitmap_zero(vf->rxq_ena, ICE_MAX_RSS_QS_PER_VF); + } else if (q_map) { + for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { + if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* Skip queue if not enabled */ + if (!test_bit(vf_q_id, vf->rxq_ena)) + continue; + + if (ice_vsi_ctrl_one_rx_ring(vsi, false, vf_q_id, + true)) { + dev_err(ice_pf_to_dev(vsi->back), "Failed to stop Rx ring %d on VSI %d\n", + vf_q_id, vsi->vsi_num); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* Clear enabled queues flag */ + clear_bit(vf_q_id, vf->rxq_ena); + } + } + + /* Clear enabled queues flag */ + if (v_ret == VIRTCHNL_STATUS_SUCCESS && ice_vf_has_no_qs_ena(vf)) + clear_bit(ICE_VF_STATE_QS_ENA, vf->vf_states); + +error_param: + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_DISABLE_QUEUES, v_ret, + NULL, 0); +} + +/** + * ice_cfg_interrupt + * @vf: pointer to the VF info + * @vsi: the VSI being configured + * @map: vector map for mapping vectors to queues + * @q_vector: structure for interrupt vector + * configure the IRQ to queue map + */ +static enum virtchnl_status_code +ice_cfg_interrupt(struct ice_vf *vf, struct ice_vsi *vsi, + struct virtchnl_vector_map *map, + struct ice_q_vector *q_vector) +{ + u16 vsi_q_id, vsi_q_id_idx; + unsigned long qmap; + + q_vector->num_ring_rx = 0; + q_vector->num_ring_tx = 0; + + qmap = map->rxq_map; + for_each_set_bit(vsi_q_id_idx, &qmap, ICE_MAX_RSS_QS_PER_VF) { + vsi_q_id = vsi_q_id_idx; + + if (!ice_vc_isvalid_q_id(vsi, vsi_q_id)) + return VIRTCHNL_STATUS_ERR_PARAM; + + q_vector->num_ring_rx++; + q_vector->rx.itr_idx = map->rxitr_idx; + vsi->rx_rings[vsi_q_id]->q_vector = q_vector; + ice_cfg_rxq_interrupt(vsi, vsi_q_id, + q_vector->vf_reg_idx, + q_vector->rx.itr_idx); + } + + qmap = map->txq_map; + for_each_set_bit(vsi_q_id_idx, &qmap, ICE_MAX_RSS_QS_PER_VF) { + vsi_q_id = vsi_q_id_idx; + + if (!ice_vc_isvalid_q_id(vsi, vsi_q_id)) + return VIRTCHNL_STATUS_ERR_PARAM; + + q_vector->num_ring_tx++; + q_vector->tx.itr_idx = map->txitr_idx; + vsi->tx_rings[vsi_q_id]->q_vector = q_vector; + ice_cfg_txq_interrupt(vsi, vsi_q_id, + q_vector->vf_reg_idx, + q_vector->tx.itr_idx); + } + + return VIRTCHNL_STATUS_SUCCESS; +} + +/** + * ice_vc_cfg_irq_map_msg + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * called from the VF to configure the IRQ to queue map + */ +int ice_vc_cfg_irq_map_msg(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + u16 num_q_vectors_mapped, vsi_id, vector_id; + struct virtchnl_irq_map_info *irqmap_info; + struct virtchnl_vector_map *map; + struct ice_vsi *vsi; + int i; + + irqmap_info = (struct virtchnl_irq_map_info *)msg; + num_q_vectors_mapped = irqmap_info->num_vectors; + + /* Check to make sure number of VF vectors mapped is not greater than + * number of VF vectors originally allocated, and check that + * there is actually at least a single VF queue vector mapped + */ + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states) || + vf->num_msix < num_q_vectors_mapped || + !num_q_vectors_mapped) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + for (i = 0; i < num_q_vectors_mapped; i++) { + struct ice_q_vector *q_vector; + + map = &irqmap_info->vecmap[i]; + + vector_id = map->vector_id; + vsi_id = map->vsi_id; + /* vector_id is always 0-based for each VF, and can never be + * larger than or equal to the max allowed interrupts per VF + */ + if (!(vector_id < vf->num_msix) || + !ice_vc_isvalid_vsi_id(vf, vsi_id) || + (!vector_id && (map->rxq_map || map->txq_map))) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* No need to map VF miscellaneous or rogue vector */ + if (!vector_id) + continue; + + /* Subtract non queue vector from vector_id passed by VF + * to get actual number of VSI queue vector array index + */ + q_vector = vsi->q_vectors[vector_id - ICE_NONQ_VECS_VF]; + if (!q_vector) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + /* lookout for the invalid queue index */ + v_ret = ice_cfg_interrupt(vf, vsi, map, q_vector); + if (v_ret) + goto error_param; + } + +error_param: + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_IRQ_MAP, v_ret, + NULL, 0); +} + +/** + * ice_vc_cfg_q_bw - Configure per queue bandwidth + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer which holds the command descriptor + * + * Configure VF queues bandwidth. + * + * Return: 0 on success or negative error value. + */ +int ice_vc_cfg_q_bw(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_queues_bw_cfg *qbw = + (struct virtchnl_queues_bw_cfg *)msg; + struct ice_vsi *vsi; + u16 i; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states) || + !ice_vc_isvalid_vsi_id(vf, qbw->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + if (qbw->num_queues > ICE_MAX_RSS_QS_PER_VF || + qbw->num_queues > min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)) { + dev_err(ice_pf_to_dev(vf->pf), "VF-%d trying to configure more than allocated number of queues: %d\n", + vf->vf_id, min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + for (i = 0; i < qbw->num_queues; i++) { + if (qbw->cfg[i].shaper.peak != 0 && vf->max_tx_rate != 0 && + qbw->cfg[i].shaper.peak > vf->max_tx_rate) { + dev_warn(ice_pf_to_dev(vf->pf), "The maximum queue %d rate limit configuration may not take effect because the maximum TX rate for VF-%d is %d\n", + qbw->cfg[i].queue_id, vf->vf_id, + vf->max_tx_rate); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + if (qbw->cfg[i].shaper.committed != 0 && vf->min_tx_rate != 0 && + qbw->cfg[i].shaper.committed < vf->min_tx_rate) { + dev_warn(ice_pf_to_dev(vf->pf), "The minimum queue %d rate limit configuration may not take effect because the minimum TX rate for VF-%d is %d\n", + qbw->cfg[i].queue_id, vf->vf_id, + vf->min_tx_rate); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + if (qbw->cfg[i].queue_id > vf->num_vf_qs) { + dev_warn(ice_pf_to_dev(vf->pf), "VF-%d trying to configure invalid queue_id\n", + vf->vf_id); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + if (qbw->cfg[i].tc >= ICE_MAX_TRAFFIC_CLASS) { + dev_warn(ice_pf_to_dev(vf->pf), "VF-%d trying to configure a traffic class higher than allowed\n", + vf->vf_id); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + } + + for (i = 0; i < qbw->num_queues; i++) { + vf->qs_bw[i].queue_id = qbw->cfg[i].queue_id; + vf->qs_bw[i].peak = qbw->cfg[i].shaper.peak; + vf->qs_bw[i].committed = qbw->cfg[i].shaper.committed; + vf->qs_bw[i].tc = qbw->cfg[i].tc; + } + + if (ice_vf_cfg_qs_bw(vf, qbw->num_queues)) + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + +err: + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_QUEUE_BW, + v_ret, NULL, 0); +} + +/** + * ice_vc_cfg_q_quanta - Configure per queue quanta + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer which holds the command descriptor + * + * Configure VF queues quanta. + * + * Return: 0 on success or negative error value. + */ +int ice_vc_cfg_q_quanta(struct ice_vf *vf, u8 *msg) +{ + u16 quanta_prof_id, quanta_size, start_qid, num_queues, end_qid, i; + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_quanta_cfg *qquanta = + (struct virtchnl_quanta_cfg *)msg; + struct ice_vsi *vsi; + int ret; + + start_qid = qquanta->queue_select.start_queue_id; + num_queues = qquanta->queue_select.num_queues; + + if (check_add_overflow(start_qid, num_queues, &end_qid)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + if (end_qid > ICE_MAX_RSS_QS_PER_VF || + end_qid > min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)) { + dev_err(ice_pf_to_dev(vf->pf), "VF-%d trying to configure more than allocated number of queues: %d\n", + vf->vf_id, min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + quanta_size = qquanta->quanta_size; + if (quanta_size > ICE_MAX_QUANTA_SIZE || + quanta_size < ICE_MIN_QUANTA_SIZE) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + if (quanta_size % 64) { + dev_err(ice_pf_to_dev(vf->pf), "quanta size should be the product of 64\n"); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + ret = ice_vf_cfg_q_quanta_profile(vf, quanta_size, + &quanta_prof_id); + if (ret) { + v_ret = VIRTCHNL_STATUS_ERR_NOT_SUPPORTED; + goto err; + } + + for (i = start_qid; i < end_qid; i++) + vsi->tx_rings[i]->quanta_prof_id = quanta_prof_id; + +err: + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_QUANTA, + v_ret, NULL, 0); +} + +/** + * ice_vc_cfg_qs_msg + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * called from the VF to configure the Rx/Tx queues + */ +int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) +{ + struct virtchnl_vsi_queue_config_info *qci = + (struct virtchnl_vsi_queue_config_info *)msg; + struct virtchnl_queue_pair_info *qpi; + struct ice_pf *pf = vf->pf; + struct ice_vsi *vsi; + int i = -1, q_idx; + bool ena_ts; + u8 act_prt; + + mutex_lock(&pf->lag_mutex); + act_prt = ice_lag_prepare_vf_reset(pf->lag); + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) + goto error_param; + + if (!ice_vc_isvalid_vsi_id(vf, qci->vsi_id)) + goto error_param; + + vsi = ice_get_vf_vsi(vf); + if (!vsi) + goto error_param; + + if (qci->num_queue_pairs > ICE_MAX_RSS_QS_PER_VF || + qci->num_queue_pairs > min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)) { + dev_err(ice_pf_to_dev(pf), "VF-%d requesting more than supported number of queues: %d\n", + vf->vf_id, min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)); + goto error_param; + } + + for (i = 0; i < qci->num_queue_pairs; i++) { + if (!qci->qpair[i].rxq.crc_disable) + continue; + + if (!(vf->driver_caps & VIRTCHNL_VF_OFFLOAD_CRC) || + vf->vlan_strip_ena) + goto error_param; + } + + for (i = 0; i < qci->num_queue_pairs; i++) { + qpi = &qci->qpair[i]; + if (qpi->txq.vsi_id != qci->vsi_id || + qpi->rxq.vsi_id != qci->vsi_id || + qpi->rxq.queue_id != qpi->txq.queue_id || + qpi->txq.headwb_enabled || + !ice_vc_isvalid_ring_len(qpi->txq.ring_len) || + !ice_vc_isvalid_ring_len(qpi->rxq.ring_len) || + !ice_vc_isvalid_q_id(vsi, qpi->txq.queue_id)) { + goto error_param; + } + + q_idx = qpi->rxq.queue_id; + + /* make sure selected "q_idx" is in valid range of queues + * for selected "vsi" + */ + if (q_idx >= vsi->alloc_txq || q_idx >= vsi->alloc_rxq) { + goto error_param; + } + + /* copy Tx queue info from VF into VSI */ + if (qpi->txq.ring_len > 0) { + vsi->tx_rings[q_idx]->dma = qpi->txq.dma_ring_addr; + vsi->tx_rings[q_idx]->count = qpi->txq.ring_len; + + /* Disable any existing queue first */ + if (ice_vf_vsi_dis_single_txq(vf, vsi, q_idx)) + goto error_param; + + /* Configure a queue with the requested settings */ + if (ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx)) { + dev_warn(ice_pf_to_dev(pf), "VF-%d failed to configure TX queue %d\n", + vf->vf_id, q_idx); + goto error_param; + } + } + + /* copy Rx queue info from VF into VSI */ + if (qpi->rxq.ring_len > 0) { + u16 max_frame_size = ice_vc_get_max_frame_size(vf); + struct ice_rx_ring *ring = vsi->rx_rings[q_idx]; + u32 rxdid; + + ring->dma = qpi->rxq.dma_ring_addr; + ring->count = qpi->rxq.ring_len; + + if (qpi->rxq.crc_disable) + ring->flags |= ICE_RX_FLAGS_CRC_STRIP_DIS; + else + ring->flags &= ~ICE_RX_FLAGS_CRC_STRIP_DIS; + + if (qpi->rxq.databuffer_size != 0 && + (qpi->rxq.databuffer_size > ((16 * 1024) - 128) || + qpi->rxq.databuffer_size < 1024)) + goto error_param; + ring->rx_buf_len = qpi->rxq.databuffer_size; + if (qpi->rxq.max_pkt_size > max_frame_size || + qpi->rxq.max_pkt_size < 64) + goto error_param; + + ring->max_frame = qpi->rxq.max_pkt_size; + /* add space for the port VLAN since the VF driver is + * not expected to account for it in the MTU + * calculation + */ + if (ice_vf_is_port_vlan_ena(vf)) + ring->max_frame += VLAN_HLEN; + + if (ice_vsi_cfg_single_rxq(vsi, q_idx)) { + dev_warn(ice_pf_to_dev(pf), "VF-%d failed to configure RX queue %d\n", + vf->vf_id, q_idx); + goto error_param; + } + + /* If Rx flex desc is supported, select RXDID for Rx + * queues. Otherwise, use legacy 32byte descriptor + * format. Legacy 16byte descriptor is not supported. + * If this RXDID is selected, return error. + */ + if (vf->driver_caps & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { + rxdid = qpi->rxq.rxdid; + if (!(BIT(rxdid) & pf->supported_rxdids)) + goto error_param; + } else { + rxdid = ICE_RXDID_LEGACY_1; + } + + ena_ts = ((vf->driver_caps & + VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) && + (vf->driver_caps & VIRTCHNL_VF_CAP_PTP) && + (qpi->rxq.flags & VIRTCHNL_PTP_RX_TSTAMP)); + + ice_write_qrxflxp_cntxt(&vsi->back->hw, + vsi->rxq_map[q_idx], rxdid, + ICE_RXDID_PRIO, ena_ts); + } + } + + ice_lag_complete_vf_reset(pf->lag, act_prt); + mutex_unlock(&pf->lag_mutex); + + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, + VIRTCHNL_STATUS_SUCCESS, NULL, 0); +error_param: + /* disable whatever we can */ + for (; i >= 0; i--) { + if (ice_vsi_ctrl_one_rx_ring(vsi, false, i, true)) + dev_err(ice_pf_to_dev(pf), "VF-%d could not disable RX queue %d\n", + vf->vf_id, i); + if (ice_vf_vsi_dis_single_txq(vf, vsi, i)) + dev_err(ice_pf_to_dev(pf), "VF-%d could not disable TX queue %d\n", + vf->vf_id, i); + } + + ice_lag_complete_vf_reset(pf->lag, act_prt); + mutex_unlock(&pf->lag_mutex); + + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, + VIRTCHNL_STATUS_ERR_PARAM, NULL, 0); +} + +/** + * ice_vc_request_qs_msg + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * VFs get a default number of queues but can use this message to request a + * different number. If the request is successful, PF will reset the VF and + * return 0. If unsuccessful, PF will send message informing VF of number of + * available queue pairs via virtchnl message response to VF. + */ +int ice_vc_request_qs_msg(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_vf_res_request *vfres = + (struct virtchnl_vf_res_request *)msg; + u16 req_queues = vfres->num_queue_pairs; + struct ice_pf *pf = vf->pf; + u16 max_allowed_vf_queues; + u16 tx_rx_queue_left; + struct device *dev; + u16 cur_queues; + + dev = ice_pf_to_dev(pf); + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + cur_queues = vf->num_vf_qs; + tx_rx_queue_left = min_t(u16, ice_get_avail_txq_count(pf), + ice_get_avail_rxq_count(pf)); + max_allowed_vf_queues = tx_rx_queue_left + cur_queues; + if (!req_queues) { + dev_err(dev, "VF %d tried to request 0 queues. Ignoring.\n", + vf->vf_id); + } else if (req_queues > ICE_MAX_RSS_QS_PER_VF) { + dev_err(dev, "VF %d tried to request more than %d queues.\n", + vf->vf_id, ICE_MAX_RSS_QS_PER_VF); + vfres->num_queue_pairs = ICE_MAX_RSS_QS_PER_VF; + } else if (req_queues > cur_queues && + req_queues - cur_queues > tx_rx_queue_left) { + dev_warn(dev, "VF %d requested %u more queues, but only %u left.\n", + vf->vf_id, req_queues - cur_queues, tx_rx_queue_left); + vfres->num_queue_pairs = min_t(u16, max_allowed_vf_queues, + ICE_MAX_RSS_QS_PER_VF); + } else { + /* request is successful, then reset VF */ + vf->num_req_qs = req_queues; + ice_reset_vf(vf, ICE_VF_RESET_NOTIFY); + dev_info(dev, "VF %d granted request of %u queues.\n", + vf->vf_id, req_queues); + return 0; + } + +error_param: + /* send the response to the VF */ + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_REQUEST_QUEUES, + v_ret, (u8 *)vfres, sizeof(*vfres)); +} + diff --git a/drivers/net/ethernet/intel/ice/virt/queues.h b/drivers/net/ethernet/intel/ice/virt/queues.h new file mode 100644 index 000000000000..c4a792cecea1 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/virt/queues.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2022, Intel Corporation. */ + +#ifndef _ICE_VIRT_QUEUES_H_ +#define _ICE_VIRT_QUEUES_H_ + +#include <linux/types.h> + +struct ice_vf; + +u16 ice_vc_get_max_frame_size(struct ice_vf *vf); +int ice_vc_ena_qs_msg(struct ice_vf *vf, u8 *msg); +int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg); +int ice_vc_cfg_irq_map_msg(struct ice_vf *vf, u8 *msg); +int ice_vc_cfg_q_bw(struct ice_vf *vf, u8 *msg); +int ice_vc_cfg_q_quanta(struct ice_vf *vf, u8 *msg); +int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg); +int ice_vc_request_qs_msg(struct ice_vf *vf, u8 *msg); + +#endif /* _ICE_VIRT_QUEUES_H_ */ diff --git a/drivers/net/ethernet/intel/ice/virt/rss.c b/drivers/net/ethernet/intel/ice/virt/rss.c new file mode 100644 index 000000000000..cbdbb32d512b --- /dev/null +++ b/drivers/net/ethernet/intel/ice/virt/rss.c @@ -0,0 +1,719 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2022, Intel Corporation. */ + +#include "rss.h" +#include "ice_vf_lib_private.h" +#include "ice.h" + +#define FIELD_SELECTOR(proto_hdr_field) \ + BIT((proto_hdr_field) & PROTO_HDR_FIELD_MASK) + +struct ice_vc_hdr_match_type { + u32 vc_hdr; /* virtchnl headers (VIRTCHNL_PROTO_HDR_XXX) */ + u32 ice_hdr; /* ice headers (ICE_FLOW_SEG_HDR_XXX) */ +}; + +static const struct ice_vc_hdr_match_type ice_vc_hdr_list[] = { + {VIRTCHNL_PROTO_HDR_NONE, ICE_FLOW_SEG_HDR_NONE}, + {VIRTCHNL_PROTO_HDR_ETH, ICE_FLOW_SEG_HDR_ETH}, + {VIRTCHNL_PROTO_HDR_S_VLAN, ICE_FLOW_SEG_HDR_VLAN}, + {VIRTCHNL_PROTO_HDR_C_VLAN, ICE_FLOW_SEG_HDR_VLAN}, + {VIRTCHNL_PROTO_HDR_IPV4, ICE_FLOW_SEG_HDR_IPV4 | + ICE_FLOW_SEG_HDR_IPV_OTHER}, + {VIRTCHNL_PROTO_HDR_IPV6, ICE_FLOW_SEG_HDR_IPV6 | + ICE_FLOW_SEG_HDR_IPV_OTHER}, + {VIRTCHNL_PROTO_HDR_TCP, ICE_FLOW_SEG_HDR_TCP}, + {VIRTCHNL_PROTO_HDR_UDP, ICE_FLOW_SEG_HDR_UDP}, + {VIRTCHNL_PROTO_HDR_SCTP, ICE_FLOW_SEG_HDR_SCTP}, + {VIRTCHNL_PROTO_HDR_PPPOE, ICE_FLOW_SEG_HDR_PPPOE}, + {VIRTCHNL_PROTO_HDR_GTPU_IP, ICE_FLOW_SEG_HDR_GTPU_IP}, + {VIRTCHNL_PROTO_HDR_GTPU_EH, ICE_FLOW_SEG_HDR_GTPU_EH}, + {VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_DWN, + ICE_FLOW_SEG_HDR_GTPU_DWN}, + {VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_UP, + ICE_FLOW_SEG_HDR_GTPU_UP}, + {VIRTCHNL_PROTO_HDR_L2TPV3, ICE_FLOW_SEG_HDR_L2TPV3}, + {VIRTCHNL_PROTO_HDR_ESP, ICE_FLOW_SEG_HDR_ESP}, + {VIRTCHNL_PROTO_HDR_AH, ICE_FLOW_SEG_HDR_AH}, + {VIRTCHNL_PROTO_HDR_PFCP, ICE_FLOW_SEG_HDR_PFCP_SESSION}, +}; + +struct ice_vc_hash_field_match_type { + u32 vc_hdr; /* virtchnl headers + * (VIRTCHNL_PROTO_HDR_XXX) + */ + u32 vc_hash_field; /* virtchnl hash fields selector + * FIELD_SELECTOR((VIRTCHNL_PROTO_HDR_ETH_XXX)) + */ + u64 ice_hash_field; /* ice hash fields + * (BIT_ULL(ICE_FLOW_FIELD_IDX_XXX)) + */ +}; + +static const struct +ice_vc_hash_field_match_type ice_vc_hash_field_list[] = { + {VIRTCHNL_PROTO_HDR_ETH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC), + BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)}, + {VIRTCHNL_PROTO_HDR_ETH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), + BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA)}, + {VIRTCHNL_PROTO_HDR_ETH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), + ICE_FLOW_HASH_ETH}, + {VIRTCHNL_PROTO_HDR_ETH, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_ETHERTYPE), + BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_TYPE)}, + {VIRTCHNL_PROTO_HDR_S_VLAN, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), + BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN)}, + {VIRTCHNL_PROTO_HDR_C_VLAN, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), + BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN)}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), + ICE_FLOW_HASH_IPV4}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), + ICE_FLOW_HASH_IPV4 | BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), + ICE_FLOW_HASH_IPV6}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), + ICE_FLOW_HASH_IPV6 | BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, + {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), + BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, + {VIRTCHNL_PROTO_HDR_TCP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT), + BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)}, + {VIRTCHNL_PROTO_HDR_TCP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), + BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)}, + {VIRTCHNL_PROTO_HDR_TCP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), + ICE_FLOW_HASH_TCP_PORT}, + {VIRTCHNL_PROTO_HDR_UDP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT), + BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)}, + {VIRTCHNL_PROTO_HDR_UDP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), + BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)}, + {VIRTCHNL_PROTO_HDR_UDP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), + ICE_FLOW_HASH_UDP_PORT}, + {VIRTCHNL_PROTO_HDR_SCTP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT), + BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)}, + {VIRTCHNL_PROTO_HDR_SCTP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), + BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)}, + {VIRTCHNL_PROTO_HDR_SCTP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), + ICE_FLOW_HASH_SCTP_PORT}, + {VIRTCHNL_PROTO_HDR_PPPOE, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PPPOE_SESS_ID), + BIT_ULL(ICE_FLOW_FIELD_IDX_PPPOE_SESS_ID)}, + {VIRTCHNL_PROTO_HDR_GTPU_IP, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_GTPU_IP_TEID), + BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID)}, + {VIRTCHNL_PROTO_HDR_L2TPV3, + FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), + BIT_ULL(ICE_FLOW_FIELD_IDX_L2TPV3_SESS_ID)}, + {VIRTCHNL_PROTO_HDR_ESP, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), + BIT_ULL(ICE_FLOW_FIELD_IDX_ESP_SPI)}, + {VIRTCHNL_PROTO_HDR_AH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), + BIT_ULL(ICE_FLOW_FIELD_IDX_AH_SPI)}, + {VIRTCHNL_PROTO_HDR_PFCP, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), + BIT_ULL(ICE_FLOW_FIELD_IDX_PFCP_SEID)}, +}; + +/** + * ice_vc_validate_pattern + * @vf: pointer to the VF info + * @proto: virtchnl protocol headers + * + * validate the pattern is supported or not. + * + * Return: true on success, false on error. + */ +bool +ice_vc_validate_pattern(struct ice_vf *vf, struct virtchnl_proto_hdrs *proto) +{ + bool is_ipv4 = false; + bool is_ipv6 = false; + bool is_udp = false; + u16 ptype = -1; + int i = 0; + + while (i < proto->count && + proto->proto_hdr[i].type != VIRTCHNL_PROTO_HDR_NONE) { + switch (proto->proto_hdr[i].type) { + case VIRTCHNL_PROTO_HDR_ETH: + ptype = ICE_PTYPE_MAC_PAY; + break; + case VIRTCHNL_PROTO_HDR_IPV4: + ptype = ICE_PTYPE_IPV4_PAY; + is_ipv4 = true; + break; + case VIRTCHNL_PROTO_HDR_IPV6: + ptype = ICE_PTYPE_IPV6_PAY; + is_ipv6 = true; + break; + case VIRTCHNL_PROTO_HDR_UDP: + if (is_ipv4) + ptype = ICE_PTYPE_IPV4_UDP_PAY; + else if (is_ipv6) + ptype = ICE_PTYPE_IPV6_UDP_PAY; + is_udp = true; + break; + case VIRTCHNL_PROTO_HDR_TCP: + if (is_ipv4) + ptype = ICE_PTYPE_IPV4_TCP_PAY; + else if (is_ipv6) + ptype = ICE_PTYPE_IPV6_TCP_PAY; + break; + case VIRTCHNL_PROTO_HDR_SCTP: + if (is_ipv4) + ptype = ICE_PTYPE_IPV4_SCTP_PAY; + else if (is_ipv6) + ptype = ICE_PTYPE_IPV6_SCTP_PAY; + break; + case VIRTCHNL_PROTO_HDR_GTPU_IP: + case VIRTCHNL_PROTO_HDR_GTPU_EH: + if (is_ipv4) + ptype = ICE_MAC_IPV4_GTPU; + else if (is_ipv6) + ptype = ICE_MAC_IPV6_GTPU; + goto out; + case VIRTCHNL_PROTO_HDR_L2TPV3: + if (is_ipv4) + ptype = ICE_MAC_IPV4_L2TPV3; + else if (is_ipv6) + ptype = ICE_MAC_IPV6_L2TPV3; + goto out; + case VIRTCHNL_PROTO_HDR_ESP: + if (is_ipv4) + ptype = is_udp ? ICE_MAC_IPV4_NAT_T_ESP : + ICE_MAC_IPV4_ESP; + else if (is_ipv6) + ptype = is_udp ? ICE_MAC_IPV6_NAT_T_ESP : + ICE_MAC_IPV6_ESP; + goto out; + case VIRTCHNL_PROTO_HDR_AH: + if (is_ipv4) + ptype = ICE_MAC_IPV4_AH; + else if (is_ipv6) + ptype = ICE_MAC_IPV6_AH; + goto out; + case VIRTCHNL_PROTO_HDR_PFCP: + if (is_ipv4) + ptype = ICE_MAC_IPV4_PFCP_SESSION; + else if (is_ipv6) + ptype = ICE_MAC_IPV6_PFCP_SESSION; + goto out; + default: + break; + } + i++; + } + +out: + return ice_hw_ptype_ena(&vf->pf->hw, ptype); +} + +/** + * ice_vc_parse_rss_cfg - parses hash fields and headers from + * a specific virtchnl RSS cfg + * @hw: pointer to the hardware + * @rss_cfg: pointer to the virtchnl RSS cfg + * @hash_cfg: pointer to the HW hash configuration + * + * Return true if all the protocol header and hash fields in the RSS cfg could + * be parsed, else return false + * + * This function parses the virtchnl RSS cfg to be the intended + * hash fields and the intended header for RSS configuration + */ +static bool ice_vc_parse_rss_cfg(struct ice_hw *hw, + struct virtchnl_rss_cfg *rss_cfg, + struct ice_rss_hash_cfg *hash_cfg) +{ + const struct ice_vc_hash_field_match_type *hf_list; + const struct ice_vc_hdr_match_type *hdr_list; + int i, hf_list_len, hdr_list_len; + u32 *addl_hdrs = &hash_cfg->addl_hdrs; + u64 *hash_flds = &hash_cfg->hash_flds; + + /* set outer layer RSS as default */ + hash_cfg->hdr_type = ICE_RSS_OUTER_HEADERS; + + if (rss_cfg->rss_algorithm == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) + hash_cfg->symm = true; + else + hash_cfg->symm = false; + + hf_list = ice_vc_hash_field_list; + hf_list_len = ARRAY_SIZE(ice_vc_hash_field_list); + hdr_list = ice_vc_hdr_list; + hdr_list_len = ARRAY_SIZE(ice_vc_hdr_list); + + for (i = 0; i < rss_cfg->proto_hdrs.count; i++) { + struct virtchnl_proto_hdr *proto_hdr = + &rss_cfg->proto_hdrs.proto_hdr[i]; + bool hdr_found = false; + int j; + + /* Find matched ice headers according to virtchnl headers. */ + for (j = 0; j < hdr_list_len; j++) { + struct ice_vc_hdr_match_type hdr_map = hdr_list[j]; + + if (proto_hdr->type == hdr_map.vc_hdr) { + *addl_hdrs |= hdr_map.ice_hdr; + hdr_found = true; + } + } + + if (!hdr_found) + return false; + + /* Find matched ice hash fields according to + * virtchnl hash fields. + */ + for (j = 0; j < hf_list_len; j++) { + struct ice_vc_hash_field_match_type hf_map = hf_list[j]; + + if (proto_hdr->type == hf_map.vc_hdr && + proto_hdr->field_selector == hf_map.vc_hash_field) { + *hash_flds |= hf_map.ice_hash_field; + break; + } + } + } + + return true; +} + +/** + * ice_vf_adv_rss_offload_ena - determine if capabilities support advanced + * RSS offloads + * @caps: VF driver negotiated capabilities + * + * Return true if VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF capability is set, + * else return false + */ +static bool ice_vf_adv_rss_offload_ena(u32 caps) +{ + return !!(caps & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF); +} + +/** + * ice_vc_handle_rss_cfg + * @vf: pointer to the VF info + * @msg: pointer to the message buffer + * @add: add a RSS config if true, otherwise delete a RSS config + * + * This function adds/deletes a RSS config + */ +int ice_vc_handle_rss_cfg(struct ice_vf *vf, u8 *msg, bool add) +{ + u32 v_opcode = add ? VIRTCHNL_OP_ADD_RSS_CFG : VIRTCHNL_OP_DEL_RSS_CFG; + struct virtchnl_rss_cfg *rss_cfg = (struct virtchnl_rss_cfg *)msg; + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct device *dev = ice_pf_to_dev(vf->pf); + struct ice_hw *hw = &vf->pf->hw; + struct ice_vsi *vsi; + + if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { + dev_dbg(dev, "VF %d attempting to configure RSS, but RSS is not supported by the PF\n", + vf->vf_id); + v_ret = VIRTCHNL_STATUS_ERR_NOT_SUPPORTED; + goto error_param; + } + + if (!ice_vf_adv_rss_offload_ena(vf->driver_caps)) { + dev_dbg(dev, "VF %d attempting to configure RSS, but Advanced RSS offload is not supported\n", + vf->vf_id); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (rss_cfg->proto_hdrs.count > VIRTCHNL_MAX_NUM_PROTO_HDRS || + rss_cfg->rss_algorithm < VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC || + rss_cfg->rss_algorithm > VIRTCHNL_RSS_ALG_XOR_SYMMETRIC) { + dev_dbg(dev, "VF %d attempting to configure RSS, but RSS configuration is not valid\n", + vf->vf_id); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_validate_pattern(vf, &rss_cfg->proto_hdrs)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (rss_cfg->rss_algorithm == VIRTCHNL_RSS_ALG_R_ASYMMETRIC) { + struct ice_vsi_ctx *ctx; + u8 lut_type, hash_type; + int status; + + lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI; + hash_type = add ? ICE_AQ_VSI_Q_OPT_RSS_HASH_XOR : + ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ; + + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); + if (!ctx) { + v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; + goto error_param; + } + + ctx->info.q_opt_rss = + FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_LUT_M, lut_type) | + FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_HASH_M, hash_type); + + /* Preserve existing queueing option setting */ + ctx->info.q_opt_rss |= (vsi->info.q_opt_rss & + ICE_AQ_VSI_Q_OPT_RSS_GBL_LUT_M); + ctx->info.q_opt_tc = vsi->info.q_opt_tc; + ctx->info.q_opt_flags = vsi->info.q_opt_rss; + + ctx->info.valid_sections = + cpu_to_le16(ICE_AQ_VSI_PROP_Q_OPT_VALID); + + status = ice_update_vsi(hw, vsi->idx, ctx, NULL); + if (status) { + dev_err(dev, "update VSI for RSS failed, err %d aq_err %s\n", + status, libie_aq_str(hw->adminq.sq_last_status)); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + } else { + vsi->info.q_opt_rss = ctx->info.q_opt_rss; + } + + kfree(ctx); + } else { + struct ice_rss_hash_cfg cfg; + + /* Only check for none raw pattern case */ + if (!ice_vc_validate_pattern(vf, &rss_cfg->proto_hdrs)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + cfg.addl_hdrs = ICE_FLOW_SEG_HDR_NONE; + cfg.hash_flds = ICE_HASH_INVALID; + cfg.hdr_type = ICE_RSS_ANY_HEADERS; + + if (!ice_vc_parse_rss_cfg(hw, rss_cfg, &cfg)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (add) { + if (ice_add_rss_cfg(hw, vsi, &cfg)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + dev_err(dev, "ice_add_rss_cfg failed for vsi = %d, v_ret = %d\n", + vsi->vsi_num, v_ret); + } + } else { + int status; + + status = ice_rem_rss_cfg(hw, vsi->idx, &cfg); + /* We just ignore -ENOENT, because if two configurations + * share the same profile remove one of them actually + * removes both, since the profile is deleted. + */ + if (status && status != -ENOENT) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + dev_err(dev, "ice_rem_rss_cfg failed for VF ID:%d, error:%d\n", + vf->vf_id, status); + } + } + } + +error_param: + return ice_vc_send_msg_to_vf(vf, v_opcode, v_ret, NULL, 0); +} + +/** + * ice_vc_config_rss_key + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * Configure the VF's RSS key + */ +int ice_vc_config_rss_key(struct ice_vf *vf, u8 *msg) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_rss_key *vrk = + (struct virtchnl_rss_key *)msg; + struct ice_vsi *vsi; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, vrk->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (vrk->key_len != ICE_VSIQF_HKEY_ARRAY_SIZE) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (ice_set_rss_key(vsi, vrk->key)) + v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; +error_param: + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_KEY, v_ret, + NULL, 0); +} + +/** + * ice_vc_config_rss_lut + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * Configure the VF's RSS LUT + */ +int ice_vc_config_rss_lut(struct ice_vf *vf, u8 *msg) +{ + struct virtchnl_rss_lut *vrl = (struct virtchnl_rss_lut *)msg; + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct ice_vsi *vsi; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, vrl->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (vrl->lut_entries != ICE_LUT_VSI_SIZE) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (ice_set_rss_lut(vsi, vrl->lut, ICE_LUT_VSI_SIZE)) + v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; +error_param: + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_LUT, v_ret, + NULL, 0); +} + +/** + * ice_vc_config_rss_hfunc + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + * + * Configure the VF's RSS Hash function + */ +int ice_vc_config_rss_hfunc(struct ice_vf *vf, u8 *msg) +{ + struct virtchnl_rss_hfunc *vrh = (struct virtchnl_rss_hfunc *)msg; + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + u8 hfunc = ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ; + struct ice_vsi *vsi; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!ice_vc_isvalid_vsi_id(vf, vrh->vsi_id)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto error_param; + } + + if (vrh->rss_algorithm == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) + hfunc = ICE_AQ_VSI_Q_OPT_RSS_HASH_SYM_TPLZ; + + if (ice_set_rss_hfunc(vsi, hfunc)) + v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; +error_param: + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_HFUNC, v_ret, + NULL, 0); +} + +/** + * ice_vc_get_rss_hashcfg - return the RSS Hash configuration + * @vf: pointer to the VF info + */ +int ice_vc_get_rss_hashcfg(struct ice_vf *vf) +{ + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct virtchnl_rss_hashcfg *vrh = NULL; + int len = 0, ret; + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { + dev_err(ice_pf_to_dev(vf->pf), "RSS not supported by PF\n"); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + len = sizeof(struct virtchnl_rss_hashcfg); + vrh = kzalloc(len, GFP_KERNEL); + if (!vrh) { + v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; + len = 0; + goto err; + } + + vrh->hashcfg = ICE_DEFAULT_RSS_HASHCFG; +err: + /* send the response back to the VF */ + ret = ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_GET_RSS_HASHCFG_CAPS, v_ret, + (u8 *)vrh, len); + kfree(vrh); + return ret; +} + +/** + * ice_vc_set_rss_hashcfg - set RSS Hash configuration bits for the VF + * @vf: pointer to the VF info + * @msg: pointer to the msg buffer + */ +int ice_vc_set_rss_hashcfg(struct ice_vf *vf, u8 *msg) +{ + struct virtchnl_rss_hashcfg *vrh = (struct virtchnl_rss_hashcfg *)msg; + enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; + struct ice_pf *pf = vf->pf; + struct ice_vsi *vsi; + struct device *dev; + int status; + + dev = ice_pf_to_dev(pf); + + if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { + dev_err(dev, "RSS not supported by PF\n"); + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + vsi = ice_get_vf_vsi(vf); + if (!vsi) { + v_ret = VIRTCHNL_STATUS_ERR_PARAM; + goto err; + } + + /* clear all previously programmed RSS configuration to allow VF drivers + * the ability to customize the RSS configuration and/or completely + * disable RSS + */ + status = ice_rem_vsi_rss_cfg(&pf->hw, vsi->idx); + if (status && !vrh->hashcfg) { + /* only report failure to clear the current RSS configuration if + * that was clearly the VF's intention (i.e. vrh->hashcfg = 0) + */ + v_ret = ice_err_to_virt_err(status); + goto err; + } else if (status) { + /* allow the VF to update the RSS configuration even on failure + * to clear the current RSS confguration in an attempt to keep + * RSS in a working state + */ + dev_warn(dev, "Failed to clear the RSS configuration for VF %u\n", + vf->vf_id); + } + + if (vrh->hashcfg) { + status = ice_add_avf_rss_cfg(&pf->hw, vsi, vrh->hashcfg); + v_ret = ice_err_to_virt_err(status); + } + + /* save the requested VF configuration */ + if (!v_ret) + vf->rss_hashcfg = vrh->hashcfg; + + /* send the response to the VF */ +err: + return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_SET_RSS_HASHCFG, v_ret, + NULL, 0); +} + diff --git a/drivers/net/ethernet/intel/ice/virt/rss.h b/drivers/net/ethernet/intel/ice/virt/rss.h new file mode 100644 index 000000000000..784d4c43ce8b --- /dev/null +++ b/drivers/net/ethernet/intel/ice/virt/rss.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2022, Intel Corporation. */ + +#ifndef _ICE_VIRT_RSS_H_ +#define _ICE_VIRT_RSS_H_ + +#include <linux/types.h> + +struct ice_vf; + +int ice_vc_handle_rss_cfg(struct ice_vf *vf, u8 *msg, bool add); +int ice_vc_config_rss_key(struct ice_vf *vf, u8 *msg); +int ice_vc_config_rss_lut(struct ice_vf *vf, u8 *msg); +int ice_vc_config_rss_hfunc(struct ice_vf *vf, u8 *msg); +int ice_vc_get_rss_hashcfg(struct ice_vf *vf); +int ice_vc_set_rss_hashcfg(struct ice_vf *vf, u8 *msg); + +#endif /* _ICE_VIRT_RSS_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/virt/virtchnl.c index 257967273079..f3f921134379 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/virt/virtchnl.c @@ -1,170 +1,20 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (C) 2022, Intel Corporation. */ -#include "ice_virtchnl.h" +#include "virtchnl.h" +#include "queues.h" +#include "rss.h" #include "ice_vf_lib_private.h" #include "ice.h" #include "ice_base.h" #include "ice_lib.h" #include "ice_fltr.h" -#include "ice_virtchnl_allowlist.h" +#include "allowlist.h" #include "ice_vf_vsi_vlan_ops.h" #include "ice_vlan.h" #include "ice_flex_pipe.h" #include "ice_dcb_lib.h" -#define FIELD_SELECTOR(proto_hdr_field) \ - BIT((proto_hdr_field) & PROTO_HDR_FIELD_MASK) - -struct ice_vc_hdr_match_type { - u32 vc_hdr; /* virtchnl headers (VIRTCHNL_PROTO_HDR_XXX) */ - u32 ice_hdr; /* ice headers (ICE_FLOW_SEG_HDR_XXX) */ -}; - -static const struct ice_vc_hdr_match_type ice_vc_hdr_list[] = { - {VIRTCHNL_PROTO_HDR_NONE, ICE_FLOW_SEG_HDR_NONE}, - {VIRTCHNL_PROTO_HDR_ETH, ICE_FLOW_SEG_HDR_ETH}, - {VIRTCHNL_PROTO_HDR_S_VLAN, ICE_FLOW_SEG_HDR_VLAN}, - {VIRTCHNL_PROTO_HDR_C_VLAN, ICE_FLOW_SEG_HDR_VLAN}, - {VIRTCHNL_PROTO_HDR_IPV4, ICE_FLOW_SEG_HDR_IPV4 | - ICE_FLOW_SEG_HDR_IPV_OTHER}, - {VIRTCHNL_PROTO_HDR_IPV6, ICE_FLOW_SEG_HDR_IPV6 | - ICE_FLOW_SEG_HDR_IPV_OTHER}, - {VIRTCHNL_PROTO_HDR_TCP, ICE_FLOW_SEG_HDR_TCP}, - {VIRTCHNL_PROTO_HDR_UDP, ICE_FLOW_SEG_HDR_UDP}, - {VIRTCHNL_PROTO_HDR_SCTP, ICE_FLOW_SEG_HDR_SCTP}, - {VIRTCHNL_PROTO_HDR_PPPOE, ICE_FLOW_SEG_HDR_PPPOE}, - {VIRTCHNL_PROTO_HDR_GTPU_IP, ICE_FLOW_SEG_HDR_GTPU_IP}, - {VIRTCHNL_PROTO_HDR_GTPU_EH, ICE_FLOW_SEG_HDR_GTPU_EH}, - {VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_DWN, - ICE_FLOW_SEG_HDR_GTPU_DWN}, - {VIRTCHNL_PROTO_HDR_GTPU_EH_PDU_UP, - ICE_FLOW_SEG_HDR_GTPU_UP}, - {VIRTCHNL_PROTO_HDR_L2TPV3, ICE_FLOW_SEG_HDR_L2TPV3}, - {VIRTCHNL_PROTO_HDR_ESP, ICE_FLOW_SEG_HDR_ESP}, - {VIRTCHNL_PROTO_HDR_AH, ICE_FLOW_SEG_HDR_AH}, - {VIRTCHNL_PROTO_HDR_PFCP, ICE_FLOW_SEG_HDR_PFCP_SESSION}, -}; - -struct ice_vc_hash_field_match_type { - u32 vc_hdr; /* virtchnl headers - * (VIRTCHNL_PROTO_HDR_XXX) - */ - u32 vc_hash_field; /* virtchnl hash fields selector - * FIELD_SELECTOR((VIRTCHNL_PROTO_HDR_ETH_XXX)) - */ - u64 ice_hash_field; /* ice hash fields - * (BIT_ULL(ICE_FLOW_FIELD_IDX_XXX)) - */ -}; - -static const struct -ice_vc_hash_field_match_type ice_vc_hash_field_list[] = { - {VIRTCHNL_PROTO_HDR_ETH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC), - BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_SA)}, - {VIRTCHNL_PROTO_HDR_ETH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), - BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_DA)}, - {VIRTCHNL_PROTO_HDR_ETH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_DST), - ICE_FLOW_HASH_ETH}, - {VIRTCHNL_PROTO_HDR_ETH, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ETH_ETHERTYPE), - BIT_ULL(ICE_FLOW_FIELD_IDX_ETH_TYPE)}, - {VIRTCHNL_PROTO_HDR_S_VLAN, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_S_VLAN_ID), - BIT_ULL(ICE_FLOW_FIELD_IDX_S_VLAN)}, - {VIRTCHNL_PROTO_HDR_C_VLAN, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_C_VLAN_ID), - BIT_ULL(ICE_FLOW_FIELD_IDX_C_VLAN)}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA)}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA)}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST), - ICE_FLOW_HASH_IPV4}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_SA) | - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_DA) | - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_DST) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), - ICE_FLOW_HASH_IPV4 | BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV4, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV4_PROT), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV4_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA)}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA)}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST), - ICE_FLOW_HASH_IPV6}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_SA) | - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_DA) | - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_SRC) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_DST) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), - ICE_FLOW_HASH_IPV6 | BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, - {VIRTCHNL_PROTO_HDR_IPV6, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_IPV6_PROT), - BIT_ULL(ICE_FLOW_FIELD_IDX_IPV6_PROT)}, - {VIRTCHNL_PROTO_HDR_TCP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT), - BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_SRC_PORT)}, - {VIRTCHNL_PROTO_HDR_TCP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), - BIT_ULL(ICE_FLOW_FIELD_IDX_TCP_DST_PORT)}, - {VIRTCHNL_PROTO_HDR_TCP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_SRC_PORT) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_TCP_DST_PORT), - ICE_FLOW_HASH_TCP_PORT}, - {VIRTCHNL_PROTO_HDR_UDP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT), - BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_SRC_PORT)}, - {VIRTCHNL_PROTO_HDR_UDP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), - BIT_ULL(ICE_FLOW_FIELD_IDX_UDP_DST_PORT)}, - {VIRTCHNL_PROTO_HDR_UDP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_SRC_PORT) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_UDP_DST_PORT), - ICE_FLOW_HASH_UDP_PORT}, - {VIRTCHNL_PROTO_HDR_SCTP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT), - BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_SRC_PORT)}, - {VIRTCHNL_PROTO_HDR_SCTP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), - BIT_ULL(ICE_FLOW_FIELD_IDX_SCTP_DST_PORT)}, - {VIRTCHNL_PROTO_HDR_SCTP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_SRC_PORT) | - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_SCTP_DST_PORT), - ICE_FLOW_HASH_SCTP_PORT}, - {VIRTCHNL_PROTO_HDR_PPPOE, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PPPOE_SESS_ID), - BIT_ULL(ICE_FLOW_FIELD_IDX_PPPOE_SESS_ID)}, - {VIRTCHNL_PROTO_HDR_GTPU_IP, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_GTPU_IP_TEID), - BIT_ULL(ICE_FLOW_FIELD_IDX_GTPU_IP_TEID)}, - {VIRTCHNL_PROTO_HDR_L2TPV3, - FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_L2TPV3_SESS_ID), - BIT_ULL(ICE_FLOW_FIELD_IDX_L2TPV3_SESS_ID)}, - {VIRTCHNL_PROTO_HDR_ESP, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_ESP_SPI), - BIT_ULL(ICE_FLOW_FIELD_IDX_ESP_SPI)}, - {VIRTCHNL_PROTO_HDR_AH, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_AH_SPI), - BIT_ULL(ICE_FLOW_FIELD_IDX_AH_SPI)}, - {VIRTCHNL_PROTO_HDR_PFCP, FIELD_SELECTOR(VIRTCHNL_PROTO_HDR_PFCP_SEID), - BIT_ULL(ICE_FLOW_FIELD_IDX_PFCP_SEID)}, -}; - /** * ice_vc_vf_broadcast - Broadcast a message to all VFs on PF * @pf: pointer to the PF structure @@ -338,28 +188,6 @@ static int ice_vc_get_ver_msg(struct ice_vf *vf, u8 *msg) } /** - * ice_vc_get_max_frame_size - get max frame size allowed for VF - * @vf: VF used to determine max frame size - * - * Max frame size is determined based on the current port's max frame size and - * whether a port VLAN is configured on this VF. The VF is not aware whether - * it's in a port VLAN so the PF needs to account for this in max frame size - * checks and sending the max frame size to the VF. - */ -static u16 ice_vc_get_max_frame_size(struct ice_vf *vf) -{ - struct ice_port_info *pi = ice_vf_get_port_info(vf); - u16 max_frame_size; - - max_frame_size = pi->phy.link_info.max_frame_size; - - if (ice_vf_is_port_vlan_ena(vf)) - max_frame_size -= VLAN_HLEN; - - return max_frame_size; -} - -/** * ice_vc_get_vlan_caps * @hw: pointer to the hw * @vf: pointer to the VF info @@ -559,488 +387,6 @@ bool ice_vc_isvalid_vsi_id(struct ice_vf *vf, u16 vsi_id) } /** - * ice_vc_isvalid_q_id - * @vsi: VSI to check queue ID against - * @qid: VSI relative queue ID - * - * check for the valid queue ID - */ -static bool ice_vc_isvalid_q_id(struct ice_vsi *vsi, u16 qid) -{ - /* allocated Tx and Rx queues should be always equal for VF VSI */ - return qid < vsi->alloc_txq; -} - -/** - * ice_vc_isvalid_ring_len - * @ring_len: length of ring - * - * check for the valid ring count, should be multiple of ICE_REQ_DESC_MULTIPLE - * or zero - */ -static bool ice_vc_isvalid_ring_len(u16 ring_len) -{ - return ring_len == 0 || - (ring_len >= ICE_MIN_NUM_DESC && - ring_len <= ICE_MAX_NUM_DESC && - !(ring_len % ICE_REQ_DESC_MULTIPLE)); -} - -/** - * ice_vc_validate_pattern - * @vf: pointer to the VF info - * @proto: virtchnl protocol headers - * - * validate the pattern is supported or not. - * - * Return: true on success, false on error. - */ -bool -ice_vc_validate_pattern(struct ice_vf *vf, struct virtchnl_proto_hdrs *proto) -{ - bool is_ipv4 = false; - bool is_ipv6 = false; - bool is_udp = false; - u16 ptype = -1; - int i = 0; - - while (i < proto->count && - proto->proto_hdr[i].type != VIRTCHNL_PROTO_HDR_NONE) { - switch (proto->proto_hdr[i].type) { - case VIRTCHNL_PROTO_HDR_ETH: - ptype = ICE_PTYPE_MAC_PAY; - break; - case VIRTCHNL_PROTO_HDR_IPV4: - ptype = ICE_PTYPE_IPV4_PAY; - is_ipv4 = true; - break; - case VIRTCHNL_PROTO_HDR_IPV6: - ptype = ICE_PTYPE_IPV6_PAY; - is_ipv6 = true; - break; - case VIRTCHNL_PROTO_HDR_UDP: - if (is_ipv4) - ptype = ICE_PTYPE_IPV4_UDP_PAY; - else if (is_ipv6) - ptype = ICE_PTYPE_IPV6_UDP_PAY; - is_udp = true; - break; - case VIRTCHNL_PROTO_HDR_TCP: - if (is_ipv4) - ptype = ICE_PTYPE_IPV4_TCP_PAY; - else if (is_ipv6) - ptype = ICE_PTYPE_IPV6_TCP_PAY; - break; - case VIRTCHNL_PROTO_HDR_SCTP: - if (is_ipv4) - ptype = ICE_PTYPE_IPV4_SCTP_PAY; - else if (is_ipv6) - ptype = ICE_PTYPE_IPV6_SCTP_PAY; - break; - case VIRTCHNL_PROTO_HDR_GTPU_IP: - case VIRTCHNL_PROTO_HDR_GTPU_EH: - if (is_ipv4) - ptype = ICE_MAC_IPV4_GTPU; - else if (is_ipv6) - ptype = ICE_MAC_IPV6_GTPU; - goto out; - case VIRTCHNL_PROTO_HDR_L2TPV3: - if (is_ipv4) - ptype = ICE_MAC_IPV4_L2TPV3; - else if (is_ipv6) - ptype = ICE_MAC_IPV6_L2TPV3; - goto out; - case VIRTCHNL_PROTO_HDR_ESP: - if (is_ipv4) - ptype = is_udp ? ICE_MAC_IPV4_NAT_T_ESP : - ICE_MAC_IPV4_ESP; - else if (is_ipv6) - ptype = is_udp ? ICE_MAC_IPV6_NAT_T_ESP : - ICE_MAC_IPV6_ESP; - goto out; - case VIRTCHNL_PROTO_HDR_AH: - if (is_ipv4) - ptype = ICE_MAC_IPV4_AH; - else if (is_ipv6) - ptype = ICE_MAC_IPV6_AH; - goto out; - case VIRTCHNL_PROTO_HDR_PFCP: - if (is_ipv4) - ptype = ICE_MAC_IPV4_PFCP_SESSION; - else if (is_ipv6) - ptype = ICE_MAC_IPV6_PFCP_SESSION; - goto out; - default: - break; - } - i++; - } - -out: - return ice_hw_ptype_ena(&vf->pf->hw, ptype); -} - -/** - * ice_vc_parse_rss_cfg - parses hash fields and headers from - * a specific virtchnl RSS cfg - * @hw: pointer to the hardware - * @rss_cfg: pointer to the virtchnl RSS cfg - * @hash_cfg: pointer to the HW hash configuration - * - * Return true if all the protocol header and hash fields in the RSS cfg could - * be parsed, else return false - * - * This function parses the virtchnl RSS cfg to be the intended - * hash fields and the intended header for RSS configuration - */ -static bool ice_vc_parse_rss_cfg(struct ice_hw *hw, - struct virtchnl_rss_cfg *rss_cfg, - struct ice_rss_hash_cfg *hash_cfg) -{ - const struct ice_vc_hash_field_match_type *hf_list; - const struct ice_vc_hdr_match_type *hdr_list; - int i, hf_list_len, hdr_list_len; - u32 *addl_hdrs = &hash_cfg->addl_hdrs; - u64 *hash_flds = &hash_cfg->hash_flds; - - /* set outer layer RSS as default */ - hash_cfg->hdr_type = ICE_RSS_OUTER_HEADERS; - - if (rss_cfg->rss_algorithm == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) - hash_cfg->symm = true; - else - hash_cfg->symm = false; - - hf_list = ice_vc_hash_field_list; - hf_list_len = ARRAY_SIZE(ice_vc_hash_field_list); - hdr_list = ice_vc_hdr_list; - hdr_list_len = ARRAY_SIZE(ice_vc_hdr_list); - - for (i = 0; i < rss_cfg->proto_hdrs.count; i++) { - struct virtchnl_proto_hdr *proto_hdr = - &rss_cfg->proto_hdrs.proto_hdr[i]; - bool hdr_found = false; - int j; - - /* Find matched ice headers according to virtchnl headers. */ - for (j = 0; j < hdr_list_len; j++) { - struct ice_vc_hdr_match_type hdr_map = hdr_list[j]; - - if (proto_hdr->type == hdr_map.vc_hdr) { - *addl_hdrs |= hdr_map.ice_hdr; - hdr_found = true; - } - } - - if (!hdr_found) - return false; - - /* Find matched ice hash fields according to - * virtchnl hash fields. - */ - for (j = 0; j < hf_list_len; j++) { - struct ice_vc_hash_field_match_type hf_map = hf_list[j]; - - if (proto_hdr->type == hf_map.vc_hdr && - proto_hdr->field_selector == hf_map.vc_hash_field) { - *hash_flds |= hf_map.ice_hash_field; - break; - } - } - } - - return true; -} - -/** - * ice_vf_adv_rss_offload_ena - determine if capabilities support advanced - * RSS offloads - * @caps: VF driver negotiated capabilities - * - * Return true if VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF capability is set, - * else return false - */ -static bool ice_vf_adv_rss_offload_ena(u32 caps) -{ - return !!(caps & VIRTCHNL_VF_OFFLOAD_ADV_RSS_PF); -} - -/** - * ice_vc_handle_rss_cfg - * @vf: pointer to the VF info - * @msg: pointer to the message buffer - * @add: add a RSS config if true, otherwise delete a RSS config - * - * This function adds/deletes a RSS config - */ -static int ice_vc_handle_rss_cfg(struct ice_vf *vf, u8 *msg, bool add) -{ - u32 v_opcode = add ? VIRTCHNL_OP_ADD_RSS_CFG : VIRTCHNL_OP_DEL_RSS_CFG; - struct virtchnl_rss_cfg *rss_cfg = (struct virtchnl_rss_cfg *)msg; - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct device *dev = ice_pf_to_dev(vf->pf); - struct ice_hw *hw = &vf->pf->hw; - struct ice_vsi *vsi; - - if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { - dev_dbg(dev, "VF %d attempting to configure RSS, but RSS is not supported by the PF\n", - vf->vf_id); - v_ret = VIRTCHNL_STATUS_ERR_NOT_SUPPORTED; - goto error_param; - } - - if (!ice_vf_adv_rss_offload_ena(vf->driver_caps)) { - dev_dbg(dev, "VF %d attempting to configure RSS, but Advanced RSS offload is not supported\n", - vf->vf_id); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (rss_cfg->proto_hdrs.count > VIRTCHNL_MAX_NUM_PROTO_HDRS || - rss_cfg->rss_algorithm < VIRTCHNL_RSS_ALG_TOEPLITZ_ASYMMETRIC || - rss_cfg->rss_algorithm > VIRTCHNL_RSS_ALG_XOR_SYMMETRIC) { - dev_dbg(dev, "VF %d attempting to configure RSS, but RSS configuration is not valid\n", - vf->vf_id); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_validate_pattern(vf, &rss_cfg->proto_hdrs)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (rss_cfg->rss_algorithm == VIRTCHNL_RSS_ALG_R_ASYMMETRIC) { - struct ice_vsi_ctx *ctx; - u8 lut_type, hash_type; - int status; - - lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI; - hash_type = add ? ICE_AQ_VSI_Q_OPT_RSS_HASH_XOR : - ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (!ctx) { - v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; - goto error_param; - } - - ctx->info.q_opt_rss = - FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_LUT_M, lut_type) | - FIELD_PREP(ICE_AQ_VSI_Q_OPT_RSS_HASH_M, hash_type); - - /* Preserve existing queueing option setting */ - ctx->info.q_opt_rss |= (vsi->info.q_opt_rss & - ICE_AQ_VSI_Q_OPT_RSS_GBL_LUT_M); - ctx->info.q_opt_tc = vsi->info.q_opt_tc; - ctx->info.q_opt_flags = vsi->info.q_opt_rss; - - ctx->info.valid_sections = - cpu_to_le16(ICE_AQ_VSI_PROP_Q_OPT_VALID); - - status = ice_update_vsi(hw, vsi->idx, ctx, NULL); - if (status) { - dev_err(dev, "update VSI for RSS failed, err %d aq_err %s\n", - status, libie_aq_str(hw->adminq.sq_last_status)); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - } else { - vsi->info.q_opt_rss = ctx->info.q_opt_rss; - } - - kfree(ctx); - } else { - struct ice_rss_hash_cfg cfg; - - /* Only check for none raw pattern case */ - if (!ice_vc_validate_pattern(vf, &rss_cfg->proto_hdrs)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - cfg.addl_hdrs = ICE_FLOW_SEG_HDR_NONE; - cfg.hash_flds = ICE_HASH_INVALID; - cfg.hdr_type = ICE_RSS_ANY_HEADERS; - - if (!ice_vc_parse_rss_cfg(hw, rss_cfg, &cfg)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (add) { - if (ice_add_rss_cfg(hw, vsi, &cfg)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - dev_err(dev, "ice_add_rss_cfg failed for vsi = %d, v_ret = %d\n", - vsi->vsi_num, v_ret); - } - } else { - int status; - - status = ice_rem_rss_cfg(hw, vsi->idx, &cfg); - /* We just ignore -ENOENT, because if two configurations - * share the same profile remove one of them actually - * removes both, since the profile is deleted. - */ - if (status && status != -ENOENT) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - dev_err(dev, "ice_rem_rss_cfg failed for VF ID:%d, error:%d\n", - vf->vf_id, status); - } - } - } - -error_param: - return ice_vc_send_msg_to_vf(vf, v_opcode, v_ret, NULL, 0); -} - -/** - * ice_vc_config_rss_key - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * Configure the VF's RSS key - */ -static int ice_vc_config_rss_key(struct ice_vf *vf, u8 *msg) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_rss_key *vrk = - (struct virtchnl_rss_key *)msg; - struct ice_vsi *vsi; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_isvalid_vsi_id(vf, vrk->vsi_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (vrk->key_len != ICE_VSIQF_HKEY_ARRAY_SIZE) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (ice_set_rss_key(vsi, vrk->key)) - v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; -error_param: - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_KEY, v_ret, - NULL, 0); -} - -/** - * ice_vc_config_rss_lut - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * Configure the VF's RSS LUT - */ -static int ice_vc_config_rss_lut(struct ice_vf *vf, u8 *msg) -{ - struct virtchnl_rss_lut *vrl = (struct virtchnl_rss_lut *)msg; - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct ice_vsi *vsi; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_isvalid_vsi_id(vf, vrl->vsi_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (vrl->lut_entries != ICE_LUT_VSI_SIZE) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (ice_set_rss_lut(vsi, vrl->lut, ICE_LUT_VSI_SIZE)) - v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; -error_param: - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_LUT, v_ret, - NULL, 0); -} - -/** - * ice_vc_config_rss_hfunc - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * Configure the VF's RSS Hash function - */ -static int ice_vc_config_rss_hfunc(struct ice_vf *vf, u8 *msg) -{ - struct virtchnl_rss_hfunc *vrh = (struct virtchnl_rss_hfunc *)msg; - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - u8 hfunc = ICE_AQ_VSI_Q_OPT_RSS_HASH_TPLZ; - struct ice_vsi *vsi; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_isvalid_vsi_id(vf, vrh->vsi_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (vrh->rss_algorithm == VIRTCHNL_RSS_ALG_TOEPLITZ_SYMMETRIC) - hfunc = ICE_AQ_VSI_Q_OPT_RSS_HASH_SYM_TPLZ; - - if (ice_set_rss_hfunc(vsi, hfunc)) - v_ret = VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR; -error_param: - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_RSS_HFUNC, v_ret, - NULL, 0); -} - -/** * ice_vc_get_qos_caps - Get current QoS caps from PF * @vf: pointer to the VF info * @@ -1122,110 +468,6 @@ err: } /** - * ice_vf_cfg_qs_bw - Configure per queue bandwidth - * @vf: pointer to the VF info - * @num_queues: number of queues to be configured - * - * Configure per queue bandwidth. - * - * Return: 0 on success or negative error value. - */ -static int ice_vf_cfg_qs_bw(struct ice_vf *vf, u16 num_queues) -{ - struct ice_hw *hw = &vf->pf->hw; - struct ice_vsi *vsi; - int ret; - u16 i; - - vsi = ice_get_vf_vsi(vf); - if (!vsi) - return -EINVAL; - - for (i = 0; i < num_queues; i++) { - u32 p_rate, min_rate; - u8 tc; - - p_rate = vf->qs_bw[i].peak; - min_rate = vf->qs_bw[i].committed; - tc = vf->qs_bw[i].tc; - if (p_rate) - ret = ice_cfg_q_bw_lmt(hw->port_info, vsi->idx, tc, - vf->qs_bw[i].queue_id, - ICE_MAX_BW, p_rate); - else - ret = ice_cfg_q_bw_dflt_lmt(hw->port_info, vsi->idx, tc, - vf->qs_bw[i].queue_id, - ICE_MAX_BW); - if (ret) - return ret; - - if (min_rate) - ret = ice_cfg_q_bw_lmt(hw->port_info, vsi->idx, tc, - vf->qs_bw[i].queue_id, - ICE_MIN_BW, min_rate); - else - ret = ice_cfg_q_bw_dflt_lmt(hw->port_info, vsi->idx, tc, - vf->qs_bw[i].queue_id, - ICE_MIN_BW); - - if (ret) - return ret; - } - - return 0; -} - -/** - * ice_vf_cfg_q_quanta_profile - Configure quanta profile - * @vf: pointer to the VF info - * @quanta_prof_idx: pointer to the quanta profile index - * @quanta_size: quanta size to be set - * - * This function chooses available quanta profile and configures the register. - * The quanta profile is evenly divided by the number of device ports, and then - * available to the specific PF and VFs. The first profile for each PF is a - * reserved default profile. Only quanta size of the rest unused profile can be - * modified. - * - * Return: 0 on success or negative error value. - */ -static int ice_vf_cfg_q_quanta_profile(struct ice_vf *vf, u16 quanta_size, - u16 *quanta_prof_idx) -{ - const u16 n_desc = calc_quanta_desc(quanta_size); - struct ice_hw *hw = &vf->pf->hw; - const u16 n_cmd = 2 * n_desc; - struct ice_pf *pf = vf->pf; - u16 per_pf, begin_id; - u8 n_used; - u32 reg; - - begin_id = (GLCOMM_QUANTA_PROF_MAX_INDEX + 1) / hw->dev_caps.num_funcs * - hw->logical_pf_id; - - if (quanta_size == ICE_DFLT_QUANTA) { - *quanta_prof_idx = begin_id; - } else { - per_pf = (GLCOMM_QUANTA_PROF_MAX_INDEX + 1) / - hw->dev_caps.num_funcs; - n_used = pf->num_quanta_prof_used; - if (n_used < per_pf) { - *quanta_prof_idx = begin_id + 1 + n_used; - pf->num_quanta_prof_used++; - } else { - return -EINVAL; - } - } - - reg = FIELD_PREP(GLCOMM_QUANTA_PROF_QUANTA_SIZE_M, quanta_size) | - FIELD_PREP(GLCOMM_QUANTA_PROF_MAX_CMD_M, n_cmd) | - FIELD_PREP(GLCOMM_QUANTA_PROF_MAX_DESC_M, n_desc); - wr32(hw, GLCOMM_QUANTA_PROF(*quanta_prof_idx), reg); - - return 0; -} - -/** * ice_vc_cfg_promiscuous_mode_msg * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -1407,757 +649,6 @@ error_param: } /** - * ice_vc_validate_vqs_bitmaps - validate Rx/Tx queue bitmaps from VIRTCHNL - * @vqs: virtchnl_queue_select structure containing bitmaps to validate - * - * Return true on successful validation, else false - */ -static bool ice_vc_validate_vqs_bitmaps(struct virtchnl_queue_select *vqs) -{ - if ((!vqs->rx_queues && !vqs->tx_queues) || - vqs->rx_queues >= BIT(ICE_MAX_RSS_QS_PER_VF) || - vqs->tx_queues >= BIT(ICE_MAX_RSS_QS_PER_VF)) - return false; - - return true; -} - -/** - * ice_vf_ena_txq_interrupt - enable Tx queue interrupt via QINT_TQCTL - * @vsi: VSI of the VF to configure - * @q_idx: VF queue index used to determine the queue in the PF's space - */ -void ice_vf_ena_txq_interrupt(struct ice_vsi *vsi, u32 q_idx) -{ - struct ice_hw *hw = &vsi->back->hw; - u32 pfq = vsi->txq_map[q_idx]; - u32 reg; - - reg = rd32(hw, QINT_TQCTL(pfq)); - - /* MSI-X index 0 in the VF's space is always for the OICR, which means - * this is most likely a poll mode VF driver, so don't enable an - * interrupt that was never configured via VIRTCHNL_OP_CONFIG_IRQ_MAP - */ - if (!(reg & QINT_TQCTL_MSIX_INDX_M)) - return; - - wr32(hw, QINT_TQCTL(pfq), reg | QINT_TQCTL_CAUSE_ENA_M); -} - -/** - * ice_vf_ena_rxq_interrupt - enable Tx queue interrupt via QINT_RQCTL - * @vsi: VSI of the VF to configure - * @q_idx: VF queue index used to determine the queue in the PF's space - */ -void ice_vf_ena_rxq_interrupt(struct ice_vsi *vsi, u32 q_idx) -{ - struct ice_hw *hw = &vsi->back->hw; - u32 pfq = vsi->rxq_map[q_idx]; - u32 reg; - - reg = rd32(hw, QINT_RQCTL(pfq)); - - /* MSI-X index 0 in the VF's space is always for the OICR, which means - * this is most likely a poll mode VF driver, so don't enable an - * interrupt that was never configured via VIRTCHNL_OP_CONFIG_IRQ_MAP - */ - if (!(reg & QINT_RQCTL_MSIX_INDX_M)) - return; - - wr32(hw, QINT_RQCTL(pfq), reg | QINT_RQCTL_CAUSE_ENA_M); -} - -/** - * ice_vc_ena_qs_msg - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * called from the VF to enable all or specific queue(s) - */ -static int ice_vc_ena_qs_msg(struct ice_vf *vf, u8 *msg) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_queue_select *vqs = - (struct virtchnl_queue_select *)msg; - struct ice_vsi *vsi; - unsigned long q_map; - u16 vf_q_id; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_validate_vqs_bitmaps(vqs)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* Enable only Rx rings, Tx rings were enabled by the FW when the - * Tx queue group list was configured and the context bits were - * programmed using ice_vsi_cfg_txqs - */ - q_map = vqs->rx_queues; - for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { - if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* Skip queue if enabled */ - if (test_bit(vf_q_id, vf->rxq_ena)) - continue; - - if (ice_vsi_ctrl_one_rx_ring(vsi, true, vf_q_id, true)) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to enable Rx ring %d on VSI %d\n", - vf_q_id, vsi->vsi_num); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - ice_vf_ena_rxq_interrupt(vsi, vf_q_id); - set_bit(vf_q_id, vf->rxq_ena); - } - - q_map = vqs->tx_queues; - for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { - if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* Skip queue if enabled */ - if (test_bit(vf_q_id, vf->txq_ena)) - continue; - - ice_vf_ena_txq_interrupt(vsi, vf_q_id); - set_bit(vf_q_id, vf->txq_ena); - } - - /* Set flag to indicate that queues are enabled */ - if (v_ret == VIRTCHNL_STATUS_SUCCESS) - set_bit(ICE_VF_STATE_QS_ENA, vf->vf_states); - -error_param: - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_ENABLE_QUEUES, v_ret, - NULL, 0); -} - -/** - * ice_vf_vsi_dis_single_txq - disable a single Tx queue - * @vf: VF to disable queue for - * @vsi: VSI for the VF - * @q_id: VF relative (0-based) queue ID - * - * Attempt to disable the Tx queue passed in. If the Tx queue was successfully - * disabled then clear q_id bit in the enabled queues bitmap and return - * success. Otherwise return error. - */ -int ice_vf_vsi_dis_single_txq(struct ice_vf *vf, struct ice_vsi *vsi, u16 q_id) -{ - struct ice_txq_meta txq_meta = { 0 }; - struct ice_tx_ring *ring; - int err; - - if (!test_bit(q_id, vf->txq_ena)) - dev_dbg(ice_pf_to_dev(vsi->back), "Queue %u on VSI %u is not enabled, but stopping it anyway\n", - q_id, vsi->vsi_num); - - ring = vsi->tx_rings[q_id]; - if (!ring) - return -EINVAL; - - ice_fill_txq_meta(vsi, ring, &txq_meta); - - err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, vf->vf_id, ring, &txq_meta); - if (err) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to stop Tx ring %d on VSI %d\n", - q_id, vsi->vsi_num); - return err; - } - - /* Clear enabled queues flag */ - clear_bit(q_id, vf->txq_ena); - - return 0; -} - -/** - * ice_vc_dis_qs_msg - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * called from the VF to disable all or specific queue(s) - */ -static int ice_vc_dis_qs_msg(struct ice_vf *vf, u8 *msg) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_queue_select *vqs = - (struct virtchnl_queue_select *)msg; - struct ice_vsi *vsi; - unsigned long q_map; - u16 vf_q_id; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states) && - !test_bit(ICE_VF_STATE_QS_ENA, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_isvalid_vsi_id(vf, vqs->vsi_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (!ice_vc_validate_vqs_bitmaps(vqs)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (vqs->tx_queues) { - q_map = vqs->tx_queues; - - for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { - if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - if (ice_vf_vsi_dis_single_txq(vf, vsi, vf_q_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - } - } - - q_map = vqs->rx_queues; - /* speed up Rx queue disable by batching them if possible */ - if (q_map && - bitmap_equal(&q_map, vf->rxq_ena, ICE_MAX_RSS_QS_PER_VF)) { - if (ice_vsi_stop_all_rx_rings(vsi)) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to stop all Rx rings on VSI %d\n", - vsi->vsi_num); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - bitmap_zero(vf->rxq_ena, ICE_MAX_RSS_QS_PER_VF); - } else if (q_map) { - for_each_set_bit(vf_q_id, &q_map, ICE_MAX_RSS_QS_PER_VF) { - if (!ice_vc_isvalid_q_id(vsi, vf_q_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* Skip queue if not enabled */ - if (!test_bit(vf_q_id, vf->rxq_ena)) - continue; - - if (ice_vsi_ctrl_one_rx_ring(vsi, false, vf_q_id, - true)) { - dev_err(ice_pf_to_dev(vsi->back), "Failed to stop Rx ring %d on VSI %d\n", - vf_q_id, vsi->vsi_num); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* Clear enabled queues flag */ - clear_bit(vf_q_id, vf->rxq_ena); - } - } - - /* Clear enabled queues flag */ - if (v_ret == VIRTCHNL_STATUS_SUCCESS && ice_vf_has_no_qs_ena(vf)) - clear_bit(ICE_VF_STATE_QS_ENA, vf->vf_states); - -error_param: - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_DISABLE_QUEUES, v_ret, - NULL, 0); -} - -/** - * ice_cfg_interrupt - * @vf: pointer to the VF info - * @vsi: the VSI being configured - * @map: vector map for mapping vectors to queues - * @q_vector: structure for interrupt vector - * configure the IRQ to queue map - */ -static enum virtchnl_status_code -ice_cfg_interrupt(struct ice_vf *vf, struct ice_vsi *vsi, - struct virtchnl_vector_map *map, - struct ice_q_vector *q_vector) -{ - u16 vsi_q_id, vsi_q_id_idx; - unsigned long qmap; - - q_vector->num_ring_rx = 0; - q_vector->num_ring_tx = 0; - - qmap = map->rxq_map; - for_each_set_bit(vsi_q_id_idx, &qmap, ICE_MAX_RSS_QS_PER_VF) { - vsi_q_id = vsi_q_id_idx; - - if (!ice_vc_isvalid_q_id(vsi, vsi_q_id)) - return VIRTCHNL_STATUS_ERR_PARAM; - - q_vector->num_ring_rx++; - q_vector->rx.itr_idx = map->rxitr_idx; - vsi->rx_rings[vsi_q_id]->q_vector = q_vector; - ice_cfg_rxq_interrupt(vsi, vsi_q_id, - q_vector->vf_reg_idx, - q_vector->rx.itr_idx); - } - - qmap = map->txq_map; - for_each_set_bit(vsi_q_id_idx, &qmap, ICE_MAX_RSS_QS_PER_VF) { - vsi_q_id = vsi_q_id_idx; - - if (!ice_vc_isvalid_q_id(vsi, vsi_q_id)) - return VIRTCHNL_STATUS_ERR_PARAM; - - q_vector->num_ring_tx++; - q_vector->tx.itr_idx = map->txitr_idx; - vsi->tx_rings[vsi_q_id]->q_vector = q_vector; - ice_cfg_txq_interrupt(vsi, vsi_q_id, - q_vector->vf_reg_idx, - q_vector->tx.itr_idx); - } - - return VIRTCHNL_STATUS_SUCCESS; -} - -/** - * ice_vc_cfg_irq_map_msg - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * called from the VF to configure the IRQ to queue map - */ -static int ice_vc_cfg_irq_map_msg(struct ice_vf *vf, u8 *msg) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - u16 num_q_vectors_mapped, vsi_id, vector_id; - struct virtchnl_irq_map_info *irqmap_info; - struct virtchnl_vector_map *map; - struct ice_vsi *vsi; - int i; - - irqmap_info = (struct virtchnl_irq_map_info *)msg; - num_q_vectors_mapped = irqmap_info->num_vectors; - - /* Check to make sure number of VF vectors mapped is not greater than - * number of VF vectors originally allocated, and check that - * there is actually at least a single VF queue vector mapped - */ - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states) || - vf->num_msix < num_q_vectors_mapped || - !num_q_vectors_mapped) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - for (i = 0; i < num_q_vectors_mapped; i++) { - struct ice_q_vector *q_vector; - - map = &irqmap_info->vecmap[i]; - - vector_id = map->vector_id; - vsi_id = map->vsi_id; - /* vector_id is always 0-based for each VF, and can never be - * larger than or equal to the max allowed interrupts per VF - */ - if (!(vector_id < vf->num_msix) || - !ice_vc_isvalid_vsi_id(vf, vsi_id) || - (!vector_id && (map->rxq_map || map->txq_map))) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* No need to map VF miscellaneous or rogue vector */ - if (!vector_id) - continue; - - /* Subtract non queue vector from vector_id passed by VF - * to get actual number of VSI queue vector array index - */ - q_vector = vsi->q_vectors[vector_id - ICE_NONQ_VECS_VF]; - if (!q_vector) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - /* lookout for the invalid queue index */ - v_ret = ice_cfg_interrupt(vf, vsi, map, q_vector); - if (v_ret) - goto error_param; - } - -error_param: - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_IRQ_MAP, v_ret, - NULL, 0); -} - -/** - * ice_vc_cfg_q_bw - Configure per queue bandwidth - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer which holds the command descriptor - * - * Configure VF queues bandwidth. - * - * Return: 0 on success or negative error value. - */ -static int ice_vc_cfg_q_bw(struct ice_vf *vf, u8 *msg) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_queues_bw_cfg *qbw = - (struct virtchnl_queues_bw_cfg *)msg; - struct ice_vsi *vsi; - u16 i; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states) || - !ice_vc_isvalid_vsi_id(vf, qbw->vsi_id)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - if (qbw->num_queues > ICE_MAX_RSS_QS_PER_VF || - qbw->num_queues > min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)) { - dev_err(ice_pf_to_dev(vf->pf), "VF-%d trying to configure more than allocated number of queues: %d\n", - vf->vf_id, min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - for (i = 0; i < qbw->num_queues; i++) { - if (qbw->cfg[i].shaper.peak != 0 && vf->max_tx_rate != 0 && - qbw->cfg[i].shaper.peak > vf->max_tx_rate) { - dev_warn(ice_pf_to_dev(vf->pf), "The maximum queue %d rate limit configuration may not take effect because the maximum TX rate for VF-%d is %d\n", - qbw->cfg[i].queue_id, vf->vf_id, - vf->max_tx_rate); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - if (qbw->cfg[i].shaper.committed != 0 && vf->min_tx_rate != 0 && - qbw->cfg[i].shaper.committed < vf->min_tx_rate) { - dev_warn(ice_pf_to_dev(vf->pf), "The minimum queue %d rate limit configuration may not take effect because the minimum TX rate for VF-%d is %d\n", - qbw->cfg[i].queue_id, vf->vf_id, - vf->min_tx_rate); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - if (qbw->cfg[i].queue_id > vf->num_vf_qs) { - dev_warn(ice_pf_to_dev(vf->pf), "VF-%d trying to configure invalid queue_id\n", - vf->vf_id); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - if (qbw->cfg[i].tc >= ICE_MAX_TRAFFIC_CLASS) { - dev_warn(ice_pf_to_dev(vf->pf), "VF-%d trying to configure a traffic class higher than allowed\n", - vf->vf_id); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - } - - for (i = 0; i < qbw->num_queues; i++) { - vf->qs_bw[i].queue_id = qbw->cfg[i].queue_id; - vf->qs_bw[i].peak = qbw->cfg[i].shaper.peak; - vf->qs_bw[i].committed = qbw->cfg[i].shaper.committed; - vf->qs_bw[i].tc = qbw->cfg[i].tc; - } - - if (ice_vf_cfg_qs_bw(vf, qbw->num_queues)) - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - -err: - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_QUEUE_BW, - v_ret, NULL, 0); -} - -/** - * ice_vc_cfg_q_quanta - Configure per queue quanta - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer which holds the command descriptor - * - * Configure VF queues quanta. - * - * Return: 0 on success or negative error value. - */ -static int ice_vc_cfg_q_quanta(struct ice_vf *vf, u8 *msg) -{ - u16 quanta_prof_id, quanta_size, start_qid, num_queues, end_qid, i; - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_quanta_cfg *qquanta = - (struct virtchnl_quanta_cfg *)msg; - struct ice_vsi *vsi; - int ret; - - start_qid = qquanta->queue_select.start_queue_id; - num_queues = qquanta->queue_select.num_queues; - - if (check_add_overflow(start_qid, num_queues, &end_qid)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - if (end_qid > ICE_MAX_RSS_QS_PER_VF || - end_qid > min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)) { - dev_err(ice_pf_to_dev(vf->pf), "VF-%d trying to configure more than allocated number of queues: %d\n", - vf->vf_id, min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - quanta_size = qquanta->quanta_size; - if (quanta_size > ICE_MAX_QUANTA_SIZE || - quanta_size < ICE_MIN_QUANTA_SIZE) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - if (quanta_size % 64) { - dev_err(ice_pf_to_dev(vf->pf), "quanta size should be the product of 64\n"); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - ret = ice_vf_cfg_q_quanta_profile(vf, quanta_size, - &quanta_prof_id); - if (ret) { - v_ret = VIRTCHNL_STATUS_ERR_NOT_SUPPORTED; - goto err; - } - - for (i = start_qid; i < end_qid; i++) - vsi->tx_rings[i]->quanta_prof_id = quanta_prof_id; - -err: - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_QUANTA, - v_ret, NULL, 0); -} - -/** - * ice_vc_cfg_qs_msg - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * called from the VF to configure the Rx/Tx queues - */ -static int ice_vc_cfg_qs_msg(struct ice_vf *vf, u8 *msg) -{ - struct virtchnl_vsi_queue_config_info *qci = - (struct virtchnl_vsi_queue_config_info *)msg; - struct virtchnl_queue_pair_info *qpi; - struct ice_pf *pf = vf->pf; - struct ice_vsi *vsi; - int i = -1, q_idx; - bool ena_ts; - u8 act_prt; - - mutex_lock(&pf->lag_mutex); - act_prt = ice_lag_prepare_vf_reset(pf->lag); - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) - goto error_param; - - if (!ice_vc_isvalid_vsi_id(vf, qci->vsi_id)) - goto error_param; - - vsi = ice_get_vf_vsi(vf); - if (!vsi) - goto error_param; - - if (qci->num_queue_pairs > ICE_MAX_RSS_QS_PER_VF || - qci->num_queue_pairs > min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)) { - dev_err(ice_pf_to_dev(pf), "VF-%d requesting more than supported number of queues: %d\n", - vf->vf_id, min_t(u16, vsi->alloc_txq, vsi->alloc_rxq)); - goto error_param; - } - - for (i = 0; i < qci->num_queue_pairs; i++) { - if (!qci->qpair[i].rxq.crc_disable) - continue; - - if (!(vf->driver_caps & VIRTCHNL_VF_OFFLOAD_CRC) || - vf->vlan_strip_ena) - goto error_param; - } - - for (i = 0; i < qci->num_queue_pairs; i++) { - qpi = &qci->qpair[i]; - if (qpi->txq.vsi_id != qci->vsi_id || - qpi->rxq.vsi_id != qci->vsi_id || - qpi->rxq.queue_id != qpi->txq.queue_id || - qpi->txq.headwb_enabled || - !ice_vc_isvalid_ring_len(qpi->txq.ring_len) || - !ice_vc_isvalid_ring_len(qpi->rxq.ring_len) || - !ice_vc_isvalid_q_id(vsi, qpi->txq.queue_id)) { - goto error_param; - } - - q_idx = qpi->rxq.queue_id; - - /* make sure selected "q_idx" is in valid range of queues - * for selected "vsi" - */ - if (q_idx >= vsi->alloc_txq || q_idx >= vsi->alloc_rxq) { - goto error_param; - } - - /* copy Tx queue info from VF into VSI */ - if (qpi->txq.ring_len > 0) { - vsi->tx_rings[q_idx]->dma = qpi->txq.dma_ring_addr; - vsi->tx_rings[q_idx]->count = qpi->txq.ring_len; - - /* Disable any existing queue first */ - if (ice_vf_vsi_dis_single_txq(vf, vsi, q_idx)) - goto error_param; - - /* Configure a queue with the requested settings */ - if (ice_vsi_cfg_single_txq(vsi, vsi->tx_rings, q_idx)) { - dev_warn(ice_pf_to_dev(pf), "VF-%d failed to configure TX queue %d\n", - vf->vf_id, q_idx); - goto error_param; - } - } - - /* copy Rx queue info from VF into VSI */ - if (qpi->rxq.ring_len > 0) { - u16 max_frame_size = ice_vc_get_max_frame_size(vf); - struct ice_rx_ring *ring = vsi->rx_rings[q_idx]; - u32 rxdid; - - ring->dma = qpi->rxq.dma_ring_addr; - ring->count = qpi->rxq.ring_len; - - if (qpi->rxq.crc_disable) - ring->flags |= ICE_RX_FLAGS_CRC_STRIP_DIS; - else - ring->flags &= ~ICE_RX_FLAGS_CRC_STRIP_DIS; - - if (qpi->rxq.databuffer_size != 0 && - (qpi->rxq.databuffer_size > ((16 * 1024) - 128) || - qpi->rxq.databuffer_size < 1024)) - goto error_param; - ring->rx_buf_len = qpi->rxq.databuffer_size; - if (qpi->rxq.max_pkt_size > max_frame_size || - qpi->rxq.max_pkt_size < 64) - goto error_param; - - ring->max_frame = qpi->rxq.max_pkt_size; - /* add space for the port VLAN since the VF driver is - * not expected to account for it in the MTU - * calculation - */ - if (ice_vf_is_port_vlan_ena(vf)) - ring->max_frame += VLAN_HLEN; - - if (ice_vsi_cfg_single_rxq(vsi, q_idx)) { - dev_warn(ice_pf_to_dev(pf), "VF-%d failed to configure RX queue %d\n", - vf->vf_id, q_idx); - goto error_param; - } - - /* If Rx flex desc is supported, select RXDID for Rx - * queues. Otherwise, use legacy 32byte descriptor - * format. Legacy 16byte descriptor is not supported. - * If this RXDID is selected, return error. - */ - if (vf->driver_caps & - VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) { - rxdid = qpi->rxq.rxdid; - if (!(BIT(rxdid) & pf->supported_rxdids)) - goto error_param; - } else { - rxdid = ICE_RXDID_LEGACY_1; - } - - ena_ts = ((vf->driver_caps & - VIRTCHNL_VF_OFFLOAD_RX_FLEX_DESC) && - (vf->driver_caps & VIRTCHNL_VF_CAP_PTP) && - (qpi->rxq.flags & VIRTCHNL_PTP_RX_TSTAMP)); - - ice_write_qrxflxp_cntxt(&vsi->back->hw, - vsi->rxq_map[q_idx], rxdid, - ICE_RXDID_PRIO, ena_ts); - } - } - - ice_lag_complete_vf_reset(pf->lag, act_prt); - mutex_unlock(&pf->lag_mutex); - - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, - VIRTCHNL_STATUS_SUCCESS, NULL, 0); -error_param: - /* disable whatever we can */ - for (; i >= 0; i--) { - if (ice_vsi_ctrl_one_rx_ring(vsi, false, i, true)) - dev_err(ice_pf_to_dev(pf), "VF-%d could not disable RX queue %d\n", - vf->vf_id, i); - if (ice_vf_vsi_dis_single_txq(vf, vsi, i)) - dev_err(ice_pf_to_dev(pf), "VF-%d could not disable TX queue %d\n", - vf->vf_id, i); - } - - ice_lag_complete_vf_reset(pf->lag, act_prt); - mutex_unlock(&pf->lag_mutex); - - ice_lag_move_new_vf_nodes(vf); - - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_CONFIG_VSI_QUEUES, - VIRTCHNL_STATUS_ERR_PARAM, NULL, 0); -} - -/** * ice_can_vf_change_mac * @vf: pointer to the VF info * @@ -2531,66 +1022,6 @@ static int ice_vc_del_mac_addr_msg(struct ice_vf *vf, u8 *msg) } /** - * ice_vc_request_qs_msg - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - * - * VFs get a default number of queues but can use this message to request a - * different number. If the request is successful, PF will reset the VF and - * return 0. If unsuccessful, PF will send message informing VF of number of - * available queue pairs via virtchnl message response to VF. - */ -static int ice_vc_request_qs_msg(struct ice_vf *vf, u8 *msg) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_vf_res_request *vfres = - (struct virtchnl_vf_res_request *)msg; - u16 req_queues = vfres->num_queue_pairs; - struct ice_pf *pf = vf->pf; - u16 max_allowed_vf_queues; - u16 tx_rx_queue_left; - struct device *dev; - u16 cur_queues; - - dev = ice_pf_to_dev(pf); - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto error_param; - } - - cur_queues = vf->num_vf_qs; - tx_rx_queue_left = min_t(u16, ice_get_avail_txq_count(pf), - ice_get_avail_rxq_count(pf)); - max_allowed_vf_queues = tx_rx_queue_left + cur_queues; - if (!req_queues) { - dev_err(dev, "VF %d tried to request 0 queues. Ignoring.\n", - vf->vf_id); - } else if (req_queues > ICE_MAX_RSS_QS_PER_VF) { - dev_err(dev, "VF %d tried to request more than %d queues.\n", - vf->vf_id, ICE_MAX_RSS_QS_PER_VF); - vfres->num_queue_pairs = ICE_MAX_RSS_QS_PER_VF; - } else if (req_queues > cur_queues && - req_queues - cur_queues > tx_rx_queue_left) { - dev_warn(dev, "VF %d requested %u more queues, but only %u left.\n", - vf->vf_id, req_queues - cur_queues, tx_rx_queue_left); - vfres->num_queue_pairs = min_t(u16, max_allowed_vf_queues, - ICE_MAX_RSS_QS_PER_VF); - } else { - /* request is successful, then reset VF */ - vf->num_req_qs = req_queues; - ice_reset_vf(vf, ICE_VF_RESET_NOTIFY); - dev_info(dev, "VF %d granted request of %u queues.\n", - vf->vf_id, req_queues); - return 0; - } - -error_param: - /* send the response to the VF */ - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_REQUEST_QUEUES, - v_ret, (u8 *)vfres, sizeof(*vfres)); -} - -/** * ice_vf_vlan_offload_ena - determine if capabilities support VLAN offloads * @caps: VF driver negotiated capabilities * @@ -2983,112 +1414,6 @@ error_param: } /** - * ice_vc_get_rss_hashcfg - return the RSS Hash configuration - * @vf: pointer to the VF info - */ -static int ice_vc_get_rss_hashcfg(struct ice_vf *vf) -{ - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct virtchnl_rss_hashcfg *vrh = NULL; - int len = 0, ret; - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - if (!test_bit(ICE_FLAG_RSS_ENA, vf->pf->flags)) { - dev_err(ice_pf_to_dev(vf->pf), "RSS not supported by PF\n"); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - len = sizeof(struct virtchnl_rss_hashcfg); - vrh = kzalloc(len, GFP_KERNEL); - if (!vrh) { - v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; - len = 0; - goto err; - } - - vrh->hashcfg = ICE_DEFAULT_RSS_HASHCFG; -err: - /* send the response back to the VF */ - ret = ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_GET_RSS_HASHCFG_CAPS, v_ret, - (u8 *)vrh, len); - kfree(vrh); - return ret; -} - -/** - * ice_vc_set_rss_hashcfg - set RSS Hash configuration bits for the VF - * @vf: pointer to the VF info - * @msg: pointer to the msg buffer - */ -static int ice_vc_set_rss_hashcfg(struct ice_vf *vf, u8 *msg) -{ - struct virtchnl_rss_hashcfg *vrh = (struct virtchnl_rss_hashcfg *)msg; - enum virtchnl_status_code v_ret = VIRTCHNL_STATUS_SUCCESS; - struct ice_pf *pf = vf->pf; - struct ice_vsi *vsi; - struct device *dev; - int status; - - dev = ice_pf_to_dev(pf); - - if (!test_bit(ICE_VF_STATE_ACTIVE, vf->vf_states)) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) { - dev_err(dev, "RSS not supported by PF\n"); - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - vsi = ice_get_vf_vsi(vf); - if (!vsi) { - v_ret = VIRTCHNL_STATUS_ERR_PARAM; - goto err; - } - - /* clear all previously programmed RSS configuration to allow VF drivers - * the ability to customize the RSS configuration and/or completely - * disable RSS - */ - status = ice_rem_vsi_rss_cfg(&pf->hw, vsi->idx); - if (status && !vrh->hashcfg) { - /* only report failure to clear the current RSS configuration if - * that was clearly the VF's intention (i.e. vrh->hashcfg = 0) - */ - v_ret = ice_err_to_virt_err(status); - goto err; - } else if (status) { - /* allow the VF to update the RSS configuration even on failure - * to clear the current RSS confguration in an attempt to keep - * RSS in a working state - */ - dev_warn(dev, "Failed to clear the RSS configuration for VF %u\n", - vf->vf_id); - } - - if (vrh->hashcfg) { - status = ice_add_avf_rss_cfg(&pf->hw, vsi, vrh->hashcfg); - v_ret = ice_err_to_virt_err(status); - } - - /* save the requested VF configuration */ - if (!v_ret) - vf->rss_hashcfg = vrh->hashcfg; - - /* send the response to the VF */ -err: - return ice_vc_send_msg_to_vf(vf, VIRTCHNL_OP_SET_RSS_HASHCFG, v_ret, - NULL, 0); -} - -/** * ice_vc_query_rxdid - query RXDID supported by DDP package * @vf: pointer to VF info * diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.h b/drivers/net/ethernet/intel/ice/virt/virtchnl.h index 71bb456e2d71..71bb456e2d71 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.h +++ b/drivers/net/ethernet/intel/ice/virt/virtchnl.h diff --git a/drivers/net/ethernet/intel/idpf/Kconfig b/drivers/net/ethernet/intel/idpf/Kconfig index 2c359a8551c7..adab2154125b 100644 --- a/drivers/net/ethernet/intel/idpf/Kconfig +++ b/drivers/net/ethernet/intel/idpf/Kconfig @@ -6,7 +6,7 @@ config IDPF depends on PCI_MSI depends on PTP_1588_CLOCK_OPTIONAL select DIMLIB - select LIBETH + select LIBETH_XDP help This driver supports Intel(R) Infrastructure Data Path Function devices. diff --git a/drivers/net/ethernet/intel/idpf/Makefile b/drivers/net/ethernet/intel/idpf/Makefile index 4ef4b2b5e37a..651ddee942bd 100644 --- a/drivers/net/ethernet/intel/idpf/Makefile +++ b/drivers/net/ethernet/intel/idpf/Makefile @@ -21,3 +21,6 @@ idpf-$(CONFIG_IDPF_SINGLEQ) += idpf_singleq_txrx.o idpf-$(CONFIG_PTP_1588_CLOCK) += idpf_ptp.o idpf-$(CONFIG_PTP_1588_CLOCK) += idpf_virtchnl_ptp.o + +idpf-y += xdp.o +idpf-y += xsk.o diff --git a/drivers/net/ethernet/intel/idpf/idpf.h b/drivers/net/ethernet/intel/idpf/idpf.h index f4c0eaf9bde3..ca4da0c89979 100644 --- a/drivers/net/ethernet/intel/idpf/idpf.h +++ b/drivers/net/ethernet/intel/idpf/idpf.h @@ -40,6 +40,7 @@ struct idpf_vport_max_q; #define IDPF_NUM_CHUNKS_PER_MSG(struct_sz, chunk_sz) \ ((IDPF_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz)) +#define IDPF_WAIT_FOR_MARKER_TIMEO 500 #define IDPF_MAX_WAIT 500 /* available message levels */ @@ -148,6 +149,7 @@ enum idpf_vport_state { * @link_speed_mbps: Link speed in mbps * @vport_idx: Relative vport index * @max_tx_hdr_size: Max header length hardware can support + * @tx_max_bufs: Max buffers that can be transmitted with scatter-gather * @state: See enum idpf_vport_state * @netstats: Packet and byte stats * @stats_lock: Lock to protect stats update @@ -159,6 +161,7 @@ struct idpf_netdev_priv { u32 link_speed_mbps; u16 vport_idx; u16 max_tx_hdr_size; + u16 tx_max_bufs; enum idpf_vport_state state; struct rtnl_link_stats64 netstats; spinlock_t stats_lock; @@ -246,16 +249,28 @@ enum idpf_vport_reset_cause { /** * enum idpf_vport_flags - Vport flags * @IDPF_VPORT_DEL_QUEUES: To send delete queues message - * @IDPF_VPORT_SW_MARKER: Indicate TX pipe drain software marker packets - * processing is done * @IDPF_VPORT_FLAGS_NBITS: Must be last */ enum idpf_vport_flags { IDPF_VPORT_DEL_QUEUES, - IDPF_VPORT_SW_MARKER, IDPF_VPORT_FLAGS_NBITS, }; +/** + * struct idpf_tstamp_stats - Tx timestamp statistics + * @stats_sync: See struct u64_stats_sync + * @packets: Number of packets successfully timestamped by the hardware + * @discarded: Number of Tx skbs discarded due to cached PHC + * being too old to correctly extend timestamp + * @flushed: Number of Tx skbs flushed due to interface closed + */ +struct idpf_tstamp_stats { + struct u64_stats_sync stats_sync; + u64_stats_t packets; + u64_stats_t discarded; + u64_stats_t flushed; +}; + struct idpf_port_stats { struct u64_stats_sync stats_sync; u64_stats_t rx_hw_csum_err; @@ -287,6 +302,10 @@ struct idpf_fsteer_fltr { * @txq_model: Split queue or single queue queuing model * @txqs: Used only in hotpath to get to the right queue very fast * @crc_enable: Enable CRC insertion offload + * @xdpsq_share: whether XDPSQ sharing is enabled + * @num_xdp_txq: number of XDPSQs + * @xdp_txq_offset: index of the first XDPSQ (== number of regular SQs) + * @xdp_prog: installed XDP program * @num_rxq: Number of allocated RX queues * @num_bufq: Number of allocated buffer queues * @rxq_desc_count: RX queue descriptor count. *MUST* have enough descriptors @@ -312,16 +331,19 @@ struct idpf_fsteer_fltr { * @num_q_vectors: Number of IRQ vectors allocated * @q_vectors: Array of queue vectors * @q_vector_idxs: Starting index of queue vectors + * @noirq_dyn_ctl: register to enable/disable the vector for NOIRQ queues + * @noirq_dyn_ctl_ena: value to write to the above to enable it + * @noirq_v_idx: ID of the NOIRQ vector * @max_mtu: device given max possible MTU * @default_mac_addr: device will give a default MAC to use * @rx_itr_profile: RX profiles for Dynamic Interrupt Moderation * @tx_itr_profile: TX profiles for Dynamic Interrupt Moderation * @port_stats: per port csum, header split, and other offload stats * @link_up: True if link is up - * @sw_marker_wq: workqueue for marker packets * @tx_tstamp_caps: Capabilities negotiated for Tx timestamping * @tstamp_config: The Tx tstamp config * @tstamp_task: Tx timestamping task + * @tstamp_stats: Tx timestamping statistics */ struct idpf_vport { u16 num_txq; @@ -335,6 +357,11 @@ struct idpf_vport { struct idpf_tx_queue **txqs; bool crc_enable; + bool xdpsq_share; + u16 num_xdp_txq; + u16 xdp_txq_offset; + struct bpf_prog *xdp_prog; + u16 num_rxq; u16 num_bufq; u32 rxq_desc_count; @@ -359,6 +386,11 @@ struct idpf_vport { u16 num_q_vectors; struct idpf_q_vector *q_vectors; u16 *q_vector_idxs; + + void __iomem *noirq_dyn_ctl; + u32 noirq_dyn_ctl_ena; + u16 noirq_v_idx; + u16 max_mtu; u8 default_mac_addr[ETH_ALEN]; u16 rx_itr_profile[IDPF_DIM_PROFILE_SLOTS]; @@ -367,11 +399,10 @@ struct idpf_vport { bool link_up; - wait_queue_head_t sw_marker_wq; - struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps; struct kernel_hwtstamp_config tstamp_config; struct work_struct tstamp_task; + struct idpf_tstamp_stats tstamp_stats; }; /** @@ -433,6 +464,7 @@ struct idpf_q_coalesce { * ethtool * @num_req_rxq_desc: Number of user requested RX queue descriptors through * ethtool + * @xdp_prog: requested XDP program to install * @user_flags: User toggled config flags * @mac_filter_list: List of MAC filters * @num_fsteer_fltrs: number of flow steering filters @@ -447,6 +479,7 @@ struct idpf_vport_user_config_data { u16 num_req_rx_qs; u32 num_req_txq_desc; u32 num_req_rxq_desc; + struct bpf_prog *xdp_prog; DECLARE_BITMAP(user_flags, __IDPF_USER_FLAGS_NBITS); struct list_head mac_filter_list; u32 num_fsteer_fltrs; @@ -676,6 +709,11 @@ static inline int idpf_is_queue_model_split(u16 q_model) q_model == VIRTCHNL2_QUEUE_MODEL_SPLIT; } +static inline bool idpf_xdp_enabled(const struct idpf_vport *vport) +{ + return vport->adapter && vport->xdp_prog; +} + #define idpf_is_cap_ena(adapter, field, flag) \ idpf_is_capability_ena(adapter, false, field, flag) #define idpf_is_cap_ena_all(adapter, field, flag) \ @@ -957,6 +995,13 @@ static inline void idpf_vport_ctrl_unlock(struct net_device *netdev) mutex_unlock(&np->adapter->vport_ctrl_lock); } +static inline bool idpf_vport_ctrl_is_locked(struct net_device *netdev) +{ + struct idpf_netdev_priv *np = netdev_priv(netdev); + + return mutex_is_locked(&np->adapter->vport_ctrl_lock); +} + void idpf_statistics_task(struct work_struct *work); void idpf_init_task(struct work_struct *work); void idpf_service_task(struct work_struct *work); diff --git a/drivers/net/ethernet/intel/idpf/idpf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_dev.c index bfa60f7d43de..3a04a6bd0d7c 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_dev.c @@ -77,7 +77,7 @@ static int idpf_intr_reg_init(struct idpf_vport *vport) int num_vecs = vport->num_q_vectors; struct idpf_vec_regs *reg_vals; int num_regs, i, err = 0; - u32 rx_itr, tx_itr; + u32 rx_itr, tx_itr, val; u16 total_vecs; total_vecs = idpf_get_reserved_vecs(vport->adapter); @@ -121,6 +121,15 @@ static int idpf_intr_reg_init(struct idpf_vport *vport) intr->tx_itr = idpf_get_reg_addr(adapter, tx_itr); } + /* Data vector for NOIRQ queues */ + + val = reg_vals[vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; + vport->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); + + val = PF_GLINT_DYN_CTL_WB_ON_ITR_M | PF_GLINT_DYN_CTL_INTENA_MSK_M | + FIELD_PREP(PF_GLINT_DYN_CTL_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); + vport->noirq_dyn_ctl_ena = val; + free_reg_vals: kfree(reg_vals); diff --git a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c index 0eb812ac19c2..a5a1eec9ade8 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ethtool.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ethtool.c @@ -1245,8 +1245,8 @@ static void idpf_get_ethtool_stats(struct net_device *netdev, * * returns pointer to rx vector */ -static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, - int q_num) +struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, + u32 q_num) { int q_grp, q_idx; @@ -1266,8 +1266,8 @@ static struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, * * returns pointer to tx vector */ -static struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, - int q_num) +struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, + u32 q_num) { int q_grp; @@ -1685,6 +1685,61 @@ unlock: return err; } +/** + * idpf_get_ts_stats - Collect HW tstamping statistics + * @netdev: network interface device structure + * @ts_stats: HW timestamping stats structure + * + * Collect HW timestamping statistics including successfully timestamped + * packets, discarded due to illegal values, flushed during releasing PTP and + * skipped due to lack of the free index. + */ +static void idpf_get_ts_stats(struct net_device *netdev, + struct ethtool_ts_stats *ts_stats) +{ + struct idpf_netdev_priv *np = netdev_priv(netdev); + struct idpf_vport *vport; + unsigned int start; + + idpf_vport_ctrl_lock(netdev); + vport = idpf_netdev_to_vport(netdev); + do { + start = u64_stats_fetch_begin(&vport->tstamp_stats.stats_sync); + ts_stats->pkts = u64_stats_read(&vport->tstamp_stats.packets); + ts_stats->lost = u64_stats_read(&vport->tstamp_stats.flushed); + ts_stats->err = u64_stats_read(&vport->tstamp_stats.discarded); + } while (u64_stats_fetch_retry(&vport->tstamp_stats.stats_sync, start)); + + if (np->state != __IDPF_VPORT_UP) + goto exit; + + for (u16 i = 0; i < vport->num_txq_grp; i++) { + struct idpf_txq_group *txq_grp = &vport->txq_grps[i]; + + for (u16 j = 0; j < txq_grp->num_txq; j++) { + struct idpf_tx_queue *txq = txq_grp->txqs[j]; + struct idpf_tx_queue_stats *stats; + u64 ts; + + if (!txq) + continue; + + stats = &txq->q_stats; + do { + start = u64_stats_fetch_begin(&txq->stats_sync); + + ts = u64_stats_read(&stats->tstamp_skipped); + } while (u64_stats_fetch_retry(&txq->stats_sync, + start)); + + ts_stats->lost += ts; + } + } + +exit: + idpf_vport_ctrl_unlock(netdev); +} + static const struct ethtool_ops idpf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE, @@ -1711,6 +1766,7 @@ static const struct ethtool_ops idpf_ethtool_ops = { .set_ringparam = idpf_set_ringparam, .get_link_ksettings = idpf_get_link_ksettings, .get_ts_info = idpf_get_ts_info, + .get_ts_stats = idpf_get_ts_stats, }; /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h index 7492d1713243..20d5af64e750 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_lan_txrx.h @@ -186,13 +186,17 @@ struct idpf_base_tx_desc { __le64 qw1; /* type_cmd_offset_bsz_l2tag1 */ }; /* read used with buffer queues */ -struct idpf_splitq_tx_compl_desc { +struct idpf_splitq_4b_tx_compl_desc { /* qid=[10:0] comptype=[13:11] rsvd=[14] gen=[15] */ __le16 qid_comptype_gen; union { __le16 q_head; /* Queue head */ __le16 compl_tag; /* Completion tag */ } q_head_compl_tag; +}; /* writeback used with completion queues */ + +struct idpf_splitq_tx_compl_desc { + struct idpf_splitq_4b_tx_compl_desc common; u8 ts[3]; u8 rsvd; /* Reserved */ }; /* writeback used with completion queues */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 513032cb5f08..8a941f0fb048 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -4,6 +4,8 @@ #include "idpf.h" #include "idpf_virtchnl.h" #include "idpf_ptp.h" +#include "xdp.h" +#include "xsk.h" static const struct net_device_ops idpf_netdev_ops; @@ -776,6 +778,7 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) np->vport_idx = vport->idx; np->vport_id = vport->vport_id; np->max_tx_hdr_size = idpf_get_max_tx_hdr_size(adapter); + np->tx_max_bufs = idpf_get_max_tx_bufs(adapter); spin_lock_init(&np->stats_lock); @@ -834,6 +837,8 @@ static int idpf_cfg_netdev(struct idpf_vport *vport) netdev->hw_features |= netdev->features | other_offloads; netdev->vlan_features |= netdev->features | other_offloads; netdev->hw_enc_features |= dflt_features | other_offloads; + idpf_xdp_set_features(vport); + idpf_set_ethtool_ops(netdev); netif_set_affinity_auto(netdev); SET_NETDEV_DEV(netdev, &adapter->pdev->dev); @@ -883,14 +888,18 @@ static void idpf_remove_features(struct idpf_vport *vport) /** * idpf_vport_stop - Disable a vport * @vport: vport to disable + * @rtnl: whether to take RTNL lock */ -static void idpf_vport_stop(struct idpf_vport *vport) +static void idpf_vport_stop(struct idpf_vport *vport, bool rtnl) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); if (np->state <= __IDPF_VPORT_DOWN) return; + if (rtnl) + rtnl_lock(); + netif_carrier_off(vport->netdev); netif_tx_disable(vport->netdev); @@ -909,9 +918,13 @@ static void idpf_vport_stop(struct idpf_vport *vport) vport->link_up = false; idpf_vport_intr_deinit(vport); + idpf_xdp_rxq_info_deinit_all(vport); idpf_vport_queues_rel(vport); idpf_vport_intr_rel(vport); np->state = __IDPF_VPORT_DOWN; + + if (rtnl) + rtnl_unlock(); } /** @@ -935,7 +948,7 @@ static int idpf_stop(struct net_device *netdev) idpf_vport_ctrl_lock(netdev); vport = idpf_netdev_to_vport(netdev); - idpf_vport_stop(vport); + idpf_vport_stop(vport, false); idpf_vport_ctrl_unlock(netdev); @@ -1028,7 +1041,7 @@ static void idpf_vport_dealloc(struct idpf_vport *vport) idpf_idc_deinit_vport_aux_device(vport->vdev_info); idpf_deinit_mac_addr(vport); - idpf_vport_stop(vport); + idpf_vport_stop(vport, true); if (!test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags)) idpf_decfg_netdev(vport); @@ -1134,7 +1147,7 @@ static struct idpf_vport *idpf_vport_alloc(struct idpf_adapter *adapter, if (!vport) return vport; - num_max_q = max(max_q->max_txq, max_q->max_rxq); + num_max_q = max(max_q->max_txq, max_q->max_rxq) + IDPF_RESERVED_VECS; if (!adapter->vport_config[idx]) { struct idpf_vport_config *vport_config; struct idpf_q_coalesce *q_coal; @@ -1308,13 +1321,13 @@ static void idpf_restore_features(struct idpf_vport *vport) */ static int idpf_set_real_num_queues(struct idpf_vport *vport) { - int err; + int err, txq = vport->num_txq - vport->num_xdp_txq; err = netif_set_real_num_rx_queues(vport->netdev, vport->num_rxq); if (err) return err; - return netif_set_real_num_tx_queues(vport->netdev, vport->num_txq); + return netif_set_real_num_tx_queues(vport->netdev, txq); } /** @@ -1369,8 +1382,9 @@ static void idpf_rx_init_buf_tail(struct idpf_vport *vport) /** * idpf_vport_open - Bring up a vport * @vport: vport to bring up + * @rtnl: whether to take RTNL lock */ -static int idpf_vport_open(struct idpf_vport *vport) +static int idpf_vport_open(struct idpf_vport *vport, bool rtnl) { struct idpf_netdev_priv *np = netdev_priv(vport->netdev); struct idpf_adapter *adapter = vport->adapter; @@ -1380,6 +1394,9 @@ static int idpf_vport_open(struct idpf_vport *vport) if (np->state != __IDPF_VPORT_DOWN) return -EBUSY; + if (rtnl) + rtnl_lock(); + /* we do not allow interface up just yet */ netif_carrier_off(vport->netdev); @@ -1387,7 +1404,7 @@ static int idpf_vport_open(struct idpf_vport *vport) if (err) { dev_err(&adapter->pdev->dev, "Failed to allocate interrupts for vport %u: %d\n", vport->vport_id, err); - return err; + goto err_rtnl_unlock; } err = idpf_vport_queues_alloc(vport); @@ -1408,35 +1425,44 @@ static int idpf_vport_open(struct idpf_vport *vport) goto queues_rel; } - err = idpf_rx_bufs_init_all(vport); + err = idpf_queue_reg_init(vport); if (err) { - dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", + dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", vport->vport_id, err); goto queues_rel; } - err = idpf_queue_reg_init(vport); + err = idpf_rx_bufs_init_all(vport); if (err) { - dev_err(&adapter->pdev->dev, "Failed to initialize queue registers for vport %u: %d\n", + dev_err(&adapter->pdev->dev, "Failed to initialize RX buffers for vport %u: %d\n", vport->vport_id, err); goto queues_rel; } idpf_rx_init_buf_tail(vport); + + err = idpf_xdp_rxq_info_init_all(vport); + if (err) { + netdev_err(vport->netdev, + "Failed to initialize XDP RxQ info for vport %u: %pe\n", + vport->vport_id, ERR_PTR(err)); + goto intr_deinit; + } + idpf_vport_intr_ena(vport); err = idpf_send_config_queues_msg(vport); if (err) { dev_err(&adapter->pdev->dev, "Failed to configure queues for vport %u, %d\n", vport->vport_id, err); - goto intr_deinit; + goto rxq_deinit; } err = idpf_send_map_unmap_queue_vector_msg(vport, true); if (err) { dev_err(&adapter->pdev->dev, "Failed to map queue vectors for vport %u: %d\n", vport->vport_id, err); - goto intr_deinit; + goto rxq_deinit; } err = idpf_send_enable_queues_msg(vport); @@ -1474,6 +1500,9 @@ static int idpf_vport_open(struct idpf_vport *vport) goto deinit_rss; } + if (rtnl) + rtnl_unlock(); + return 0; deinit_rss: @@ -1484,6 +1513,8 @@ disable_queues: idpf_send_disable_queues_msg(vport); unmap_queue_vectors: idpf_send_map_unmap_queue_vector_msg(vport, false); +rxq_deinit: + idpf_xdp_rxq_info_deinit_all(vport); intr_deinit: idpf_vport_intr_deinit(vport); queues_rel: @@ -1491,6 +1522,10 @@ queues_rel: intr_rel: idpf_vport_intr_rel(vport); +err_rtnl_unlock: + if (rtnl) + rtnl_unlock(); + return err; } @@ -1547,8 +1582,6 @@ void idpf_init_task(struct work_struct *work) index = vport->idx; vport_config = adapter->vport_config[index]; - init_waitqueue_head(&vport->sw_marker_wq); - spin_lock_init(&vport_config->mac_filter_list_lock); INIT_LIST_HEAD(&vport_config->user_config.mac_filter_list); @@ -1571,7 +1604,7 @@ void idpf_init_task(struct work_struct *work) np = netdev_priv(vport->netdev); np->state = __IDPF_VPORT_DOWN; if (test_and_clear_bit(IDPF_VPORT_UP_REQUESTED, vport_config->flags)) - idpf_vport_open(vport); + idpf_vport_open(vport, true); /* Spawn and return 'idpf_init_task' work queue until all the * default vports are created @@ -1961,7 +1994,7 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, idpf_send_delete_queues_msg(vport); } else { set_bit(IDPF_VPORT_DEL_QUEUES, vport->flags); - idpf_vport_stop(vport); + idpf_vport_stop(vport, false); } idpf_deinit_rss(vport); @@ -1991,7 +2024,7 @@ int idpf_initiate_soft_reset(struct idpf_vport *vport, goto err_open; if (current_state == __IDPF_VPORT_UP) - err = idpf_vport_open(vport); + err = idpf_vport_open(vport, false); goto free_vport; @@ -2001,7 +2034,7 @@ err_reset: err_open: if (current_state == __IDPF_VPORT_UP) - idpf_vport_open(vport); + idpf_vport_open(vport, false); free_vport: kfree(new_vport); @@ -2239,7 +2272,7 @@ static int idpf_open(struct net_device *netdev) if (err) goto unlock; - err = idpf_vport_open(vport); + err = idpf_vport_open(vport, false); unlock: idpf_vport_ctrl_unlock(netdev); @@ -2272,6 +2305,92 @@ static int idpf_change_mtu(struct net_device *netdev, int new_mtu) } /** + * idpf_chk_tso_segment - Check skb is not using too many buffers + * @skb: send buffer + * @max_bufs: maximum number of buffers + * + * For TSO we need to count the TSO header and segment payload separately. As + * such we need to check cases where we have max_bufs-1 fragments or more as we + * can potentially require max_bufs+1 DMA transactions, 1 for the TSO header, 1 + * for the segment payload in the first descriptor, and another max_buf-1 for + * the fragments. + * + * Returns true if the packet needs to be software segmented by core stack. + */ +static bool idpf_chk_tso_segment(const struct sk_buff *skb, + unsigned int max_bufs) +{ + const struct skb_shared_info *shinfo = skb_shinfo(skb); + const skb_frag_t *frag, *stale; + int nr_frags, sum; + + /* no need to check if number of frags is less than max_bufs - 1 */ + nr_frags = shinfo->nr_frags; + if (nr_frags < (max_bufs - 1)) + return false; + + /* We need to walk through the list and validate that each group + * of max_bufs-2 fragments totals at least gso_size. + */ + nr_frags -= max_bufs - 2; + frag = &shinfo->frags[0]; + + /* Initialize size to the negative value of gso_size minus 1. We use + * this as the worst case scenario in which the frag ahead of us only + * provides one byte which is why we are limited to max_bufs-2 + * descriptors for a single transmit as the header and previous + * fragment are already consuming 2 descriptors. + */ + sum = 1 - shinfo->gso_size; + + /* Add size of frags 0 through 4 to create our initial sum */ + sum += skb_frag_size(frag++); + sum += skb_frag_size(frag++); + sum += skb_frag_size(frag++); + sum += skb_frag_size(frag++); + sum += skb_frag_size(frag++); + + /* Walk through fragments adding latest fragment, testing it, and + * then removing stale fragments from the sum. + */ + for (stale = &shinfo->frags[0];; stale++) { + int stale_size = skb_frag_size(stale); + + sum += skb_frag_size(frag++); + + /* The stale fragment may present us with a smaller + * descriptor than the actual fragment size. To account + * for that we need to remove all the data on the front and + * figure out what the remainder would be in the last + * descriptor associated with the fragment. + */ + if (stale_size > IDPF_TX_MAX_DESC_DATA) { + int align_pad = -(skb_frag_off(stale)) & + (IDPF_TX_MAX_READ_REQ_SIZE - 1); + + sum -= align_pad; + stale_size -= align_pad; + + do { + sum -= IDPF_TX_MAX_DESC_DATA_ALIGNED; + stale_size -= IDPF_TX_MAX_DESC_DATA_ALIGNED; + } while (stale_size > IDPF_TX_MAX_DESC_DATA); + } + + /* if sum is negative we failed to make sufficient progress */ + if (sum < 0) + return true; + + if (!nr_frags--) + break; + + sum -= stale_size; + } + + return false; +} + +/** * idpf_features_check - Validate packet conforms to limits * @skb: skb buffer * @netdev: This port's netdev @@ -2292,12 +2411,15 @@ static netdev_features_t idpf_features_check(struct sk_buff *skb, if (skb->ip_summed != CHECKSUM_PARTIAL) return features; - /* We cannot support GSO if the MSS is going to be less than - * 88 bytes. If it is then we need to drop support for GSO. - */ - if (skb_is_gso(skb) && - (skb_shinfo(skb)->gso_size < IDPF_TX_TSO_MIN_MSS)) - features &= ~NETIF_F_GSO_MASK; + if (skb_is_gso(skb)) { + /* We cannot support GSO if the MSS is going to be less than + * 88 bytes. If it is then we need to drop support for GSO. + */ + if (skb_shinfo(skb)->gso_size < IDPF_TX_TSO_MIN_MSS) + features &= ~NETIF_F_GSO_MASK; + else if (idpf_chk_tso_segment(skb, np->tx_max_bufs)) + features &= ~NETIF_F_GSO_MASK; + } /* Ensure MACLEN is <= 126 bytes (63 words) and not an odd size */ len = skb_network_offset(skb); @@ -2495,4 +2617,7 @@ static const struct net_device_ops idpf_netdev_ops = { .ndo_tx_timeout = idpf_tx_timeout, .ndo_hwtstamp_get = idpf_hwtstamp_get, .ndo_hwtstamp_set = idpf_hwtstamp_set, + .ndo_bpf = idpf_xdp, + .ndo_xdp_xmit = idpf_xdp_xmit, + .ndo_xsk_wakeup = idpf_xsk_wakeup, }; diff --git a/drivers/net/ethernet/intel/idpf/idpf_main.c b/drivers/net/ethernet/intel/idpf/idpf_main.c index dfe9126f1f4a..8c46481d2e1f 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_main.c +++ b/drivers/net/ethernet/intel/idpf/idpf_main.c @@ -9,6 +9,7 @@ MODULE_DESCRIPTION(DRV_SUMMARY); MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL"); /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_ptp.c b/drivers/net/ethernet/intel/idpf/idpf_ptp.c index ee21f2ff0cad..142823af1f9e 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_ptp.c +++ b/drivers/net/ethernet/intel/idpf/idpf_ptp.c @@ -618,8 +618,13 @@ u64 idpf_ptp_extend_ts(struct idpf_vport *vport, u64 in_tstamp) discard_time = ptp->cached_phc_jiffies + 2 * HZ; - if (time_is_before_jiffies(discard_time)) + if (time_is_before_jiffies(discard_time)) { + u64_stats_update_begin(&vport->tstamp_stats.stats_sync); + u64_stats_inc(&vport->tstamp_stats.discarded); + u64_stats_update_end(&vport->tstamp_stats.stats_sync); + return 0; + } return idpf_ptp_tstamp_extend_32b_to_64b(ptp->cached_phc_time, lower_32_bits(in_tstamp)); @@ -853,10 +858,14 @@ static void idpf_ptp_release_vport_tstamp(struct idpf_vport *vport) /* Remove list with latches in use */ head = &vport->tx_tstamp_caps->latches_in_use; + u64_stats_update_begin(&vport->tstamp_stats.stats_sync); list_for_each_entry_safe(ptp_tx_tstamp, tmp, head, list_member) { + u64_stats_inc(&vport->tstamp_stats.flushed); + list_del(&ptp_tx_tstamp->list_member); kfree(ptp_tx_tstamp); } + u64_stats_update_end(&vport->tstamp_stats.stats_sync); spin_unlock_bh(&vport->tx_tstamp_caps->latches_lock); diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index b19b462e0bb6..61e613066140 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -1,8 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ -#include <net/libeth/rx.h> -#include <net/libeth/tx.h> +#include <net/libeth/xdp.h> #include "idpf.h" @@ -655,7 +654,7 @@ static void idpf_rx_singleq_csum(struct idpf_rx_queue *rxq, bool ipv4, ipv6; /* check if Rx checksum is enabled */ - if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) + if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) return; /* check if HW has decoded the packet and checksum */ @@ -794,7 +793,7 @@ static void idpf_rx_singleq_base_hash(struct idpf_rx_queue *rx_q, { u64 mask, qw1; - if (!libeth_rx_pt_has_hash(rx_q->netdev, decoded)) + if (!libeth_rx_pt_has_hash(rx_q->xdp_rxq.dev, decoded)) return; mask = VIRTCHNL2_RX_BASE_DESC_FLTSTAT_RSS_HASH_M; @@ -822,7 +821,7 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, const union virtchnl2_rx_desc *rx_desc, struct libeth_rx_pt decoded) { - if (!libeth_rx_pt_has_hash(rx_q->netdev, decoded)) + if (!libeth_rx_pt_has_hash(rx_q->xdp_rxq.dev, decoded)) return; if (FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_STATUS0_RSS_VALID_M, @@ -834,7 +833,7 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, } /** - * idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx + * __idpf_rx_singleq_process_skb_fields - Populate skb header fields from Rx * descriptor * @rx_q: Rx ring being processed * @skb: pointer to current skb being populated @@ -846,17 +845,14 @@ static void idpf_rx_singleq_flex_hash(struct idpf_rx_queue *rx_q, * other fields within the skb. */ static void -idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, - struct sk_buff *skb, - const union virtchnl2_rx_desc *rx_desc, - u16 ptype) +__idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, + struct sk_buff *skb, + const union virtchnl2_rx_desc *rx_desc, + u16 ptype) { struct libeth_rx_pt decoded = rx_q->rx_ptype_lkup[ptype]; struct libeth_rx_csum csum_bits; - /* modifies the skb - consumes the enet header */ - skb->protocol = eth_type_trans(skb, rx_q->netdev); - /* Check if we're using base mode descriptor IDs */ if (rx_q->rxdids == VIRTCHNL2_RXDID_1_32B_BASE_M) { idpf_rx_singleq_base_hash(rx_q, skb, rx_desc, decoded); @@ -867,7 +863,6 @@ idpf_rx_singleq_process_skb_fields(struct idpf_rx_queue *rx_q, } idpf_rx_singleq_csum(rx_q, skb, csum_bits, decoded); - skb_record_rx_queue(skb, rx_q->idx); } /** @@ -1003,6 +998,32 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q, idpf_rx_singleq_extract_flex_fields(rx_desc, fields); } +static bool +idpf_rx_singleq_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *rs) +{ + struct libeth_rqe_info fields; + struct idpf_rx_queue *rxq; + + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); + + idpf_rx_singleq_extract_fields(rxq, xdp->desc, &fields); + __idpf_rx_singleq_process_skb_fields(rxq, skb, xdp->desc, + fields.ptype); + + return true; +} + +static void idpf_xdp_run_pass(struct libeth_xdp_buff *xdp, + struct napi_struct *napi, + struct libeth_rq_napi_stats *rs, + const union virtchnl2_rx_desc *desc) +{ + libeth_xdp_run_pass(xdp, NULL, napi, rs, desc, NULL, + idpf_rx_singleq_process_skb_fields); +} + /** * idpf_rx_singleq_clean - Reclaim resources after receive completes * @rx_q: rx queue to clean @@ -1012,14 +1033,15 @@ idpf_rx_singleq_extract_fields(const struct idpf_rx_queue *rx_q, */ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) { - unsigned int total_rx_bytes = 0, total_rx_pkts = 0; - struct sk_buff *skb = rx_q->skb; + struct libeth_rq_napi_stats rs = { }; u16 ntc = rx_q->next_to_clean; + LIBETH_XDP_ONSTACK_BUFF(xdp); u16 cleaned_count = 0; - bool failure = false; + + libeth_xdp_init_buff(xdp, &rx_q->xdp, &rx_q->xdp_rxq); /* Process Rx packets bounded by budget */ - while (likely(total_rx_pkts < (unsigned int)budget)) { + while (likely(rs.packets < budget)) { struct libeth_rqe_info fields = { }; union virtchnl2_rx_desc *rx_desc; struct idpf_rx_buf *rx_buf; @@ -1046,73 +1068,41 @@ static int idpf_rx_singleq_clean(struct idpf_rx_queue *rx_q, int budget) idpf_rx_singleq_extract_fields(rx_q, rx_desc, &fields); rx_buf = &rx_q->rx_buf[ntc]; - if (!libeth_rx_sync_for_cpu(rx_buf, fields.len)) - goto skip_data; - - if (skb) - idpf_rx_add_frag(rx_buf, skb, fields.len); - else - skb = idpf_rx_build_skb(rx_buf, fields.len); - - /* exit if we failed to retrieve a buffer */ - if (!skb) - break; - -skip_data: + libeth_xdp_process_buff(xdp, rx_buf, fields.len); rx_buf->netmem = 0; IDPF_SINGLEQ_BUMP_RING_IDX(rx_q, ntc); cleaned_count++; /* skip if it is non EOP desc */ - if (idpf_rx_singleq_is_non_eop(rx_desc) || unlikely(!skb)) + if (idpf_rx_singleq_is_non_eop(rx_desc) || + unlikely(!xdp->data)) continue; #define IDPF_RXD_ERR_S FIELD_PREP(VIRTCHNL2_RX_BASE_DESC_QW1_ERROR_M, \ VIRTCHNL2_RX_BASE_DESC_ERROR_RXE_M) if (unlikely(idpf_rx_singleq_test_staterr(rx_desc, IDPF_RXD_ERR_S))) { - dev_kfree_skb_any(skb); - skb = NULL; - continue; - } - - /* pad skb if needed (to make valid ethernet frame) */ - if (eth_skb_pad(skb)) { - skb = NULL; + libeth_xdp_return_buff_slow(xdp); continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - - /* protocol */ - idpf_rx_singleq_process_skb_fields(rx_q, skb, rx_desc, - fields.ptype); - - /* send completed skb up the stack */ - napi_gro_receive(rx_q->pp->p.napi, skb); - skb = NULL; - - /* update budget accounting */ - total_rx_pkts++; + idpf_xdp_run_pass(xdp, rx_q->pp->p.napi, &rs, rx_desc); } - rx_q->skb = skb; - rx_q->next_to_clean = ntc; + libeth_xdp_save_buff(&rx_q->xdp, xdp); page_pool_nid_changed(rx_q->pp, numa_mem_id()); if (cleaned_count) - failure = idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count); + idpf_rx_singleq_buf_hw_alloc_all(rx_q, cleaned_count); u64_stats_update_begin(&rx_q->stats_sync); - u64_stats_add(&rx_q->q_stats.packets, total_rx_pkts); - u64_stats_add(&rx_q->q_stats.bytes, total_rx_bytes); + u64_stats_add(&rx_q->q_stats.packets, rs.packets); + u64_stats_add(&rx_q->q_stats.bytes, rs.bytes); u64_stats_update_end(&rx_q->stats_sync); - /* guarantee a trip back through this routine if there was a failure */ - return failure ? budget : (int)total_rx_pkts; + return rs.packets; } /** diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index eaad52a83b04..828f7c444d30 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -1,18 +1,37 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright (C) 2023 Intel Corporation */ -#include <net/libeth/rx.h> -#include <net/libeth/tx.h> - #include "idpf.h" #include "idpf_ptp.h" #include "idpf_virtchnl.h" +#include "xdp.h" +#include "xsk.h" #define idpf_tx_buf_next(buf) (*(u32 *)&(buf)->priv) LIBETH_SQE_CHECK_PRIV(u32); -static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, - unsigned int count); +/** + * idpf_chk_linearize - Check if skb exceeds max descriptors per packet + * @skb: send buffer + * @max_bufs: maximum scatter gather buffers for single packet + * @count: number of buffers this packet needs + * + * Make sure we don't exceed maximum scatter gather buffers for a single + * packet. + * TSO case has been handled earlier from idpf_features_check(). + */ +static bool idpf_chk_linearize(const struct sk_buff *skb, + unsigned int max_bufs, + unsigned int count) +{ + if (likely(count <= max_bufs)) + return false; + + if (skb_is_gso(skb)) + return false; + + return true; +} /** * idpf_tx_timeout - Respond to a Tx Hang @@ -35,26 +54,39 @@ void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) } } -/** - * idpf_tx_buf_rel_all - Free any empty Tx buffers - * @txq: queue to be cleaned - */ -static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) +static void idpf_tx_buf_clean(struct idpf_tx_queue *txq) { struct libeth_sq_napi_stats ss = { }; + struct xdp_frame_bulk bq; struct libeth_cq_pp cp = { .dev = txq->dev, + .bq = &bq, .ss = &ss, }; - u32 i; + xdp_frame_bulk_init(&bq); + + /* Free all the Tx buffer sk_buffs */ + for (u32 i = 0; i < txq->buf_pool_size; i++) + libeth_tx_complete_any(&txq->tx_buf[i], &cp); + + xdp_flush_frame_bulk(&bq); +} + +/** + * idpf_tx_buf_rel_all - Free any empty Tx buffers + * @txq: queue to be cleaned + */ +static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) +{ /* Buffers already cleared, nothing to do */ if (!txq->tx_buf) return; - /* Free all the Tx buffer sk_buffs */ - for (i = 0; i < txq->buf_pool_size; i++) - libeth_tx_complete(&txq->tx_buf[i], &cp); + if (idpf_queue_has(XSK, txq)) + idpf_xsksq_clean(txq); + else + idpf_tx_buf_clean(txq); kfree(txq->tx_buf); txq->tx_buf = NULL; @@ -68,13 +100,22 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) */ static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) { + bool xdp = idpf_queue_has(XDP, txq); + + if (xdp) + libeth_xdpsq_deinit_timer(txq->timer); + idpf_tx_buf_rel_all(txq); - netdev_tx_reset_subqueue(txq->netdev, txq->idx); + + if (!xdp) + netdev_tx_reset_subqueue(txq->netdev, txq->idx); + + idpf_xsk_clear_queue(txq, VIRTCHNL2_QUEUE_TYPE_TX); if (!txq->desc_ring) return; - if (txq->refillq) + if (!xdp && txq->refillq) kfree(txq->refillq->ring); dmam_free_coherent(txq->dev, txq->size, txq->desc_ring, txq->dma); @@ -91,12 +132,14 @@ static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) */ static void idpf_compl_desc_rel(struct idpf_compl_queue *complq) { + idpf_xsk_clear_queue(complq, VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); + if (!complq->comp) return; dma_free_coherent(complq->netdev->dev.parent, complq->size, - complq->comp, complq->dma); - complq->comp = NULL; + complq->desc_ring, complq->dma); + complq->desc_ring = NULL; complq->next_to_use = 0; complq->next_to_clean = 0; } @@ -183,6 +226,8 @@ static int idpf_tx_desc_alloc(const struct idpf_vport *vport, tx_q->next_to_clean = 0; idpf_queue_set(GEN_CHK, tx_q); + idpf_xsk_setup_queue(vport, tx_q, VIRTCHNL2_QUEUE_TYPE_TX); + if (!idpf_queue_has(FLOW_SCH_EN, tx_q)) return 0; @@ -226,18 +271,25 @@ err_alloc: static int idpf_compl_desc_alloc(const struct idpf_vport *vport, struct idpf_compl_queue *complq) { - complq->size = array_size(complq->desc_count, sizeof(*complq->comp)); + u32 desc_size; - complq->comp = dma_alloc_coherent(complq->netdev->dev.parent, - complq->size, &complq->dma, - GFP_KERNEL); - if (!complq->comp) + desc_size = idpf_queue_has(FLOW_SCH_EN, complq) ? + sizeof(*complq->comp) : sizeof(*complq->comp_4b); + complq->size = array_size(complq->desc_count, desc_size); + + complq->desc_ring = dma_alloc_coherent(complq->netdev->dev.parent, + complq->size, &complq->dma, + GFP_KERNEL); + if (!complq->desc_ring) return -ENOMEM; complq->next_to_use = 0; complq->next_to_clean = 0; idpf_queue_set(GEN_CHK, complq); + idpf_xsk_setup_queue(vport, complq, + VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); + return 0; } @@ -337,6 +389,11 @@ static void idpf_rx_buf_rel_bufq(struct idpf_buf_queue *bufq) if (!bufq->buf) return; + if (idpf_queue_has(XSK, bufq)) { + idpf_xskfq_rel(bufq); + return; + } + /* Free all the bufs allocated and given to hw on Rx queue */ for (u32 i = 0; i < bufq->desc_count; i++) idpf_rx_page_rel(&bufq->buf[i]); @@ -385,14 +442,14 @@ static void idpf_rx_desc_rel(struct idpf_rx_queue *rxq, struct device *dev, if (!rxq) return; - if (rxq->skb) { - dev_kfree_skb_any(rxq->skb); - rxq->skb = NULL; - } + if (!idpf_queue_has(XSK, rxq)) + libeth_xdp_return_stash(&rxq->xdp); if (!idpf_is_queue_model_split(model)) idpf_rx_buf_rel_all(rxq); + idpf_xsk_clear_queue(rxq, VIRTCHNL2_QUEUE_TYPE_RX); + rxq->next_to_alloc = 0; rxq->next_to_clean = 0; rxq->next_to_use = 0; @@ -415,6 +472,7 @@ static void idpf_rx_desc_rel_bufq(struct idpf_buf_queue *bufq, return; idpf_rx_buf_rel_bufq(bufq); + idpf_xsk_clear_queue(bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); bufq->next_to_alloc = 0; bufq->next_to_clean = 0; @@ -497,6 +555,7 @@ static int idpf_rx_hdr_buf_alloc_all(struct idpf_buf_queue *bufq) struct libeth_fq fq = { .count = bufq->desc_count, .type = LIBETH_FQE_HDR, + .xdp = idpf_xdp_enabled(bufq->q_vector->vport), .nid = idpf_q_vector_to_mem(bufq->q_vector), }; int ret; @@ -696,10 +755,14 @@ static int idpf_rx_bufs_init(struct idpf_buf_queue *bufq, .count = bufq->desc_count, .type = type, .hsplit = idpf_queue_has(HSPLIT_EN, bufq), + .xdp = idpf_xdp_enabled(bufq->q_vector->vport), .nid = idpf_q_vector_to_mem(bufq->q_vector), }; int ret; + if (idpf_queue_has(XSK, bufq)) + return idpf_xskfq_init(bufq); + ret = libeth_rx_fq_create(&fq, &bufq->q_vector->napi); if (ret) return ret; @@ -723,6 +786,8 @@ int idpf_rx_bufs_init_all(struct idpf_vport *vport) bool split = idpf_is_queue_model_split(vport->rxq_model); int i, j, err; + idpf_xdp_copy_prog_to_rqs(vport, vport->xdp_prog); + for (i = 0; i < vport->num_rxq_grp; i++) { struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; u32 truesize = 0; @@ -793,6 +858,8 @@ static int idpf_rx_desc_alloc(const struct idpf_vport *vport, rxq->next_to_use = 0; idpf_queue_set(GEN_CHK, rxq); + idpf_xsk_setup_queue(vport, rxq, VIRTCHNL2_QUEUE_TYPE_RX); + return 0; } @@ -818,9 +885,10 @@ static int idpf_bufq_desc_alloc(const struct idpf_vport *vport, bufq->next_to_alloc = 0; bufq->next_to_clean = 0; bufq->next_to_use = 0; - idpf_queue_set(GEN_CHK, bufq); + idpf_xsk_setup_queue(vport, bufq, VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); + return 0; } @@ -886,6 +954,341 @@ err_out: return err; } +static int idpf_init_queue_set(const struct idpf_queue_set *qs) +{ + const struct idpf_vport *vport = qs->vport; + bool splitq; + int err; + + splitq = idpf_is_queue_model_split(vport->rxq_model); + + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + struct idpf_buf_queue *bufq; + + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + err = idpf_rx_desc_alloc(vport, q->rxq); + if (err) + break; + + err = idpf_xdp_rxq_info_init(q->rxq); + if (err) + break; + + if (!splitq) + err = idpf_rx_bufs_init_singleq(q->rxq); + + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + bufq = q->bufq; + + err = idpf_bufq_desc_alloc(vport, bufq); + if (err) + break; + + for (u32 j = 0; j < bufq->q_vector->num_bufq; j++) { + struct idpf_buf_queue * const *bufqs; + enum libeth_fqe_type type; + u32 ts; + + bufqs = bufq->q_vector->bufq; + if (bufqs[j] != bufq) + continue; + + if (j) { + type = LIBETH_FQE_SHORT; + ts = bufqs[j - 1]->truesize >> 1; + } else { + type = LIBETH_FQE_MTU; + ts = 0; + } + + bufq->truesize = ts; + + err = idpf_rx_bufs_init(bufq, type); + break; + } + + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + err = idpf_tx_desc_alloc(vport, q->txq); + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + err = idpf_compl_desc_alloc(vport, q->complq); + break; + default: + continue; + } + + if (err) + return err; + } + + return 0; +} + +static void idpf_clean_queue_set(const struct idpf_queue_set *qs) +{ + const struct idpf_vport *vport = qs->vport; + struct device *dev = vport->netdev->dev.parent; + + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + idpf_xdp_rxq_info_deinit(q->rxq, vport->rxq_model); + idpf_rx_desc_rel(q->rxq, dev, vport->rxq_model); + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + idpf_rx_desc_rel_bufq(q->bufq, dev); + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + idpf_tx_desc_rel(q->txq); + + if (idpf_queue_has(XDP, q->txq)) { + q->txq->pending = 0; + q->txq->xdp_tx = 0; + } else { + q->txq->txq_grp->num_completions_pending = 0; + } + + writel(q->txq->next_to_use, q->txq->tail); + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + idpf_compl_desc_rel(q->complq); + q->complq->num_completions = 0; + break; + default: + break; + } + } +} + +static void idpf_qvec_ena_irq(struct idpf_q_vector *qv) +{ + if (qv->num_txq) { + u32 itr; + + if (IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode)) + itr = qv->vport->tx_itr_profile[qv->tx_dim.profile_ix]; + else + itr = qv->tx_itr_value; + + idpf_vport_intr_write_itr(qv, itr, true); + } + + if (qv->num_rxq) { + u32 itr; + + if (IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode)) + itr = qv->vport->rx_itr_profile[qv->rx_dim.profile_ix]; + else + itr = qv->rx_itr_value; + + idpf_vport_intr_write_itr(qv, itr, false); + } + + if (qv->num_txq || qv->num_rxq) + idpf_vport_intr_update_itr_ena_irq(qv); +} + +/** + * idpf_vector_to_queue_set - create a queue set associated with the given + * queue vector + * @qv: queue vector corresponding to the queue pair + * + * Returns a pointer to a dynamically allocated array of pointers to all + * queues associated with a given queue vector (@qv). + * Please note that the caller is responsible to free the memory allocated + * by this function using kfree(). + * + * Return: &idpf_queue_set on success, %NULL in case of error. + */ +static struct idpf_queue_set * +idpf_vector_to_queue_set(struct idpf_q_vector *qv) +{ + bool xdp = qv->vport->xdp_txq_offset && !qv->num_xsksq; + struct idpf_vport *vport = qv->vport; + struct idpf_queue_set *qs; + u32 num; + + num = qv->num_rxq + qv->num_bufq + qv->num_txq + qv->num_complq; + num += xdp ? qv->num_rxq * 2 : qv->num_xsksq * 2; + if (!num) + return NULL; + + qs = idpf_alloc_queue_set(vport, num); + if (!qs) + return NULL; + + num = 0; + + for (u32 i = 0; i < qv->num_bufq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + qs->qs[num++].bufq = qv->bufq[i]; + } + + for (u32 i = 0; i < qv->num_rxq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_RX; + qs->qs[num++].rxq = qv->rx[i]; + } + + for (u32 i = 0; i < qv->num_txq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[num++].txq = qv->tx[i]; + } + + for (u32 i = 0; i < qv->num_complq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[num++].complq = qv->complq[i]; + } + + if (!vport->xdp_txq_offset) + goto finalize; + + if (xdp) { + for (u32 i = 0; i < qv->num_rxq; i++) { + u32 idx = vport->xdp_txq_offset + qv->rx[i]->idx; + + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[num++].txq = vport->txqs[idx]; + + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[num++].complq = vport->txqs[idx]->complq; + } + } else { + for (u32 i = 0; i < qv->num_xsksq; i++) { + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[num++].txq = qv->xsksq[i]; + + qs->qs[num].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[num++].complq = qv->xsksq[i]->complq; + } + } + +finalize: + if (num != qs->num) { + kfree(qs); + return NULL; + } + + return qs; +} + +static int idpf_qp_enable(const struct idpf_queue_set *qs, u32 qid) +{ + struct idpf_vport *vport = qs->vport; + struct idpf_q_vector *q_vector; + int err; + + q_vector = idpf_find_rxq_vec(vport, qid); + + err = idpf_init_queue_set(qs); + if (err) { + netdev_err(vport->netdev, "Could not initialize queues in pair %u: %pe\n", + qid, ERR_PTR(err)); + return err; + } + + if (!vport->xdp_txq_offset) + goto config; + + q_vector->xsksq = kcalloc(DIV_ROUND_UP(vport->num_rxq_grp, + vport->num_q_vectors), + sizeof(*q_vector->xsksq), GFP_KERNEL); + if (!q_vector->xsksq) + return -ENOMEM; + + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + + if (q->type != VIRTCHNL2_QUEUE_TYPE_TX) + continue; + + if (!idpf_queue_has(XSK, q->txq)) + continue; + + idpf_xsk_init_wakeup(q_vector); + + q->txq->q_vector = q_vector; + q_vector->xsksq[q_vector->num_xsksq++] = q->txq; + } + +config: + err = idpf_send_config_queue_set_msg(qs); + if (err) { + netdev_err(vport->netdev, "Could not configure queues in pair %u: %pe\n", + qid, ERR_PTR(err)); + return err; + } + + err = idpf_send_enable_queue_set_msg(qs); + if (err) { + netdev_err(vport->netdev, "Could not enable queues in pair %u: %pe\n", + qid, ERR_PTR(err)); + return err; + } + + napi_enable(&q_vector->napi); + idpf_qvec_ena_irq(q_vector); + + netif_start_subqueue(vport->netdev, qid); + + return 0; +} + +static int idpf_qp_disable(const struct idpf_queue_set *qs, u32 qid) +{ + struct idpf_vport *vport = qs->vport; + struct idpf_q_vector *q_vector; + int err; + + q_vector = idpf_find_rxq_vec(vport, qid); + netif_stop_subqueue(vport->netdev, qid); + + writel(0, q_vector->intr_reg.dyn_ctl); + napi_disable(&q_vector->napi); + + err = idpf_send_disable_queue_set_msg(qs); + if (err) { + netdev_err(vport->netdev, "Could not disable queues in pair %u: %pe\n", + qid, ERR_PTR(err)); + return err; + } + + idpf_clean_queue_set(qs); + + kfree(q_vector->xsksq); + q_vector->num_xsksq = 0; + + return 0; +} + +/** + * idpf_qp_switch - enable or disable queues associated with queue pair + * @vport: vport to switch the pair for + * @qid: index of the queue pair to switch + * @en: whether to enable or disable the pair + * + * Return: 0 on success, -errno on failure. + */ +int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en) +{ + struct idpf_q_vector *q_vector = idpf_find_rxq_vec(vport, qid); + struct idpf_queue_set *qs __free(kfree) = NULL; + + if (idpf_find_txq_vec(vport, qid) != q_vector) + return -EINVAL; + + qs = idpf_vector_to_queue_set(q_vector); + if (!qs) + return -ENOMEM; + + return en ? idpf_qp_enable(qs, qid) : idpf_qp_disable(qs, qid); +} + /** * idpf_txq_group_rel - Release all resources for txq groups * @vport: vport to release txq groups on @@ -1001,8 +1404,12 @@ static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport) */ void idpf_vport_queues_rel(struct idpf_vport *vport) { + idpf_xdp_copy_prog_to_rqs(vport, NULL); + idpf_tx_desc_rel_all(vport); idpf_rx_desc_rel_all(vport); + + idpf_xdpsqs_put(vport); idpf_vport_queue_grp_rel_all(vport); kfree(vport->txqs); @@ -1076,6 +1483,18 @@ void idpf_vport_init_num_qs(struct idpf_vport *vport, if (idpf_is_queue_model_split(vport->rxq_model)) vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq); + vport->xdp_prog = config_data->xdp_prog; + if (idpf_xdp_enabled(vport)) { + vport->xdp_txq_offset = config_data->num_req_tx_qs; + vport->num_xdp_txq = le16_to_cpu(vport_msg->num_tx_q) - + vport->xdp_txq_offset; + vport->xdpsq_share = libeth_xdpsq_shared(vport->num_xdp_txq); + } else { + vport->xdp_txq_offset = 0; + vport->num_xdp_txq = 0; + vport->xdpsq_share = false; + } + /* Adjust number of buffer queues per Rx queue group. */ if (!idpf_is_queue_model_split(vport->rxq_model)) { vport->num_bufqs_per_qgrp = 0; @@ -1147,22 +1566,17 @@ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0; int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0; u16 num_req_tx_qs = 0, num_req_rx_qs = 0; + struct idpf_vport_user_config_data *user; struct idpf_vport_config *vport_config; u16 num_txq_grps, num_rxq_grps; - u32 num_qs; + u32 num_qs, num_xdpsq; vport_config = adapter->vport_config[vport_idx]; if (vport_config) { num_req_tx_qs = vport_config->user_config.num_req_tx_qs; num_req_rx_qs = vport_config->user_config.num_req_rx_qs; } else { - int num_cpus; - - /* Restrict num of queues to cpus online as a default - * configuration to give best performance. User can always - * override to a max number of queues via ethtool. - */ - num_cpus = num_online_cpus(); + u32 num_cpus = netif_get_num_default_rss_queues(); dflt_splitq_txq_grps = min_t(int, max_q->max_txq, num_cpus); dflt_singleq_txqs = min_t(int, max_q->max_txq, num_cpus); @@ -1197,6 +1611,24 @@ int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx, vport_msg->num_rx_bufq = 0; } + if (!vport_config) + return 0; + + user = &vport_config->user_config; + user->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q); + user->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q); + + if (vport_config->user_config.xdp_prog) + num_xdpsq = libeth_xdpsq_num(user->num_req_rx_qs, + user->num_req_tx_qs, + vport_config->max_q.max_txq); + else + num_xdpsq = 0; + + vport_msg->num_tx_q = cpu_to_le16(user->num_req_tx_qs + num_xdpsq); + if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) + vport_msg->num_tx_complq = vport_msg->num_tx_q; + return 0; } @@ -1246,14 +1678,13 @@ static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport, static void idpf_rxq_set_descids(const struct idpf_vport *vport, struct idpf_rx_queue *q) { - if (idpf_is_queue_model_split(vport->rxq_model)) { - q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M; - } else { - if (vport->base_rxd) - q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; - else - q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; - } + if (idpf_is_queue_model_split(vport->rxq_model)) + return; + + if (vport->base_rxd) + q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M; + else + q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M; } /** @@ -1301,6 +1732,7 @@ static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq) q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter); q->netdev = vport->netdev; q->txq_grp = tx_qgrp; + q->rel_q_id = j; if (!split) { q->clean_budget = vport->compln_clean_budget; @@ -1461,7 +1893,6 @@ skip_splitq_rx_init: setup_rxq: q->desc_count = vport->rxq_desc_count; q->rx_ptype_lkup = vport->rx_ptype_lkup; - q->netdev = vport->netdev; q->bufq_sets = rx_qgrp->splitq.bufq_sets; q->idx = (i * num_rxq) + j; q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK; @@ -1522,15 +1953,19 @@ int idpf_vport_queues_alloc(struct idpf_vport *vport) if (err) goto err_out; - err = idpf_tx_desc_alloc_all(vport); + err = idpf_vport_init_fast_path_txqs(vport); if (err) goto err_out; - err = idpf_rx_desc_alloc_all(vport); + err = idpf_xdpsqs_get(vport); if (err) goto err_out; - err = idpf_vport_init_fast_path_txqs(vport); + err = idpf_tx_desc_alloc_all(vport); + if (err) + goto err_out; + + err = idpf_rx_desc_alloc_all(vport); if (err) goto err_out; @@ -1543,32 +1978,6 @@ err_out: } /** - * idpf_tx_handle_sw_marker - Handle queue marker packet - * @tx_q: tx queue to handle software marker - */ -static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) -{ - struct idpf_netdev_priv *priv = netdev_priv(tx_q->netdev); - struct idpf_vport *vport = priv->vport; - int i; - - idpf_queue_clear(SW_MARKER, tx_q); - /* Hardware must write marker packets to all queues associated with - * completion queues. So check if all queues received marker packets - */ - for (i = 0; i < vport->num_txq; i++) - /* If we're still waiting on any other TXQ marker completions, - * just return now since we cannot wake up the marker_wq yet. - */ - if (idpf_queue_has(SW_MARKER, vport->txqs[i])) - return; - - /* Drain complete */ - set_bit(IDPF_VPORT_SW_MARKER, vport->flags); - wake_up(&vport->sw_marker_wq); -} - -/** * idpf_tx_read_tstamp - schedule a work to read Tx timestamp value * @txq: queue to read the timestamp from * @skb: socket buffer to provide Tx timestamp value @@ -1745,7 +2154,7 @@ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, /* RS completion contains queue head for queue based scheduling or * completion tag for flow based scheduling. */ - u16 rs_compl_val = le16_to_cpu(desc->q_head_compl_tag.q_head); + u16 rs_compl_val = le16_to_cpu(desc->common.q_head_compl_tag.q_head); if (!idpf_queue_has(FLOW_SCH_EN, txq)) { idpf_tx_splitq_clean(txq, rs_compl_val, budget, cleaned, false); @@ -1780,19 +2189,19 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, do { struct libeth_sq_napi_stats cleaned_stats = { }; struct idpf_tx_queue *tx_q; + __le16 hw_head; int rel_tx_qid; - u16 hw_head; u8 ctype; /* completion type */ u16 gen; /* if the descriptor isn't done, no work yet to do */ - gen = le16_get_bits(tx_desc->qid_comptype_gen, + gen = le16_get_bits(tx_desc->common.qid_comptype_gen, IDPF_TXD_COMPLQ_GEN_M); if (idpf_queue_has(GEN_CHK, complq) != gen) break; /* Find necessary info of TX queue to clean buffers */ - rel_tx_qid = le16_get_bits(tx_desc->qid_comptype_gen, + rel_tx_qid = le16_get_bits(tx_desc->common.qid_comptype_gen, IDPF_TXD_COMPLQ_QID_M); if (rel_tx_qid >= complq->txq_grp->num_txq || !complq->txq_grp->txqs[rel_tx_qid]) { @@ -1802,22 +2211,19 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, tx_q = complq->txq_grp->txqs[rel_tx_qid]; /* Determine completion type */ - ctype = le16_get_bits(tx_desc->qid_comptype_gen, + ctype = le16_get_bits(tx_desc->common.qid_comptype_gen, IDPF_TXD_COMPLQ_COMPL_TYPE_M); switch (ctype) { case IDPF_TXD_COMPLT_RE: - hw_head = le16_to_cpu(tx_desc->q_head_compl_tag.q_head); + hw_head = tx_desc->common.q_head_compl_tag.q_head; - idpf_tx_splitq_clean(tx_q, hw_head, budget, - &cleaned_stats, true); + idpf_tx_splitq_clean(tx_q, le16_to_cpu(hw_head), + budget, &cleaned_stats, true); break; case IDPF_TXD_COMPLT_RS: idpf_tx_handle_rs_completion(tx_q, tx_desc, &cleaned_stats, budget); break; - case IDPF_TXD_COMPLT_SW_MARKER: - idpf_tx_handle_sw_marker(tx_q); - break; default: netdev_err(tx_q->netdev, "Unknown TX completion type: %d\n", ctype); @@ -1890,6 +2296,69 @@ fetch_next_desc: } /** + * idpf_wait_for_sw_marker_completion - wait for SW marker of disabled Tx queue + * @txq: disabled Tx queue + * + * When Tx queue is requested for disabling, the CP sends a special completion + * descriptor called "SW marker", meaning the queue is ready to be destroyed. + * If, for some reason, the marker is not received within 500 ms, break the + * polling to not hang the driver. + */ +void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq) +{ + struct idpf_compl_queue *complq; + unsigned long timeout; + bool flow, gen_flag; + u32 ntc; + + if (!idpf_queue_has(SW_MARKER, txq)) + return; + + complq = idpf_queue_has(XDP, txq) ? txq->complq : txq->txq_grp->complq; + ntc = complq->next_to_clean; + + flow = idpf_queue_has(FLOW_SCH_EN, complq); + gen_flag = idpf_queue_has(GEN_CHK, complq); + + timeout = jiffies + msecs_to_jiffies(IDPF_WAIT_FOR_MARKER_TIMEO); + + do { + struct idpf_splitq_4b_tx_compl_desc *tx_desc; + struct idpf_tx_queue *target; + u32 ctype_gen, id; + + tx_desc = flow ? &complq->comp[ntc].common : + &complq->comp_4b[ntc]; + ctype_gen = le16_to_cpu(tx_desc->qid_comptype_gen); + + if (!!(ctype_gen & IDPF_TXD_COMPLQ_GEN_M) != gen_flag) { + usleep_range(500, 1000); + continue; + } + + if (FIELD_GET(IDPF_TXD_COMPLQ_COMPL_TYPE_M, ctype_gen) != + IDPF_TXD_COMPLT_SW_MARKER) + goto next; + + id = FIELD_GET(IDPF_TXD_COMPLQ_QID_M, ctype_gen); + target = complq->txq_grp->txqs[id]; + + idpf_queue_clear(SW_MARKER, target); + if (target == txq) + break; + +next: + if (unlikely(++ntc == complq->desc_count)) { + ntc = 0; + gen_flag = !gen_flag; + } + } while (time_before(jiffies, timeout)); + + idpf_queue_assign(GEN_CHK, complq, gen_flag); + complq->next_to_clean = ntc; +} + +/** * idpf_tx_splitq_build_ctb - populate command tag and size for queue * based scheduling descriptors * @desc: descriptor to populate @@ -2397,111 +2866,6 @@ int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off) return 1; } -/** - * __idpf_chk_linearize - Check skb is not using too many buffers - * @skb: send buffer - * @max_bufs: maximum number of buffers - * - * For TSO we need to count the TSO header and segment payload separately. As - * such we need to check cases where we have max_bufs-1 fragments or more as we - * can potentially require max_bufs+1 DMA transactions, 1 for the TSO header, 1 - * for the segment payload in the first descriptor, and another max_buf-1 for - * the fragments. - */ -static bool __idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs) -{ - const struct skb_shared_info *shinfo = skb_shinfo(skb); - const skb_frag_t *frag, *stale; - int nr_frags, sum; - - /* no need to check if number of frags is less than max_bufs - 1 */ - nr_frags = shinfo->nr_frags; - if (nr_frags < (max_bufs - 1)) - return false; - - /* We need to walk through the list and validate that each group - * of max_bufs-2 fragments totals at least gso_size. - */ - nr_frags -= max_bufs - 2; - frag = &shinfo->frags[0]; - - /* Initialize size to the negative value of gso_size minus 1. We use - * this as the worst case scenario in which the frag ahead of us only - * provides one byte which is why we are limited to max_bufs-2 - * descriptors for a single transmit as the header and previous - * fragment are already consuming 2 descriptors. - */ - sum = 1 - shinfo->gso_size; - - /* Add size of frags 0 through 4 to create our initial sum */ - sum += skb_frag_size(frag++); - sum += skb_frag_size(frag++); - sum += skb_frag_size(frag++); - sum += skb_frag_size(frag++); - sum += skb_frag_size(frag++); - - /* Walk through fragments adding latest fragment, testing it, and - * then removing stale fragments from the sum. - */ - for (stale = &shinfo->frags[0];; stale++) { - int stale_size = skb_frag_size(stale); - - sum += skb_frag_size(frag++); - - /* The stale fragment may present us with a smaller - * descriptor than the actual fragment size. To account - * for that we need to remove all the data on the front and - * figure out what the remainder would be in the last - * descriptor associated with the fragment. - */ - if (stale_size > IDPF_TX_MAX_DESC_DATA) { - int align_pad = -(skb_frag_off(stale)) & - (IDPF_TX_MAX_READ_REQ_SIZE - 1); - - sum -= align_pad; - stale_size -= align_pad; - - do { - sum -= IDPF_TX_MAX_DESC_DATA_ALIGNED; - stale_size -= IDPF_TX_MAX_DESC_DATA_ALIGNED; - } while (stale_size > IDPF_TX_MAX_DESC_DATA); - } - - /* if sum is negative we failed to make sufficient progress */ - if (sum < 0) - return true; - - if (!nr_frags--) - break; - - sum -= stale_size; - } - - return false; -} - -/** - * idpf_chk_linearize - Check if skb exceeds max descriptors per packet - * @skb: send buffer - * @max_bufs: maximum scatter gather buffers for single packet - * @count: number of buffers this packet needs - * - * Make sure we don't exceed maximum scatter gather buffers for a single - * packet. We have to do some special checking around the boundary (max_bufs-1) - * if TSO is on since we need count the TSO header and payload separately. - * E.g.: a packet with 7 fragments can require 9 DMA transactions; 1 for TSO - * header, 1 for segment payload, and then 7 for the fragments. - */ -static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, - unsigned int count) -{ - if (likely(count < max_bufs)) - return false; - if (skb_is_gso(skb)) - return __idpf_chk_linearize(skb, max_bufs); - - return count > max_bufs; -} /** * idpf_tx_splitq_get_ctx_desc - grab next desc and update buffer ring @@ -2758,10 +3122,11 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, */ netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev) { - struct idpf_vport *vport = idpf_netdev_to_vport(netdev); + const struct idpf_vport *vport = idpf_netdev_to_vport(netdev); struct idpf_tx_queue *tx_q; - if (unlikely(skb_get_queue_mapping(skb) >= vport->num_txq)) { + if (unlikely(skb_get_queue_mapping(skb) >= + vport->num_txq - vport->num_xdp_txq)) { dev_kfree_skb_any(skb); return NETDEV_TX_OK; @@ -2798,7 +3163,7 @@ idpf_rx_hash(const struct idpf_rx_queue *rxq, struct sk_buff *skb, { u32 hash; - if (!libeth_rx_pt_has_hash(rxq->netdev, decoded)) + if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, decoded)) return; hash = le16_to_cpu(rx_desc->hash1) | @@ -2824,7 +3189,7 @@ static void idpf_rx_csum(struct idpf_rx_queue *rxq, struct sk_buff *skb, bool ipv4, ipv6; /* check if Rx checksum is enabled */ - if (!libeth_rx_pt_has_checksum(rxq->netdev, decoded)) + if (!libeth_rx_pt_has_checksum(rxq->xdp_rxq.dev, decoded)) return; /* check if HW has decoded the packet and checksum */ @@ -2996,7 +3361,7 @@ idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq, } /** - * idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor + * __idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor * @rxq: Rx descriptor ring packet is being transacted on * @skb: pointer to current skb being populated * @rx_desc: Receive descriptor @@ -3006,8 +3371,8 @@ idpf_rx_hwtstamp(const struct idpf_rx_queue *rxq, * other fields within the skb. */ static int -idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, - const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) +__idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc) { struct libeth_rx_csum csum_bits; struct libeth_rx_pt decoded; @@ -3023,9 +3388,6 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, if (idpf_queue_has(PTP, rxq)) idpf_rx_hwtstamp(rxq, rx_desc, skb); - skb->protocol = eth_type_trans(skb, rxq->netdev); - skb_record_rx_queue(skb, rxq->idx); - if (le16_get_bits(rx_desc->hdrlen_flags, VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M)) return idpf_rx_rsc(rxq, skb, rx_desc, decoded); @@ -3036,25 +3398,24 @@ idpf_rx_process_skb_fields(struct idpf_rx_queue *rxq, struct sk_buff *skb, return 0; } -/** - * idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag - * @rx_buf: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: packet length from rx_desc - * - * This function will add the data contained in rx_buf->page to the skb. - * It will just attach the page as a frag to the skb. - * The function will then update the page offset. - */ -void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, - unsigned int size) +bool idpf_rx_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *rs) { - u32 hr = netmem_get_pp(rx_buf->netmem)->p.offset; + struct idpf_rx_queue *rxq; + + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, rx_buf->netmem, - rx_buf->offset + hr, size, rx_buf->truesize); + return !__idpf_rx_process_skb_fields(rxq, skb, xdp->desc); } +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_RUN(static idpf_xdp_run_pass, idpf_xdp_run_prog, + idpf_xdp_tx_flush_bulk, idpf_rx_process_skb_fields); +LIBETH_XDP_DEFINE_FINALIZE(static idpf_xdp_finalize_rx, idpf_xdp_tx_flush_bulk, + idpf_xdp_tx_finalize); +LIBETH_XDP_DEFINE_END(); + /** * idpf_rx_hsplit_wa - handle header buffer overflows and split errors * @hdr: Rx buffer for the headers @@ -3097,36 +3458,6 @@ static u32 idpf_rx_hsplit_wa(const struct libeth_fqe *hdr, } /** - * idpf_rx_build_skb - Allocate skb and populate it from header buffer - * @buf: Rx buffer to pull data from - * @size: the length of the packet - * - * This function allocates an skb. It then populates it with the page data from - * the current receive descriptor, taking care to set up the skb correctly. - */ -struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size) -{ - struct page *buf_page = __netmem_to_page(buf->netmem); - u32 hr = pp_page_to_nmdesc(buf_page)->pp->p.offset; - struct sk_buff *skb; - void *va; - - va = page_address(buf_page) + buf->offset; - prefetch(va + hr); - - skb = napi_build_skb(va, buf->truesize); - if (unlikely(!skb)) - return NULL; - - skb_mark_for_recycle(skb); - - skb_reserve(skb, hr); - __skb_put(skb, size); - - return skb; -} - -/** * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor * status and error fields * @stat_err_field: field from descriptor to test bits in @@ -3167,13 +3498,18 @@ static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_de */ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) { - int total_rx_bytes = 0, total_rx_pkts = 0; struct idpf_buf_queue *rx_bufq = NULL; - struct sk_buff *skb = rxq->skb; + struct libeth_rq_napi_stats rs = { }; u16 ntc = rxq->next_to_clean; + LIBETH_XDP_ONSTACK_BUFF(xdp); + LIBETH_XDP_ONSTACK_BULK(bq); + + libeth_xdp_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, + rxq->xdpsqs, rxq->num_xdp_txq); + libeth_xdp_init_buff(xdp, &rxq->xdp, &rxq->xdp_rxq); /* Process Rx packets bounded by budget */ - while (likely(total_rx_pkts < budget)) { + while (likely(rs.packets < budget)) { struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; struct libeth_fqe *hdr, *rx_buf = NULL; struct idpf_sw_queue *refillq = NULL; @@ -3187,18 +3523,14 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) /* get the Rx desc from Rx queue based on 'next_to_clean' */ rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; - /* This memory barrier is needed to keep us from reading - * any other fields out of the rx_desc - */ - dma_rmb(); - /* if the descriptor isn't done, no work yet to do */ gen_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id, VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M); - if (idpf_queue_has(GEN_CHK, rxq) != gen_id) break; + dma_rmb(); + rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M, rx_desc->rxdid_ucast); if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) { @@ -3243,7 +3575,7 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) hdr = &rx_bufq->hdr_buf[buf_id]; - if (unlikely(!hdr_len && !skb)) { + if (unlikely(!hdr_len && !xdp->data)) { hdr_len = idpf_rx_hsplit_wa(hdr, rx_buf, pkt_len); /* If failed, drop both buffers by setting len to 0 */ pkt_len -= hdr_len ? : pkt_len; @@ -3253,75 +3585,37 @@ static int idpf_rx_splitq_clean(struct idpf_rx_queue *rxq, int budget) u64_stats_update_end(&rxq->stats_sync); } - if (libeth_rx_sync_for_cpu(hdr, hdr_len)) { - skb = idpf_rx_build_skb(hdr, hdr_len); - if (!skb) - break; - - u64_stats_update_begin(&rxq->stats_sync); - u64_stats_inc(&rxq->q_stats.hsplit_pkts); - u64_stats_update_end(&rxq->stats_sync); - } + if (libeth_xdp_process_buff(xdp, hdr, hdr_len)) + rs.hsplit++; hdr->netmem = 0; payload: - if (!libeth_rx_sync_for_cpu(rx_buf, pkt_len)) - goto skip_data; - - if (skb) - idpf_rx_add_frag(rx_buf, skb, pkt_len); - else - skb = idpf_rx_build_skb(rx_buf, pkt_len); - - /* exit if we failed to retrieve a buffer */ - if (!skb) - break; - -skip_data: + libeth_xdp_process_buff(xdp, rx_buf, pkt_len); rx_buf->netmem = 0; idpf_post_buf_refill(refillq, buf_id); IDPF_RX_BUMP_NTC(rxq, ntc); /* skip if it is non EOP desc */ - if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!skb)) + if (!idpf_rx_splitq_is_eop(rx_desc) || unlikely(!xdp->data)) continue; - /* pad skb if needed (to make valid ethernet frame) */ - if (eth_skb_pad(skb)) { - skb = NULL; - continue; - } - - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - - /* protocol */ - if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc))) { - dev_kfree_skb_any(skb); - skb = NULL; - continue; - } - - /* send completed skb up the stack */ - napi_gro_receive(rxq->napi, skb); - skb = NULL; - - /* update budget accounting */ - total_rx_pkts++; + idpf_xdp_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); } + idpf_xdp_finalize_rx(&bq); + rxq->next_to_clean = ntc; + libeth_xdp_save_buff(&rxq->xdp, xdp); - rxq->skb = skb; u64_stats_update_begin(&rxq->stats_sync); - u64_stats_add(&rxq->q_stats.packets, total_rx_pkts); - u64_stats_add(&rxq->q_stats.bytes, total_rx_bytes); + u64_stats_add(&rxq->q_stats.packets, rs.packets); + u64_stats_add(&rxq->q_stats.bytes, rs.bytes); + u64_stats_add(&rxq->q_stats.hsplit_pkts, rs.hsplit); u64_stats_update_end(&rxq->stats_sync); - /* guarantee a trip back through this routine if there was a failure */ - return total_rx_pkts; + return rs.packets; } /** @@ -3464,7 +3758,7 @@ static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq, struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data; q_vector->total_events++; - napi_schedule(&q_vector->napi); + napi_schedule_irqoff(&q_vector->napi); return IRQ_HANDLED; } @@ -3505,6 +3799,8 @@ void idpf_vport_intr_rel(struct idpf_vport *vport) for (u32 v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) { struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx]; + kfree(q_vector->xsksq); + q_vector->xsksq = NULL; kfree(q_vector->complq); q_vector->complq = NULL; kfree(q_vector->bufq); @@ -3519,6 +3815,20 @@ void idpf_vport_intr_rel(struct idpf_vport *vport) vport->q_vectors = NULL; } +static void idpf_q_vector_set_napi(struct idpf_q_vector *q_vector, bool link) +{ + struct napi_struct *napi = link ? &q_vector->napi : NULL; + struct net_device *dev = q_vector->vport->netdev; + + for (u32 i = 0; i < q_vector->num_rxq; i++) + netif_queue_set_napi(dev, q_vector->rx[i]->idx, + NETDEV_QUEUE_TYPE_RX, napi); + + for (u32 i = 0; i < q_vector->num_txq; i++) + netif_queue_set_napi(dev, q_vector->tx[i]->idx, + NETDEV_QUEUE_TYPE_TX, napi); +} + /** * idpf_vport_intr_rel_irq - Free the IRQ association with the OS * @vport: main vport structure @@ -3539,6 +3849,7 @@ static void idpf_vport_intr_rel_irq(struct idpf_vport *vport) vidx = vport->q_vector_idxs[vector]; irq_num = adapter->msix_entries[vidx].vector; + idpf_q_vector_set_napi(q_vector, false); kfree(free_irq(irq_num, q_vector)); } } @@ -3552,6 +3863,8 @@ static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport) struct idpf_q_vector *q_vector = vport->q_vectors; int q_idx; + writel(0, vport->noirq_dyn_ctl); + for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) writel(0, q_vector[q_idx].intr_reg.dyn_ctl); } @@ -3726,6 +4039,8 @@ static int idpf_vport_intr_req_irq(struct idpf_vport *vport) "Request_irq failed, error: %d\n", err); goto free_q_irqs; } + + idpf_q_vector_set_napi(q_vector, true); } return 0; @@ -3793,6 +4108,8 @@ static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport) if (qv->num_txq || qv->num_rxq) idpf_vport_intr_update_itr_ena_irq(qv); } + + writel(vport->noirq_dyn_ctl_ena, vport->noirq_dyn_ctl); } /** @@ -3942,7 +4259,9 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, struct idpf_rx_queue *rxq = q_vec->rx[i]; int pkts_cleaned_per_q; - pkts_cleaned_per_q = idpf_rx_splitq_clean(rxq, budget_per_q); + pkts_cleaned_per_q = idpf_queue_has(XSK, rxq) ? + idpf_xskrq_poll(rxq, budget_per_q) : + idpf_rx_splitq_clean(rxq, budget_per_q); /* if we clean as many as budgeted, we must not be done */ if (pkts_cleaned_per_q >= budget_per_q) clean_complete = false; @@ -3952,8 +4271,10 @@ static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget, nid = numa_mem_id(); - for (i = 0; i < q_vec->num_bufq; i++) - idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); + for (i = 0; i < q_vec->num_bufq; i++) { + if (!idpf_queue_has(XSK, q_vec->bufq[i])) + idpf_rx_clean_refillq_all(q_vec->bufq[i], nid); + } return clean_complete; } @@ -3967,7 +4288,7 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) { struct idpf_q_vector *q_vector = container_of(napi, struct idpf_q_vector, napi); - bool clean_complete; + bool clean_complete = true; int work_done = 0; /* Handle case where we are called by netpoll with a budget of 0 */ @@ -3977,8 +4298,13 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) return 0; } - clean_complete = idpf_rx_splitq_clean_all(q_vector, budget, &work_done); - clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done); + for (u32 i = 0; i < q_vector->num_xsksq; i++) + clean_complete &= idpf_xsk_xmit(q_vector->xsksq[i]); + + clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, + &work_done); + clean_complete &= idpf_rx_splitq_clean_all(q_vector, budget, + &work_done); /* If work not completed, return budget and polling will return */ if (!clean_complete) { @@ -3986,20 +4312,12 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) return budget; } - /* Switch to poll mode in the tear-down path after sending disable - * queues virtchnl message, as the interrupts will be disabled after - * that. - */ - if (unlikely(q_vector->num_txq && idpf_queue_has(POLL_MODE, - q_vector->tx[0]))) - return budget; - work_done = min_t(int, work_done, budget - 1); /* Exit the polling mode, but don't re-enable interrupts if stack might * poll us due to busy-polling */ - if (likely(napi_complete_done(napi, work_done))) + if (napi_complete_done(napi, work_done)) idpf_vport_intr_update_itr_ena_irq(q_vector); else idpf_vport_intr_set_wb_on_itr(q_vector); @@ -4015,8 +4333,8 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) */ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) { + u16 num_txq_grp = vport->num_txq_grp - vport->num_xdp_txq; bool split = idpf_is_queue_model_split(vport->rxq_model); - u16 num_txq_grp = vport->num_txq_grp; struct idpf_rxq_group *rx_qgrp; struct idpf_txq_group *tx_qgrp; u32 i, qv_idx, q_index; @@ -4092,6 +4410,21 @@ static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport) qv_idx++; } + + for (i = 0; i < vport->num_xdp_txq; i++) { + struct idpf_tx_queue *xdpsq; + struct idpf_q_vector *qv; + + xdpsq = vport->txqs[vport->xdp_txq_offset + i]; + if (!idpf_queue_has(XSK, xdpsq)) + continue; + + qv = idpf_find_rxq_vec(vport, i); + idpf_xsk_init_wakeup(qv); + + xdpsq->q_vector = qv; + qv->xsksq[qv->num_xsksq++] = xdpsq; + } } /** @@ -4112,6 +4445,8 @@ static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) for (i = 0; i < vport->num_q_vectors; i++) vport->q_vectors[i].v_idx = vport->q_vector_idxs[i]; + vport->noirq_v_idx = vport->q_vector_idxs[i]; + return 0; } @@ -4125,6 +4460,8 @@ static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport) for (i = 0; i < vport->num_q_vectors; i++) vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]]; + vport->noirq_v_idx = vecids[vport->q_vector_idxs[i]]; + kfree(vecids); return 0; @@ -4225,6 +4562,15 @@ int idpf_vport_intr_alloc(struct idpf_vport *vport) GFP_KERNEL); if (!q_vector->complq) goto error; + + if (!vport->xdp_txq_offset) + continue; + + q_vector->xsksq = kcalloc(rxqs_per_vector, + sizeof(*q_vector->xsksq), + GFP_KERNEL); + if (!q_vector->xsksq) + goto error; } return 0; diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 52753dff381c..75b977094741 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -7,8 +7,10 @@ #include <linux/dim.h> #include <net/libeth/cache.h> -#include <net/tcp.h> +#include <net/libeth/types.h> #include <net/netdev_queues.h> +#include <net/tcp.h> +#include <net/xdp.h> #include "idpf_lan_txrx.h" #include "virtchnl2_lan_desc.h" @@ -58,6 +60,8 @@ #define IDPF_MBX_Q_VEC 1 #define IDPF_MIN_Q_VEC 1 #define IDPF_MIN_RDMA_VEC 2 +/* Data vector for NOIRQ queues */ +#define IDPF_RESERVED_VECS 1 #define IDPF_DFLT_TX_Q_DESC_COUNT 512 #define IDPF_DFLT_TX_COMPLQ_DESC_COUNT 512 @@ -137,6 +141,8 @@ do { \ #define IDPF_TX_FLAGS_TUNNEL BIT(3) #define IDPF_TX_FLAGS_TSYN BIT(4) +struct libeth_rq_napi_stats; + union idpf_tx_flex_desc { struct idpf_flex_tx_desc q; /* queue based scheduling */ struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ @@ -275,11 +281,13 @@ struct idpf_ptype_state { * bit and Q_RFL_GEN is the SW bit. * @__IDPF_Q_FLOW_SCH_EN: Enable flow scheduling * @__IDPF_Q_SW_MARKER: Used to indicate TX queue marker completions - * @__IDPF_Q_POLL_MODE: Enable poll mode * @__IDPF_Q_CRC_EN: enable CRC offload in singleq mode * @__IDPF_Q_HSPLIT_EN: enable header split on Rx (splitq) * @__IDPF_Q_PTP: indicates whether the Rx timestamping is enabled for the * queue + * @__IDPF_Q_NOIRQ: queue is polling-driven and has no interrupt + * @__IDPF_Q_XDP: this is an XDP queue + * @__IDPF_Q_XSK: the queue has an XSk pool installed * @__IDPF_Q_FLAGS_NBITS: Must be last */ enum idpf_queue_flags_t { @@ -287,10 +295,12 @@ enum idpf_queue_flags_t { __IDPF_Q_RFL_GEN_CHK, __IDPF_Q_FLOW_SCH_EN, __IDPF_Q_SW_MARKER, - __IDPF_Q_POLL_MODE, __IDPF_Q_CRC_EN, __IDPF_Q_HSPLIT_EN, __IDPF_Q_PTP, + __IDPF_Q_NOIRQ, + __IDPF_Q_XDP, + __IDPF_Q_XSK, __IDPF_Q_FLAGS_NBITS, }; @@ -357,14 +367,17 @@ struct idpf_intr_reg { * @num_txq: Number of TX queues * @num_bufq: Number of buffer queues * @num_complq: number of completion queues + * @num_xsksq: number of XSk send queues * @rx: Array of RX queues to service * @tx: Array of TX queues to service * @bufq: Array of buffer queues to service * @complq: array of completion queues + * @xsksq: array of XSk send queues * @intr_reg: See struct idpf_intr_reg - * @napi: napi handler + * @csd: XSk wakeup CSD * @total_events: Number of interrupts processed * @wb_on_itr: whether WB on ITR is enabled + * @napi: napi handler * @tx_dim: Data for TX net_dim algorithm * @tx_itr_value: TX interrupt throttling rate * @tx_intr_mode: Dynamic ITR or not @@ -383,19 +396,24 @@ struct idpf_q_vector { u16 num_txq; u16 num_bufq; u16 num_complq; + u16 num_xsksq; struct idpf_rx_queue **rx; struct idpf_tx_queue **tx; struct idpf_buf_queue **bufq; struct idpf_compl_queue **complq; + struct idpf_tx_queue **xsksq; struct idpf_intr_reg intr_reg; __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); - struct napi_struct napi; + call_single_data_t csd; + u16 total_events; bool wb_on_itr; + struct napi_struct napi; + struct dim tx_dim; u16 tx_itr_value; bool tx_intr_mode; @@ -412,8 +430,8 @@ struct idpf_q_vector { __cacheline_group_end_aligned(cold); }; -libeth_cacheline_set_assert(struct idpf_q_vector, 120, - 24 + sizeof(struct napi_struct) + +libeth_cacheline_set_assert(struct idpf_q_vector, 136, + 56 + sizeof(struct napi_struct) + 2 * sizeof(struct dim), 8); @@ -461,21 +479,26 @@ struct idpf_tx_queue_stats { * @desc_ring: virtual descriptor ring address * @bufq_sets: Pointer to the array of buffer queues in splitq mode * @napi: NAPI instance corresponding to this queue (splitq) + * @xdp_prog: attached XDP program * @rx_buf: See struct &libeth_fqe * @pp: Page pool pointer in singleq mode - * @netdev: &net_device corresponding to this queue * @tail: Tail offset. Used for both queue models single and split. * @flags: See enum idpf_queue_flags_t * @idx: For RX queue, it is used to index to total RX queue across groups and * used for skb reporting. * @desc_count: Number of descriptors + * @num_xdp_txq: total number of XDP Tx queues + * @xdpsqs: shortcut for XDP Tx queues array * @rxdids: Supported RX descriptor ids + * @truesize: data buffer truesize in singleq * @rx_ptype_lkup: LUT of Rx ptypes + * @xdp_rxq: XDP queue info * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at - * @skb: Pointer to the skb - * @truesize: data buffer truesize in singleq + * @xdp: XDP buffer with the current frame + * @xsk: current XDP buffer in XSk mode + * @pool: XSk pool if installed * @cached_phc_time: Cached PHC time for the Rx queue * @stats_sync: See struct u64_stats_sync * @q_stats: See union idpf_rx_queue_stats @@ -500,30 +523,44 @@ struct idpf_rx_queue { struct { struct idpf_bufq_set *bufq_sets; struct napi_struct *napi; + struct bpf_prog __rcu *xdp_prog; }; struct { struct libeth_fqe *rx_buf; struct page_pool *pp; + void __iomem *tail; }; }; - struct net_device *netdev; - void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); u16 idx; u16 desc_count; - u32 rxdids; + u32 num_xdp_txq; + union { + struct idpf_tx_queue **xdpsqs; + struct { + u32 rxdids; + u32 truesize; + }; + }; const struct libeth_rx_pt *rx_ptype_lkup; + + struct xdp_rxq_info xdp_rxq; __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); - u16 next_to_use; - u16 next_to_clean; - u16 next_to_alloc; + u32 next_to_use; + u32 next_to_clean; + u32 next_to_alloc; - struct sk_buff *skb; - u32 truesize; + union { + struct libeth_xdp_buff_stash xdp; + struct { + struct libeth_xdp_buff *xsk; + struct xsk_buff_pool *pool; + }; + }; u64 cached_phc_time; struct u64_stats_sync stats_sync; @@ -543,8 +580,11 @@ struct idpf_rx_queue { u16 rx_max_pkt_size; __cacheline_group_end_aligned(cold); }; -libeth_cacheline_set_assert(struct idpf_rx_queue, 64, - 88 + sizeof(struct u64_stats_sync), +libeth_cacheline_set_assert(struct idpf_rx_queue, + ALIGN(64, __alignof(struct xdp_rxq_info)) + + sizeof(struct xdp_rxq_info), + 96 + offsetof(struct idpf_rx_queue, q_stats) - + offsetofend(struct idpf_rx_queue, cached_phc_time), 32); /** @@ -556,33 +596,16 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * @desc_ring: virtual descriptor ring address * @tx_buf: See struct idpf_tx_buf * @txq_grp: See struct idpf_txq_group + * @complq: corresponding completion queue in XDP mode * @dev: Device back pointer for DMA mapping + * @pool: corresponding XSk pool if installed * @tail: Tail offset. Used for both queue models single and split * @flags: See enum idpf_queue_flags_t * @idx: For TX queue, it is used as index to map between TX queue group and * hot path TX pointers stored in vport. Used in both singleq/splitq. * @desc_count: Number of descriptors * @tx_min_pkt_len: Min supported packet length - * @compl_tag_gen_s: Completion tag generation bit - * The format of the completion tag will change based on the TXQ - * descriptor ring size so that we can maintain roughly the same level - * of "uniqueness" across all descriptor sizes. For example, if the - * TXQ descriptor ring size is 64 (the minimum size supported), the - * completion tag will be formatted as below: - * 15 6 5 0 - * -------------------------------- - * | GEN=0-1023 |IDX = 0-63| - * -------------------------------- - * - * This gives us 64*1024 = 65536 possible unique values. Similarly, if - * the TXQ descriptor ring size is 8160 (the maximum size supported), - * the completion tag will be formatted as below: - * 15 13 12 0 - * -------------------------------- - * |GEN | IDX = 0-8159 | - * -------------------------------- - * - * This gives us 8*8160 = 65280 possible unique values. + * @thresh: XDP queue cleaning threshold * @netdev: &net_device corresponding to this queue * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean @@ -599,6 +622,10 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * @clean_budget: singleq only, queue cleaning budget * @cleaned_pkts: Number of packets cleaned for the above said case * @refillq: Pointer to refill queue + * @pending: number of pending descriptors to send in QB + * @xdp_tx: number of pending &xdp_buff or &xdp_frame buffers + * @timer: timer for XDP Tx queue cleanup + * @xdp_lock: lock for XDP Tx queues sharing * @cached_tstamp_caps: Tx timestamp capabilities negotiated with the CP * @tstamp_task: Work that handles Tx timestamp read * @stats_sync: See struct u64_stats_sync @@ -608,6 +635,7 @@ libeth_cacheline_set_assert(struct idpf_rx_queue, 64, * @dma: Physical address of ring * @q_vector: Backreference to associated vector * @buf_pool_size: Total number of idpf_tx_buf + * @rel_q_id: relative virtchnl queue index */ struct idpf_tx_queue { __cacheline_group_begin_aligned(read_mostly); @@ -620,32 +648,53 @@ struct idpf_tx_queue { void *desc_ring; }; struct libeth_sqe *tx_buf; - struct idpf_txq_group *txq_grp; - struct device *dev; + union { + struct idpf_txq_group *txq_grp; + struct idpf_compl_queue *complq; + }; + union { + struct device *dev; + struct xsk_buff_pool *pool; + }; void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); u16 idx; u16 desc_count; - u16 tx_min_pkt_len; + union { + u16 tx_min_pkt_len; + u32 thresh; + }; struct net_device *netdev; __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); - u16 next_to_use; - u16 next_to_clean; - u16 last_re; - u16 tx_max_bufs; + u32 next_to_use; + u32 next_to_clean; union { - u32 cleaned_bytes; - u32 clean_budget; - }; - u16 cleaned_pkts; + struct { + u16 last_re; + u16 tx_max_bufs; + + union { + u32 cleaned_bytes; + u32 clean_budget; + }; + u16 cleaned_pkts; - struct idpf_sw_queue *refillq; + struct idpf_sw_queue *refillq; + }; + struct { + u32 pending; + u32 xdp_tx; + + struct libeth_xdpsq_timer *timer; + struct libeth_xdpsq_lock xdp_lock; + }; + }; struct idpf_ptp_vport_tx_tstamp_caps *cached_tstamp_caps; struct work_struct *tstamp_task; @@ -660,26 +709,36 @@ struct idpf_tx_queue { dma_addr_t dma; struct idpf_q_vector *q_vector; + u32 buf_pool_size; + u32 rel_q_id; __cacheline_group_end_aligned(cold); }; libeth_cacheline_set_assert(struct idpf_tx_queue, 64, - 104 + sizeof(struct u64_stats_sync), + 104 + + offsetof(struct idpf_tx_queue, cached_tstamp_caps) - + offsetofend(struct idpf_tx_queue, timer) + + offsetof(struct idpf_tx_queue, q_stats) - + offsetofend(struct idpf_tx_queue, tstamp_task), 32); /** * struct idpf_buf_queue - software structure representing a buffer queue * @split_buf: buffer descriptor array - * @hdr_buf: &libeth_fqe for header buffers - * @hdr_pp: &page_pool for header buffers * @buf: &libeth_fqe for data buffers * @pp: &page_pool for data buffers + * @xsk_buf: &xdp_buff for XSk Rx buffers + * @pool: &xsk_buff_pool on XSk queues + * @hdr_buf: &libeth_fqe for header buffers + * @hdr_pp: &page_pool for header buffers * @tail: Tail offset * @flags: See enum idpf_queue_flags_t * @desc_count: Number of descriptors + * @thresh: refill threshold in XSk * @next_to_use: Next descriptor to use * @next_to_clean: Next descriptor to clean * @next_to_alloc: RX buffer to allocate at + * @pending: number of buffers to refill (Xsk) * @hdr_truesize: truesize for buffer headers * @truesize: truesize for data buffers * @q_id: Queue id @@ -693,14 +752,24 @@ libeth_cacheline_set_assert(struct idpf_tx_queue, 64, struct idpf_buf_queue { __cacheline_group_begin_aligned(read_mostly); struct virtchnl2_splitq_rx_buf_desc *split_buf; + union { + struct { + struct libeth_fqe *buf; + struct page_pool *pp; + }; + struct { + struct libeth_xdp_buff **xsk_buf; + struct xsk_buff_pool *pool; + }; + }; struct libeth_fqe *hdr_buf; struct page_pool *hdr_pp; - struct libeth_fqe *buf; - struct page_pool *pp; void __iomem *tail; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); u32 desc_count; + + u32 thresh; __cacheline_group_end_aligned(read_mostly); __cacheline_group_begin_aligned(read_write); @@ -708,6 +777,7 @@ struct idpf_buf_queue { u32 next_to_clean; u32 next_to_alloc; + u32 pending; u32 hdr_truesize; u32 truesize; __cacheline_group_end_aligned(read_write); @@ -728,7 +798,9 @@ libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32); /** * struct idpf_compl_queue - software structure representing a completion queue - * @comp: completion descriptor array + * @comp: 8-byte completion descriptor array + * @comp_4b: 4-byte completion descriptor array + * @desc_ring: virtual descriptor ring address * @txq_grp: See struct idpf_txq_group * @flags: See enum idpf_queue_flags_t * @desc_count: Number of descriptors @@ -748,7 +820,12 @@ libeth_cacheline_set_assert(struct idpf_buf_queue, 64, 24, 32); */ struct idpf_compl_queue { __cacheline_group_begin_aligned(read_mostly); - struct idpf_splitq_tx_compl_desc *comp; + union { + struct idpf_splitq_tx_compl_desc *comp; + struct idpf_splitq_4b_tx_compl_desc *comp_4b; + + void *desc_ring; + }; struct idpf_txq_group *txq_grp; DECLARE_BITMAP(flags, __IDPF_Q_FLAGS_NBITS); @@ -1012,9 +1089,13 @@ int idpf_config_rss(struct idpf_vport *vport); int idpf_init_rss(struct idpf_vport *vport); void idpf_deinit_rss(struct idpf_vport *vport); int idpf_rx_bufs_init_all(struct idpf_vport *vport); -void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb, - unsigned int size); -struct sk_buff *idpf_rx_build_skb(const struct libeth_fqe *buf, u32 size); + +struct idpf_q_vector *idpf_find_rxq_vec(const struct idpf_vport *vport, + u32 q_num); +struct idpf_q_vector *idpf_find_txq_vec(const struct idpf_vport *vport, + u32 q_num); +int idpf_qp_switch(struct idpf_vport *vport, u32 qid, bool en); + void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, bool xmit_more); unsigned int idpf_size_to_txd_count(unsigned int size); @@ -1027,6 +1108,11 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, netdev_tx_t idpf_tx_start(struct sk_buff *skb, struct net_device *netdev); bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, u16 cleaned_count); +bool idpf_rx_process_skb_fields(struct sk_buff *skb, + const struct libeth_xdp_buff *xdp, + struct libeth_rq_napi_stats *rs); int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); +void idpf_wait_for_sw_marker_completion(const struct idpf_tx_queue *txq); + #endif /* !_IDPF_TXRX_H_ */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c index 259d50fded67..4cc58c83688c 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c @@ -76,7 +76,7 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport) int num_vecs = vport->num_q_vectors; struct idpf_vec_regs *reg_vals; int num_regs, i, err = 0; - u32 rx_itr, tx_itr; + u32 rx_itr, tx_itr, val; u16 total_vecs; total_vecs = idpf_get_reserved_vecs(vport->adapter); @@ -120,6 +120,15 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport) intr->tx_itr = idpf_get_reg_addr(adapter, tx_itr); } + /* Data vector for NOIRQ queues */ + + val = reg_vals[vport->q_vector_idxs[i] - IDPF_MBX_Q_VEC].dyn_ctl_reg; + vport->noirq_dyn_ctl = idpf_get_reg_addr(adapter, val); + + val = VF_INT_DYN_CTLN_WB_ON_ITR_M | VF_INT_DYN_CTLN_INTENA_MSK_M | + FIELD_PREP(VF_INT_DYN_CTLN_ITR_INDX_M, IDPF_NO_ITR_UPDATE_IDX); + vport->noirq_dyn_ctl_ena = val; + free_reg_vals: kfree(reg_vals); diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c index 6330d4a0ae07..cbb5fa30f5a0 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.c @@ -702,9 +702,9 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter) /* If post failed clear the only buffer we supplied */ if (post_err) { if (dma_mem) - dmam_free_coherent(&adapter->pdev->dev, - dma_mem->size, dma_mem->va, - dma_mem->pa); + dma_free_coherent(&adapter->pdev->dev, + dma_mem->size, dma_mem->va, + dma_mem->pa); break; } @@ -716,34 +716,145 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter) return err; } +struct idpf_chunked_msg_params { + u32 (*prepare_msg)(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num); + + const void *chunks; + u32 num_chunks; + + u32 chunk_sz; + u32 config_sz; + + u32 vc_op; +}; + +struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num) +{ + struct idpf_queue_set *qp; + + qp = kzalloc(struct_size(qp, qs, num), GFP_KERNEL); + if (!qp) + return NULL; + + qp->vport = vport; + qp->num = num; + + return qp; +} + /** - * idpf_wait_for_marker_event - wait for software marker response + * idpf_send_chunked_msg - send VC message consisting of chunks * @vport: virtual port data structure + * @params: message params * - * Returns 0 success, negative on failure. - **/ -static int idpf_wait_for_marker_event(struct idpf_vport *vport) + * Helper function for preparing a message describing queues to be enabled + * or disabled. + * + * Return: the total size of the prepared message. + */ +static int idpf_send_chunked_msg(struct idpf_vport *vport, + const struct idpf_chunked_msg_params *params) { - int event; - int i; + struct idpf_vc_xn_params xn_params = { + .vc_op = params->vc_op, + .timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC, + }; + const void *pos = params->chunks; + u32 num_chunks, num_msgs, buf_sz; + void *buf __free(kfree) = NULL; + u32 totqs = params->num_chunks; + + num_chunks = min(IDPF_NUM_CHUNKS_PER_MSG(params->config_sz, + params->chunk_sz), totqs); + num_msgs = DIV_ROUND_UP(totqs, num_chunks); - for (i = 0; i < vport->num_txq; i++) - idpf_queue_set(SW_MARKER, vport->txqs[i]); + buf_sz = params->config_sz + num_chunks * params->chunk_sz; + buf = kzalloc(buf_sz, GFP_KERNEL); + if (!buf) + return -ENOMEM; - event = wait_event_timeout(vport->sw_marker_wq, - test_and_clear_bit(IDPF_VPORT_SW_MARKER, - vport->flags), - msecs_to_jiffies(500)); + xn_params.send_buf.iov_base = buf; - for (i = 0; i < vport->num_txq; i++) - idpf_queue_clear(POLL_MODE, vport->txqs[i]); + for (u32 i = 0; i < num_msgs; i++) { + ssize_t reply_sz; - if (event) - return 0; + memset(buf, 0, buf_sz); + xn_params.send_buf.iov_len = buf_sz; + + if (params->prepare_msg(vport, buf, pos, num_chunks) != buf_sz) + return -EINVAL; + + reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); + if (reply_sz < 0) + return reply_sz; + + pos += num_chunks * params->chunk_sz; + totqs -= num_chunks; + + num_chunks = min(num_chunks, totqs); + buf_sz = params->config_sz + num_chunks * params->chunk_sz; + } + + return 0; +} + +/** + * idpf_wait_for_marker_event_set - wait for software marker response for + * selected Tx queues + * @qs: set of the Tx queues + * + * Return: 0 success, -errno on failure. + */ +static int idpf_wait_for_marker_event_set(const struct idpf_queue_set *qs) +{ + struct idpf_tx_queue *txq; + bool markers_rcvd = true; + + for (u32 i = 0; i < qs->num; i++) { + switch (qs->qs[i].type) { + case VIRTCHNL2_QUEUE_TYPE_TX: + txq = qs->qs[i].txq; + + idpf_queue_set(SW_MARKER, txq); + idpf_wait_for_sw_marker_completion(txq); + markers_rcvd &= !idpf_queue_has(SW_MARKER, txq); + break; + default: + break; + } + } + + if (!markers_rcvd) { + netdev_warn(qs->vport->netdev, + "Failed to receive marker packets\n"); + return -ETIMEDOUT; + } + + return 0; +} + +/** + * idpf_wait_for_marker_event - wait for software marker response + * @vport: virtual port data structure + * + * Return: 0 success, negative on failure. + **/ +static int idpf_wait_for_marker_event(struct idpf_vport *vport) +{ + struct idpf_queue_set *qs __free(kfree) = NULL; + + qs = idpf_alloc_queue_set(vport, vport->num_txq); + if (!qs) + return -ENOMEM; - dev_warn(&vport->adapter->pdev->dev, "Failed to receive marker packets\n"); + for (u32 i = 0; i < qs->num; i++) { + qs->qs[i].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[i].txq = vport->txqs[i]; + } - return -ETIMEDOUT; + return idpf_wait_for_marker_event_set(qs); } /** @@ -1061,21 +1172,35 @@ int idpf_vport_alloc_max_qs(struct idpf_adapter *adapter, struct idpf_avail_queue_info *avail_queues = &adapter->avail_queues; struct virtchnl2_get_capabilities *caps = &adapter->caps; u16 default_vports = idpf_get_default_vports(adapter); - int max_rx_q, max_tx_q; + u32 max_rx_q, max_tx_q, max_buf_q, max_compl_q; mutex_lock(&adapter->queue_lock); + /* Caps are device-wide. Give each vport an equal piece */ max_rx_q = le16_to_cpu(caps->max_rx_q) / default_vports; max_tx_q = le16_to_cpu(caps->max_tx_q) / default_vports; - if (adapter->num_alloc_vports < default_vports) { - max_q->max_rxq = min_t(u16, max_rx_q, IDPF_MAX_Q); - max_q->max_txq = min_t(u16, max_tx_q, IDPF_MAX_Q); - } else { - max_q->max_rxq = IDPF_MIN_Q; - max_q->max_txq = IDPF_MIN_Q; + max_buf_q = le16_to_cpu(caps->max_rx_bufq) / default_vports; + max_compl_q = le16_to_cpu(caps->max_tx_complq) / default_vports; + + if (adapter->num_alloc_vports >= default_vports) { + max_rx_q = IDPF_MIN_Q; + max_tx_q = IDPF_MIN_Q; } - max_q->max_bufq = max_q->max_rxq * IDPF_MAX_BUFQS_PER_RXQ_GRP; - max_q->max_complq = max_q->max_txq; + + /* + * Harmonize the numbers. The current implementation always creates + * `IDPF_MAX_BUFQS_PER_RXQ_GRP` buffer queues for each Rx queue and + * one completion queue for each Tx queue for best performance. + * If less buffer or completion queues is available, cap the number + * of the corresponding Rx/Tx queues. + */ + max_rx_q = min(max_rx_q, max_buf_q / IDPF_MAX_BUFQS_PER_RXQ_GRP); + max_tx_q = min(max_tx_q, max_compl_q); + + max_q->max_rxq = max_rx_q; + max_q->max_txq = max_tx_q; + max_q->max_bufq = max_rx_q * IDPF_MAX_BUFQS_PER_RXQ_GRP; + max_q->max_complq = max_tx_q; if (avail_queues->avail_rxq < max_q->max_rxq || avail_queues->avail_txq < max_q->max_txq || @@ -1506,7 +1631,7 @@ int idpf_send_destroy_vport_msg(struct idpf_vport *vport) xn_params.vc_op = VIRTCHNL2_OP_DESTROY_VPORT; xn_params.send_buf.iov_base = &v_id; xn_params.send_buf.iov_len = sizeof(v_id); - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); return reply_sz < 0 ? reply_sz : 0; @@ -1554,236 +1679,368 @@ int idpf_send_disable_vport_msg(struct idpf_vport *vport) xn_params.vc_op = VIRTCHNL2_OP_DISABLE_VPORT; xn_params.send_buf.iov_base = &v_id; xn_params.send_buf.iov_len = sizeof(v_id); - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); return reply_sz < 0 ? reply_sz : 0; } /** - * idpf_send_config_tx_queues_msg - Send virtchnl config tx queues message + * idpf_fill_txq_config_chunk - fill chunk describing the Tx queue + * @vport: virtual port data structure + * @q: Tx queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_txq_config_chunk(const struct idpf_vport *vport, + const struct idpf_tx_queue *q, + struct virtchnl2_txq_info *qi) +{ + u32 val; + + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->txq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + qi->relative_queue_id = cpu_to_le16(q->rel_q_id); + + if (!idpf_is_queue_model_split(vport->txq_model)) { + qi->sched_mode = cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); + return; + } + + if (idpf_queue_has(XDP, q)) + val = q->complq->q_id; + else + val = q->txq_grp->complq->q_id; + + qi->tx_compl_queue_id = cpu_to_le16(val); + + if (idpf_queue_has(FLOW_SCH_EN, q)) + val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + else + val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; + + qi->sched_mode = cpu_to_le16(val); +} + +/** + * idpf_fill_complq_config_chunk - fill chunk describing the completion queue + * @vport: virtual port data structure + * @q: completion queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_complq_config_chunk(const struct idpf_vport *vport, + const struct idpf_compl_queue *q, + struct virtchnl2_txq_info *qi) +{ + u32 val; + + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->txq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + + if (idpf_queue_has(FLOW_SCH_EN, q)) + val = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; + else + val = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; + + qi->sched_mode = cpu_to_le16(val); +} + +/** + * idpf_prepare_cfg_txqs_msg - prepare message to configure selected Tx queues * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the tx queue + * @num_chunks: number of chunks in the message * - * Send config tx queues virtchnl message. Returns 0 on success, negative on - * failure. + * Helper function for preparing the message describing configuration of + * Tx queues. + * + * Return: the total size of the prepared message. */ -static int idpf_send_config_tx_queues_msg(struct idpf_vport *vport) +static u32 idpf_prepare_cfg_txqs_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_config_tx_queues *ctq = buf; + + ctq->vport_id = cpu_to_le32(vport->vport_id); + ctq->num_qinfo = cpu_to_le16(num_chunks); + memcpy(ctq->qinfo, pos, num_chunks * sizeof(*ctq->qinfo)); + + return struct_size(ctq, qinfo, num_chunks); +} + +/** + * idpf_send_config_tx_queue_set_msg - send virtchnl config Tx queues + * message for selected queues + * @qs: set of the Tx queues to configure + * + * Send config queues virtchnl message for queues contained in the @qs array. + * The @qs array can contain Tx queues (or completion queues) only. + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_config_tx_queue_set_msg(const struct idpf_queue_set *qs) { - struct virtchnl2_config_tx_queues *ctq __free(kfree) = NULL; struct virtchnl2_txq_info *qi __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - u32 config_sz, chunk_sz, buf_sz; - int totqs, num_msgs, num_chunks; - ssize_t reply_sz; - int i, k = 0; + struct idpf_chunked_msg_params params = { + .vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES, + .prepare_msg = idpf_prepare_cfg_txqs_msg, + .config_sz = sizeof(struct virtchnl2_config_tx_queues), + .chunk_sz = sizeof(*qi), + }; - totqs = vport->num_txq + vport->num_complq; - qi = kcalloc(totqs, sizeof(struct virtchnl2_txq_info), GFP_KERNEL); + qi = kcalloc(qs->num, sizeof(*qi), GFP_KERNEL); if (!qi) return -ENOMEM; - /* Populate the queue info buffer with all queue context info */ - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - int j, sched_mode; - - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { - qi[k].queue_id = - cpu_to_le32(tx_qgrp->txqs[j]->q_id); - qi[k].model = - cpu_to_le16(vport->txq_model); - qi[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - qi[k].ring_len = - cpu_to_le16(tx_qgrp->txqs[j]->desc_count); - qi[k].dma_ring_addr = - cpu_to_le64(tx_qgrp->txqs[j]->dma); - if (idpf_is_queue_model_split(vport->txq_model)) { - struct idpf_tx_queue *q = tx_qgrp->txqs[j]; - - qi[k].tx_compl_queue_id = - cpu_to_le16(tx_qgrp->complq->q_id); - qi[k].relative_queue_id = cpu_to_le16(j); - - if (idpf_queue_has(FLOW_SCH_EN, q)) - qi[k].sched_mode = - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_FLOW); - else - qi[k].sched_mode = - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); - } else { - qi[k].sched_mode = - cpu_to_le16(VIRTCHNL2_TXQ_SCHED_MODE_QUEUE); - } - } + params.chunks = qi; - if (!idpf_is_queue_model_split(vport->txq_model)) - continue; + for (u32 i = 0; i < qs->num; i++) { + if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX) + idpf_fill_txq_config_chunk(qs->vport, qs->qs[i].txq, + &qi[params.num_chunks++]); + else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION) + idpf_fill_complq_config_chunk(qs->vport, + qs->qs[i].complq, + &qi[params.num_chunks++]); + } - qi[k].queue_id = cpu_to_le32(tx_qgrp->complq->q_id); - qi[k].model = cpu_to_le16(vport->txq_model); - qi[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); - qi[k].ring_len = cpu_to_le16(tx_qgrp->complq->desc_count); - qi[k].dma_ring_addr = cpu_to_le64(tx_qgrp->complq->dma); + return idpf_send_chunked_msg(qs->vport, ¶ms); +} - if (idpf_queue_has(FLOW_SCH_EN, tx_qgrp->complq)) - sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_FLOW; - else - sched_mode = VIRTCHNL2_TXQ_SCHED_MODE_QUEUE; - qi[k].sched_mode = cpu_to_le16(sched_mode); +/** + * idpf_send_config_tx_queues_msg - send virtchnl config Tx queues message + * @vport: virtual port data structure + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_config_tx_queues_msg(struct idpf_vport *vport) +{ + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 totqs = vport->num_txq + vport->num_complq; + u32 k = 0; - k++; + qs = idpf_alloc_queue_set(vport, totqs); + if (!qs) + return -ENOMEM; + + /* Populate the queue info buffer with all queue context info */ + for (u32 i = 0; i < vport->num_txq_grp; i++) { + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[k++].txq = tx_qgrp->txqs[j]; + } + + if (idpf_is_queue_model_split(vport->txq_model)) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[k++].complq = tx_qgrp->complq; + } } /* Make sure accounting agrees */ if (k != totqs) return -EINVAL; - /* Chunk up the queue contexts into multiple messages to avoid - * sending a control queue message buffer that is too large - */ - config_sz = sizeof(struct virtchnl2_config_tx_queues); - chunk_sz = sizeof(struct virtchnl2_txq_info); + return idpf_send_config_tx_queue_set_msg(qs); +} - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - totqs); - num_msgs = DIV_ROUND_UP(totqs, num_chunks); +/** + * idpf_fill_rxq_config_chunk - fill chunk describing the Rx queue + * @vport: virtual port data structure + * @q: Rx queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_rxq_config_chunk(const struct idpf_vport *vport, + struct idpf_rx_queue *q, + struct virtchnl2_rxq_info *qi) +{ + const struct idpf_bufq_set *sets; + + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->rxq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + qi->max_pkt_size = cpu_to_le32(q->rx_max_pkt_size); + qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); + qi->qflags = cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); + if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); + + if (!idpf_is_queue_model_split(vport->rxq_model)) { + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); + qi->desc_ids = cpu_to_le64(q->rxdids); - buf_sz = struct_size(ctq, qinfo, num_chunks); - ctq = kzalloc(buf_sz, GFP_KERNEL); - if (!ctq) - return -ENOMEM; + return; + } - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_TX_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + sets = q->bufq_sets; - for (i = 0, k = 0; i < num_msgs; i++) { - memset(ctq, 0, buf_sz); - ctq->vport_id = cpu_to_le32(vport->vport_id); - ctq->num_qinfo = cpu_to_le16(num_chunks); - memcpy(ctq->qinfo, &qi[k], chunk_sz * num_chunks); + /* + * In splitq mode, RxQ buffer size should be set to that of the first + * buffer queue associated with this RxQ. + */ + q->rx_buf_size = sets[0].bufq.rx_buf_size; + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); - xn_params.send_buf.iov_base = ctq; - xn_params.send_buf.iov_len = buf_sz; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + qi->rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); + if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { + qi->bufq2_ena = IDPF_BUFQ2_ENA; + qi->rx_bufq2_id = cpu_to_le16(sets[1].bufq.q_id); + } - k += num_chunks; - totqs -= num_chunks; - num_chunks = min(num_chunks, totqs); - /* Recalculate buffer size */ - buf_sz = struct_size(ctq, qinfo, num_chunks); + q->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; + + if (idpf_queue_has(HSPLIT_EN, q)) { + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); + qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); } - return 0; + qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); } /** - * idpf_send_config_rx_queues_msg - Send virtchnl config rx queues message + * idpf_fill_bufq_config_chunk - fill chunk describing the buffer queue * @vport: virtual port data structure + * @q: buffer queue to be inserted into VC chunk + * @qi: pointer to the buffer containing the VC chunk + */ +static void idpf_fill_bufq_config_chunk(const struct idpf_vport *vport, + const struct idpf_buf_queue *q, + struct virtchnl2_rxq_info *qi) +{ + qi->queue_id = cpu_to_le32(q->q_id); + qi->model = cpu_to_le16(vport->rxq_model); + qi->type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); + qi->ring_len = cpu_to_le16(q->desc_count); + qi->dma_ring_addr = cpu_to_le64(q->dma); + qi->data_buffer_size = cpu_to_le32(q->rx_buf_size); + qi->rx_buffer_low_watermark = cpu_to_le16(q->rx_buffer_low_watermark); + qi->desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); + qi->buffer_notif_stride = IDPF_RX_BUF_STRIDE; + if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) + qi->qflags = cpu_to_le16(VIRTCHNL2_RXQ_RSC); + + if (idpf_queue_has(HSPLIT_EN, q)) { + qi->qflags |= cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); + qi->hdr_buffer_size = cpu_to_le16(q->rx_hbuf_size); + } +} + +/** + * idpf_prepare_cfg_rxqs_msg - prepare message to configure selected Rx queues + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the rx queue + * @num_chunks: number of chunks in the message * - * Send config rx queues virtchnl message. Returns 0 on success, negative on - * failure. + * Helper function for preparing the message describing configuration of + * Rx queues. + * + * Return: the total size of the prepared message. */ -static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) +static u32 idpf_prepare_cfg_rxqs_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_config_rx_queues *crq = buf; + + crq->vport_id = cpu_to_le32(vport->vport_id); + crq->num_qinfo = cpu_to_le16(num_chunks); + memcpy(crq->qinfo, pos, num_chunks * sizeof(*crq->qinfo)); + + return struct_size(crq, qinfo, num_chunks); +} + +/** + * idpf_send_config_rx_queue_set_msg - send virtchnl config Rx queues message + * for selected queues. + * @qs: set of the Rx queues to configure + * + * Send config queues virtchnl message for queues contained in the @qs array. + * The @qs array can contain Rx queues (or buffer queues) only. + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_config_rx_queue_set_msg(const struct idpf_queue_set *qs) { - struct virtchnl2_config_rx_queues *crq __free(kfree) = NULL; struct virtchnl2_rxq_info *qi __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - u32 config_sz, chunk_sz, buf_sz; - int totqs, num_msgs, num_chunks; - ssize_t reply_sz; - int i, k = 0; + struct idpf_chunked_msg_params params = { + .vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES, + .prepare_msg = idpf_prepare_cfg_rxqs_msg, + .config_sz = sizeof(struct virtchnl2_config_rx_queues), + .chunk_sz = sizeof(*qi), + }; - totqs = vport->num_rxq + vport->num_bufq; - qi = kcalloc(totqs, sizeof(struct virtchnl2_rxq_info), GFP_KERNEL); + qi = kcalloc(qs->num, sizeof(*qi), GFP_KERNEL); if (!qi) return -ENOMEM; - /* Populate the queue info buffer with all queue context info */ - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; - u16 num_rxq; - int j; - - if (!idpf_is_queue_model_split(vport->rxq_model)) - goto setup_rxqs; - - for (j = 0; j < vport->num_bufqs_per_qgrp; j++, k++) { - struct idpf_buf_queue *bufq = - &rx_qgrp->splitq.bufq_sets[j].bufq; - - qi[k].queue_id = cpu_to_le32(bufq->q_id); - qi[k].model = cpu_to_le16(vport->rxq_model); - qi[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); - qi[k].desc_ids = cpu_to_le64(VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M); - qi[k].ring_len = cpu_to_le16(bufq->desc_count); - qi[k].dma_ring_addr = cpu_to_le64(bufq->dma); - qi[k].data_buffer_size = cpu_to_le32(bufq->rx_buf_size); - qi[k].buffer_notif_stride = IDPF_RX_BUF_STRIDE; - qi[k].rx_buffer_low_watermark = - cpu_to_le16(bufq->rx_buffer_low_watermark); - if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) - qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); - } + params.chunks = qi; -setup_rxqs: - if (idpf_is_queue_model_split(vport->rxq_model)) - num_rxq = rx_qgrp->splitq.num_rxq_sets; - else - num_rxq = rx_qgrp->singleq.num_rxq; + for (u32 i = 0; i < qs->num; i++) { + if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX) + idpf_fill_rxq_config_chunk(qs->vport, qs->qs[i].rxq, + &qi[params.num_chunks++]); + else if (qs->qs[i].type == VIRTCHNL2_QUEUE_TYPE_RX_BUFFER) + idpf_fill_bufq_config_chunk(qs->vport, qs->qs[i].bufq, + &qi[params.num_chunks++]); + } - for (j = 0; j < num_rxq; j++, k++) { - const struct idpf_bufq_set *sets; - struct idpf_rx_queue *rxq; + return idpf_send_chunked_msg(qs->vport, ¶ms); +} - if (!idpf_is_queue_model_split(vport->rxq_model)) { - rxq = rx_qgrp->singleq.rxqs[j]; - goto common_qi_fields; - } +/** + * idpf_send_config_rx_queues_msg - send virtchnl config Rx queues message + * @vport: virtual port data structure + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_config_rx_queues_msg(struct idpf_vport *vport) +{ + bool splitq = idpf_is_queue_model_split(vport->rxq_model); + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 totqs = vport->num_rxq + vport->num_bufq; + u32 k = 0; - rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; - sets = rxq->bufq_sets; + qs = idpf_alloc_queue_set(vport, totqs); + if (!qs) + return -ENOMEM; - /* In splitq mode, RXQ buffer size should be - * set to that of the first buffer queue - * associated with this RXQ. - */ - rxq->rx_buf_size = sets[0].bufq.rx_buf_size; + /* Populate the queue info buffer with all queue context info */ + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; - qi[k].rx_bufq1_id = cpu_to_le16(sets[0].bufq.q_id); - if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP) { - qi[k].bufq2_ena = IDPF_BUFQ2_ENA; - qi[k].rx_bufq2_id = - cpu_to_le16(sets[1].bufq.q_id); - } - qi[k].rx_buffer_low_watermark = - cpu_to_le16(rxq->rx_buffer_low_watermark); - if (idpf_is_feature_ena(vport, NETIF_F_GRO_HW)) - qi[k].qflags |= cpu_to_le16(VIRTCHNL2_RXQ_RSC); - - rxq->rx_hbuf_size = sets[0].bufq.rx_hbuf_size; - - if (idpf_queue_has(HSPLIT_EN, rxq)) { - qi[k].qflags |= - cpu_to_le16(VIRTCHNL2_RXQ_HDR_SPLIT); - qi[k].hdr_buffer_size = - cpu_to_le16(rxq->rx_hbuf_size); - } + if (!splitq) { + num_rxq = rx_qgrp->singleq.num_rxq; + goto rxq; + } + + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; + } + + num_rxq = rx_qgrp->splitq.num_rxq_sets; + +rxq: + for (u32 j = 0; j < num_rxq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; -common_qi_fields: - qi[k].queue_id = cpu_to_le32(rxq->q_id); - qi[k].model = cpu_to_le16(vport->rxq_model); - qi[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - qi[k].ring_len = cpu_to_le16(rxq->desc_count); - qi[k].dma_ring_addr = cpu_to_le64(rxq->dma); - qi[k].max_pkt_size = cpu_to_le32(rxq->rx_max_pkt_size); - qi[k].data_buffer_size = cpu_to_le32(rxq->rx_buf_size); - qi[k].qflags |= - cpu_to_le16(VIRTCHNL2_RX_DESC_SIZE_32BYTE); - qi[k].desc_ids = cpu_to_le64(rxq->rxdids); + if (splitq) + qs->qs[k++].rxq = + &rx_qgrp->splitq.rxq_sets[j]->rxq; + else + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; } } @@ -1791,317 +2048,395 @@ common_qi_fields: if (k != totqs) return -EINVAL; - /* Chunk up the queue contexts into multiple messages to avoid - * sending a control queue message buffer that is too large - */ - config_sz = sizeof(struct virtchnl2_config_rx_queues); - chunk_sz = sizeof(struct virtchnl2_rxq_info); + return idpf_send_config_rx_queue_set_msg(qs); +} - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - totqs); - num_msgs = DIV_ROUND_UP(totqs, num_chunks); +/** + * idpf_prepare_ena_dis_qs_msg - prepare message to enable/disable selected + * queues + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the queue + * @num_chunks: number of chunks in the message + * + * Helper function for preparing the message describing queues to be enabled + * or disabled. + * + * Return: the total size of the prepared message. + */ +static u32 idpf_prepare_ena_dis_qs_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_del_ena_dis_queues *eq = buf; + + eq->vport_id = cpu_to_le32(vport->vport_id); + eq->chunks.num_chunks = cpu_to_le16(num_chunks); + memcpy(eq->chunks.chunks, pos, + num_chunks * sizeof(*eq->chunks.chunks)); + + return struct_size(eq, chunks.chunks, num_chunks); +} - buf_sz = struct_size(crq, qinfo, num_chunks); - crq = kzalloc(buf_sz, GFP_KERNEL); - if (!crq) +/** + * idpf_send_ena_dis_queue_set_msg - send virtchnl enable or disable queues + * message for selected queues + * @qs: set of the queues to enable or disable + * @en: whether to enable or disable queues + * + * Send enable or disable queues virtchnl message for queues contained + * in the @qs array. + * The @qs array can contain pointers to both Rx and Tx queues. + * + * Return: 0 on success, -errno on failure. + */ +static int idpf_send_ena_dis_queue_set_msg(const struct idpf_queue_set *qs, + bool en) +{ + struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; + struct idpf_chunked_msg_params params = { + .vc_op = en ? VIRTCHNL2_OP_ENABLE_QUEUES : + VIRTCHNL2_OP_DISABLE_QUEUES, + .prepare_msg = idpf_prepare_ena_dis_qs_msg, + .config_sz = sizeof(struct virtchnl2_del_ena_dis_queues), + .chunk_sz = sizeof(*qc), + .num_chunks = qs->num, + }; + + qc = kcalloc(qs->num, sizeof(*qc), GFP_KERNEL); + if (!qc) return -ENOMEM; - xn_params.vc_op = VIRTCHNL2_OP_CONFIG_RX_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; + params.chunks = qc; - for (i = 0, k = 0; i < num_msgs; i++) { - memset(crq, 0, buf_sz); - crq->vport_id = cpu_to_le32(vport->vport_id); - crq->num_qinfo = cpu_to_le16(num_chunks); - memcpy(crq->qinfo, &qi[k], chunk_sz * num_chunks); + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + u32 qid; - xn_params.send_buf.iov_base = crq; - xn_params.send_buf.iov_len = buf_sz; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + qc[i].type = cpu_to_le32(q->type); + qc[i].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); - k += num_chunks; - totqs -= num_chunks; - num_chunks = min(num_chunks, totqs); - /* Recalculate buffer size */ - buf_sz = struct_size(crq, qinfo, num_chunks); + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + qid = q->rxq->q_id; + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + qid = q->txq->q_id; + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + qid = q->bufq->q_id; + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + qid = q->complq->q_id; + break; + default: + return -EINVAL; + } + + qc[i].start_queue_id = cpu_to_le32(qid); } - return 0; + return idpf_send_chunked_msg(qs->vport, ¶ms); } /** - * idpf_send_ena_dis_queues_msg - Send virtchnl enable or disable - * queues message + * idpf_send_ena_dis_queues_msg - send virtchnl enable or disable queues + * message * @vport: virtual port data structure - * @ena: if true enable, false disable + * @en: whether to enable or disable queues * - * Send enable or disable queues virtchnl message. Returns 0 on success, - * negative on failure. + * Return: 0 on success, -errno on failure. */ -static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool ena) +static int idpf_send_ena_dis_queues_msg(struct idpf_vport *vport, bool en) { - struct virtchnl2_del_ena_dis_queues *eq __free(kfree) = NULL; - struct virtchnl2_queue_chunk *qc __free(kfree) = NULL; - u32 num_msgs, num_chunks, num_txq, num_rxq, num_q; - struct idpf_vc_xn_params xn_params = {}; - struct virtchnl2_queue_chunks *qcs; - u32 config_sz, chunk_sz, buf_sz; - ssize_t reply_sz; - int i, j, k = 0; + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 num_txq, num_q, k = 0; + bool split; num_txq = vport->num_txq + vport->num_complq; - num_rxq = vport->num_rxq + vport->num_bufq; - num_q = num_txq + num_rxq; - buf_sz = sizeof(struct virtchnl2_queue_chunk) * num_q; - qc = kzalloc(buf_sz, GFP_KERNEL); - if (!qc) + num_q = num_txq + vport->num_rxq + vport->num_bufq; + + qs = idpf_alloc_queue_set(vport, num_q); + if (!qs) return -ENOMEM; - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + split = idpf_is_queue_model_split(vport->txq_model); - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { - qc[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - qc[k].start_queue_id = cpu_to_le32(tx_qgrp->txqs[j]->q_id); - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); - } - } - if (vport->num_txq != k) - return -EINVAL; + for (u32 i = 0; i < vport->num_txq_grp; i++) { + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - if (!idpf_is_queue_model_split(vport->txq_model)) - goto setup_rx; + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[k++].txq = tx_qgrp->txqs[j]; + } - for (i = 0; i < vport->num_txq_grp; i++, k++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; + if (!split) + continue; - qc[k].type = cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION); - qc[k].start_queue_id = cpu_to_le32(tx_qgrp->complq->q_id); - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION; + qs->qs[k++].complq = tx_qgrp->complq; } - if (vport->num_complq != (k - vport->num_txq)) + + if (k != num_txq) return -EINVAL; -setup_rx: - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + split = idpf_is_queue_model_split(vport->rxq_model); - if (idpf_is_queue_model_split(vport->rxq_model)) + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; + + if (split) num_rxq = rx_qgrp->splitq.num_rxq_sets; else num_rxq = rx_qgrp->singleq.num_rxq; - for (j = 0; j < num_rxq; j++, k++) { - if (idpf_is_queue_model_split(vport->rxq_model)) { - qc[k].start_queue_id = - cpu_to_le32(rx_qgrp->splitq.rxq_sets[j]->rxq.q_id); - qc[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - } else { - qc[k].start_queue_id = - cpu_to_le32(rx_qgrp->singleq.rxqs[j]->q_id); - qc[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - } - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); - } - } - if (vport->num_rxq != k - (vport->num_txq + vport->num_complq)) - return -EINVAL; - - if (!idpf_is_queue_model_split(vport->rxq_model)) - goto send_msg; + for (u32 j = 0; j < num_rxq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + if (split) + qs->qs[k++].rxq = + &rx_qgrp->splitq.rxq_sets[j]->rxq; + else + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; + } - for (j = 0; j < vport->num_bufqs_per_qgrp; j++, k++) { - const struct idpf_buf_queue *q; + if (!split) + continue; - q = &rx_qgrp->splitq.bufq_sets[j].bufq; - qc[k].type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX_BUFFER); - qc[k].start_queue_id = cpu_to_le32(q->q_id); - qc[k].num_queues = cpu_to_le32(IDPF_NUMQ_PER_CHUNK); + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX_BUFFER; + qs->qs[k++].bufq = &rx_qgrp->splitq.bufq_sets[j].bufq; } } - if (vport->num_bufq != k - (vport->num_txq + - vport->num_complq + - vport->num_rxq)) + + if (k != num_q) return -EINVAL; -send_msg: - /* Chunk up the queue info into multiple messages */ - config_sz = sizeof(struct virtchnl2_del_ena_dis_queues); - chunk_sz = sizeof(struct virtchnl2_queue_chunk); + return idpf_send_ena_dis_queue_set_msg(qs, en); +} + +/** + * idpf_prep_map_unmap_queue_set_vector_msg - prepare message to map or unmap + * queue set to the interrupt vector + * @vport: virtual port data structure + * @buf: buffer containing the message + * @pos: pointer to the first chunk describing the vector mapping + * @num_chunks: number of chunks in the message + * + * Helper function for preparing the message describing mapping queues to + * q_vectors. + * + * Return: the total size of the prepared message. + */ +static u32 +idpf_prep_map_unmap_queue_set_vector_msg(const struct idpf_vport *vport, + void *buf, const void *pos, + u32 num_chunks) +{ + struct virtchnl2_queue_vector_maps *vqvm = buf; + + vqvm->vport_id = cpu_to_le32(vport->vport_id); + vqvm->num_qv_maps = cpu_to_le16(num_chunks); + memcpy(vqvm->qv_maps, pos, num_chunks * sizeof(*vqvm->qv_maps)); - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - num_q); - num_msgs = DIV_ROUND_UP(num_q, num_chunks); + return struct_size(vqvm, qv_maps, num_chunks); +} - buf_sz = struct_size(eq, chunks.chunks, num_chunks); - eq = kzalloc(buf_sz, GFP_KERNEL); - if (!eq) +/** + * idpf_send_map_unmap_queue_set_vector_msg - send virtchnl map or unmap + * queue set vector message + * @qs: set of the queues to map or unmap + * @map: true for map and false for unmap + * + * Return: 0 on success, -errno on failure. + */ +static int +idpf_send_map_unmap_queue_set_vector_msg(const struct idpf_queue_set *qs, + bool map) +{ + struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; + struct idpf_chunked_msg_params params = { + .vc_op = map ? VIRTCHNL2_OP_MAP_QUEUE_VECTOR : + VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR, + .prepare_msg = idpf_prep_map_unmap_queue_set_vector_msg, + .config_sz = sizeof(struct virtchnl2_queue_vector_maps), + .chunk_sz = sizeof(*vqv), + .num_chunks = qs->num, + }; + bool split; + + vqv = kcalloc(qs->num, sizeof(*vqv), GFP_KERNEL); + if (!vqv) return -ENOMEM; - if (ena) { - xn_params.vc_op = VIRTCHNL2_OP_ENABLE_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - } else { - xn_params.vc_op = VIRTCHNL2_OP_DISABLE_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; - } + params.chunks = vqv; - for (i = 0, k = 0; i < num_msgs; i++) { - memset(eq, 0, buf_sz); - eq->vport_id = cpu_to_le32(vport->vport_id); - eq->chunks.num_chunks = cpu_to_le16(num_chunks); - qcs = &eq->chunks; - memcpy(qcs->chunks, &qc[k], chunk_sz * num_chunks); + split = idpf_is_queue_model_split(qs->vport->txq_model); - xn_params.send_buf.iov_base = eq; - xn_params.send_buf.iov_len = buf_sz; - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; + for (u32 i = 0; i < qs->num; i++) { + const struct idpf_queue_ptr *q = &qs->qs[i]; + const struct idpf_q_vector *vec; + u32 qid, v_idx, itr_idx; - k += num_chunks; - num_q -= num_chunks; - num_chunks = min(num_chunks, num_q); - /* Recalculate buffer size */ - buf_sz = struct_size(eq, chunks.chunks, num_chunks); + vqv[i].queue_type = cpu_to_le32(q->type); + + switch (q->type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + qid = q->rxq->q_id; + + if (idpf_queue_has(NOIRQ, q->rxq)) + vec = NULL; + else + vec = q->rxq->q_vector; + + if (vec) { + v_idx = vec->v_idx; + itr_idx = vec->rx_itr_idx; + } else { + v_idx = qs->vport->noirq_v_idx; + itr_idx = VIRTCHNL2_ITR_IDX_0; + } + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + qid = q->txq->q_id; + + if (idpf_queue_has(NOIRQ, q->txq)) + vec = NULL; + else if (idpf_queue_has(XDP, q->txq)) + vec = q->txq->complq->q_vector; + else if (split) + vec = q->txq->txq_grp->complq->q_vector; + else + vec = q->txq->q_vector; + + if (vec) { + v_idx = vec->v_idx; + itr_idx = vec->tx_itr_idx; + } else { + v_idx = qs->vport->noirq_v_idx; + itr_idx = VIRTCHNL2_ITR_IDX_1; + } + break; + default: + return -EINVAL; + } + + vqv[i].queue_id = cpu_to_le32(qid); + vqv[i].vector_id = cpu_to_le16(v_idx); + vqv[i].itr_idx = cpu_to_le32(itr_idx); } - return 0; + return idpf_send_chunked_msg(qs->vport, ¶ms); } /** - * idpf_send_map_unmap_queue_vector_msg - Send virtchnl map or unmap queue - * vector message + * idpf_send_map_unmap_queue_vector_msg - send virtchnl map or unmap queue + * vector message * @vport: virtual port data structure * @map: true for map and false for unmap * - * Send map or unmap queue vector virtchnl message. Returns 0 on success, - * negative on failure. + * Return: 0 on success, -errno on failure. */ int idpf_send_map_unmap_queue_vector_msg(struct idpf_vport *vport, bool map) { - struct virtchnl2_queue_vector_maps *vqvm __free(kfree) = NULL; - struct virtchnl2_queue_vector *vqv __free(kfree) = NULL; - struct idpf_vc_xn_params xn_params = {}; - u32 config_sz, chunk_sz, buf_sz; - u32 num_msgs, num_chunks, num_q; - ssize_t reply_sz; - int i, j, k = 0; + struct idpf_queue_set *qs __free(kfree) = NULL; + u32 num_q = vport->num_txq + vport->num_rxq; + u32 k = 0; - num_q = vport->num_txq + vport->num_rxq; - - buf_sz = sizeof(struct virtchnl2_queue_vector) * num_q; - vqv = kzalloc(buf_sz, GFP_KERNEL); - if (!vqv) + qs = idpf_alloc_queue_set(vport, num_q); + if (!qs) return -ENOMEM; - for (i = 0; i < vport->num_txq_grp; i++) { - struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - - for (j = 0; j < tx_qgrp->num_txq; j++, k++) { - vqv[k].queue_type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_TX); - vqv[k].queue_id = cpu_to_le32(tx_qgrp->txqs[j]->q_id); + for (u32 i = 0; i < vport->num_txq_grp; i++) { + const struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i]; - if (idpf_is_queue_model_split(vport->txq_model)) { - vqv[k].vector_id = - cpu_to_le16(tx_qgrp->complq->q_vector->v_idx); - vqv[k].itr_idx = - cpu_to_le32(tx_qgrp->complq->q_vector->tx_itr_idx); - } else { - vqv[k].vector_id = - cpu_to_le16(tx_qgrp->txqs[j]->q_vector->v_idx); - vqv[k].itr_idx = - cpu_to_le32(tx_qgrp->txqs[j]->q_vector->tx_itr_idx); - } + for (u32 j = 0; j < tx_qgrp->num_txq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_TX; + qs->qs[k++].txq = tx_qgrp->txqs[j]; } } - if (vport->num_txq != k) + if (k != vport->num_txq) return -EINVAL; - for (i = 0; i < vport->num_rxq_grp; i++) { - struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; - u16 num_rxq; + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; if (idpf_is_queue_model_split(vport->rxq_model)) num_rxq = rx_qgrp->splitq.num_rxq_sets; else num_rxq = rx_qgrp->singleq.num_rxq; - for (j = 0; j < num_rxq; j++, k++) { - struct idpf_rx_queue *rxq; + for (u32 j = 0; j < num_rxq; j++) { + qs->qs[k].type = VIRTCHNL2_QUEUE_TYPE_RX; if (idpf_is_queue_model_split(vport->rxq_model)) - rxq = &rx_qgrp->splitq.rxq_sets[j]->rxq; + qs->qs[k++].rxq = + &rx_qgrp->splitq.rxq_sets[j]->rxq; else - rxq = rx_qgrp->singleq.rxqs[j]; - - vqv[k].queue_type = - cpu_to_le32(VIRTCHNL2_QUEUE_TYPE_RX); - vqv[k].queue_id = cpu_to_le32(rxq->q_id); - vqv[k].vector_id = cpu_to_le16(rxq->q_vector->v_idx); - vqv[k].itr_idx = cpu_to_le32(rxq->q_vector->rx_itr_idx); + qs->qs[k++].rxq = rx_qgrp->singleq.rxqs[j]; } } - if (idpf_is_queue_model_split(vport->txq_model)) { - if (vport->num_rxq != k - vport->num_complq) - return -EINVAL; - } else { - if (vport->num_rxq != k - vport->num_txq) - return -EINVAL; - } + if (k != num_q) + return -EINVAL; - /* Chunk up the vector info into multiple messages */ - config_sz = sizeof(struct virtchnl2_queue_vector_maps); - chunk_sz = sizeof(struct virtchnl2_queue_vector); + return idpf_send_map_unmap_queue_set_vector_msg(qs, map); +} - num_chunks = min_t(u32, IDPF_NUM_CHUNKS_PER_MSG(config_sz, chunk_sz), - num_q); - num_msgs = DIV_ROUND_UP(num_q, num_chunks); +/** + * idpf_send_enable_queue_set_msg - send enable queues virtchnl message for + * selected queues + * @qs: set of the queues + * + * Send enable queues virtchnl message for queues contained in the @qs array. + * + * Return: 0 on success, -errno on failure. + */ +int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs) +{ + return idpf_send_ena_dis_queue_set_msg(qs, true); +} - buf_sz = struct_size(vqvm, qv_maps, num_chunks); - vqvm = kzalloc(buf_sz, GFP_KERNEL); - if (!vqvm) - return -ENOMEM; +/** + * idpf_send_disable_queue_set_msg - send disable queues virtchnl message for + * selected queues + * @qs: set of the queues + * + * Return: 0 on success, -errno on failure. + */ +int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs) +{ + int err; - if (map) { - xn_params.vc_op = VIRTCHNL2_OP_MAP_QUEUE_VECTOR; - xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; - } else { - xn_params.vc_op = VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR; - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; - } + err = idpf_send_ena_dis_queue_set_msg(qs, false); + if (err) + return err; - for (i = 0, k = 0; i < num_msgs; i++) { - memset(vqvm, 0, buf_sz); - xn_params.send_buf.iov_base = vqvm; - xn_params.send_buf.iov_len = buf_sz; - vqvm->vport_id = cpu_to_le32(vport->vport_id); - vqvm->num_qv_maps = cpu_to_le16(num_chunks); - memcpy(vqvm->qv_maps, &vqv[k], chunk_sz * num_chunks); + return idpf_wait_for_marker_event_set(qs); +} - reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); - if (reply_sz < 0) - return reply_sz; +/** + * idpf_send_config_queue_set_msg - send virtchnl config queues message for + * selected queues + * @qs: set of the queues + * + * Send config queues virtchnl message for queues contained in the @qs array. + * The @qs array can contain both Rx or Tx queues. + * + * Return: 0 on success, -errno on failure. + */ +int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs) +{ + int err; - k += num_chunks; - num_q -= num_chunks; - num_chunks = min(num_chunks, num_q); - /* Recalculate buffer size */ - buf_sz = struct_size(vqvm, qv_maps, num_chunks); - } + err = idpf_send_config_tx_queue_set_msg(qs); + if (err) + return err; - return 0; + return idpf_send_config_rx_queue_set_msg(qs); } /** @@ -2125,24 +2460,12 @@ int idpf_send_enable_queues_msg(struct idpf_vport *vport) */ int idpf_send_disable_queues_msg(struct idpf_vport *vport) { - int err, i; + int err; err = idpf_send_ena_dis_queues_msg(vport, false); if (err) return err; - /* switch to poll mode as interrupts will be disabled after disable - * queues virtchnl message is sent - */ - for (i = 0; i < vport->num_txq; i++) - idpf_queue_set(POLL_MODE, vport->txqs[i]); - - /* schedule the napi to receive all the marker packets */ - local_bh_disable(); - for (i = 0; i < vport->num_q_vectors; i++) - napi_schedule(&vport->q_vectors[i].napi); - local_bh_enable(); - return idpf_wait_for_marker_event(vport); } @@ -2207,7 +2530,7 @@ int idpf_send_delete_queues_msg(struct idpf_vport *vport) num_chunks); xn_params.vc_op = VIRTCHNL2_OP_DEL_QUEUES; - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; xn_params.send_buf.iov_base = eq; xn_params.send_buf.iov_len = buf_size; reply_sz = idpf_vc_xn_exec(vport->adapter, &xn_params); @@ -2371,7 +2694,7 @@ int idpf_send_dealloc_vectors_msg(struct idpf_adapter *adapter) xn_params.vc_op = VIRTCHNL2_OP_DEALLOC_VECTORS; xn_params.send_buf.iov_base = vcs; xn_params.send_buf.iov_len = buf_size; - xn_params.timeout_ms = IDPF_VC_XN_MIN_TIMEOUT_MSEC; + xn_params.timeout_ms = IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC; reply_sz = idpf_vc_xn_exec(adapter, &xn_params); if (reply_sz < 0) return reply_sz; @@ -3285,9 +3608,17 @@ int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) { struct idpf_vector_info vec_info; int num_alloc_vecs; + u32 req; vec_info.num_curr_vecs = vport->num_q_vectors; - vec_info.num_req_vecs = max(vport->num_txq, vport->num_rxq); + if (vec_info.num_curr_vecs) + vec_info.num_curr_vecs += IDPF_RESERVED_VECS; + + /* XDPSQs are all bound to the NOIRQ vector from IDPF_RESERVED_VECS */ + req = max(vport->num_txq - vport->num_xdp_txq, vport->num_rxq) + + IDPF_RESERVED_VECS; + vec_info.num_req_vecs = req; + vec_info.default_vport = vport->default_vport; vec_info.index = vport->idx; @@ -3300,7 +3631,7 @@ int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport) return -EINVAL; } - vport->num_q_vectors = num_alloc_vecs; + vport->num_q_vectors = num_alloc_vecs - IDPF_RESERVED_VECS; return 0; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h index 86f30f0db07a..eac3d15daa42 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl.h @@ -4,7 +4,8 @@ #ifndef _IDPF_VIRTCHNL_H_ #define _IDPF_VIRTCHNL_H_ -#define IDPF_VC_XN_MIN_TIMEOUT_MSEC 2000 +#include "virtchnl2.h" + #define IDPF_VC_XN_DEFAULT_TIMEOUT_MSEC (60 * 1000) #define IDPF_VC_XN_IDX_M GENMASK(7, 0) #define IDPF_VC_XN_SALT_M GENMASK(15, 8) @@ -115,6 +116,33 @@ int idpf_recv_mb_msg(struct idpf_adapter *adapter); int idpf_send_mb_msg(struct idpf_adapter *adapter, u32 op, u16 msg_size, u8 *msg, u16 cookie); +struct idpf_queue_ptr { + enum virtchnl2_queue_type type; + union { + struct idpf_rx_queue *rxq; + struct idpf_tx_queue *txq; + struct idpf_buf_queue *bufq; + struct idpf_compl_queue *complq; + }; +}; + +struct idpf_queue_set { + struct idpf_vport *vport; + + u32 num; + struct idpf_queue_ptr qs[] __counted_by(num); +}; + +struct idpf_queue_set *idpf_alloc_queue_set(struct idpf_vport *vport, u32 num); + +int idpf_send_enable_queue_set_msg(const struct idpf_queue_set *qs); +int idpf_send_disable_queue_set_msg(const struct idpf_queue_set *qs); +int idpf_send_config_queue_set_msg(const struct idpf_queue_set *qs); + +int idpf_send_disable_queues_msg(struct idpf_vport *vport); +int idpf_send_config_queues_msg(struct idpf_vport *vport); +int idpf_send_enable_queues_msg(struct idpf_vport *vport); + void idpf_vport_init(struct idpf_vport *vport, struct idpf_vport_max_q *max_q); u32 idpf_get_vport_id(struct idpf_vport *vport); int idpf_send_create_vport_msg(struct idpf_adapter *adapter, @@ -131,9 +159,6 @@ void idpf_vport_dealloc_max_qs(struct idpf_adapter *adapter, int idpf_send_add_queues_msg(const struct idpf_vport *vport, u16 num_tx_q, u16 num_complq, u16 num_rx_q, u16 num_rx_bufq); int idpf_send_delete_queues_msg(struct idpf_vport *vport); -int idpf_send_enable_queues_msg(struct idpf_vport *vport); -int idpf_send_disable_queues_msg(struct idpf_vport *vport); -int idpf_send_config_queues_msg(struct idpf_vport *vport); int idpf_vport_alloc_vec_indexes(struct idpf_vport *vport); int idpf_get_vec_ids(struct idpf_adapter *adapter, diff --git a/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c b/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c index 4f1fb0cefe51..8a2e0f8c5e36 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c +++ b/drivers/net/ethernet/intel/idpf/idpf_virtchnl_ptp.c @@ -521,6 +521,10 @@ idpf_ptp_get_tstamp_value(struct idpf_vport *vport, list_add(&ptp_tx_tstamp->list_member, &tx_tstamp_caps->latches_free); + u64_stats_update_begin(&vport->tstamp_stats.stats_sync); + u64_stats_inc(&vport->tstamp_stats.packets); + u64_stats_update_end(&vport->tstamp_stats.stats_sync); + return 0; } diff --git a/drivers/net/ethernet/intel/idpf/xdp.c b/drivers/net/ethernet/intel/idpf/xdp.c new file mode 100644 index 000000000000..21ce25b0567f --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xdp.c @@ -0,0 +1,486 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include "idpf.h" +#include "idpf_virtchnl.h" +#include "xdp.h" +#include "xsk.h" + +static int idpf_rxq_for_each(const struct idpf_vport *vport, + int (*fn)(struct idpf_rx_queue *rxq, void *arg), + void *arg) +{ + bool splitq = idpf_is_queue_model_split(vport->rxq_model); + + if (!vport->rxq_grps) + return -ENETDOWN; + + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i]; + u32 num_rxq; + + if (splitq) + num_rxq = rx_qgrp->splitq.num_rxq_sets; + else + num_rxq = rx_qgrp->singleq.num_rxq; + + for (u32 j = 0; j < num_rxq; j++) { + struct idpf_rx_queue *q; + int err; + + if (splitq) + q = &rx_qgrp->splitq.rxq_sets[j]->rxq; + else + q = rx_qgrp->singleq.rxqs[j]; + + err = fn(q, arg); + if (err) + return err; + } + } + + return 0; +} + +static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg) +{ + const struct idpf_vport *vport = rxq->q_vector->vport; + bool split = idpf_is_queue_model_split(vport->rxq_model); + int err; + + err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx, + rxq->q_vector->napi.napi_id, + rxq->rx_buf_size); + if (err) + return err; + + if (idpf_queue_has(XSK, rxq)) { + err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + rxq->pool); + if (err) + goto unreg; + } else { + const struct page_pool *pp; + + pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp; + xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp); + } + + if (!split) + return 0; + + rxq->xdpsqs = &vport->txqs[vport->xdp_txq_offset]; + rxq->num_xdp_txq = vport->num_xdp_txq; + + return 0; + +unreg: + xdp_rxq_info_unreg(&rxq->xdp_rxq); + + return err; +} + +int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq) +{ + return __idpf_xdp_rxq_info_init(rxq, NULL); +} + +int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport) +{ + return idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_init, NULL); +} + +static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg) +{ + if (idpf_is_queue_model_split((size_t)arg)) { + rxq->xdpsqs = NULL; + rxq->num_xdp_txq = 0; + } + + if (!idpf_queue_has(XSK, rxq)) + xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq); + + xdp_rxq_info_unreg(&rxq->xdp_rxq); + + return 0; +} + +void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model) +{ + __idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model); +} + +void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport) +{ + idpf_rxq_for_each(vport, __idpf_xdp_rxq_info_deinit, + (void *)(size_t)vport->rxq_model); +} + +static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg) +{ + struct bpf_prog *prog = arg; + struct bpf_prog *old; + + if (prog) + bpf_prog_inc(prog); + + old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held()); + if (old) + bpf_prog_put(old); + + return 0; +} + +void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, + struct bpf_prog *xdp_prog) +{ + idpf_rxq_for_each(vport, idpf_xdp_rxq_assign_prog, xdp_prog); +} + +static void idpf_xdp_tx_timer(struct work_struct *work); + +int idpf_xdpsqs_get(const struct idpf_vport *vport) +{ + struct libeth_xdpsq_timer **timers __free(kvfree) = NULL; + struct net_device *dev; + u32 sqs; + + if (!idpf_xdp_enabled(vport)) + return 0; + + timers = kvcalloc(vport->num_xdp_txq, sizeof(*timers), GFP_KERNEL); + if (!timers) + return -ENOMEM; + + for (u32 i = 0; i < vport->num_xdp_txq; i++) { + timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL, + cpu_to_mem(i)); + if (!timers[i]) { + for (int j = i - 1; j >= 0; j--) + kfree(timers[j]); + + return -ENOMEM; + } + } + + dev = vport->netdev; + sqs = vport->xdp_txq_offset; + + for (u32 i = sqs; i < vport->num_txq; i++) { + struct idpf_tx_queue *xdpsq = vport->txqs[i]; + + xdpsq->complq = xdpsq->txq_grp->complq; + kfree(xdpsq->refillq); + xdpsq->refillq = NULL; + + idpf_queue_clear(FLOW_SCH_EN, xdpsq); + idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq); + idpf_queue_set(NOIRQ, xdpsq); + idpf_queue_set(XDP, xdpsq); + idpf_queue_set(XDP, xdpsq->complq); + + xdpsq->timer = timers[i - sqs]; + libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share); + libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock, + idpf_xdp_tx_timer); + + xdpsq->pending = 0; + xdpsq->xdp_tx = 0; + xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count); + } + + return 0; +} + +void idpf_xdpsqs_put(const struct idpf_vport *vport) +{ + struct net_device *dev; + u32 sqs; + + if (!idpf_xdp_enabled(vport)) + return; + + dev = vport->netdev; + sqs = vport->xdp_txq_offset; + + for (u32 i = sqs; i < vport->num_txq; i++) { + struct idpf_tx_queue *xdpsq = vport->txqs[i]; + + if (!idpf_queue_has_clear(XDP, xdpsq)) + continue; + + libeth_xdpsq_deinit_timer(xdpsq->timer); + libeth_xdpsq_put(&xdpsq->xdp_lock, dev); + + kfree(xdpsq->timer); + xdpsq->refillq = NULL; + idpf_queue_clear(NOIRQ, xdpsq); + } +} + +static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc, + bool gen) +{ + u32 val; + +#ifdef __LIBETH_WORD_ACCESS + val = *(const u32 *)desc; +#else + val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) | + le16_to_cpu(desc->qid_comptype_gen); +#endif + if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen) + return -ENODATA; + + if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) != + FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M, + IDPF_TXD_COMPLT_RS))) + return -EINVAL; + + return upper_16_bits(val); +} + +u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget) +{ + struct idpf_compl_queue *cq = xdpsq->complq; + u32 tx_ntc = xdpsq->next_to_clean; + u32 tx_cnt = xdpsq->desc_count; + u32 ntc = cq->next_to_clean; + u32 cnt = cq->desc_count; + u32 done_frames; + bool gen; + + gen = idpf_queue_has(GEN_CHK, cq); + + for (done_frames = 0; done_frames < budget; ) { + int ret; + + ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen); + if (ret >= 0) { + done_frames = ret > tx_ntc ? ret - tx_ntc : + ret + tx_cnt - tx_ntc; + goto next; + } + + switch (ret) { + case -ENODATA: + goto out; + case -EINVAL: + break; + } + +next: + if (unlikely(++ntc == cnt)) { + ntc = 0; + gen = !gen; + idpf_queue_change(GEN_CHK, cq); + } + } + +out: + cq->next_to_clean = ntc; + + return done_frames; +} + +static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget) +{ + struct libeth_xdpsq_napi_stats ss = { }; + struct idpf_tx_queue *xdpsq = _xdpsq; + u32 tx_ntc = xdpsq->next_to_clean; + u32 tx_cnt = xdpsq->desc_count; + struct xdp_frame_bulk bq; + struct libeth_cq_pp cp = { + .dev = xdpsq->dev, + .bq = &bq, + .xss = &ss, + .napi = true, + }; + u32 done_frames; + + done_frames = idpf_xdpsq_poll(xdpsq, budget); + if (unlikely(!done_frames)) + return 0; + + xdp_frame_bulk_init(&bq); + + for (u32 i = 0; likely(i < done_frames); i++) { + libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp); + + if (unlikely(++tx_ntc == tx_cnt)) + tx_ntc = 0; + } + + xdp_flush_frame_bulk(&bq); + + xdpsq->next_to_clean = tx_ntc; + xdpsq->pending -= done_frames; + xdpsq->xdp_tx -= cp.xdp_tx; + + return done_frames; +} + +static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) +{ + struct idpf_tx_queue *xdpsq = _xdpsq; + u32 free; + + libeth_xdpsq_lock(&xdpsq->xdp_lock); + + free = xdpsq->desc_count - xdpsq->pending; + if (free < xdpsq->thresh) + free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh); + + *sq = (struct libeth_xdpsq){ + .sqes = xdpsq->tx_buf, + .descs = xdpsq->desc_ring, + .count = xdpsq->desc_count, + .lock = &xdpsq->xdp_lock, + .ntu = &xdpsq->next_to_use, + .pending = &xdpsq->pending, + .xdp_tx = &xdpsq->xdp_tx, + }; + + return free; +} + +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete); +LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep, + idpf_xdp_tx_xmit); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep, + idpf_xdp_tx_xmit); +LIBETH_XDP_DEFINE_END(); + +int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, + u32 flags) +{ + const struct idpf_netdev_priv *np = netdev_priv(dev); + const struct idpf_vport *vport = np->vport; + + if (unlikely(!netif_carrier_ok(dev) || !vport->link_up)) + return -ENETDOWN; + + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, + &vport->txqs[vport->xdp_txq_offset], + vport->num_xdp_txq, + idpf_xdp_xmit_flush_bulk, + idpf_xdp_tx_finalize); +} + +static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash, + enum xdp_rss_hash_type *rss_type) +{ + const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx; + struct idpf_xdp_rx_desc desc __uninitialized; + const struct idpf_rx_queue *rxq; + struct libeth_rx_pt pt; + + rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq); + + idpf_xdp_get_qw0(&desc, xdp->desc); + + pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)]; + if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt)) + return -ENODATA; + + idpf_xdp_get_qw2(&desc, xdp->desc); + + return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc), + pt); +} + +static const struct xdp_metadata_ops idpf_xdpmo = { + .xmo_rx_hash = idpf_xdpmo_rx_hash, +}; + +void idpf_xdp_set_features(const struct idpf_vport *vport) +{ + if (!idpf_is_queue_model_split(vport->rxq_model)) + return; + + libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo, + idpf_get_max_tx_bufs(vport->adapter), + libeth_xsktmo); +} + +static int idpf_xdp_setup_prog(struct idpf_vport *vport, + const struct netdev_bpf *xdp) +{ + const struct idpf_netdev_priv *np = netdev_priv(vport->netdev); + struct bpf_prog *old, *prog = xdp->prog; + struct idpf_vport_config *cfg; + int ret; + + cfg = vport->adapter->vport_config[vport->idx]; + + if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) || + !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) || + !!vport->xdp_prog == !!prog) { + if (np->state == __IDPF_VPORT_UP) + idpf_xdp_copy_prog_to_rqs(vport, prog); + + old = xchg(&vport->xdp_prog, prog); + if (old) + bpf_prog_put(old); + + cfg->user_config.xdp_prog = prog; + + return 0; + } + + if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) { + NL_SET_ERR_MSG_MOD(xdp->extack, + "No Tx queues available for XDP, please decrease the number of regular SQs"); + return -ENOSPC; + } + + old = cfg->user_config.xdp_prog; + cfg->user_config.xdp_prog = prog; + + ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE); + if (ret) { + NL_SET_ERR_MSG_MOD(xdp->extack, + "Could not reopen the vport after XDP setup"); + + cfg->user_config.xdp_prog = old; + old = prog; + } + + if (old) + bpf_prog_put(old); + + libeth_xdp_set_redirect(vport->netdev, vport->xdp_prog); + + return ret; +} + +int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp) +{ + struct idpf_vport *vport; + int ret; + + idpf_vport_ctrl_lock(dev); + vport = idpf_netdev_to_vport(dev); + + if (!idpf_is_queue_model_split(vport->txq_model)) + goto notsupp; + + switch (xdp->command) { + case XDP_SETUP_PROG: + ret = idpf_xdp_setup_prog(vport, xdp); + break; + case XDP_SETUP_XSK_POOL: + ret = idpf_xsk_pool_setup(vport, xdp); + break; + default: +notsupp: + ret = -EOPNOTSUPP; + break; + } + + idpf_vport_ctrl_unlock(dev); + + return ret; +} diff --git a/drivers/net/ethernet/intel/idpf/xdp.h b/drivers/net/ethernet/intel/idpf/xdp.h new file mode 100644 index 000000000000..479f5ef3c604 --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xdp.h @@ -0,0 +1,175 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef _IDPF_XDP_H_ +#define _IDPF_XDP_H_ + +#include <net/libeth/xdp.h> + +#include "idpf_txrx.h" + +int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq); +int idpf_xdp_rxq_info_init_all(const struct idpf_vport *vport); +void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model); +void idpf_xdp_rxq_info_deinit_all(const struct idpf_vport *vport); +void idpf_xdp_copy_prog_to_rqs(const struct idpf_vport *vport, + struct bpf_prog *xdp_prog); + +int idpf_xdpsqs_get(const struct idpf_vport *vport); +void idpf_xdpsqs_put(const struct idpf_vport *vport); + +u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget); +bool idpf_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags); + +/** + * idpf_xdp_tx_xmit - produce a single HW Tx descriptor out of XDP desc + * @desc: XDP descriptor to pull the DMA address and length from + * @i: descriptor index on the queue to fill + * @sq: XDP queue to produce the HW Tx descriptor on + * @priv: &xsk_tx_metadata_ops on XSk xmit or %NULL + */ +static inline void idpf_xdp_tx_xmit(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, u64 priv) +{ + struct idpf_flex_tx_desc *tx_desc = sq->descs; + u32 cmd; + + cmd = FIELD_PREP(IDPF_FLEX_TXD_QW1_DTYPE_M, + IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2); + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd |= FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, + IDPF_TX_DESC_CMD_EOP); + if (priv && (desc.flags & LIBETH_XDP_TX_CSUM)) + cmd |= FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, + IDPF_TX_FLEX_DESC_CMD_CS_EN); + + tx_desc = &tx_desc[i]; + tx_desc->buf_addr = cpu_to_le64(desc.addr); +#ifdef __LIBETH_WORD_ACCESS + *(u64 *)&tx_desc->qw1 = ((u64)desc.len << 48) | cmd; +#else + tx_desc->qw1.buf_size = cpu_to_le16(desc.len); + tx_desc->qw1.cmd_dtype = cpu_to_le16(cmd); +#endif +} + +static inline void idpf_xdpsq_set_rs(const struct idpf_tx_queue *xdpsq) +{ + u32 ntu, cmd; + + ntu = xdpsq->next_to_use; + if (unlikely(!ntu)) + ntu = xdpsq->desc_count; + + cmd = FIELD_PREP(IDPF_FLEX_TXD_QW1_CMD_M, IDPF_TX_DESC_CMD_RS); +#ifdef __LIBETH_WORD_ACCESS + *(u64 *)&xdpsq->flex_tx[ntu - 1].q.qw1 |= cmd; +#else + xdpsq->flex_tx[ntu - 1].q.qw1.cmd_dtype |= cpu_to_le16(cmd); +#endif +} + +static inline void idpf_xdpsq_update_tail(const struct idpf_tx_queue *xdpsq) +{ + dma_wmb(); + writel_relaxed(xdpsq->next_to_use, xdpsq->tail); +} + +/** + * idpf_xdp_tx_finalize - finalize sending over XDPSQ + * @_xdpsq: XDP Tx queue + * @sent: whether any frames were sent + * @flush: whether to update RS bit and the tail register + * + * Set the RS bit ("end of batch"), bump the tail, and queue the cleanup timer. + * To be called after a NAPI polling loop, at the end of .ndo_xdp_xmit() etc. + */ +static inline void idpf_xdp_tx_finalize(void *_xdpsq, bool sent, bool flush) +{ + struct idpf_tx_queue *xdpsq = _xdpsq; + + if ((!flush || unlikely(!sent)) && + likely(xdpsq->desc_count - 1 != xdpsq->pending)) + return; + + libeth_xdpsq_lock(&xdpsq->xdp_lock); + + idpf_xdpsq_set_rs(xdpsq); + idpf_xdpsq_update_tail(xdpsq); + + libeth_xdpsq_queue_timer(xdpsq->timer); + + libeth_xdpsq_unlock(&xdpsq->xdp_lock); +} + +struct idpf_xdp_rx_desc { + aligned_u64 qw0; +#define IDPF_XDP_RX_BUFQ BIT_ULL(47) +#define IDPF_XDP_RX_GEN BIT_ULL(46) +#define IDPF_XDP_RX_LEN GENMASK_ULL(45, 32) +#define IDPF_XDP_RX_PT GENMASK_ULL(25, 16) + + aligned_u64 qw1; +#define IDPF_XDP_RX_BUF GENMASK_ULL(47, 32) +#define IDPF_XDP_RX_EOP BIT_ULL(1) + + aligned_u64 qw2; +#define IDPF_XDP_RX_HASH GENMASK_ULL(31, 0) + + aligned_u64 qw3; +} __aligned(4 * sizeof(u64)); +static_assert(sizeof(struct idpf_xdp_rx_desc) == + sizeof(struct virtchnl2_rx_flex_desc_adv_nic_3)); + +#define idpf_xdp_rx_bufq(desc) !!((desc)->qw0 & IDPF_XDP_RX_BUFQ) +#define idpf_xdp_rx_gen(desc) !!((desc)->qw0 & IDPF_XDP_RX_GEN) +#define idpf_xdp_rx_len(desc) FIELD_GET(IDPF_XDP_RX_LEN, (desc)->qw0) +#define idpf_xdp_rx_pt(desc) FIELD_GET(IDPF_XDP_RX_PT, (desc)->qw0) +#define idpf_xdp_rx_buf(desc) FIELD_GET(IDPF_XDP_RX_BUF, (desc)->qw1) +#define idpf_xdp_rx_eop(desc) !!((desc)->qw1 & IDPF_XDP_RX_EOP) +#define idpf_xdp_rx_hash(desc) FIELD_GET(IDPF_XDP_RX_HASH, (desc)->qw2) + +static inline void +idpf_xdp_get_qw0(struct idpf_xdp_rx_desc *desc, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) +{ +#ifdef __LIBETH_WORD_ACCESS + desc->qw0 = ((const typeof(desc))rxd)->qw0; +#else + desc->qw0 = ((u64)le16_to_cpu(rxd->pktlen_gen_bufq_id) << 32) | + ((u64)le16_to_cpu(rxd->ptype_err_fflags0) << 16); +#endif +} + +static inline void +idpf_xdp_get_qw1(struct idpf_xdp_rx_desc *desc, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) +{ +#ifdef __LIBETH_WORD_ACCESS + desc->qw1 = ((const typeof(desc))rxd)->qw1; +#else + desc->qw1 = ((u64)le16_to_cpu(rxd->buf_id) << 32) | + rxd->status_err0_qw1; +#endif +} + +static inline void +idpf_xdp_get_qw2(struct idpf_xdp_rx_desc *desc, + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rxd) +{ +#ifdef __LIBETH_WORD_ACCESS + desc->qw2 = ((const typeof(desc))rxd)->qw2; +#else + desc->qw2 = ((u64)rxd->hash3 << 24) | + ((u64)rxd->ff2_mirrid_hash2.hash2 << 16) | + le16_to_cpu(rxd->hash1); +#endif +} + +void idpf_xdp_set_features(const struct idpf_vport *vport); + +int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp); +int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, + u32 flags); + +#endif /* _IDPF_XDP_H_ */ diff --git a/drivers/net/ethernet/intel/idpf/xsk.c b/drivers/net/ethernet/intel/idpf/xsk.c new file mode 100644 index 000000000000..fd2cc43ab43c --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xsk.c @@ -0,0 +1,633 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2025 Intel Corporation */ + +#include <net/libeth/xsk.h> + +#include "idpf.h" +#include "xdp.h" +#include "xsk.h" + +static void idpf_xsk_tx_timer(struct work_struct *work); + +static void idpf_xsk_setup_rxq(const struct idpf_vport *vport, + struct idpf_rx_queue *rxq) +{ + struct xsk_buff_pool *pool; + + pool = xsk_get_pool_from_qid(vport->netdev, rxq->idx); + if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) + return; + + rxq->pool = pool; + + idpf_queue_set(XSK, rxq); +} + +static void idpf_xsk_setup_bufq(const struct idpf_vport *vport, + struct idpf_buf_queue *bufq) +{ + struct xsk_buff_pool *pool; + u32 qid = U32_MAX; + + for (u32 i = 0; i < vport->num_rxq_grp; i++) { + const struct idpf_rxq_group *grp = &vport->rxq_grps[i]; + + for (u32 j = 0; j < vport->num_bufqs_per_qgrp; j++) { + if (&grp->splitq.bufq_sets[j].bufq == bufq) { + qid = grp->splitq.rxq_sets[0]->rxq.idx; + goto setup; + } + } + } + +setup: + pool = xsk_get_pool_from_qid(vport->netdev, qid); + if (!pool || !pool->dev || !xsk_buff_can_alloc(pool, 1)) + return; + + bufq->pool = pool; + + idpf_queue_set(XSK, bufq); +} + +static void idpf_xsk_setup_txq(const struct idpf_vport *vport, + struct idpf_tx_queue *txq) +{ + struct xsk_buff_pool *pool; + u32 qid; + + idpf_queue_clear(XSK, txq); + + if (!idpf_queue_has(XDP, txq)) + return; + + qid = txq->idx - vport->xdp_txq_offset; + + pool = xsk_get_pool_from_qid(vport->netdev, qid); + if (!pool || !pool->dev) + return; + + txq->pool = pool; + libeth_xdpsq_init_timer(txq->timer, txq, &txq->xdp_lock, + idpf_xsk_tx_timer); + + idpf_queue_assign(NOIRQ, txq, xsk_uses_need_wakeup(pool)); + idpf_queue_set(XSK, txq); +} + +static void idpf_xsk_setup_complq(const struct idpf_vport *vport, + struct idpf_compl_queue *complq) +{ + const struct xsk_buff_pool *pool; + u32 qid; + + idpf_queue_clear(XSK, complq); + + if (!idpf_queue_has(XDP, complq)) + return; + + qid = complq->txq_grp->txqs[0]->idx - vport->xdp_txq_offset; + + pool = xsk_get_pool_from_qid(vport->netdev, qid); + if (!pool || !pool->dev) + return; + + idpf_queue_set(XSK, complq); +} + +void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, + enum virtchnl2_queue_type type) +{ + if (!idpf_xdp_enabled(vport)) + return; + + switch (type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + idpf_xsk_setup_rxq(vport, q); + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + idpf_xsk_setup_bufq(vport, q); + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + idpf_xsk_setup_txq(vport, q); + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + idpf_xsk_setup_complq(vport, q); + break; + default: + break; + } +} + +void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type) +{ + struct idpf_compl_queue *complq; + struct idpf_buf_queue *bufq; + struct idpf_rx_queue *rxq; + struct idpf_tx_queue *txq; + + switch (type) { + case VIRTCHNL2_QUEUE_TYPE_RX: + rxq = q; + if (!idpf_queue_has_clear(XSK, rxq)) + return; + + rxq->pool = NULL; + break; + case VIRTCHNL2_QUEUE_TYPE_RX_BUFFER: + bufq = q; + if (!idpf_queue_has_clear(XSK, bufq)) + return; + + bufq->pool = NULL; + break; + case VIRTCHNL2_QUEUE_TYPE_TX: + txq = q; + if (!idpf_queue_has_clear(XSK, txq)) + return; + + idpf_queue_set(NOIRQ, txq); + txq->dev = txq->netdev->dev.parent; + break; + case VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION: + complq = q; + idpf_queue_clear(XSK, complq); + break; + default: + break; + } +} + +void idpf_xsk_init_wakeup(struct idpf_q_vector *qv) +{ + libeth_xsk_init_wakeup(&qv->csd, &qv->napi); +} + +void idpf_xsksq_clean(struct idpf_tx_queue *xdpsq) +{ + struct libeth_xdpsq_napi_stats ss = { }; + u32 ntc = xdpsq->next_to_clean; + struct xdp_frame_bulk bq; + struct libeth_cq_pp cp = { + .dev = xdpsq->pool->dev, + .bq = &bq, + .xss = &ss, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&bq); + + while (ntc != xdpsq->next_to_use) { + struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; + + if (sqe->type) + libeth_xdp_complete_tx(sqe, &cp); + else + xsk_frames++; + + if (unlikely(++ntc == xdpsq->desc_count)) + ntc = 0; + } + + xdp_flush_frame_bulk(&bq); + + if (xsk_frames) + xsk_tx_completed(xdpsq->pool, xsk_frames); +} + +static noinline u32 idpf_xsksq_complete_slow(struct idpf_tx_queue *xdpsq, + u32 done) +{ + struct libeth_xdpsq_napi_stats ss = { }; + u32 ntc = xdpsq->next_to_clean; + u32 cnt = xdpsq->desc_count; + struct xdp_frame_bulk bq; + struct libeth_cq_pp cp = { + .dev = xdpsq->pool->dev, + .bq = &bq, + .xss = &ss, + .napi = true, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&bq); + + for (u32 i = 0; likely(i < done); i++) { + struct libeth_sqe *sqe = &xdpsq->tx_buf[ntc]; + + if (sqe->type) + libeth_xdp_complete_tx(sqe, &cp); + else + xsk_frames++; + + if (unlikely(++ntc == cnt)) + ntc = 0; + } + + xdp_flush_frame_bulk(&bq); + + xdpsq->next_to_clean = ntc; + xdpsq->xdp_tx -= cp.xdp_tx; + + return xsk_frames; +} + +static __always_inline u32 idpf_xsksq_complete(void *_xdpsq, u32 budget) +{ + struct idpf_tx_queue *xdpsq = _xdpsq; + u32 tx_ntc = xdpsq->next_to_clean; + u32 tx_cnt = xdpsq->desc_count; + u32 done_frames; + u32 xsk_frames; + + done_frames = idpf_xdpsq_poll(xdpsq, budget); + if (unlikely(!done_frames)) + return 0; + + if (likely(!xdpsq->xdp_tx)) { + tx_ntc += done_frames; + if (tx_ntc >= tx_cnt) + tx_ntc -= tx_cnt; + + xdpsq->next_to_clean = tx_ntc; + xsk_frames = done_frames; + + goto finalize; + } + + xsk_frames = idpf_xsksq_complete_slow(xdpsq, done_frames); + if (xsk_frames) +finalize: + xsk_tx_completed(xdpsq->pool, xsk_frames); + + xdpsq->pending -= done_frames; + + return done_frames; +} + +static u32 idpf_xsk_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq) +{ + struct idpf_tx_queue *xdpsq = _xdpsq; + u32 free; + + libeth_xdpsq_lock(&xdpsq->xdp_lock); + + free = xdpsq->desc_count - xdpsq->pending; + if (free < xdpsq->thresh) + free += idpf_xsksq_complete(xdpsq, xdpsq->thresh); + + *sq = (struct libeth_xdpsq){ + .pool = xdpsq->pool, + .sqes = xdpsq->tx_buf, + .descs = xdpsq->desc_ring, + .count = xdpsq->desc_count, + .lock = &xdpsq->xdp_lock, + .ntu = &xdpsq->next_to_use, + .pending = &xdpsq->pending, + .xdp_tx = &xdpsq->xdp_tx, + }; + + return free; +} + +static u32 idpf_xsk_xmit_prep(void *_xdpsq, struct libeth_xdpsq *sq) +{ + struct idpf_tx_queue *xdpsq = _xdpsq; + + *sq = (struct libeth_xdpsq){ + .pool = xdpsq->pool, + .sqes = xdpsq->tx_buf, + .descs = xdpsq->desc_ring, + .count = xdpsq->desc_count, + .lock = &xdpsq->xdp_lock, + .ntu = &xdpsq->next_to_use, + .pending = &xdpsq->pending, + }; + + /* + * The queue is cleaned, the budget is already known, optimize out + * the second min() by passing the type limit. + */ + return U32_MAX; +} + +bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq) +{ + u32 free; + + libeth_xdpsq_lock(&xsksq->xdp_lock); + + free = xsksq->desc_count - xsksq->pending; + if (free < xsksq->thresh) + free += idpf_xsksq_complete(xsksq, xsksq->thresh); + + return libeth_xsk_xmit_do_bulk(xsksq->pool, xsksq, + min(free - 1, xsksq->thresh), + libeth_xsktmo, idpf_xsk_xmit_prep, + idpf_xdp_tx_xmit, idpf_xdp_tx_finalize); +} + +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_TIMER(static idpf_xsk_tx_timer, idpf_xsksq_complete); +LIBETH_XSK_DEFINE_FLUSH_TX(static idpf_xsk_tx_flush_bulk, idpf_xsk_tx_prep, + idpf_xdp_tx_xmit); +LIBETH_XSK_DEFINE_RUN(static idpf_xsk_run_pass, idpf_xsk_run_prog, + idpf_xsk_tx_flush_bulk, idpf_rx_process_skb_fields); +LIBETH_XSK_DEFINE_FINALIZE(static idpf_xsk_finalize_rx, idpf_xsk_tx_flush_bulk, + idpf_xdp_tx_finalize); +LIBETH_XDP_DEFINE_END(); + +static void idpf_xskfqe_init(const struct libeth_xskfq_fp *fq, u32 i) +{ + struct virtchnl2_splitq_rx_buf_desc *desc = fq->descs; + + desc = &desc[i]; +#ifdef __LIBETH_WORD_ACCESS + *(u64 *)&desc->qword0 = i; +#else + desc->qword0.buf_id = cpu_to_le16(i); +#endif + desc->pkt_addr = cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); +} + +static bool idpf_xskfq_refill_thresh(struct idpf_buf_queue *bufq, u32 count) +{ + struct libeth_xskfq_fp fq = { + .pool = bufq->pool, + .fqes = bufq->xsk_buf, + .descs = bufq->split_buf, + .ntu = bufq->next_to_use, + .count = bufq->desc_count, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, count, idpf_xskfqe_init); + writel(fq.ntu, bufq->tail); + + bufq->next_to_use = fq.ntu; + bufq->pending -= done; + + return done == count; +} + +static bool idpf_xskfq_refill(struct idpf_buf_queue *bufq) +{ + u32 count, rx_thresh = bufq->thresh; + + count = ALIGN_DOWN(bufq->pending - 1, rx_thresh); + + for (u32 i = 0; i < count; i += rx_thresh) { + if (unlikely(!idpf_xskfq_refill_thresh(bufq, rx_thresh))) + return false; + } + + return true; +} + +int idpf_xskfq_init(struct idpf_buf_queue *bufq) +{ + struct libeth_xskfq fq = { + .pool = bufq->pool, + .count = bufq->desc_count, + .nid = idpf_q_vector_to_mem(bufq->q_vector), + }; + int ret; + + ret = libeth_xskfq_create(&fq); + if (ret) + return ret; + + bufq->xsk_buf = fq.fqes; + bufq->pending = fq.pending; + bufq->thresh = fq.thresh; + bufq->rx_buf_size = fq.buf_len; + + if (!idpf_xskfq_refill(bufq)) + netdev_err(bufq->pool->netdev, + "failed to allocate XSk buffers for qid %d\n", + bufq->pool->queue_id); + + bufq->next_to_alloc = bufq->next_to_use; + + idpf_queue_clear(HSPLIT_EN, bufq); + bufq->rx_hbuf_size = 0; + + return 0; +} + +void idpf_xskfq_rel(struct idpf_buf_queue *bufq) +{ + struct libeth_xskfq fq = { + .fqes = bufq->xsk_buf, + }; + + libeth_xskfq_destroy(&fq); + + bufq->rx_buf_size = fq.buf_len; + bufq->thresh = fq.thresh; + bufq->pending = fq.pending; +} + +struct idpf_xskfq_refill_set { + struct { + struct idpf_buf_queue *q; + u32 buf_id; + u32 pending; + } bufqs[IDPF_MAX_BUFQS_PER_RXQ_GRP]; +}; + +static bool idpf_xskfq_refill_set(const struct idpf_xskfq_refill_set *set) +{ + bool ret = true; + + for (u32 i = 0; i < ARRAY_SIZE(set->bufqs); i++) { + struct idpf_buf_queue *bufq = set->bufqs[i].q; + u32 ntc; + + if (!bufq) + continue; + + ntc = set->bufqs[i].buf_id; + if (unlikely(++ntc == bufq->desc_count)) + ntc = 0; + + bufq->next_to_clean = ntc; + bufq->pending += set->bufqs[i].pending; + + if (bufq->pending > bufq->thresh) + ret &= idpf_xskfq_refill(bufq); + } + + return ret; +} + +int idpf_xskrq_poll(struct idpf_rx_queue *rxq, u32 budget) +{ + struct idpf_xskfq_refill_set set = { }; + struct libeth_rq_napi_stats rs = { }; + bool wake, gen, fail = false; + u32 ntc = rxq->next_to_clean; + struct libeth_xdp_buff *xdp; + LIBETH_XDP_ONSTACK_BULK(bq); + u32 cnt = rxq->desc_count; + + wake = xsk_uses_need_wakeup(rxq->pool); + if (wake) + xsk_clear_rx_need_wakeup(rxq->pool); + + gen = idpf_queue_has(GEN_CHK, rxq); + + libeth_xsk_tx_init_bulk(&bq, rxq->xdp_prog, rxq->xdp_rxq.dev, + rxq->xdpsqs, rxq->num_xdp_txq); + xdp = rxq->xsk; + + while (likely(rs.packets < budget)) { + const struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc; + struct idpf_xdp_rx_desc desc __uninitialized; + struct idpf_buf_queue *bufq; + u32 bufq_id, buf_id; + + rx_desc = &rxq->rx[ntc].flex_adv_nic_3_wb; + + idpf_xdp_get_qw0(&desc, rx_desc); + if (idpf_xdp_rx_gen(&desc) != gen) + break; + + dma_rmb(); + + bufq_id = idpf_xdp_rx_bufq(&desc); + bufq = set.bufqs[bufq_id].q; + if (!bufq) { + bufq = &rxq->bufq_sets[bufq_id].bufq; + set.bufqs[bufq_id].q = bufq; + } + + idpf_xdp_get_qw1(&desc, rx_desc); + buf_id = idpf_xdp_rx_buf(&desc); + + set.bufqs[bufq_id].buf_id = buf_id; + set.bufqs[bufq_id].pending++; + + xdp = libeth_xsk_process_buff(xdp, bufq->xsk_buf[buf_id], + idpf_xdp_rx_len(&desc)); + + if (unlikely(++ntc == cnt)) { + ntc = 0; + gen = !gen; + idpf_queue_change(GEN_CHK, rxq); + } + + if (!idpf_xdp_rx_eop(&desc) || unlikely(!xdp)) + continue; + + fail = !idpf_xsk_run_pass(xdp, &bq, rxq->napi, &rs, rx_desc); + xdp = NULL; + + if (fail) + break; + } + + idpf_xsk_finalize_rx(&bq); + + rxq->next_to_clean = ntc; + rxq->xsk = xdp; + + fail |= !idpf_xskfq_refill_set(&set); + + u64_stats_update_begin(&rxq->stats_sync); + u64_stats_add(&rxq->q_stats.packets, rs.packets); + u64_stats_add(&rxq->q_stats.bytes, rs.bytes); + u64_stats_update_end(&rxq->stats_sync); + + if (!wake) + return unlikely(fail) ? budget : rs.packets; + + if (unlikely(fail)) + xsk_set_rx_need_wakeup(rxq->pool); + + return rs.packets; +} + +int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *bpf) +{ + struct xsk_buff_pool *pool = bpf->xsk.pool; + u32 qid = bpf->xsk.queue_id; + bool restart; + int ret; + + if (pool && !IS_ALIGNED(xsk_pool_get_rx_frame_size(pool), + LIBETH_RX_BUF_STRIDE)) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: HW doesn't support frames sizes not aligned to %u (qid %u: %u)", + netdev_name(vport->netdev), + LIBETH_RX_BUF_STRIDE, qid, + xsk_pool_get_rx_frame_size(pool)); + return -EINVAL; + } + + restart = idpf_xdp_enabled(vport) && netif_running(vport->netdev); + if (!restart) + goto pool; + + ret = idpf_qp_switch(vport, qid, false); + if (ret) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: failed to disable queue pair %u: %pe", + netdev_name(vport->netdev), qid, + ERR_PTR(ret)); + return ret; + } + +pool: + ret = libeth_xsk_setup_pool(vport->netdev, qid, pool); + if (ret) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: failed to configure XSk pool for pair %u: %pe", + netdev_name(vport->netdev), qid, + ERR_PTR(ret)); + return ret; + } + + if (!restart) + return 0; + + ret = idpf_qp_switch(vport, qid, true); + if (ret) { + NL_SET_ERR_MSG_FMT_MOD(bpf->extack, + "%s: failed to enable queue pair %u: %pe", + netdev_name(vport->netdev), qid, + ERR_PTR(ret)); + goto err_dis; + } + + return 0; + +err_dis: + libeth_xsk_setup_pool(vport->netdev, qid, false); + + return ret; +} + +int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) +{ + const struct idpf_netdev_priv *np = netdev_priv(dev); + const struct idpf_vport *vport = np->vport; + struct idpf_q_vector *q_vector; + + if (unlikely(idpf_vport_ctrl_is_locked(dev))) + return -EBUSY; + + if (unlikely(!vport->link_up)) + return -ENETDOWN; + + if (unlikely(!vport->num_xdp_txq)) + return -ENXIO; + + q_vector = idpf_find_rxq_vec(vport, qid); + if (unlikely(!q_vector->xsksq)) + return -ENXIO; + + libeth_xsk_wakeup(&q_vector->csd, qid); + + return 0; +} diff --git a/drivers/net/ethernet/intel/idpf/xsk.h b/drivers/net/ethernet/intel/idpf/xsk.h new file mode 100644 index 000000000000..b622d08c03e8 --- /dev/null +++ b/drivers/net/ethernet/intel/idpf/xsk.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef _IDPF_XSK_H_ +#define _IDPF_XSK_H_ + +#include <linux/types.h> + +enum virtchnl2_queue_type; +struct idpf_buf_queue; +struct idpf_q_vector; +struct idpf_rx_queue; +struct idpf_tx_queue; +struct idpf_vport; +struct net_device; +struct netdev_bpf; + +void idpf_xsk_setup_queue(const struct idpf_vport *vport, void *q, + enum virtchnl2_queue_type type); +void idpf_xsk_clear_queue(void *q, enum virtchnl2_queue_type type); +void idpf_xsk_init_wakeup(struct idpf_q_vector *qv); + +int idpf_xskfq_init(struct idpf_buf_queue *bufq); +void idpf_xskfq_rel(struct idpf_buf_queue *bufq); +void idpf_xsksq_clean(struct idpf_tx_queue *xdpq); + +int idpf_xskrq_poll(struct idpf_rx_queue *rxq, u32 budget); +bool idpf_xsk_xmit(struct idpf_tx_queue *xsksq); + +int idpf_xsk_pool_setup(struct idpf_vport *vport, struct netdev_bpf *xdp); +int idpf_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags); + +#endif /* !_IDPF_XSK_H_ */ diff --git a/drivers/net/ethernet/intel/igb/e1000_82575.c b/drivers/net/ethernet/intel/igb/e1000_82575.c index 64dfc362d1dc..44a85ad749a4 100644 --- a/drivers/net/ethernet/intel/igb/e1000_82575.c +++ b/drivers/net/ethernet/intel/igb/e1000_82575.c @@ -2372,7 +2372,7 @@ static s32 igb_validate_nvm_checksum_with_offset(struct e1000_hw *hw, checksum += nvm_data; } - if (checksum != (u16) NVM_SUM) { + if (checksum != NVM_SUM) { hw_dbg("NVM Checksum Invalid\n"); ret_val = -E1000_ERR_NVM; goto out; @@ -2406,7 +2406,7 @@ static s32 igb_update_nvm_checksum_with_offset(struct e1000_hw *hw, u16 offset) } checksum += nvm_data; } - checksum = (u16) NVM_SUM - checksum; + checksum = NVM_SUM - checksum; ret_val = hw->nvm.ops.write(hw, (NVM_CHECKSUM_REG + offset), 1, &checksum); if (ret_val) diff --git a/drivers/net/ethernet/intel/igb/e1000_i210.c b/drivers/net/ethernet/intel/igb/e1000_i210.c index 503b239868e8..9db29b231d6a 100644 --- a/drivers/net/ethernet/intel/igb/e1000_i210.c +++ b/drivers/net/ethernet/intel/igb/e1000_i210.c @@ -602,7 +602,7 @@ static s32 igb_update_nvm_checksum_i210(struct e1000_hw *hw) } checksum += nvm_data; } - checksum = (u16) NVM_SUM - checksum; + checksum = NVM_SUM - checksum; ret_val = igb_write_nvm_srwr(hw, NVM_CHECKSUM_REG, 1, &checksum); if (ret_val) { diff --git a/drivers/net/ethernet/intel/igb/e1000_nvm.c b/drivers/net/ethernet/intel/igb/e1000_nvm.c index 2dcd64d6dec3..c8638502c2be 100644 --- a/drivers/net/ethernet/intel/igb/e1000_nvm.c +++ b/drivers/net/ethernet/intel/igb/e1000_nvm.c @@ -636,7 +636,7 @@ s32 igb_validate_nvm_checksum(struct e1000_hw *hw) checksum += nvm_data; } - if (checksum != (u16) NVM_SUM) { + if (checksum != NVM_SUM) { hw_dbg("NVM Checksum Invalid\n"); ret_val = -E1000_ERR_NVM; goto out; @@ -668,7 +668,7 @@ s32 igb_update_nvm_checksum(struct e1000_hw *hw) } checksum += nvm_data; } - checksum = (u16) NVM_SUM - checksum; + checksum = NVM_SUM - checksum; ret_val = hw->nvm.ops.write(hw, NVM_CHECKSUM_REG, 1, &checksum); if (ret_val) hw_dbg("NVM Write Error while updating checksum.\n"); diff --git a/drivers/net/ethernet/intel/igb/igb.h b/drivers/net/ethernet/intel/igb/igb.h index c3f4f7cd264e..0fff1df81b7b 100644 --- a/drivers/net/ethernet/intel/igb/igb.h +++ b/drivers/net/ethernet/intel/igb/igb.h @@ -217,7 +217,7 @@ static inline int igb_skb_pad(void) #define IGB_MASTER_SLAVE e1000_ms_hw_default #endif -#define IGB_MNG_VLAN_NONE -1 +#define IGB_MNG_VLAN_NONE 0xFFFF enum igb_tx_flags { /* cmd_type flags */ diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 7b8f32c5169a..f8a208c84f15 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -920,11 +920,11 @@ static int igb_set_ringparam(struct net_device *netdev, } if (adapter->num_tx_queues > adapter->num_rx_queues) - temp_ring = vmalloc(array_size(sizeof(struct igb_ring), - adapter->num_tx_queues)); + temp_ring = vmalloc_array(adapter->num_tx_queues, + sizeof(struct igb_ring)); else - temp_ring = vmalloc(array_size(sizeof(struct igb_ring), - adapter->num_rx_queues)); + temp_ring = vmalloc_array(adapter->num_rx_queues, + sizeof(struct igb_ring)); if (!temp_ring) { err = -ENOMEM; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 453deb6d14b3..85f9589cc568 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -1531,8 +1531,7 @@ static void igb_update_mng_vlan(struct igb_adapter *adapter) adapter->mng_vlan_id = IGB_MNG_VLAN_NONE; } - if ((old_vid != (u16)IGB_MNG_VLAN_NONE) && - (vid != old_vid) && + if (old_vid != IGB_MNG_VLAN_NONE && vid != old_vid && !test_bit(old_vid, adapter->active_vlans)) { /* remove VID from filter table */ igb_vfta_set(hw, vid, pf_id, false, true); diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c index 773895c663fd..9c08ebfad804 100644 --- a/drivers/net/ethernet/intel/igbvf/ethtool.c +++ b/drivers/net/ethernet/intel/igbvf/ethtool.c @@ -30,11 +30,12 @@ static const struct igbvf_stats igbvf_gstrings_stats[] = { { "rx_bytes", IGBVF_STAT(stats.gorc, stats.base_gorc) }, { "tx_bytes", IGBVF_STAT(stats.gotc, stats.base_gotc) }, { "multicast", IGBVF_STAT(stats.mprc, stats.base_mprc) }, - { "lbrx_bytes", IGBVF_STAT(stats.gorlbc, stats.base_gorlbc) }, { "lbrx_packets", IGBVF_STAT(stats.gprlbc, stats.base_gprlbc) }, + { "lbtx_packets", IGBVF_STAT(stats.gptlbc, stats.base_gptlbc) }, + { "lbrx_bytes", IGBVF_STAT(stats.gorlbc, stats.base_gorlbc) }, + { "lbtx_bytes", IGBVF_STAT(stats.gotlbc, stats.base_gotlbc) }, { "tx_restart_queue", IGBVF_STAT(restart_queue, zero_base) }, { "tx_timeout_count", IGBVF_STAT(tx_timeout_count, zero_base) }, - { "rx_long_byte_count", IGBVF_STAT(stats.gorc, stats.base_gorc) }, { "rx_csum_offload_good", IGBVF_STAT(hw_csum_good, zero_base) }, { "rx_csum_offload_errors", IGBVF_STAT(hw_csum_err, zero_base) }, { "rx_header_split", IGBVF_STAT(rx_hdr_split, zero_base) }, diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index ecb35b693ce5..f3e7218ba6f3 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -627,11 +627,11 @@ igc_ethtool_set_ringparam(struct net_device *netdev, } if (adapter->num_tx_queues > adapter->num_rx_queues) - temp_ring = vmalloc(array_size(sizeof(struct igc_ring), - adapter->num_tx_queues)); + temp_ring = vmalloc_array(adapter->num_tx_queues, + sizeof(struct igc_ring)); else - temp_ring = vmalloc(array_size(sizeof(struct igc_ring), - adapter->num_rx_queues)); + temp_ring = vmalloc_array(adapter->num_rx_queues, + sizeof(struct igc_ring)); if (!temp_ring) { err = -ENOMEM; diff --git a/drivers/net/ethernet/intel/igc/igc_i225.c b/drivers/net/ethernet/intel/igc/igc_i225.c index 0dd61719f1ed..5226d10cc95b 100644 --- a/drivers/net/ethernet/intel/igc/igc_i225.c +++ b/drivers/net/ethernet/intel/igc/igc_i225.c @@ -435,7 +435,7 @@ static s32 igc_update_nvm_checksum_i225(struct igc_hw *hw) } checksum += nvm_data; } - checksum = (u16)NVM_SUM - checksum; + checksum = NVM_SUM - checksum; ret_val = igc_write_nvm_srwr(hw, NVM_CHECKSUM_REG, 1, &checksum); if (ret_val) { diff --git a/drivers/net/ethernet/intel/igc/igc_nvm.c b/drivers/net/ethernet/intel/igc/igc_nvm.c index efd121c03967..a47b8d39238c 100644 --- a/drivers/net/ethernet/intel/igc/igc_nvm.c +++ b/drivers/net/ethernet/intel/igc/igc_nvm.c @@ -123,7 +123,7 @@ s32 igc_validate_nvm_checksum(struct igc_hw *hw) checksum += nvm_data; } - if (checksum != (u16)NVM_SUM) { + if (checksum != NVM_SUM) { hw_dbg("NVM Checksum Invalid\n"); ret_val = -IGC_ERR_NVM; goto out; @@ -155,7 +155,7 @@ s32 igc_update_nvm_checksum(struct igc_hw *hw) } checksum += nvm_data; } - checksum = (u16)NVM_SUM - checksum; + checksum = NVM_SUM - checksum; ret_val = hw->nvm.ops.write(hw, NVM_CHECKSUM_REG, 1, &checksum); if (ret_val) hw_dbg("NVM Write Error while updating checksum.\n"); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c index 4ff19426ab74..3ea6765f9c5d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_common.c @@ -1739,9 +1739,9 @@ int ixgbe_calc_eeprom_checksum_generic(struct ixgbe_hw *hw) } } - checksum = (u16)IXGBE_EEPROM_SUM - checksum; + checksum = IXGBE_EEPROM_SUM - checksum; - return (int)checksum; + return checksum; } /** diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c index bfeef5b0b99d..c2f8189a0738 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.c @@ -774,7 +774,7 @@ static void ixgbe_parse_vf_func_caps(struct ixgbe_hw *hw, * from parsing capabilities and use this to calculate the number of resources * per PF based on the max value passed in. * - * Return: the number of resources per PF or 0, if no PH are available. + * Return: the number of resources per PF or 0, if no PFs are available. */ static u32 ixgbe_get_num_per_func(struct ixgbe_hw *hw, u32 max) { @@ -1953,6 +1953,16 @@ int ixgbe_identify_phy_e610(struct ixgbe_hw *hw) phy_type_low & IXGBE_PHY_TYPE_LOW_1G_SGMII || phy_type_high & IXGBE_PHY_TYPE_HIGH_1G_USXGMII) hw->phy.speeds_supported |= IXGBE_LINK_SPEED_1GB_FULL; + if (phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_T || + phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_X || + phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_KX || + phy_type_high & IXGBE_PHY_TYPE_HIGH_2500M_SGMII || + phy_type_high & IXGBE_PHY_TYPE_HIGH_2500M_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_2_5GB_FULL; + if (phy_type_low & IXGBE_PHY_TYPE_LOW_5GBASE_T || + phy_type_low & IXGBE_PHY_TYPE_LOW_5GBASE_KR || + phy_type_high & IXGBE_PHY_TYPE_HIGH_5G_USXGMII) + hw->phy.speeds_supported |= IXGBE_LINK_SPEED_5GB_FULL; if (phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_T || phy_type_low & IXGBE_PHY_TYPE_LOW_10G_SFI_DA || phy_type_low & IXGBE_PHY_TYPE_LOW_10GBASE_SR || @@ -1963,31 +1973,10 @@ int ixgbe_identify_phy_e610(struct ixgbe_hw *hw) phy_type_high & IXGBE_PHY_TYPE_HIGH_10G_USXGMII) hw->phy.speeds_supported |= IXGBE_LINK_SPEED_10GB_FULL; - /* 2.5 and 5 Gbps link speeds must be excluded from the - * auto-negotiation set used during driver initialization due to - * compatibility issues with certain switches. Those issues do not - * exist in case of E610 2.5G SKU device (0x57b1). - */ - if (!hw->phy.autoneg_advertised && - hw->device_id != IXGBE_DEV_ID_E610_2_5G_T) - hw->phy.autoneg_advertised = hw->phy.speeds_supported; - - if (phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_T || - phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_X || - phy_type_low & IXGBE_PHY_TYPE_LOW_2500BASE_KX || - phy_type_high & IXGBE_PHY_TYPE_HIGH_2500M_SGMII || - phy_type_high & IXGBE_PHY_TYPE_HIGH_2500M_USXGMII) - hw->phy.speeds_supported |= IXGBE_LINK_SPEED_2_5GB_FULL; - - if (!hw->phy.autoneg_advertised && - hw->device_id == IXGBE_DEV_ID_E610_2_5G_T) + /* Initialize autoneg speeds */ + if (!hw->phy.autoneg_advertised) hw->phy.autoneg_advertised = hw->phy.speeds_supported; - if (phy_type_low & IXGBE_PHY_TYPE_LOW_5GBASE_T || - phy_type_low & IXGBE_PHY_TYPE_LOW_5GBASE_KR || - phy_type_high & IXGBE_PHY_TYPE_HIGH_5G_USXGMII) - hw->phy.speeds_supported |= IXGBE_LINK_SPEED_5GB_FULL; - /* Set PHY ID */ memcpy(&hw->phy.id, pcaps.phy_id_oui, sizeof(u32)); @@ -3008,50 +2997,71 @@ static int ixgbe_get_nvm_srev(struct ixgbe_hw *hw, * Searches through the Option ROM flash contents to locate the CIVD data for * the image. * - * Return: the exit code of the operation. + * Return: -ENOMEM when cannot allocate memory, -EDOM for checksum violation, + * -ENODATA when cannot find proper data, -EIO for faulty read or + * 0 on success. + * + * On success @civd stores collected data. */ static int ixgbe_get_orom_civd_data(struct ixgbe_hw *hw, enum ixgbe_bank_select bank, struct ixgbe_orom_civd_info *civd) { - struct ixgbe_orom_civd_info tmp; + u32 orom_size = hw->flash.banks.orom_size; + u8 *orom_data; u32 offset; int err; + orom_data = kzalloc(orom_size, GFP_KERNEL); + if (!orom_data) + return -ENOMEM; + + err = ixgbe_read_flash_module(hw, bank, + IXGBE_E610_SR_1ST_OROM_BANK_PTR, 0, + orom_data, orom_size); + if (err) { + err = -EIO; + goto cleanup; + } + /* The CIVD section is located in the Option ROM aligned to 512 bytes. * The first 4 bytes must contain the ASCII characters "$CIV". * A simple modulo 256 sum of all of the bytes of the structure must * equal 0. */ - for (offset = 0; (offset + SZ_512) <= hw->flash.banks.orom_size; - offset += SZ_512) { + for (offset = 0; offset + SZ_512 <= orom_size; offset += SZ_512) { + struct ixgbe_orom_civd_info *tmp; u8 sum = 0; u32 i; - err = ixgbe_read_flash_module(hw, bank, - IXGBE_E610_SR_1ST_OROM_BANK_PTR, - offset, - (u8 *)&tmp, sizeof(tmp)); - if (err) - return err; + BUILD_BUG_ON(sizeof(*tmp) > SZ_512); + + tmp = (struct ixgbe_orom_civd_info *)&orom_data[offset]; /* Skip forward until we find a matching signature */ - if (memcmp(IXGBE_OROM_CIV_SIGNATURE, tmp.signature, - sizeof(tmp.signature))) + if (memcmp(IXGBE_OROM_CIV_SIGNATURE, tmp->signature, + sizeof(tmp->signature))) continue; /* Verify that the simple checksum is zero */ - for (i = 0; i < sizeof(tmp); i++) - sum += ((u8 *)&tmp)[i]; + for (i = 0; i < sizeof(*tmp); i++) + sum += ((u8 *)tmp)[i]; + + if (sum) { + err = -EDOM; + goto cleanup; + } - if (sum) - return -EDOM; + *civd = *tmp; + err = 0; - *civd = tmp; - return 0; + goto cleanup; } - return -ENODATA; + err = -ENODATA; +cleanup: + kfree(orom_data); + return err; } /** @@ -3911,6 +3921,38 @@ static int ixgbe_read_pba_string_e610(struct ixgbe_hw *hw, u8 *pba_num, return err; } +static int __fwlog_send_cmd(void *priv, struct libie_aq_desc *desc, void *buf, + u16 size) +{ + struct ixgbe_hw *hw = priv; + + return ixgbe_aci_send_cmd(hw, desc, buf, size); +} + +int ixgbe_fwlog_init(struct ixgbe_hw *hw) +{ + struct ixgbe_adapter *adapter = hw->back; + struct libie_fwlog_api api = { + .pdev = adapter->pdev, + .send_cmd = __fwlog_send_cmd, + .debugfs_root = adapter->ixgbe_dbg_adapter, + .priv = hw, + }; + + if (hw->mac.type != ixgbe_mac_e610) + return -EOPNOTSUPP; + + return libie_fwlog_init(&hw->fwlog, &api); +} + +void ixgbe_fwlog_deinit(struct ixgbe_hw *hw) +{ + if (hw->mac.type != ixgbe_mac_e610) + return; + + libie_fwlog_deinit(&hw->fwlog); +} + static const struct ixgbe_mac_operations mac_ops_e610 = { .init_hw = ixgbe_init_hw_generic, .start_hw = ixgbe_start_hw_e610, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h index 782c489b0fa7..11916b979d28 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_e610.h @@ -96,5 +96,7 @@ int ixgbe_aci_update_nvm(struct ixgbe_hw *hw, u16 module_typeid, bool last_command, u8 command_flags); int ixgbe_nvm_write_activate(struct ixgbe_hw *hw, u16 cmd_flags, u8 *response_flags); +int ixgbe_fwlog_init(struct ixgbe_hw *hw); +void ixgbe_fwlog_deinit(struct ixgbe_hw *hw); #endif /* _IXGBE_E610_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 1a2f1bdb91aa..2d660e9edb80 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1278,7 +1278,7 @@ static int ixgbe_set_ringparam(struct net_device *netdev, /* allocate temporary buffer to store rings in */ i = max_t(int, adapter->num_tx_queues + adapter->num_xdp_queues, adapter->num_rx_queues); - temp_ring = vmalloc(array_size(i, sizeof(struct ixgbe_ring))); + temp_ring = vmalloc_array(i, sizeof(struct ixgbe_ring)); if (!temp_ring) { err = -ENOMEM; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 6218bdb7f941..90d4e57b1c93 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -172,6 +172,7 @@ static int debug = -1; module_param(debug, int, 0); MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); +MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); MODULE_LICENSE("GPL v2"); @@ -3355,6 +3356,10 @@ static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter) e_crit(drv, "%s\n", ixgbe_overheat_msg); ixgbe_down(adapter); break; + case libie_aqc_opc_fw_logs_event: + libie_get_fwlog_data(&hw->fwlog, event.msg_buf, + le16_to_cpu(event.desc.datalen)); + break; default: e_warn(hw, "unknown FW async event captured\n"); break; @@ -11999,6 +12004,10 @@ skip_sriov: ixgbe_devlink_init_regions(adapter); devl_register(adapter->devlink); devl_unlock(adapter->devlink); + + if (ixgbe_fwlog_init(hw)) + e_dev_info("Firmware logging not supported\n"); + return 0; err_netdev: @@ -12056,6 +12065,7 @@ static void ixgbe_remove(struct pci_dev *pdev) devl_lock(adapter->devlink); devl_unregister(adapter->devlink); ixgbe_devlink_destroy_regions(adapter); + ixgbe_fwlog_deinit(&adapter->hw); ixgbe_dbg_adapter_exit(adapter); set_bit(__IXGBE_REMOVING, &adapter->state); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index 36577091cd9e..b1bfeb21537a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -7,6 +7,7 @@ #include <linux/types.h> #include <linux/mdio.h> #include <linux/netdevice.h> +#include <linux/net/intel/libie/fwlog.h> #include "ixgbe_type_e610.h" /* Device IDs */ @@ -3752,6 +3753,7 @@ struct ixgbe_hw { struct ixgbe_flash_info flash; struct ixgbe_hw_dev_caps dev_caps; struct ixgbe_hw_func_caps func_caps; + struct libie_fwlog fwlog; }; struct ixgbe_info { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c index c2353aed0120..e67e2feb045b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x540.c @@ -373,9 +373,9 @@ static int ixgbe_calc_eeprom_checksum_X540(struct ixgbe_hw *hw) } } - checksum = (u16)IXGBE_EEPROM_SUM - checksum; + checksum = IXGBE_EEPROM_SUM - checksum; - return (int)checksum; + return checksum; } /** diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c index bfa647086c70..76d2fa3ef518 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_x550.c @@ -1060,9 +1060,9 @@ static int ixgbe_calc_checksum_X550(struct ixgbe_hw *hw, u16 *buffer, return status; } - checksum = (u16)IXGBE_EEPROM_SUM - checksum; + checksum = IXGBE_EEPROM_SUM - checksum; - return (int)checksum; + return checksum; } /** ixgbe_calc_eeprom_checksum_X550 - Calculates and returns the checksum @@ -1163,7 +1163,7 @@ static int ixgbe_validate_eeprom_checksum_X550(struct ixgbe_hw *hw, return status; } -/** ixgbe_write_ee_hostif_X550 - Write EEPROM word using hostif +/** ixgbe_write_ee_hostif_data_X550 - Write EEPROM word using hostif * @hw: pointer to hardware structure * @offset: offset of word in the EEPROM to write * @data: word write to the EEPROM @@ -2318,7 +2318,7 @@ static int ixgbe_get_link_capabilities_X550em(struct ixgbe_hw *hw, } /** - * ixgbe_get_lasi_ext_t_x550em - Determime external Base T PHY interrupt cause + * ixgbe_get_lasi_ext_t_x550em - Determine external Base T PHY interrupt cause * @hw: pointer to hardware structure * @lsc: pointer to boolean flag which indicates whether external Base T * PHY interrupt is lsc @@ -2628,7 +2628,7 @@ static int ixgbe_ext_phy_t_x550em_get_link(struct ixgbe_hw *hw, bool *link_up) } /** ixgbe_setup_internal_phy_t_x550em - Configure KR PHY to X557 link - * @hw: point to hardware structure + * @hw: pointer to hardware structure * * Configures the link between the integrated KR PHY and the external X557 PHY * The driver will call this function when it gets a link status change @@ -2745,7 +2745,7 @@ static int ixgbe_led_off_t_x550em(struct ixgbe_hw *hw, u32 led_idx) if (led_idx >= IXGBE_X557_MAX_LED_INDEX) return -EINVAL; - /* To turn on the LED, set mode to ON. */ + /* To turn off the LED, set mode to OFF. */ hw->phy.ops.read_reg(hw, IXGBE_X557_LED_PROVISIONING + led_idx, MDIO_MMD_VEND1, &phy_data); phy_data &= ~IXGBE_X557_LED_MANUAL_SET_MASK; @@ -2812,7 +2812,7 @@ int ixgbe_set_fw_drv_ver_x550(struct ixgbe_hw *hw, u8 maj, u8 min, return ret_val; } -/** ixgbe_get_lcd_x550em - Determine lowest common denominator +/** ixgbe_get_lcd_t_x550em - Determine lowest common denominator * @hw: pointer to hardware structure * @lcd_speed: pointer to lowest common link speed * diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index 7ac53171b041..bebad564188e 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -276,9 +276,9 @@ static int ixgbevf_set_ringparam(struct net_device *netdev, } if (new_tx_count != adapter->tx_ring_count) { - tx_ring = vmalloc(array_size(sizeof(*tx_ring), - adapter->num_tx_queues + - adapter->num_xdp_queues)); + tx_ring = vmalloc_array(adapter->num_tx_queues + + adapter->num_xdp_queues, + sizeof(*tx_ring)); if (!tx_ring) { err = -ENOMEM; goto clear_reset; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 535d0f71f521..28e25641b167 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -4323,7 +4323,7 @@ static int ixgbevf_resume(struct device *dev_d) struct pci_dev *pdev = to_pci_dev(dev_d); struct net_device *netdev = pci_get_drvdata(pdev); struct ixgbevf_adapter *adapter = netdev_priv(netdev); - u32 err; + int err; adapter->hw.hw_addr = adapter->io_addr; smp_mb__before_atomic(); diff --git a/drivers/net/ethernet/intel/libie/Kconfig b/drivers/net/ethernet/intel/libie/Kconfig index e6072758e3d8..70831c7e336e 100644 --- a/drivers/net/ethernet/intel/libie/Kconfig +++ b/drivers/net/ethernet/intel/libie/Kconfig @@ -14,3 +14,12 @@ config LIBIE_ADMINQ help Helper functions used by Intel Ethernet drivers for administration queue command interface (aka adminq). + +config LIBIE_FWLOG + tristate + select LIBIE_ADMINQ + help + Library to support firmware logging on device that have support + for it. Firmware logging is using admin queue interface to communicate + with the device. Debugfs is a user interface used to config logging + and dump all collected logs. diff --git a/drivers/net/ethernet/intel/libie/Makefile b/drivers/net/ethernet/intel/libie/Makefile index e98f00b865d3..db57fc6780ea 100644 --- a/drivers/net/ethernet/intel/libie/Makefile +++ b/drivers/net/ethernet/intel/libie/Makefile @@ -8,3 +8,7 @@ libie-y := rx.o obj-$(CONFIG_LIBIE_ADMINQ) += libie_adminq.o libie_adminq-y := adminq.o + +obj-$(CONFIG_LIBIE_FWLOG) += libie_fwlog.o + +libie_fwlog-y := fwlog.o diff --git a/drivers/net/ethernet/intel/libie/fwlog.c b/drivers/net/ethernet/intel/libie/fwlog.c new file mode 100644 index 000000000000..f39cc11cb7c5 --- /dev/null +++ b/drivers/net/ethernet/intel/libie/fwlog.c @@ -0,0 +1,1115 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2022, Intel Corporation. */ + +#include <linux/debugfs.h> +#include <linux/export.h> +#include <linux/fs.h> +#include <linux/net/intel/libie/fwlog.h> +#include <linux/pci.h> +#include <linux/random.h> +#include <linux/vmalloc.h> + +#define DEFAULT_SYMBOL_NAMESPACE "LIBIE_FWLOG" + +/* create a define that has an extra module that doesn't really exist. this + * is so we can add a module 'all' to easily enable/disable all the modules + */ +#define LIBIE_NR_FW_LOG_MODULES (LIBIE_AQC_FW_LOG_ID_MAX + 1) + +/* the ordering in this array is important. it matches the ordering of the + * values in the FW so the index is the same value as in + * libie_aqc_fw_logging_mod + */ +static const char * const libie_fwlog_module_string[] = { + "general", + "ctrl", + "link", + "link_topo", + "dnl", + "i2c", + "sdp", + "mdio", + "adminq", + "hdma", + "lldp", + "dcbx", + "dcb", + "xlr", + "nvm", + "auth", + "vpd", + "iosf", + "parser", + "sw", + "scheduler", + "txq", + "rsvd", + "post", + "watchdog", + "task_dispatch", + "mng", + "synce", + "health", + "tsdrv", + "pfreg", + "mdlver", + "all", +}; + +/* the ordering in this array is important. it matches the ordering of the + * values in the FW so the index is the same value as in libie_fwlog_level + */ +static const char * const libie_fwlog_level_string[] = { + "none", + "error", + "warning", + "normal", + "verbose", +}; + +static const char * const libie_fwlog_log_size[] = { + "128K", + "256K", + "512K", + "1M", + "2M", +}; + +static bool libie_fwlog_ring_empty(struct libie_fwlog_ring *rings) +{ + return rings->head == rings->tail; +} + +static void libie_fwlog_ring_increment(u16 *item, u16 size) +{ + *item = (*item + 1) & (size - 1); +} + +static int libie_fwlog_alloc_ring_buffs(struct libie_fwlog_ring *rings) +{ + int i, nr_bytes; + u8 *mem; + + nr_bytes = rings->size * LIBIE_AQ_MAX_BUF_LEN; + mem = vzalloc(nr_bytes); + if (!mem) + return -ENOMEM; + + for (i = 0; i < rings->size; i++) { + struct libie_fwlog_data *ring = &rings->rings[i]; + + ring->data_size = LIBIE_AQ_MAX_BUF_LEN; + ring->data = mem; + mem += LIBIE_AQ_MAX_BUF_LEN; + } + + return 0; +} + +static void libie_fwlog_free_ring_buffs(struct libie_fwlog_ring *rings) +{ + int i; + + for (i = 0; i < rings->size; i++) { + struct libie_fwlog_data *ring = &rings->rings[i]; + + /* the first ring is the base memory for the whole range so + * free it + */ + if (!i) + vfree(ring->data); + + ring->data = NULL; + ring->data_size = 0; + } +} + +#define LIBIE_FWLOG_INDEX_TO_BYTES(n) ((128 * 1024) << (n)) +/** + * libie_fwlog_realloc_rings - reallocate the FW log rings + * @fwlog: pointer to the fwlog structure + * @index: the new index to use to allocate memory for the log data + * + */ +static void libie_fwlog_realloc_rings(struct libie_fwlog *fwlog, int index) +{ + struct libie_fwlog_ring ring; + int status, ring_size; + + /* convert the number of bytes into a number of 4K buffers. externally + * the driver presents the interface to the FW log data as a number of + * bytes because that's easy for users to understand. internally the + * driver uses a ring of buffers because the driver doesn't know where + * the beginning and end of any line of log data is so the driver has + * to overwrite data as complete blocks. when the data is returned to + * the user the driver knows that the data is correct and the FW log + * can be correctly parsed by the tools + */ + ring_size = LIBIE_FWLOG_INDEX_TO_BYTES(index) / LIBIE_AQ_MAX_BUF_LEN; + if (ring_size == fwlog->ring.size) + return; + + /* allocate space for the new rings and buffers then release the + * old rings and buffers. that way if we don't have enough + * memory then we at least have what we had before + */ + ring.rings = kcalloc(ring_size, sizeof(*ring.rings), GFP_KERNEL); + if (!ring.rings) + return; + + ring.size = ring_size; + + status = libie_fwlog_alloc_ring_buffs(&ring); + if (status) { + dev_warn(&fwlog->pdev->dev, "Unable to allocate memory for FW log ring data buffers\n"); + libie_fwlog_free_ring_buffs(&ring); + kfree(ring.rings); + return; + } + + libie_fwlog_free_ring_buffs(&fwlog->ring); + kfree(fwlog->ring.rings); + + fwlog->ring.rings = ring.rings; + fwlog->ring.size = ring.size; + fwlog->ring.index = index; + fwlog->ring.head = 0; + fwlog->ring.tail = 0; +} + +/** + * libie_fwlog_supported - Cached for whether FW supports FW logging or not + * @fwlog: pointer to the fwlog structure + * + * This will always return false if called before libie_init_hw(), so it must be + * called after libie_init_hw(). + */ +static bool libie_fwlog_supported(struct libie_fwlog *fwlog) +{ + return fwlog->supported; +} + +/** + * libie_aq_fwlog_set - Set FW logging configuration AQ command (0xFF30) + * @fwlog: pointer to the fwlog structure + * @entries: entries to configure + * @num_entries: number of @entries + * @options: options from libie_fwlog_cfg->options structure + * @log_resolution: logging resolution + */ +static int +libie_aq_fwlog_set(struct libie_fwlog *fwlog, + struct libie_fwlog_module_entry *entries, u16 num_entries, + u16 options, u16 log_resolution) +{ + struct libie_aqc_fw_log_cfg_resp *fw_modules; + struct libie_aq_desc desc = {0}; + struct libie_aqc_fw_log *cmd; + int status; + int i; + + fw_modules = kcalloc(num_entries, sizeof(*fw_modules), GFP_KERNEL); + if (!fw_modules) + return -ENOMEM; + + for (i = 0; i < num_entries; i++) { + fw_modules[i].module_identifier = + cpu_to_le16(entries[i].module_id); + fw_modules[i].log_level = entries[i].log_level; + } + + desc.opcode = cpu_to_le16(libie_aqc_opc_fw_logs_config); + desc.flags = cpu_to_le16(LIBIE_AQ_FLAG_SI) | + cpu_to_le16(LIBIE_AQ_FLAG_RD); + + cmd = libie_aq_raw(&desc); + + cmd->cmd_flags = LIBIE_AQC_FW_LOG_CONF_SET_VALID; + cmd->ops.cfg.log_resolution = cpu_to_le16(log_resolution); + cmd->ops.cfg.mdl_cnt = cpu_to_le16(num_entries); + + if (options & LIBIE_FWLOG_OPTION_ARQ_ENA) + cmd->cmd_flags |= LIBIE_AQC_FW_LOG_CONF_AQ_EN; + if (options & LIBIE_FWLOG_OPTION_UART_ENA) + cmd->cmd_flags |= LIBIE_AQC_FW_LOG_CONF_UART_EN; + + status = fwlog->send_cmd(fwlog->priv, &desc, fw_modules, + sizeof(*fw_modules) * num_entries); + + kfree(fw_modules); + + return status; +} + +/** + * libie_fwlog_set - Set the firmware logging settings + * @fwlog: pointer to the fwlog structure + * @cfg: config used to set firmware logging + * + * This function should be called whenever the driver needs to set the firmware + * logging configuration. It can be called on initialization, reset, or during + * runtime. + * + * If the PF wishes to receive FW logging then it must register via + * libie_fwlog_register. Note, that libie_fwlog_register does not need to be called + * for init. + */ +static int libie_fwlog_set(struct libie_fwlog *fwlog, + struct libie_fwlog_cfg *cfg) +{ + if (!libie_fwlog_supported(fwlog)) + return -EOPNOTSUPP; + + return libie_aq_fwlog_set(fwlog, cfg->module_entries, + LIBIE_AQC_FW_LOG_ID_MAX, cfg->options, + cfg->log_resolution); +} + +/** + * libie_aq_fwlog_register - Register PF for firmware logging events (0xFF31) + * @fwlog: pointer to the fwlog structure + * @reg: true to register and false to unregister + */ +static int libie_aq_fwlog_register(struct libie_fwlog *fwlog, bool reg) +{ + struct libie_aq_desc desc = {0}; + struct libie_aqc_fw_log *cmd; + + desc.opcode = cpu_to_le16(libie_aqc_opc_fw_logs_register); + desc.flags = cpu_to_le16(LIBIE_AQ_FLAG_SI); + cmd = libie_aq_raw(&desc); + + if (reg) + cmd->cmd_flags = LIBIE_AQC_FW_LOG_AQ_REGISTER; + + return fwlog->send_cmd(fwlog->priv, &desc, NULL, 0); +} + +/** + * libie_fwlog_register - Register the PF for firmware logging + * @fwlog: pointer to the fwlog structure + * + * After this call the PF will start to receive firmware logging based on the + * configuration set in libie_fwlog_set. + */ +static int libie_fwlog_register(struct libie_fwlog *fwlog) +{ + int status; + + if (!libie_fwlog_supported(fwlog)) + return -EOPNOTSUPP; + + status = libie_aq_fwlog_register(fwlog, true); + if (status) + dev_dbg(&fwlog->pdev->dev, "Failed to register for firmware logging events over ARQ\n"); + else + fwlog->cfg.options |= LIBIE_FWLOG_OPTION_IS_REGISTERED; + + return status; +} + +/** + * libie_fwlog_unregister - Unregister the PF from firmware logging + * @fwlog: pointer to the fwlog structure + */ +static int libie_fwlog_unregister(struct libie_fwlog *fwlog) +{ + int status; + + if (!libie_fwlog_supported(fwlog)) + return -EOPNOTSUPP; + + status = libie_aq_fwlog_register(fwlog, false); + if (status) + dev_dbg(&fwlog->pdev->dev, "Failed to unregister from firmware logging events over ARQ\n"); + else + fwlog->cfg.options &= ~LIBIE_FWLOG_OPTION_IS_REGISTERED; + + return status; +} + +/** + * libie_fwlog_print_module_cfg - print current FW logging module configuration + * @cfg: pointer to the fwlog cfg structure + * @module: module to print + * @s: the seq file to put data into + */ +static void +libie_fwlog_print_module_cfg(struct libie_fwlog_cfg *cfg, int module, + struct seq_file *s) +{ + struct libie_fwlog_module_entry *entry; + + if (module != LIBIE_AQC_FW_LOG_ID_MAX) { + entry = &cfg->module_entries[module]; + + seq_printf(s, "\tModule: %s, Log Level: %s\n", + libie_fwlog_module_string[entry->module_id], + libie_fwlog_level_string[entry->log_level]); + } else { + int i; + + for (i = 0; i < LIBIE_AQC_FW_LOG_ID_MAX; i++) { + entry = &cfg->module_entries[i]; + + seq_printf(s, "\tModule: %s, Log Level: %s\n", + libie_fwlog_module_string[entry->module_id], + libie_fwlog_level_string[entry->log_level]); + } + } +} + +static int libie_find_module_by_dentry(struct dentry **modules, struct dentry *d) +{ + int i, module; + + module = -1; + /* find the module based on the dentry */ + for (i = 0; i < LIBIE_NR_FW_LOG_MODULES; i++) { + if (d == modules[i]) { + module = i; + break; + } + } + + return module; +} + +/** + * libie_debugfs_module_show - read from 'module' file + * @s: the opened file + * @v: pointer to the offset + */ +static int libie_debugfs_module_show(struct seq_file *s, void *v) +{ + struct libie_fwlog *fwlog = s->private; + const struct file *filp = s->file; + struct dentry *dentry; + int module; + + dentry = file_dentry(filp); + + module = libie_find_module_by_dentry(fwlog->debugfs_modules, dentry); + if (module < 0) { + dev_info(&fwlog->pdev->dev, "unknown module\n"); + return -EINVAL; + } + + libie_fwlog_print_module_cfg(&fwlog->cfg, module, s); + + return 0; +} + +static int libie_debugfs_module_open(struct inode *inode, struct file *filp) +{ + return single_open(filp, libie_debugfs_module_show, inode->i_private); +} + +/** + * libie_debugfs_module_write - write into 'module' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +libie_debugfs_module_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct libie_fwlog *fwlog = file_inode(filp)->i_private; + struct dentry *dentry = file_dentry(filp); + struct device *dev = &fwlog->pdev->dev; + char user_val[16], *cmd_buf; + int module, log_level, cnt; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 8) + return -EINVAL; + + cmd_buf = memdup_user_nul(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + module = libie_find_module_by_dentry(fwlog->debugfs_modules, dentry); + if (module < 0) { + dev_info(dev, "unknown module\n"); + return -EINVAL; + } + + cnt = sscanf(cmd_buf, "%s", user_val); + if (cnt != 1) + return -EINVAL; + + log_level = sysfs_match_string(libie_fwlog_level_string, user_val); + if (log_level < 0) { + dev_info(dev, "unknown log level '%s'\n", user_val); + return -EINVAL; + } + + if (module != LIBIE_AQC_FW_LOG_ID_MAX) { + fwlog->cfg.module_entries[module].log_level = log_level; + } else { + /* the module 'all' is a shortcut so that we can set + * all of the modules to the same level quickly + */ + int i; + + for (i = 0; i < LIBIE_AQC_FW_LOG_ID_MAX; i++) + fwlog->cfg.module_entries[i].log_level = log_level; + } + + return count; +} + +static const struct file_operations libie_debugfs_module_fops = { + .owner = THIS_MODULE, + .open = libie_debugfs_module_open, + .read = seq_read, + .release = single_release, + .write = libie_debugfs_module_write, +}; + +/** + * libie_debugfs_nr_messages_read - read from 'nr_messages' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t libie_debugfs_nr_messages_read(struct file *filp, + char __user *buffer, size_t count, + loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + char buff[32] = {}; + + snprintf(buff, sizeof(buff), "%d\n", + fwlog->cfg.log_resolution); + + return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); +} + +/** + * libie_debugfs_nr_messages_write - write into 'nr_messages' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +libie_debugfs_nr_messages_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + struct device *dev = &fwlog->pdev->dev; + char user_val[8], *cmd_buf; + s16 nr_messages; + ssize_t ret; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 4) + return -EINVAL; + + cmd_buf = memdup_user_nul(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + ret = sscanf(cmd_buf, "%s", user_val); + if (ret != 1) + return -EINVAL; + + ret = kstrtos16(user_val, 0, &nr_messages); + if (ret) + return ret; + + if (nr_messages < LIBIE_AQC_FW_LOG_MIN_RESOLUTION || + nr_messages > LIBIE_AQC_FW_LOG_MAX_RESOLUTION) { + dev_err(dev, "Invalid FW log number of messages %d, value must be between %d - %d\n", + nr_messages, LIBIE_AQC_FW_LOG_MIN_RESOLUTION, + LIBIE_AQC_FW_LOG_MAX_RESOLUTION); + return -EINVAL; + } + + fwlog->cfg.log_resolution = nr_messages; + + return count; +} + +static const struct file_operations libie_debugfs_nr_messages_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = libie_debugfs_nr_messages_read, + .write = libie_debugfs_nr_messages_write, +}; + +/** + * libie_debugfs_enable_read - read from 'enable' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t libie_debugfs_enable_read(struct file *filp, + char __user *buffer, size_t count, + loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + char buff[32] = {}; + + snprintf(buff, sizeof(buff), "%u\n", + (u16)(fwlog->cfg.options & + LIBIE_FWLOG_OPTION_IS_REGISTERED) >> 3); + + return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); +} + +/** + * libie_debugfs_enable_write - write into 'enable' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +libie_debugfs_enable_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + char user_val[8], *cmd_buf; + bool enable; + ssize_t ret; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 2) + return -EINVAL; + + cmd_buf = memdup_user_nul(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + ret = sscanf(cmd_buf, "%s", user_val); + if (ret != 1) + return -EINVAL; + + ret = kstrtobool(user_val, &enable); + if (ret) + goto enable_write_error; + + if (enable) + fwlog->cfg.options |= LIBIE_FWLOG_OPTION_ARQ_ENA; + else + fwlog->cfg.options &= ~LIBIE_FWLOG_OPTION_ARQ_ENA; + + ret = libie_fwlog_set(fwlog, &fwlog->cfg); + if (ret) + goto enable_write_error; + + if (enable) + ret = libie_fwlog_register(fwlog); + else + ret = libie_fwlog_unregister(fwlog); + + if (ret) + goto enable_write_error; + + /* if we get here, nothing went wrong; return count since we didn't + * really write anything + */ + ret = (ssize_t)count; + +enable_write_error: + /* This function always consumes all of the written input, or produces + * an error. Check and enforce this. Otherwise, the write operation + * won't complete properly. + */ + if (WARN_ON(ret != (ssize_t)count && ret >= 0)) + ret = -EIO; + + return ret; +} + +static const struct file_operations libie_debugfs_enable_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = libie_debugfs_enable_read, + .write = libie_debugfs_enable_write, +}; + +/** + * libie_debugfs_log_size_read - read from 'log_size' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t libie_debugfs_log_size_read(struct file *filp, + char __user *buffer, size_t count, + loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + char buff[32] = {}; + int index; + + index = fwlog->ring.index; + snprintf(buff, sizeof(buff), "%s\n", libie_fwlog_log_size[index]); + + return simple_read_from_buffer(buffer, count, ppos, buff, strlen(buff)); +} + +/** + * libie_debugfs_log_size_write - write into 'log_size' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +libie_debugfs_log_size_write(struct file *filp, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + struct device *dev = &fwlog->pdev->dev; + char user_val[8], *cmd_buf; + ssize_t ret; + int index; + + /* don't allow partial writes or invalid input */ + if (*ppos != 0 || count > 5) + return -EINVAL; + + cmd_buf = memdup_user_nul(buf, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + + ret = sscanf(cmd_buf, "%s", user_val); + if (ret != 1) + return -EINVAL; + + index = sysfs_match_string(libie_fwlog_log_size, user_val); + if (index < 0) { + dev_info(dev, "Invalid log size '%s'. The value must be one of 128K, 256K, 512K, 1M, 2M\n", + user_val); + ret = -EINVAL; + goto log_size_write_error; + } else if (fwlog->cfg.options & LIBIE_FWLOG_OPTION_IS_REGISTERED) { + dev_info(dev, "FW logging is currently running. Please disable FW logging to change log_size\n"); + ret = -EINVAL; + goto log_size_write_error; + } + + /* free all the buffers and the tracking info and resize */ + libie_fwlog_realloc_rings(fwlog, index); + + /* if we get here, nothing went wrong; return count since we didn't + * really write anything + */ + ret = (ssize_t)count; + +log_size_write_error: + /* This function always consumes all of the written input, or produces + * an error. Check and enforce this. Otherwise, the write operation + * won't complete properly. + */ + if (WARN_ON(ret != (ssize_t)count && ret >= 0)) + ret = -EIO; + + return ret; +} + +static const struct file_operations libie_debugfs_log_size_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = libie_debugfs_log_size_read, + .write = libie_debugfs_log_size_write, +}; + +/** + * libie_debugfs_data_read - read from 'data' file + * @filp: the opened file + * @buffer: where to write the data for the user to read + * @count: the size of the user's buffer + * @ppos: file position offset + */ +static ssize_t libie_debugfs_data_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + int data_copied = 0; + bool done = false; + + if (libie_fwlog_ring_empty(&fwlog->ring)) + return 0; + + while (!libie_fwlog_ring_empty(&fwlog->ring) && !done) { + struct libie_fwlog_data *log; + u16 cur_buf_len; + + log = &fwlog->ring.rings[fwlog->ring.head]; + cur_buf_len = log->data_size; + if (cur_buf_len >= count) { + done = true; + continue; + } + + if (copy_to_user(buffer, log->data, cur_buf_len)) { + /* if there is an error then bail and return whatever + * the driver has copied so far + */ + done = true; + continue; + } + + data_copied += cur_buf_len; + buffer += cur_buf_len; + count -= cur_buf_len; + *ppos += cur_buf_len; + libie_fwlog_ring_increment(&fwlog->ring.head, fwlog->ring.size); + } + + return data_copied; +} + +/** + * libie_debugfs_data_write - write into 'data' file + * @filp: the opened file + * @buf: where to find the user's data + * @count: the length of the user's data + * @ppos: file position offset + */ +static ssize_t +libie_debugfs_data_write(struct file *filp, const char __user *buf, size_t count, + loff_t *ppos) +{ + struct libie_fwlog *fwlog = filp->private_data; + struct device *dev = &fwlog->pdev->dev; + ssize_t ret; + + /* don't allow partial writes */ + if (*ppos != 0) + return 0; + + /* any value is allowed to clear the buffer so no need to even look at + * what the value is + */ + if (!(fwlog->cfg.options & LIBIE_FWLOG_OPTION_IS_REGISTERED)) { + fwlog->ring.head = 0; + fwlog->ring.tail = 0; + } else { + dev_info(dev, "Can't clear FW log data while FW log running\n"); + ret = -EINVAL; + goto nr_buffs_write_error; + } + + /* if we get here, nothing went wrong; return count since we didn't + * really write anything + */ + ret = (ssize_t)count; + +nr_buffs_write_error: + /* This function always consumes all of the written input, or produces + * an error. Check and enforce this. Otherwise, the write operation + * won't complete properly. + */ + if (WARN_ON(ret != (ssize_t)count && ret >= 0)) + ret = -EIO; + + return ret; +} + +static const struct file_operations libie_debugfs_data_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = libie_debugfs_data_read, + .write = libie_debugfs_data_write, +}; + +/** + * libie_debugfs_fwlog_init - setup the debugfs directory + * @fwlog: pointer to the fwlog structure + * @root: debugfs root entry on which fwlog director will be registered + */ +static void libie_debugfs_fwlog_init(struct libie_fwlog *fwlog, + struct dentry *root) +{ + struct dentry *fw_modules_dir; + struct dentry **fw_modules; + int i; + + /* allocate space for this first because if it fails then we don't + * need to unwind + */ + fw_modules = kcalloc(LIBIE_NR_FW_LOG_MODULES, sizeof(*fw_modules), + GFP_KERNEL); + if (!fw_modules) + return; + + fwlog->debugfs = debugfs_create_dir("fwlog", root); + if (IS_ERR(fwlog->debugfs)) + goto err_create_module_files; + + fw_modules_dir = debugfs_create_dir("modules", fwlog->debugfs); + if (IS_ERR(fw_modules_dir)) + goto err_create_module_files; + + for (i = 0; i < LIBIE_NR_FW_LOG_MODULES; i++) { + fw_modules[i] = debugfs_create_file(libie_fwlog_module_string[i], + 0600, fw_modules_dir, fwlog, + &libie_debugfs_module_fops); + if (IS_ERR(fw_modules[i])) + goto err_create_module_files; + } + + debugfs_create_file("nr_messages", 0600, fwlog->debugfs, fwlog, + &libie_debugfs_nr_messages_fops); + + fwlog->debugfs_modules = fw_modules; + + debugfs_create_file("enable", 0600, fwlog->debugfs, fwlog, + &libie_debugfs_enable_fops); + + debugfs_create_file("log_size", 0600, fwlog->debugfs, fwlog, + &libie_debugfs_log_size_fops); + + debugfs_create_file("data", 0600, fwlog->debugfs, fwlog, + &libie_debugfs_data_fops); + + return; + +err_create_module_files: + debugfs_remove_recursive(fwlog->debugfs); + kfree(fw_modules); +} + +static bool libie_fwlog_ring_full(struct libie_fwlog_ring *rings) +{ + u16 head, tail; + + head = rings->head; + tail = rings->tail; + + if (head < tail && (tail - head == (rings->size - 1))) + return true; + else if (head > tail && (tail == (head - 1))) + return true; + + return false; +} + +/** + * libie_aq_fwlog_get - Get the current firmware logging configuration (0xFF32) + * @fwlog: pointer to the fwlog structure + * @cfg: firmware logging configuration to populate + */ +static int libie_aq_fwlog_get(struct libie_fwlog *fwlog, + struct libie_fwlog_cfg *cfg) +{ + struct libie_aqc_fw_log_cfg_resp *fw_modules; + struct libie_aq_desc desc = {0}; + struct libie_aqc_fw_log *cmd; + u16 module_id_cnt; + int status; + void *buf; + int i; + + memset(cfg, 0, sizeof(*cfg)); + + buf = kzalloc(LIBIE_AQ_MAX_BUF_LEN, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + desc.opcode = cpu_to_le16(libie_aqc_opc_fw_logs_query); + desc.flags = cpu_to_le16(LIBIE_AQ_FLAG_SI); + cmd = libie_aq_raw(&desc); + + cmd->cmd_flags = LIBIE_AQC_FW_LOG_AQ_QUERY; + + status = fwlog->send_cmd(fwlog->priv, &desc, buf, LIBIE_AQ_MAX_BUF_LEN); + if (status) { + dev_dbg(&fwlog->pdev->dev, "Failed to get FW log configuration\n"); + goto status_out; + } + + module_id_cnt = le16_to_cpu(cmd->ops.cfg.mdl_cnt); + if (module_id_cnt < LIBIE_AQC_FW_LOG_ID_MAX) { + dev_dbg(&fwlog->pdev->dev, "FW returned less than the expected number of FW log module IDs\n"); + } else if (module_id_cnt > LIBIE_AQC_FW_LOG_ID_MAX) { + dev_dbg(&fwlog->pdev->dev, "FW returned more than expected number of FW log module IDs, setting module_id_cnt to software expected max %u\n", + LIBIE_AQC_FW_LOG_ID_MAX); + module_id_cnt = LIBIE_AQC_FW_LOG_ID_MAX; + } + + cfg->log_resolution = le16_to_cpu(cmd->ops.cfg.log_resolution); + if (cmd->cmd_flags & LIBIE_AQC_FW_LOG_CONF_AQ_EN) + cfg->options |= LIBIE_FWLOG_OPTION_ARQ_ENA; + if (cmd->cmd_flags & LIBIE_AQC_FW_LOG_CONF_UART_EN) + cfg->options |= LIBIE_FWLOG_OPTION_UART_ENA; + if (cmd->cmd_flags & LIBIE_AQC_FW_LOG_QUERY_REGISTERED) + cfg->options |= LIBIE_FWLOG_OPTION_IS_REGISTERED; + + fw_modules = (struct libie_aqc_fw_log_cfg_resp *)buf; + + for (i = 0; i < module_id_cnt; i++) { + struct libie_aqc_fw_log_cfg_resp *fw_module = &fw_modules[i]; + + cfg->module_entries[i].module_id = + le16_to_cpu(fw_module->module_identifier); + cfg->module_entries[i].log_level = fw_module->log_level; + } + +status_out: + kfree(buf); + return status; +} + +/** + * libie_fwlog_set_supported - Set if FW logging is supported by FW + * @fwlog: pointer to the fwlog structure + * + * If FW returns success to the libie_aq_fwlog_get call then it supports FW + * logging, else it doesn't. Set the fwlog_supported flag accordingly. + * + * This function is only meant to be called during driver init to determine if + * the FW support FW logging. + */ +static void libie_fwlog_set_supported(struct libie_fwlog *fwlog) +{ + struct libie_fwlog_cfg *cfg; + int status; + + fwlog->supported = false; + + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) + return; + + status = libie_aq_fwlog_get(fwlog, cfg); + if (status) + dev_dbg(&fwlog->pdev->dev, "libie_aq_fwlog_get failed, FW logging is not supported on this version of FW, status %d\n", + status); + else + fwlog->supported = true; + + kfree(cfg); +} + +/** + * libie_fwlog_init - Initialize FW logging configuration + * @fwlog: pointer to the fwlog structure + * @api: api structure to init fwlog + * + * This function should be called on driver initialization during + * libie_init_hw(). + */ +int libie_fwlog_init(struct libie_fwlog *fwlog, struct libie_fwlog_api *api) +{ + fwlog->api = *api; + libie_fwlog_set_supported(fwlog); + + if (libie_fwlog_supported(fwlog)) { + int status; + + /* read the current config from the FW and store it */ + status = libie_aq_fwlog_get(fwlog, &fwlog->cfg); + if (status) + return status; + + fwlog->ring.rings = kcalloc(LIBIE_FWLOG_RING_SIZE_DFLT, + sizeof(*fwlog->ring.rings), + GFP_KERNEL); + if (!fwlog->ring.rings) { + dev_warn(&fwlog->pdev->dev, "Unable to allocate memory for FW log rings\n"); + return -ENOMEM; + } + + fwlog->ring.size = LIBIE_FWLOG_RING_SIZE_DFLT; + fwlog->ring.index = LIBIE_FWLOG_RING_SIZE_INDEX_DFLT; + + status = libie_fwlog_alloc_ring_buffs(&fwlog->ring); + if (status) { + dev_warn(&fwlog->pdev->dev, "Unable to allocate memory for FW log ring data buffers\n"); + libie_fwlog_free_ring_buffs(&fwlog->ring); + kfree(fwlog->ring.rings); + return status; + } + + libie_debugfs_fwlog_init(fwlog, api->debugfs_root); + } else { + dev_warn(&fwlog->pdev->dev, "FW logging is not supported in this NVM image. Please update the NVM to get FW log support\n"); + } + + return 0; +} +EXPORT_SYMBOL_GPL(libie_fwlog_init); + +/** + * libie_fwlog_deinit - unroll FW logging configuration + * @fwlog: pointer to the fwlog structure + * + * This function should be called in libie_deinit_hw(). + */ +void libie_fwlog_deinit(struct libie_fwlog *fwlog) +{ + int status; + + /* make sure FW logging is disabled to not put the FW in a weird state + * for the next driver load + */ + fwlog->cfg.options &= ~LIBIE_FWLOG_OPTION_ARQ_ENA; + status = libie_fwlog_set(fwlog, &fwlog->cfg); + if (status) + dev_warn(&fwlog->pdev->dev, "Unable to turn off FW logging, status: %d\n", + status); + + kfree(fwlog->debugfs_modules); + + fwlog->debugfs_modules = NULL; + + status = libie_fwlog_unregister(fwlog); + if (status) + dev_warn(&fwlog->pdev->dev, "Unable to unregister FW logging, status: %d\n", + status); + + if (fwlog->ring.rings) { + libie_fwlog_free_ring_buffs(&fwlog->ring); + kfree(fwlog->ring.rings); + } +} +EXPORT_SYMBOL_GPL(libie_fwlog_deinit); + +/** + * libie_get_fwlog_data - copy the FW log data from ARQ event + * @fwlog: fwlog that the FW log event is associated with + * @buf: event buffer pointer + * @len: len of event descriptor + */ +void libie_get_fwlog_data(struct libie_fwlog *fwlog, u8 *buf, u16 len) +{ + struct libie_fwlog_data *log; + + log = &fwlog->ring.rings[fwlog->ring.tail]; + + memset(log->data, 0, PAGE_SIZE); + log->data_size = len; + + memcpy(log->data, buf, log->data_size); + libie_fwlog_ring_increment(&fwlog->ring.tail, fwlog->ring.size); + + if (libie_fwlog_ring_full(&fwlog->ring)) { + /* the rings are full so bump the head to create room */ + libie_fwlog_ring_increment(&fwlog->ring.head, fwlog->ring.size); + } +} +EXPORT_SYMBOL_GPL(libie_get_fwlog_data); + +void libie_fwlog_reregister(struct libie_fwlog *fwlog) +{ + if (!(fwlog->cfg.options & LIBIE_FWLOG_OPTION_IS_REGISTERED)) + return; + + if (libie_fwlog_register(fwlog)) + fwlog->cfg.options &= ~LIBIE_FWLOG_OPTION_IS_REGISTERED; +} +EXPORT_SYMBOL_GPL(libie_fwlog_reregister); + +MODULE_DESCRIPTION("Intel(R) Ethernet common library"); +MODULE_LICENSE("GPL"); |