From 7829451c695e5b795fef95d72cd258e78d52f62d Mon Sep 17 00:00:00 2001 From: Hariprasad Shenai Date: Thu, 11 Aug 2016 21:06:23 +0530 Subject: [PATCH 1/2] cxgb4: Add control net_device for configuring PCIe VF Issue: For instance, the current APIs assume a 1-to-1 mapping of Network Ports, Physical Functions and the SR-IOV Virtual Functions of those Physical Functions. This is not the case with our cards where any Virtual Function can be hooked up to any Port -- or any number of Ports the current Linux APIs also assume only 1 Network Interface/Port can be accessed per Virtual Function. Another issue is that these APIs assume that the Administrative Driver is attached to the Physical Function Associated with a Virtual Function. This is not the case with our card where all administration is performed by a Driver which is not attached to any of the Physical Functions which have SR-IOV PCI Capabilities. Another consequence of these assumptions is the inability to utilize all of the cards SR-IOV resources. For instance, our cards have SR-IOV Capabilities on Physical Functions 0..3 and the administrative Driver attaches to Physical Function 4. Each of the Physical Functions 0..3 can support up to 16 Virtual Functions. With the current Linux APIs, a 2-Port card would only be able to use the Virtual Functions on Physical Function 0..1 and not allow the Virtual Functions on Physical Functions 2..3 to be used since there are no Ports 2..3 on a 2-Port card. Fix: Since the control node is always the netdevice for all VF ACL commands. Created a dummy netdevice for each Physical Function from 0 to 3 through which one could control their VFs. The device won't be associated with any port, since it doesn't need to transmit/receive. Its purely used for VF management purpose only. The device will be registered only when VF for a particular PF is configured using PCI sysfs interface and unregistered while pci_disable_sriov() for the PF is called. Signed-off-by: Hariprasad Shenai Signed-off-by: David S. Miller --- .../net/ethernet/chelsio/cxgb4/cxgb4_main.c | 121 +++++++++++++++--- 1 file changed, 103 insertions(+), 18 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index c45de49dc963..b7e932ec6894 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -3139,6 +3139,24 @@ static const struct net_device_ops cxgb4_netdev_ops = { }; +static const struct net_device_ops cxgb4_mgmt_netdev_ops = { +}; + +static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) +{ + struct adapter *adapter = netdev2adap(dev); + + strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); + strlcpy(info->version, cxgb4_driver_version, + sizeof(info->version)); + strlcpy(info->bus_info, pci_name(adapter->pdev), + sizeof(info->bus_info)); +} + +static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { + .get_drvinfo = get_drvinfo, +}; + void t4_fatal_err(struct adapter *adap) { t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0); @@ -4836,19 +4854,12 @@ static int get_chip_type(struct pci_dev *pdev, u32 pl_rev) #ifdef CONFIG_PCI_IOV static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) { + struct adapter *adap = pci_get_drvdata(pdev); int err = 0; int current_vfs = pci_num_vf(pdev); u32 pcie_fw; - void __iomem *regs; - regs = pci_ioremap_bar(pdev, 0); - if (!regs) { - dev_err(&pdev->dev, "cannot map device registers\n"); - return -ENOMEM; - } - - pcie_fw = readl(regs + PCIE_FW_A); - iounmap(regs); + pcie_fw = readl(adap->regs + PCIE_FW_A); /* Check if cxgb4 is the MASTER and fw is initialized */ if (!(pcie_fw & PCIE_FW_INIT_F) || !(pcie_fw & PCIE_FW_MASTER_VLD_F) || @@ -4875,6 +4886,8 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) */ if (!num_vfs) { pci_disable_sriov(pdev); + if (adap->port[0]->reg_state == NETREG_REGISTERED) + unregister_netdev(adap->port[0]); return num_vfs; } @@ -4882,6 +4895,12 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) err = pci_enable_sriov(pdev, num_vfs); if (err) return err; + + if (adap->port[0]->reg_state == NETREG_UNINITIALIZED) { + err = register_netdev(adap->port[0]); + if (err < 0) + pr_info("Unable to register VF mgmt netdev\n"); + } } return num_vfs; } @@ -4893,9 +4912,14 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) struct port_info *pi; bool highdma = false; struct adapter *adapter = NULL; + struct net_device *netdev; +#ifdef CONFIG_PCI_IOV + char name[IFNAMSIZ]; +#endif void __iomem *regs; u32 whoami, pl_rev; enum chip_type chip; + static int adap_idx = 1; printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); @@ -4930,7 +4954,9 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); if (func != ent->driver_data) { +#ifndef CONFIG_PCI_IOV iounmap(regs); +#endif pci_disable_device(pdev); pci_save_state(pdev); /* to restore SR-IOV later */ goto sriov; @@ -4962,6 +4988,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) err = -ENOMEM; goto out_unmap_bar0; } + adap_idx++; adapter->workq = create_singlethread_workqueue("cxgb4"); if (!adapter->workq) { @@ -5048,8 +5075,6 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) T6_STATMODE_V(0))); for_each_port(adapter, i) { - struct net_device *netdev; - netdev = alloc_etherdev_mq(sizeof(struct port_info), MAX_ETH_QSETS); if (!netdev) { @@ -5217,6 +5242,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) attach_ulds(adapter); print_adapter_info(adapter); + return 0; sriov: #ifdef CONFIG_PCI_IOV @@ -5230,9 +5256,58 @@ sriov: "instantiated %u virtual functions\n", num_vf[func]); } -#endif + + adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); + if (!adapter) { + err = -ENOMEM; + goto free_pci_region; + } + + snprintf(name, IFNAMSIZ, "mgmtpf%d%d", adap_idx, func); + netdev = alloc_netdev(0, name, NET_NAME_UNKNOWN, ether_setup); + if (!netdev) { + err = -ENOMEM; + goto free_adapter; + } + + adapter->pdev = pdev; + adapter->pdev_dev = &pdev->dev; + adapter->name = pci_name(pdev); + adapter->mbox = func; + adapter->pf = func; + adapter->regs = regs; + adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + + (sizeof(struct mbox_cmd) * + T4_OS_LOG_MBOX_CMDS), + GFP_KERNEL); + if (!adapter->mbox_log) { + err = -ENOMEM; + goto free_netdevice; + } + pi = netdev_priv(netdev); + pi->adapter = adapter; + SET_NETDEV_DEV(netdev, &pdev->dev); + pci_set_drvdata(pdev, adapter); + + adapter->port[0] = netdev; + netdev->netdev_ops = &cxgb4_mgmt_netdev_ops; + netdev->ethtool_ops = &cxgb4_mgmt_ethtool_ops; + return 0; + free_netdevice: + free_netdev(adapter->port[0]); + free_adapter: + kfree(adapter); + free_pci_region: + iounmap(regs); + pci_disable_sriov(pdev); + pci_release_regions(pdev); + return err; +#else + return 0; +#endif + out_free_dev: free_some_resources(adapter); out_unmap_bar: @@ -5258,12 +5333,12 @@ static void remove_one(struct pci_dev *pdev) { struct adapter *adapter = pci_get_drvdata(pdev); -#ifdef CONFIG_PCI_IOV - pci_disable_sriov(pdev); + if (!adapter) { + pci_release_regions(pdev); + return; + } -#endif - - if (adapter) { + if (adapter->pf == 4) { int i; /* Tear down per-adapter Work Queue first since it can contain @@ -5312,8 +5387,18 @@ static void remove_one(struct pci_dev *pdev) kfree(adapter->mbox_log); synchronize_rcu(); kfree(adapter); - } else + } +#ifdef CONFIG_PCI_IOV + else { + if (adapter->port[0]->reg_state == NETREG_REGISTERED) + unregister_netdev(adapter->port[0]); + free_netdev(adapter->port[0]); + iounmap(adapter->regs); + kfree(adapter); + pci_disable_sriov(pdev); pci_release_regions(pdev); + } +#endif } static struct pci_driver cxgb4_driver = { From 858aa65c5110b624bfdab6a891d53826b0dd45aa Mon Sep 17 00:00:00 2001 From: Hariprasad Shenai Date: Thu, 11 Aug 2016 21:06:24 +0530 Subject: [PATCH 2/2] cxgb4/cxgb4vf: Add set VF mac address support Add ndo_set_vf_mac support which allows to set the MAC address for cxgb4vf interfaces from the host Signed-off-by: Hariprasad Shenai Signed-off-by: David S. Miller --- drivers/net/ethernet/chelsio/cxgb4/cxgb4.h | 3 + .../net/ethernet/chelsio/cxgb4/cxgb4_main.c | 24 ++++++- drivers/net/ethernet/chelsio/cxgb4/t4_hw.c | 41 ++++++++++++ .../ethernet/chelsio/cxgb4vf/cxgb4vf_main.c | 24 +++++++ .../ethernet/chelsio/cxgb4vf/t4vf_common.h | 3 + .../net/ethernet/chelsio/cxgb4vf/t4vf_hw.c | 63 +++++++++++++++++-- 6 files changed, 151 insertions(+), 7 deletions(-) diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h index 2e2aa9fec9bb..bcfa51226b46 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -1521,4 +1521,7 @@ void t4_idma_monitor_init(struct adapter *adapter, void t4_idma_monitor(struct adapter *adapter, struct sge_idma_monitor_state *idma, int hz, int ticks); +int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf, + unsigned int naddr, u8 *addr); + #endif /* __CXGB4_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index b7e932ec6894..2bb804c93688 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -3078,6 +3078,26 @@ static int cxgb_change_mtu(struct net_device *dev, int new_mtu) return ret; } +#ifdef CONFIG_PCI_IOV +static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac) +{ + struct port_info *pi = netdev_priv(dev); + struct adapter *adap = pi->adapter; + + /* verify MAC addr is valid */ + if (!is_valid_ether_addr(mac)) { + dev_err(pi->adapter->pdev_dev, + "Invalid Ethernet address %pM for VF %d\n", + mac, vf); + return -EINVAL; + } + + dev_info(pi->adapter->pdev_dev, + "Setting MAC %pM on VF %d\n", mac, vf); + return t4_set_vf_mac_acl(adap, vf + 1, 1, mac); +} +#endif + static int cxgb_set_mac_addr(struct net_device *dev, void *p) { int ret; @@ -3136,10 +3156,12 @@ static const struct net_device_ops cxgb4_netdev_ops = { #ifdef CONFIG_NET_RX_BUSY_POLL .ndo_busy_poll = cxgb_busy_poll, #endif - }; static const struct net_device_ops cxgb4_mgmt_netdev_ops = { +#ifdef CONFIG_PCI_IOV + .ndo_set_vf_mac = cxgb_set_vf_mac, +#endif }; static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c index dc92c80a75f4..2a476cc4e073 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c +++ b/drivers/net/ethernet/chelsio/cxgb4/t4_hw.c @@ -8264,3 +8264,44 @@ void t4_idma_monitor(struct adapter *adapter, t4_sge_decode_idma_state(adapter, idma->idma_state[i]); } } + +/** + * t4_set_vf_mac - Set MAC address for the specified VF + * @adapter: The adapter + * @vf: one of the VFs instantiated by the specified PF + * @naddr: the number of MAC addresses + * @addr: the MAC address(es) to be set to the specified VF + */ +int t4_set_vf_mac_acl(struct adapter *adapter, unsigned int vf, + unsigned int naddr, u8 *addr) +{ + struct fw_acl_mac_cmd cmd; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_WRITE_F | + FW_ACL_MAC_CMD_PFN_V(adapter->pf) | + FW_ACL_MAC_CMD_VFN_V(vf)); + + /* Note: Do not enable the ACL */ + cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd)); + cmd.nmac = naddr; + + switch (adapter->pf) { + case 3: + memcpy(cmd.macaddr3, addr, sizeof(cmd.macaddr3)); + break; + case 2: + memcpy(cmd.macaddr2, addr, sizeof(cmd.macaddr2)); + break; + case 1: + memcpy(cmd.macaddr1, addr, sizeof(cmd.macaddr1)); + break; + case 0: + memcpy(cmd.macaddr0, addr, sizeof(cmd.macaddr0)); + break; + } + + return t4_wr_mbox(adapter, adapter->mbox, &cmd, sizeof(cmd), &cmd); +} diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index e116bb8d1729..f2951bf68992 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -2777,6 +2777,7 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, struct adapter *adapter; struct port_info *pi; struct net_device *netdev; + unsigned int pf; /* * Print our driver banner the first time we're called to initialize a @@ -2903,8 +2904,11 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, * Allocate our "adapter ports" and stitch everything together. */ pmask = adapter->params.vfres.pmask; + pf = t4vf_get_pf_from_vf(adapter); for_each_port(adapter, pidx) { int port_id, viid; + u8 mac[ETH_ALEN]; + unsigned int naddr = 1; /* * We simplistically allocate our virtual interfaces @@ -2975,6 +2979,26 @@ static int cxgb4vf_pci_probe(struct pci_dev *pdev, pidx); goto err_free_dev; } + + err = t4vf_get_vf_mac_acl(adapter, pf, &naddr, mac); + if (err) { + dev_err(&pdev->dev, + "unable to determine MAC ACL address, " + "continuing anyway.. (status %d)\n", err); + } else if (naddr && adapter->params.vfres.nvi == 1) { + struct sockaddr addr; + + ether_addr_copy(addr.sa_data, mac); + err = cxgb4vf_set_mac_addr(netdev, &addr); + if (err) { + dev_err(&pdev->dev, + "unable to set MAC address %pM\n", + mac); + goto err_free_dev; + } + dev_info(&pdev->dev, + "Using assigned MAC ACL: %pM\n", mac); + } } /* See what interrupts we'll be using. If we've been configured to diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h index 8ee541431e8b..8067424ad4a8 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h +++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_common.h @@ -347,6 +347,7 @@ int t4vf_bar2_sge_qregs(struct adapter *adapter, u64 *pbar2_qoffset, unsigned int *pbar2_qid); +unsigned int t4vf_get_pf_from_vf(struct adapter *); int t4vf_get_sge_params(struct adapter *); int t4vf_get_vpd_params(struct adapter *); int t4vf_get_dev_params(struct adapter *); @@ -381,5 +382,7 @@ int t4vf_eth_eq_free(struct adapter *, unsigned int); int t4vf_handle_fw_rpl(struct adapter *, const __be64 *); int t4vf_prep_adapter(struct adapter *); +int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf, + unsigned int *naddr, u8 *addr); #endif /* __T4VF_COMMON_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c index 427bfa71388b..879f4c52b3d5 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/t4vf_hw.c @@ -639,6 +639,15 @@ int t4vf_bar2_sge_qregs(struct adapter *adapter, return 0; } +unsigned int t4vf_get_pf_from_vf(struct adapter *adapter) +{ + u32 whoami; + + whoami = t4_read_reg(adapter, T4VF_PL_BASE_ADDR + PL_VF_WHOAMI_A); + return (CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? + SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami)); +} + /** * t4vf_get_sge_params - retrieve adapter Scatter gather Engine parameters * @adapter: the adapter @@ -716,7 +725,6 @@ int t4vf_get_sge_params(struct adapter *adapter) * read. */ if (!is_t4(adapter->params.chip)) { - u32 whoami; unsigned int pf, s_hps, s_qpp; params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_REG) | @@ -740,11 +748,7 @@ int t4vf_get_sge_params(struct adapter *adapter) * register we just read. Do it once here so other code in * the driver can just use it. */ - whoami = t4_read_reg(adapter, - T4VF_PL_BASE_ADDR + PL_VF_WHOAMI_A); - pf = CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5 ? - SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); - + pf = t4vf_get_pf_from_vf(adapter); s_hps = (HOSTPAGESIZEPF0_S + (HOSTPAGESIZEPF1_S - HOSTPAGESIZEPF0_S) * pf); sge_params->sge_vf_hps = @@ -1807,3 +1811,50 @@ int t4vf_prep_adapter(struct adapter *adapter) return 0; } + +/** + * t4vf_get_vf_mac_acl - Get the MAC address to be set to + * the VI of this VF. + * @adapter: The adapter + * @pf: The pf associated with vf + * @naddr: the number of ACL MAC addresses returned in addr + * @addr: Placeholder for MAC addresses + * + * Find the MAC address to be set to the VF's VI. The requested MAC address + * is from the host OS via callback in the PF driver. + */ +int t4vf_get_vf_mac_acl(struct adapter *adapter, unsigned int pf, + unsigned int *naddr, u8 *addr) +{ + struct fw_acl_mac_cmd cmd; + int ret; + + memset(&cmd, 0, sizeof(cmd)); + cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_ACL_MAC_CMD) | + FW_CMD_REQUEST_F | + FW_CMD_READ_F); + cmd.en_to_len16 = cpu_to_be32((unsigned int)FW_LEN16(cmd)); + ret = t4vf_wr_mbox(adapter, &cmd, sizeof(cmd), &cmd); + if (ret) + return ret; + + if (cmd.nmac < *naddr) + *naddr = cmd.nmac; + + switch (pf) { + case 3: + memcpy(addr, cmd.macaddr3, sizeof(cmd.macaddr3)); + break; + case 2: + memcpy(addr, cmd.macaddr2, sizeof(cmd.macaddr2)); + break; + case 1: + memcpy(addr, cmd.macaddr1, sizeof(cmd.macaddr1)); + break; + case 0: + memcpy(addr, cmd.macaddr0, sizeof(cmd.macaddr0)); + break; + } + + return ret; +}