diff options
Diffstat (limited to 'drivers/net/ethernet/netronome')
46 files changed, 4236 insertions, 361 deletions
diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig index ae0c46ba7546d..66f15b05b65e0 100644 --- a/drivers/net/ethernet/netronome/Kconfig +++ b/drivers/net/ethernet/netronome/Kconfig @@ -36,6 +36,19 @@ config NFP_APP_FLOWER either directly, with Open vSwitch, or any other way. Note that TC Flower offload requires specific FW to work. +config NFP_APP_ABM_NIC + bool "NFP4000/NFP6000 Advanced buffer management NIC support" + depends on NFP + depends on NET_SWITCHDEV + default y + help + Enable driver support for Advanced buffer management NIC on NFP. + ABM NIC allows advanced configuration of queuing and scheduling + of packets, including ECN marking. Say Y, if you are planning to + use one of the NFP4000 and NFP6000 platforms which support this + functionality. + Code will be built into the nfp.ko driver. + config NFP_DEBUG bool "Debug support for Netronome(R) NFP4000/NFP6000 NIC drivers" depends on NFP diff --git a/drivers/net/ethernet/netronome/nfp/Makefile b/drivers/net/ethernet/netronome/nfp/Makefile index d5866d708dfa3..4afb10375397b 100644 --- a/drivers/net/ethernet/netronome/nfp/Makefile +++ b/drivers/net/ethernet/netronome/nfp/Makefile @@ -30,12 +30,14 @@ nfp-objs := \ nfp_net_sriov.o \ nfp_netvf_main.o \ nfp_port.o \ + nfp_shared_buf.o \ nic/main.o ifeq ($(CONFIG_NFP_APP_FLOWER),y) nfp-objs += \ flower/action.o \ flower/cmsg.o \ + flower/lag_conf.o \ flower/main.o \ flower/match.o \ flower/metadata.o \ @@ -52,4 +54,10 @@ nfp-objs += \ bpf/jit.o endif +ifeq ($(CONFIG_NFP_APP_ABM_NIC),y) +nfp-objs += \ + abm/ctrl.o \ + abm/main.o +endif + nfp-$(CONFIG_NFP_DEBUG) += nfp_net_debugfs.o diff --git a/drivers/net/ethernet/netronome/nfp/abm/ctrl.c b/drivers/net/ethernet/netronome/nfp/abm/ctrl.c new file mode 100644 index 0000000000000..b157ccd8c80f7 --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/abm/ctrl.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +/* + * Copyright (C) 2018 Netronome Systems, Inc. + * + * This software is dual licensed under the GNU General License Version 2, + * June 1991 as shown in the file COPYING in the top-level directory of this + * source tree or the BSD 2-Clause License provided below. You have the + * option to license this software under the complete terms of either license. + * + * The BSD 2-Clause License: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/kernel.h> + +#include "../nfpcore/nfp_cpp.h" +#include "../nfpcore/nfp_nffw.h" +#include "../nfp_app.h" +#include "../nfp_abi.h" +#include "../nfp_main.h" +#include "../nfp_net.h" +#include "main.h" + +#define NFP_QLVL_SYM_NAME "_abi_nfd_out_q_lvls_%u" +#define NFP_QLVL_STRIDE 16 +#define NFP_QLVL_BLOG_BYTES 0 +#define NFP_QLVL_BLOG_PKTS 4 +#define NFP_QLVL_THRS 8 + +#define NFP_QMSTAT_SYM_NAME "_abi_nfdqm%u_stats" +#define NFP_QMSTAT_STRIDE 32 +#define NFP_QMSTAT_NON_STO 0 +#define NFP_QMSTAT_STO 8 +#define NFP_QMSTAT_DROP 16 +#define NFP_QMSTAT_ECN 24 + +static unsigned long long +nfp_abm_q_lvl_thrs(struct nfp_abm_link *alink, unsigned int queue) +{ + return alink->abm->q_lvls->addr + + (alink->queue_base + queue) * NFP_QLVL_STRIDE + NFP_QLVL_THRS; +} + +static int +nfp_abm_ctrl_stat(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, + unsigned int stride, unsigned int offset, unsigned int i, + bool is_u64, u64 *res) +{ + struct nfp_cpp *cpp = alink->abm->app->cpp; + u32 val32, mur; + u64 val, addr; + int err; + + mur = NFP_CPP_ATOMIC_RD(sym->target, sym->domain); + + addr = sym->addr + (alink->queue_base + i) * stride + offset; + if (is_u64) + err = nfp_cpp_readq(cpp, mur, addr, &val); + else + err = nfp_cpp_readl(cpp, mur, addr, &val32); + if (err) { + nfp_err(cpp, + "RED offload reading stat failed on vNIC %d queue %d\n", + alink->id, i); + return err; + } + + *res = is_u64 ? val : val32; + return 0; +} + +static int +nfp_abm_ctrl_stat_all(struct nfp_abm_link *alink, const struct nfp_rtsym *sym, + unsigned int stride, unsigned int offset, bool is_u64, + u64 *res) +{ + u64 val, sum = 0; + unsigned int i; + int err; + + for (i = 0; i < alink->vnic->max_rx_rings; i++) { + err = nfp_abm_ctrl_stat(alink, sym, stride, offset, i, + is_u64, &val); + if (err) + return err; + sum += val; + } + + *res = sum; + return 0; +} + +int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int i, u32 val) +{ + struct nfp_cpp *cpp = alink->abm->app->cpp; + u32 muw; + int err; + + muw = NFP_CPP_ATOMIC_WR(alink->abm->q_lvls->target, + alink->abm->q_lvls->domain); + + err = nfp_cpp_writel(cpp, muw, nfp_abm_q_lvl_thrs(alink, i), val); + if (err) { + nfp_err(cpp, "RED offload setting level failed on vNIC %d queue %d\n", + alink->id, i); + return err; + } + + return 0; +} + +int nfp_abm_ctrl_set_all_q_lvls(struct nfp_abm_link *alink, u32 val) +{ + int i, err; + + for (i = 0; i < alink->vnic->max_rx_rings; i++) { + err = nfp_abm_ctrl_set_q_lvl(alink, i, val); + if (err) + return err; + } + + return 0; +} + +u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int i) +{ + u64 val; + + if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, NFP_QMSTAT_STRIDE, + NFP_QMSTAT_NON_STO, i, true, &val)) + return 0; + return val; +} + +u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int i) +{ + u64 val; + + if (nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, NFP_QMSTAT_STRIDE, + NFP_QMSTAT_STO, i, true, &val)) + return 0; + return val; +} + +int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int i, + struct nfp_alink_stats *stats) +{ + int err; + + stats->tx_pkts = nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i)); + stats->tx_bytes = nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i) + 8); + + err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, + NFP_QLVL_STRIDE, NFP_QLVL_BLOG_BYTES, + i, false, &stats->backlog_bytes); + if (err) + return err; + + err = nfp_abm_ctrl_stat(alink, alink->abm->q_lvls, + NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS, + i, false, &stats->backlog_pkts); + if (err) + return err; + + err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, + i, true, &stats->drops); + if (err) + return err; + + return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, + i, true, &stats->overlimits); +} + +int nfp_abm_ctrl_read_stats(struct nfp_abm_link *alink, + struct nfp_alink_stats *stats) +{ + u64 pkts = 0, bytes = 0; + int i, err; + + for (i = 0; i < alink->vnic->max_rx_rings; i++) { + pkts += nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i)); + bytes += nn_readq(alink->vnic, NFP_NET_CFG_RXR_STATS(i) + 8); + } + stats->tx_pkts = pkts; + stats->tx_bytes = bytes; + + err = nfp_abm_ctrl_stat_all(alink, alink->abm->q_lvls, + NFP_QLVL_STRIDE, NFP_QLVL_BLOG_BYTES, + false, &stats->backlog_bytes); + if (err) + return err; + + err = nfp_abm_ctrl_stat_all(alink, alink->abm->q_lvls, + NFP_QLVL_STRIDE, NFP_QLVL_BLOG_PKTS, + false, &stats->backlog_pkts); + if (err) + return err; + + err = nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, + true, &stats->drops); + if (err) + return err; + + return nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, + true, &stats->overlimits); +} + +int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i, + struct nfp_alink_xstats *xstats) +{ + int err; + + err = nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, + i, true, &xstats->pdrop); + if (err) + return err; + + return nfp_abm_ctrl_stat(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, + i, true, &xstats->ecn_marked); +} + +int nfp_abm_ctrl_read_xstats(struct nfp_abm_link *alink, + struct nfp_alink_xstats *xstats) +{ + int err; + + err = nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_DROP, + true, &xstats->pdrop); + if (err) + return err; + + return nfp_abm_ctrl_stat_all(alink, alink->abm->qm_stats, + NFP_QMSTAT_STRIDE, NFP_QMSTAT_ECN, + true, &xstats->ecn_marked); +} + +int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm) +{ + return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_ENABLE, + NULL, 0, NULL, 0); +} + +int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm) +{ + return nfp_mbox_cmd(abm->app->pf, NFP_MBOX_PCIE_ABM_DISABLE, + NULL, 0, NULL, 0); +} + +void nfp_abm_ctrl_read_params(struct nfp_abm_link *alink) +{ + alink->queue_base = nn_readl(alink->vnic, NFP_NET_CFG_START_RXQ); + alink->queue_base /= alink->vnic->stride_rx; +} + +static const struct nfp_rtsym * +nfp_abm_ctrl_find_rtsym(struct nfp_pf *pf, const char *name, unsigned int size) +{ + const struct nfp_rtsym *sym; + + sym = nfp_rtsym_lookup(pf->rtbl, name); + if (!sym) { + nfp_err(pf->cpp, "Symbol '%s' not found\n", name); + return ERR_PTR(-ENOENT); + } + if (sym->size != size) { + nfp_err(pf->cpp, + "Symbol '%s' wrong size: expected %u got %llu\n", + name, size, sym->size); + return ERR_PTR(-EINVAL); + } + + return sym; +} + +static const struct nfp_rtsym * +nfp_abm_ctrl_find_q_rtsym(struct nfp_pf *pf, const char *name, + unsigned int size) +{ + return nfp_abm_ctrl_find_rtsym(pf, name, size * NFP_NET_MAX_RX_RINGS); +} + +int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm) +{ + struct nfp_pf *pf = abm->app->pf; + const struct nfp_rtsym *sym; + unsigned int pf_id; + char pf_symbol[64]; + + pf_id = nfp_cppcore_pcie_unit(pf->cpp); + abm->pf_id = pf_id; + + snprintf(pf_symbol, sizeof(pf_symbol), NFP_QLVL_SYM_NAME, pf_id); + sym = nfp_abm_ctrl_find_q_rtsym(pf, pf_symbol, NFP_QLVL_STRIDE); + if (IS_ERR(sym)) + return PTR_ERR(sym); + abm->q_lvls = sym; + + snprintf(pf_symbol, sizeof(pf_symbol), NFP_QMSTAT_SYM_NAME, pf_id); + sym = nfp_abm_ctrl_find_q_rtsym(pf, pf_symbol, NFP_QMSTAT_STRIDE); + if (IS_ERR(sym)) + return PTR_ERR(sym); + abm->qm_stats = sym; + + return 0; +} diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.c b/drivers/net/ethernet/netronome/nfp/abm/main.c new file mode 100644 index 0000000000000..1561c2724c26a --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/abm/main.c @@ -0,0 +1,765 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +/* + * Copyright (C) 2018 Netronome Systems, Inc. + * + * This software is dual licensed under the GNU General License Version 2, + * June 1991 as shown in the file COPYING in the top-level directory of this + * source tree or the BSD 2-Clause License provided below. You have the + * option to license this software under the complete terms of either license. + * + * The BSD 2-Clause License: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/bitfield.h> +#include <linux/etherdevice.h> +#include <linux/lockdep.h> +#include <linux/netdevice.h> +#include <linux/rcupdate.h> +#include <linux/slab.h> +#include <net/pkt_cls.h> +#include <net/pkt_sched.h> +#include <net/red.h> + +#include "../nfpcore/nfp.h" +#include "../nfpcore/nfp_cpp.h" +#include "../nfpcore/nfp_nsp.h" +#include "../nfp_app.h" +#include "../nfp_main.h" +#include "../nfp_net.h" +#include "../nfp_net_repr.h" +#include "../nfp_port.h" +#include "main.h" + +static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) +{ + return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | + FIELD_PREP(NFP_ABM_PORTID_ID, id); +} + +static int +__nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink, + u32 handle, unsigned int qs, u32 init_val) +{ + struct nfp_port *port = nfp_port_from_netdev(netdev); + int ret; + + ret = nfp_abm_ctrl_set_all_q_lvls(alink, init_val); + memset(alink->qdiscs, 0, sizeof(*alink->qdiscs) * alink->num_qdiscs); + + alink->parent = handle; + alink->num_qdiscs = qs; + port->tc_offload_cnt = qs; + + return ret; +} + +static void +nfp_abm_reset_root(struct net_device *netdev, struct nfp_abm_link *alink, + u32 handle, unsigned int qs) +{ + __nfp_abm_reset_root(netdev, alink, handle, qs, ~0); +} + +static int +nfp_abm_red_find(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) +{ + unsigned int i = TC_H_MIN(opt->parent) - 1; + + if (opt->parent == TC_H_ROOT) + i = 0; + else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent)) + i = TC_H_MIN(opt->parent) - 1; + else + return -EOPNOTSUPP; + + if (i >= alink->num_qdiscs || opt->handle != alink->qdiscs[i].handle) + return -EOPNOTSUPP; + + return i; +} + +static void +nfp_abm_red_destroy(struct net_device *netdev, struct nfp_abm_link *alink, + u32 handle) +{ + unsigned int i; + + for (i = 0; i < alink->num_qdiscs; i++) + if (handle == alink->qdiscs[i].handle) + break; + if (i == alink->num_qdiscs) + return; + + if (alink->parent == TC_H_ROOT) { + nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0); + } else { + nfp_abm_ctrl_set_q_lvl(alink, i, ~0); + memset(&alink->qdiscs[i], 0, sizeof(*alink->qdiscs)); + } +} + +static int +nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_red_qopt_offload *opt) +{ + bool existing; + int i, err; + + i = nfp_abm_red_find(alink, opt); + existing = i >= 0; + + if (opt->set.min != opt->set.max || !opt->set.is_ecn) { + nfp_warn(alink->abm->app->cpp, + "RED offload failed - unsupported parameters\n"); + err = -EINVAL; + goto err_destroy; + } + + if (existing) { + if (alink->parent == TC_H_ROOT) + err = nfp_abm_ctrl_set_all_q_lvls(alink, opt->set.min); + else + err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min); + if (err) + goto err_destroy; + return 0; + } + + if (opt->parent == TC_H_ROOT) { + i = 0; + err = __nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 1, + opt->set.min); + } else if (TC_H_MAJ(alink->parent) == TC_H_MAJ(opt->parent)) { + i = TC_H_MIN(opt->parent) - 1; + err = nfp_abm_ctrl_set_q_lvl(alink, i, opt->set.min); + } else { + return -EINVAL; + } + /* Set the handle to try full clean up, in case IO failed */ + alink->qdiscs[i].handle = opt->handle; + if (err) + goto err_destroy; + + if (opt->parent == TC_H_ROOT) + err = nfp_abm_ctrl_read_stats(alink, &alink->qdiscs[i].stats); + else + err = nfp_abm_ctrl_read_q_stats(alink, i, + &alink->qdiscs[i].stats); + if (err) + goto err_destroy; + + if (opt->parent == TC_H_ROOT) + err = nfp_abm_ctrl_read_xstats(alink, + &alink->qdiscs[i].xstats); + else + err = nfp_abm_ctrl_read_q_xstats(alink, i, + &alink->qdiscs[i].xstats); + if (err) + goto err_destroy; + + alink->qdiscs[i].stats.backlog_pkts = 0; + alink->qdiscs[i].stats.backlog_bytes = 0; + + return 0; +err_destroy: + /* If the qdisc keeps on living, but we can't offload undo changes */ + if (existing) { + opt->set.qstats->qlen -= alink->qdiscs[i].stats.backlog_pkts; + opt->set.qstats->backlog -= + alink->qdiscs[i].stats.backlog_bytes; + } + nfp_abm_red_destroy(netdev, alink, opt->handle); + + return err; +} + +static void +nfp_abm_update_stats(struct nfp_alink_stats *new, struct nfp_alink_stats *old, + struct tc_qopt_offload_stats *stats) +{ + _bstats_update(stats->bstats, new->tx_bytes - old->tx_bytes, + new->tx_pkts - old->tx_pkts); + stats->qstats->qlen += new->backlog_pkts - old->backlog_pkts; + stats->qstats->backlog += new->backlog_bytes - old->backlog_bytes; + stats->qstats->overlimits += new->overlimits - old->overlimits; + stats->qstats->drops += new->drops - old->drops; +} + +static int +nfp_abm_red_stats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) +{ + struct nfp_alink_stats *prev_stats; + struct nfp_alink_stats stats; + int i, err; + + i = nfp_abm_red_find(alink, opt); + if (i < 0) + return i; + prev_stats = &alink->qdiscs[i].stats; + + if (alink->parent == TC_H_ROOT) + err = nfp_abm_ctrl_read_stats(alink, &stats); + else + err = nfp_abm_ctrl_read_q_stats(alink, i, &stats); + if (err) + return err; + + nfp_abm_update_stats(&stats, prev_stats, &opt->stats); + + *prev_stats = stats; + + return 0; +} + +static int +nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) +{ + struct nfp_alink_xstats *prev_xstats; + struct nfp_alink_xstats xstats; + int i, err; + + i = nfp_abm_red_find(alink, opt); + if (i < 0) + return i; + prev_xstats = &alink->qdiscs[i].xstats; + + if (alink->parent == TC_H_ROOT) + err = nfp_abm_ctrl_read_xstats(alink, &xstats); + else + err = nfp_abm_ctrl_read_q_xstats(alink, i, &xstats); + if (err) + return err; + + opt->xstats->forced_mark += xstats.ecn_marked - prev_xstats->ecn_marked; + opt->xstats->pdrop += xstats.pdrop - prev_xstats->pdrop; + + *prev_xstats = xstats; + + return 0; +} + +static int +nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_red_qopt_offload *opt) +{ + switch (opt->command) { + case TC_RED_REPLACE: + return nfp_abm_red_replace(netdev, alink, opt); + case TC_RED_DESTROY: + nfp_abm_red_destroy(netdev, alink, opt->handle); + return 0; + case TC_RED_STATS: + return nfp_abm_red_stats(alink, opt); + case TC_RED_XSTATS: + return nfp_abm_red_xstats(alink, opt); + default: + return -EOPNOTSUPP; + } +} + +static int +nfp_abm_mq_stats(struct nfp_abm_link *alink, struct tc_mq_qopt_offload *opt) +{ + struct nfp_alink_stats stats; + unsigned int i; + int err; + + for (i = 0; i < alink->num_qdiscs; i++) { + if (alink->qdiscs[i].handle == TC_H_UNSPEC) + continue; + + err = nfp_abm_ctrl_read_q_stats(alink, i, &stats); + if (err) + return err; + + nfp_abm_update_stats(&stats, &alink->qdiscs[i].stats, + &opt->stats); + } + + return 0; +} + +static int +nfp_abm_setup_tc_mq(struct net_device *netdev, struct nfp_abm_link *alink, + struct tc_mq_qopt_offload *opt) +{ + switch (opt->command) { + case TC_MQ_CREATE: + nfp_abm_reset_root(netdev, alink, opt->handle, + alink->total_queues); + return 0; + case TC_MQ_DESTROY: + if (opt->handle == alink->parent) + nfp_abm_reset_root(netdev, alink, TC_H_ROOT, 0); + return 0; + case TC_MQ_STATS: + return nfp_abm_mq_stats(alink, opt); + default: + return -EOPNOTSUPP; + } +} + +static int +nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, + enum tc_setup_type type, void *type_data) +{ + struct nfp_repr *repr = netdev_priv(netdev); + struct nfp_port *port; + + port = nfp_port_from_netdev(netdev); + if (!port || port->type != NFP_PORT_PF_PORT) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_QDISC_MQ: + return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); + case TC_SETUP_QDISC_RED: + return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); + default: + return -EOPNOTSUPP; + } +} + +static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id) +{ + enum nfp_repr_type rtype; + struct nfp_reprs *reprs; + u8 port; + + rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); + port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); + + reprs = rcu_dereference(app->reprs[rtype]); + if (!reprs) + return NULL; + + if (port >= reprs->num_reprs) + return NULL; + + return rcu_dereference(reprs->reprs[port]); +} + +static int +nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, + enum nfp_port_type ptype) +{ + struct net_device *netdev; + enum nfp_repr_type rtype; + struct nfp_reprs *reprs; + struct nfp_repr *repr; + struct nfp_port *port; + unsigned int txqs; + int err; + + if (ptype == NFP_PORT_PHYS_PORT) { + rtype = NFP_REPR_TYPE_PHYS_PORT; + txqs = 1; + } else { + rtype = NFP_REPR_TYPE_PF; + txqs = alink->vnic->max_rx_rings; + } + + netdev = nfp_repr_alloc_mqs(app, txqs, 1); + if (!netdev) + return -ENOMEM; + repr = netdev_priv(netdev); + repr->app_priv = alink; + + port = nfp_port_alloc(app, ptype, netdev); + if (IS_ERR(port)) { + err = PTR_ERR(port); + goto err_free_repr; + } + + if (ptype == NFP_PORT_PHYS_PORT) { + port->eth_forced = true; + err = nfp_port_init_phy_port(app->pf, app, port, alink->id); + if (err) + goto err_free_port; + } else { + port->pf_id = alink->abm->pf_id; + port->pf_split = app->pf->max_data_vnics > 1; + port->pf_split_id = alink->id; + port->vnic = alink->vnic->dp.ctrl_bar; + } + + SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); + eth_hw_addr_random(netdev); + + err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), + port, alink->vnic->dp.netdev); + if (err) + goto err_free_port; + + reprs = nfp_reprs_get_locked(app, rtype); + WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); + rcu_assign_pointer(reprs->reprs[alink->id], netdev); + + nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", + ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", + alink->id, netdev->name); + + return 0; + +err_free_port: + nfp_port_free(port); +err_free_repr: + nfp_repr_free(netdev); + return err; +} + +static void +nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, + enum nfp_repr_type rtype) +{ + struct net_device *netdev; + struct nfp_reprs *reprs; + + reprs = nfp_reprs_get_locked(app, rtype); + netdev = nfp_repr_get_locked(app, reprs, alink->id); + if (!netdev) + return; + rcu_assign_pointer(reprs->reprs[alink->id], NULL); + synchronize_rcu(); + /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ + nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); +} + +static void +nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) +{ + nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); + nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); +} + +static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) +{ + struct nfp_pf *pf = abm->app->pf; + struct nfp_net *nn; + + list_for_each_entry(nn, &pf->vnics, vnic_list) + nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); +} + +static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) +{ + struct nfp_abm *abm = app->priv; + + return abm->eswitch_mode; +} + +static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) +{ + nfp_abm_kill_reprs_all(abm); + nfp_abm_ctrl_qm_disable(abm); + + abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; + return 0; +} + +static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) +{ + if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) + WARN_ON(nfp_abm_eswitch_set_legacy(abm)); +} + +static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) +{ + struct nfp_app *app = abm->app; + struct nfp_pf *pf = app->pf; + struct nfp_net *nn; + int err; + + err = nfp_abm_ctrl_qm_enable(abm); + if (err) + return err; + + list_for_each_entry(nn, &pf->vnics, vnic_list) { + struct nfp_abm_link *alink = nn->app_priv; + + err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); + if (err) + goto err_kill_all_reprs; + + err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); + if (err) + goto err_kill_all_reprs; + } + + abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; + return 0; + +err_kill_all_reprs: + nfp_abm_kill_reprs_all(abm); + nfp_abm_ctrl_qm_disable(abm); + return err; +} + +static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) +{ + struct nfp_abm *abm = app->priv; + + if (abm->eswitch_mode == mode) + return 0; + + switch (mode) { + case DEVLINK_ESWITCH_MODE_LEGACY: + return nfp_abm_eswitch_set_legacy(abm); + case DEVLINK_ESWITCH_MODE_SWITCHDEV: + return nfp_abm_eswitch_set_switchdev(abm); + default: + return -EINVAL; + } +} + +static void +nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, + unsigned int id) +{ + struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; + u8 mac_addr[ETH_ALEN]; + const char *mac_str; + char name[32]; + + if (id > pf->eth_tbl->count) { + nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); + eth_hw_addr_random(nn->dp.netdev); + return; + } + + snprintf(name, sizeof(name), "eth%u.mac.pf%u", + eth_port->eth_index, abm->pf_id); + + mac_str = nfp_hwinfo_lookup(pf->hwinfo, name); + if (!mac_str) { + nfp_warn(pf->cpp, "Can't lookup persistent MAC address (%s)\n", + name); + eth_hw_addr_random(nn->dp.netdev); + return; + } + + if (sscanf(mac_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", + &mac_addr[0], &mac_addr[1], &mac_addr[2], + &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { + nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", + mac_str); + eth_hw_addr_random(nn->dp.netdev); + return; + } + + ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); + ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); +} + +static int +nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) +{ + struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; + struct nfp_abm *abm = app->priv; + struct nfp_abm_link *alink; + int err; + + alink = kzalloc(sizeof(*alink), GFP_KERNEL); + if (!alink) + return -ENOMEM; + nn->app_priv = alink; + alink->abm = abm; + alink->vnic = nn; + alink->id = id; + alink->parent = TC_H_ROOT; + alink->total_queues = alink->vnic->max_rx_rings; + alink->qdiscs = kvzalloc(sizeof(*alink->qdiscs) * alink->total_queues, + GFP_KERNEL); + if (!alink->qdiscs) { + err = -ENOMEM; + goto err_free_alink; + } + + /* This is a multi-host app, make sure MAC/PHY is up, but don't + * make the MAC/PHY state follow the state of any of the ports. + */ + err = nfp_eth_set_configured(app->cpp, eth_port->index, true); + if (err < 0) + goto err_free_qdiscs; + + netif_keep_dst(nn->dp.netdev); + + nfp_abm_vnic_set_mac(app->pf, abm, nn, id); + nfp_abm_ctrl_read_params(alink); + + return 0; + +err_free_qdiscs: + kvfree(alink->qdiscs); +err_free_alink: + kfree(alink); + return err; +} + +static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) +{ + struct nfp_abm_link *alink = nn->app_priv; + + nfp_abm_kill_reprs(alink->abm, alink); + kvfree(alink->qdiscs); + kfree(alink); +} + +static u64 * +nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data) +{ + struct nfp_repr *repr = netdev_priv(port->netdev); + struct nfp_abm_link *alink; + unsigned int i; + + if (port->type != NFP_PORT_PF_PORT) + return data; + alink = repr->app_priv; + for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { + *data++ = nfp_abm_ctrl_stat_non_sto(alink, i); + *data++ = nfp_abm_ctrl_stat_sto(alink, i); + } + return data; +} + +static int +nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port) +{ + struct nfp_repr *repr = netdev_priv(port->netdev); + struct nfp_abm_link *alink; + + if (port->type != NFP_PORT_PF_PORT) + return 0; + alink = repr->app_priv; + return alink->vnic->dp.num_r_vecs * 2; +} + +static u8 * +nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port, + u8 *data) +{ + struct nfp_repr *repr = netdev_priv(port->netdev); + struct nfp_abm_link *alink; + unsigned int i; + + if (port->type != NFP_PORT_PF_PORT) + return data; + alink = repr->app_priv; + for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { + data = nfp_pr_et(data, "q%u_no_wait", i); + data = nfp_pr_et(data, "q%u_delayed", i); + } + return data; +} + +static int nfp_abm_init(struct nfp_app *app) +{ + struct nfp_pf *pf = app->pf; + struct nfp_reprs *reprs; + struct nfp_abm *abm; + int err; + + if (!pf->eth_tbl) { + nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); + return -EINVAL; + } + if (pf->max_data_vnics != pf->eth_tbl->count) { + nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", + pf->max_data_vnics, pf->eth_tbl->count); + return -EINVAL; + } + if (!pf->mac_stats_bar) { + nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); + return -EINVAL; + } + + abm = kzalloc(sizeof(*abm), GFP_KERNEL); + if (!abm) + return -ENOMEM; + app->priv = abm; + abm->app = app; + + err = nfp_abm_ctrl_find_addrs(abm); + if (err) + goto err_free_abm; + + /* We start in legacy mode, make sure advanced queuing is disabled */ + err = nfp_abm_ctrl_qm_disable(abm); + if (err) + goto err_free_abm; + + err = -ENOMEM; + reprs = nfp_reprs_alloc(pf->max_data_vnics); + if (!reprs) + goto err_free_abm; + RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); + + reprs = nfp_reprs_alloc(pf->max_data_vnics); + if (!reprs) + goto err_free_phys; + RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); + + return 0; + +err_free_phys: + nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); +err_free_abm: + kfree(abm); + app->priv = NULL; + return err; +} + +static void nfp_abm_clean(struct nfp_app *app) +{ + struct nfp_abm *abm = app->priv; + + nfp_abm_eswitch_clean_up(abm); + nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); + nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); + kfree(abm); + app->priv = NULL; +} + +const struct nfp_app_type app_abm = { + .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, + .name = "abm", + + .init = nfp_abm_init, + .clean = nfp_abm_clean, + + .vnic_alloc = nfp_abm_vnic_alloc, + .vnic_free = nfp_abm_vnic_free, + + .port_get_stats = nfp_abm_port_get_stats, + .port_get_stats_count = nfp_abm_port_get_stats_count, + .port_get_stats_strings = nfp_abm_port_get_stats_strings, + + .setup_tc = nfp_abm_setup_tc, + + .eswitch_mode_get = nfp_abm_eswitch_mode_get, + .eswitch_mode_set = nfp_abm_eswitch_mode_set, + + .repr_get = nfp_abm_repr_get, +}; diff --git a/drivers/net/ethernet/netronome/nfp/abm/main.h b/drivers/net/ethernet/netronome/nfp/abm/main.h new file mode 100644 index 0000000000000..934a70835473e --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/abm/main.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * Copyright (C) 2018 Netronome Systems, Inc. + * + * This software is dual licensed under the GNU General License Version 2, + * June 1991 as shown in the file COPYING in the top-level directory of this + * source tree or the BSD 2-Clause License provided below. You have the + * option to license this software under the complete terms of either license. + * + * The BSD 2-Clause License: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __NFP_ABM_H__ +#define __NFP_ABM_H__ 1 + +#include <net/devlink.h> + +struct nfp_app; +struct nfp_net; + +#define NFP_ABM_PORTID_TYPE GENMASK(23, 16) +#define NFP_ABM_PORTID_ID GENMASK(7, 0) + +/** + * struct nfp_abm - ABM NIC app structure + * @app: back pointer to nfp_app + * @pf_id: ID of our PF link + * @eswitch_mode: devlink eswitch mode, advanced functions only visible + * in switchdev mode + * @q_lvls: queue level control area + * @qm_stats: queue statistics symbol + */ +struct nfp_abm { + struct nfp_app *app; + unsigned int pf_id; + enum devlink_eswitch_mode eswitch_mode; + const struct nfp_rtsym *q_lvls; + const struct nfp_rtsym *qm_stats; +}; + +/** + * struct nfp_alink_stats - ABM NIC statistics + * @tx_pkts: number of TXed packets + * @tx_bytes: number of TXed bytes + * @backlog_pkts: momentary backlog length (packets) + * @backlog_bytes: momentary backlog length (bytes) + * @overlimits: number of ECN marked TXed packets (accumulative) + * @drops: number of tail-dropped packets (accumulative) + */ +struct nfp_alink_stats { + u64 tx_pkts; + u64 tx_bytes; + u64 backlog_pkts; + u64 backlog_bytes; + u64 overlimits; + u64 drops; +}; + +/** + * struct nfp_alink_xstats - extended ABM NIC statistics + * @ecn_marked: number of ECN marked TXed packets + * @pdrop: number of hard drops due to queue limit + */ +struct nfp_alink_xstats { + u64 ecn_marked; + u64 pdrop; +}; + +/** + * struct nfp_red_qdisc - representation of single RED Qdisc + * @handle: handle of currently offloaded RED Qdisc + * @stats: statistics from last refresh + * @xstats: base of extended statistics + */ +struct nfp_red_qdisc { + u32 handle; + struct nfp_alink_stats stats; + struct nfp_alink_xstats xstats; +}; + +/** + * struct nfp_abm_link - port tuple of a ABM NIC + * @abm: back pointer to nfp_abm + * @vnic: data vNIC + * @id: id of the data vNIC + * @queue_base: id of base to host queue within PCIe (not QC idx) + * @total_queues: number of PF queues + * @parent: handle of expected parent, i.e. handle of MQ, or TC_H_ROOT + * @num_qdiscs: number of currently used qdiscs + * @qdiscs: array of qdiscs + */ +struct nfp_abm_link { + struct nfp_abm *abm; + struct nfp_net *vnic; + unsigned int id; + unsigned int queue_base; + unsigned int total_queues; + u32 parent; + unsigned int num_qdiscs; + struct nfp_red_qdisc *qdiscs; +}; + +void nfp_abm_ctrl_read_params(struct nfp_abm_link *alink); +int nfp_abm_ctrl_find_addrs(struct nfp_abm *abm); +int nfp_abm_ctrl_set_all_q_lvls(struct nfp_abm_link *alink, u32 val); +int nfp_abm_ctrl_set_q_lvl(struct nfp_abm_link *alink, unsigned int i, + u32 val); +int nfp_abm_ctrl_read_stats(struct nfp_abm_link *alink, + struct nfp_alink_stats *stats); +int nfp_abm_ctrl_read_q_stats(struct nfp_abm_link *alink, unsigned int i, + struct nfp_alink_stats *stats); +int nfp_abm_ctrl_read_xstats(struct nfp_abm_link *alink, + struct nfp_alink_xstats *xstats); +int nfp_abm_ctrl_read_q_xstats(struct nfp_abm_link *alink, unsigned int i, + struct nfp_alink_xstats *xstats); +u64 nfp_abm_ctrl_stat_non_sto(struct nfp_abm_link *alink, unsigned int i); +u64 nfp_abm_ctrl_stat_sto(struct nfp_abm_link *alink, unsigned int i); +int nfp_abm_ctrl_qm_enable(struct nfp_abm *abm); +int nfp_abm_ctrl_qm_disable(struct nfp_abm *abm); +#endif diff --git a/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c b/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c index 7e298148ca26b..cb87fccb9f6a2 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/cmsg.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Netronome Systems, Inc. + * Copyright (C) 2017-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -102,6 +102,15 @@ nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n) return nfp_bpf_cmsg_alloc(bpf, size); } +static u8 nfp_bpf_cmsg_get_type(struct sk_buff *skb) +{ + struct cmsg_hdr *hdr; + + hdr = (struct cmsg_hdr *)skb->data; + + return hdr->type; +} + static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb) { struct cmsg_hdr *hdr; @@ -431,6 +440,11 @@ void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb) goto err_free; } + if (nfp_bpf_cmsg_get_type(skb) == CMSG_TYPE_BPF_EVENT) { + nfp_bpf_event_output(bpf, skb); + return; + } + nfp_ctrl_lock(bpf->app->ctrl); tag = nfp_bpf_cmsg_get_tag(skb); diff --git a/drivers/net/ethernet/netronome/nfp/bpf/fw.h b/drivers/net/ethernet/netronome/nfp/bpf/fw.h index 39639ac28b017..4c7972e3db63e 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/fw.h +++ b/drivers/net/ethernet/netronome/nfp/bpf/fw.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Netronome Systems, Inc. + * Copyright (C) 2017-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -37,11 +37,20 @@ #include <linux/bitops.h> #include <linux/types.h> +/* Kernel's enum bpf_reg_type is not uABI so people may change it breaking + * our FW ABI. In that case we will do translation in the driver. + */ +#define NFP_BPF_SCALAR_VALUE 1 +#define NFP_BPF_MAP_VALUE 4 +#define NFP_BPF_STACK 6 +#define NFP_BPF_PACKET_DATA 8 + enum bpf_cap_tlv_type { NFP_BPF_CAP_TYPE_FUNC = 1, NFP_BPF_CAP_TYPE_ADJUST_HEAD = 2, NFP_BPF_CAP_TYPE_MAPS = 3, NFP_BPF_CAP_TYPE_RANDOM = 4, + NFP_BPF_CAP_TYPE_QUEUE_SELECT = 5, }; struct nfp_bpf_cap_tlv_func { @@ -81,6 +90,7 @@ enum nfp_bpf_cmsg_type { CMSG_TYPE_MAP_DELETE = 5, CMSG_TYPE_MAP_GETNEXT = 6, CMSG_TYPE_MAP_GETFIRST = 7, + CMSG_TYPE_BPF_EVENT = 8, __CMSG_TYPE_MAP_MAX, }; @@ -155,4 +165,13 @@ struct cmsg_reply_map_op { __be32 resv; struct cmsg_key_value_pair elem[0]; }; + +struct cmsg_bpf_event { + struct cmsg_hdr hdr; + __be32 cpu_id; + __be64 map_ptr; + __be32 data_size; + __be32 pkt_size; + u8 data[0]; +}; #endif diff --git a/drivers/net/ethernet/netronome/nfp/bpf/jit.c b/drivers/net/ethernet/netronome/nfp/bpf/jit.c index 29b4e5f8c102d..8a92088df0d7b 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/jit.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/jit.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2017 Netronome Systems, Inc. + * Copyright (C) 2016-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -42,6 +42,7 @@ #include "main.h" #include "../nfp_asm.h" +#include "../nfp_net_ctrl.h" /* --- NFP prog --- */ /* Foreach "multiple" entries macros provide pos and next<n> pointers. @@ -211,6 +212,60 @@ emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer) } static void +__emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer, + bool set, bool src_lmextn) +{ + u16 addr_lo, addr_hi; + u64 insn; + + addr_lo = addr & (OP_BR_BIT_ADDR_LO >> __bf_shf(OP_BR_BIT_ADDR_LO)); + addr_hi = addr != addr_lo; + + insn = OP_BR_BIT_BASE | + FIELD_PREP(OP_BR_BIT_A_SRC, areg) | + FIELD_PREP(OP_BR_BIT_B_SRC, breg) | + FIELD_PREP(OP_BR_BIT_BV, set) | + FIELD_PREP(OP_BR_BIT_DEFBR, defer) | + FIELD_PREP(OP_BR_BIT_ADDR_LO, addr_lo) | + FIELD_PREP(OP_BR_BIT_ADDR_HI, addr_hi) | + FIELD_PREP(OP_BR_BIT_SRC_LMEXTN, src_lmextn); + + nfp_prog_push(nfp_prog, insn); +} + +static void +emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, + u8 defer, bool set, enum nfp_relo_type relo) +{ + struct nfp_insn_re_regs reg; + int err; + + /* NOTE: The bit to test is specified as an rotation amount, such that + * the bit to test will be placed on the MSB of the result when + * doing a rotate right. For bit X, we need right rotate X + 1. + */ + bit += 1; + + err = swreg_to_restricted(reg_none(), src, reg_imm(bit), ®, false); + if (err) { + nfp_prog->error = err; + return; + } + + __emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set, + reg.src_lmextn); + + nfp_prog->prog[nfp_prog->prog_len - 1] |= + FIELD_PREP(OP_RELO_TYPE, relo); +} + +static void +emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer) +{ + emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL); +} + +static void __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi, enum immed_width width, bool invert, enum immed_shift shift, bool wr_both, @@ -309,6 +364,19 @@ emit_shf(struct nfp_prog *nfp_prog, swreg dst, } static void +emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst, + swreg lreg, enum shf_op op, swreg rreg, enum shf_sc sc) +{ + if (sc == SHF_SC_R_ROT) { + pr_err("indirect shift is not allowed on rotation\n"); + nfp_prog->error = -EFAULT; + return; + } + + emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0); +} + +static void __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab, u16 areg, enum alu_op op, u16 breg, bool swap, bool wr_both, bool dst_lmextn, bool src_lmextn) @@ -1214,45 +1282,83 @@ wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, return 0; } -static int -wrp_cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, - enum br_mask br_mask, bool swap) +static const struct jmp_code_map { + enum br_mask br_mask; + bool swap; +} jmp_code_map[] = { + [BPF_JGT >> 4] = { BR_BLO, true }, + [BPF_JGE >> 4] = { BR_BHS, false }, + [BPF_JLT >> 4] = { BR_BLO, false }, + [BPF_JLE >> 4] = { BR_BHS, true }, + [BPF_JSGT >> 4] = { BR_BLT, true }, + [BPF_JSGE >> 4] = { BR_BGE, false }, + [BPF_JSLT >> 4] = { BR_BLT, false }, + [BPF_JSLE >> 4] = { BR_BGE, true }, +}; + +static const struct jmp_code_map *nfp_jmp_code_get(struct nfp_insn_meta *meta) +{ + unsigned int op; + + op = BPF_OP(meta->insn.code) >> 4; + /* br_mask of 0 is BR_BEQ which we don't use in jump code table */ + if (WARN_ONCE(op >= ARRAY_SIZE(jmp_code_map) || + !jmp_code_map[op].br_mask, + "no code found for jump instruction")) + return NULL; + + return &jmp_code_map[op]; +} + +static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { const struct bpf_insn *insn = &meta->insn; u64 imm = insn->imm; /* sign extend */ + const struct jmp_code_map *code; + enum alu_op alu_op, carry_op; u8 reg = insn->dst_reg * 2; swreg tmp_reg; + code = nfp_jmp_code_get(meta); + if (!code) + return -EINVAL; + + alu_op = meta->jump_neg_op ? ALU_OP_ADD : ALU_OP_SUB; + carry_op = meta->jump_neg_op ? ALU_OP_ADD_C : ALU_OP_SUB_C; + tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog)); - if (!swap) - emit_alu(nfp_prog, reg_none(), reg_a(reg), ALU_OP_SUB, tmp_reg); + if (!code->swap) + emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg); else - emit_alu(nfp_prog, reg_none(), tmp_reg, ALU_OP_SUB, reg_a(reg)); + emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg)); tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog)); - if (!swap) + if (!code->swap) emit_alu(nfp_prog, reg_none(), - reg_a(reg + 1), ALU_OP_SUB_C, tmp_reg); + reg_a(reg + 1), carry_op, tmp_reg); else emit_alu(nfp_prog, reg_none(), - tmp_reg, ALU_OP_SUB_C, reg_a(reg + 1)); + tmp_reg, carry_op, reg_a(reg + 1)); - emit_br(nfp_prog, br_mask, insn->off, 0); + emit_br(nfp_prog, code->br_mask, insn->off, 0); return 0; } -static int -wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, - enum br_mask br_mask, bool swap) +static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { const struct bpf_insn *insn = &meta->insn; + const struct jmp_code_map *code; u8 areg, breg; + code = nfp_jmp_code_get(meta); + if (!code) + return -EINVAL; + areg = insn->dst_reg * 2; breg = insn->src_reg * 2; - if (swap) { + if (code->swap) { areg ^= breg; breg ^= areg; areg ^= breg; @@ -1261,7 +1367,7 @@ wrp_cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg)); emit_alu(nfp_prog, reg_none(), reg_a(areg + 1), ALU_OP_SUB_C, reg_b(breg + 1)); - emit_br(nfp_prog, br_mask, insn->off, 0); + emit_br(nfp_prog, code->br_mask, insn->off, 0); return 0; } @@ -1357,15 +1463,9 @@ static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) static int map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { - struct bpf_offloaded_map *offmap; - struct nfp_bpf_map *nfp_map; bool load_lm_ptr; u32 ret_tgt; s64 lm_off; - swreg tid; - - offmap = (struct bpf_offloaded_map *)meta->arg1.map_ptr; - nfp_map = offmap->dev_priv; /* We only have to reload LM0 if the key is not at start of stack */ lm_off = nfp_prog->stack_depth; @@ -1378,17 +1478,12 @@ map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) if (meta->func_id == BPF_FUNC_map_update_elem) emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2); - /* Load map ID into a register, it should actually fit as an immediate - * but in case it doesn't deal with it here, not in the delay slots. - */ - tid = ur_load_imm_any(nfp_prog, nfp_map->tid, imm_a(nfp_prog)); - emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, 2, RELO_BR_HELPER); ret_tgt = nfp_prog_current_offset(nfp_prog) + 2; /* Load map ID into A0 */ - wrp_mov(nfp_prog, reg_a(0), tid); + wrp_mov(nfp_prog, reg_a(0), reg_a(2)); /* Load the return address into B0 */ wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); @@ -1400,7 +1495,7 @@ map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) if (!load_lm_ptr) return 0; - emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0); + emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0); wrp_nops(nfp_prog, 3); return 0; @@ -1418,6 +1513,63 @@ nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return 0; } +static int +nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + swreg ptr_type; + u32 ret_tgt; + + ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog)); + + ret_tgt = nfp_prog_current_offset(nfp_prog) + 3; + + emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id, + 2, RELO_BR_HELPER); + + /* Load ptr type into A1 */ + wrp_mov(nfp_prog, reg_a(1), ptr_type); + + /* Load the return address into B0 */ + wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL); + + if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt)) + return -EINVAL; + + return 0; +} + +static int +nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + u32 jmp_tgt; + + jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5; + + /* Make sure the queue id fits into FW field */ + emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2), + ALU_OP_AND_NOT_B, reg_imm(0xff)); + emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2); + + /* Set the 'queue selected' bit and the queue value */ + emit_shf(nfp_prog, pv_qsel_set(nfp_prog), + pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1), + SHF_SC_L_SHF, PKT_VEL_QSEL_SET_BIT); + emit_ld_field(nfp_prog, + pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2), + SHF_SC_NONE, 0); + /* Delay slots end here, we will jump over next instruction if queue + * value fits into the field. + */ + emit_ld_field(nfp_prog, + pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX), + SHF_SC_NONE, 0); + + if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt)) + return -EINVAL; + + return 0; +} + /* --- Callbacks --- */ static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { @@ -1544,26 +1696,142 @@ static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return 0; } +/* Pseudo code: + * if shift_amt >= 32 + * dst_high = dst_low << shift_amt[4:0] + * dst_low = 0; + * else + * dst_high = (dst_high, dst_low) >> (32 - shift_amt) + * dst_low = dst_low << shift_amt + * + * The indirect shift will use the same logic at runtime. + */ +static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) +{ + if (shift_amt < 32) { + emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), + SHF_OP_NONE, reg_b(dst), SHF_SC_R_DSHF, + 32 - shift_amt); + emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, + reg_b(dst), SHF_SC_L_SHF, shift_amt); + } else if (shift_amt == 32) { + wrp_reg_mov(nfp_prog, dst + 1, dst); + wrp_immed(nfp_prog, reg_both(dst), 0); + } else if (shift_amt > 32) { + emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, + reg_b(dst), SHF_SC_L_SHF, shift_amt - 32); + wrp_immed(nfp_prog, reg_both(dst), 0); + } + + return 0; +} + static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { const struct bpf_insn *insn = &meta->insn; u8 dst = insn->dst_reg * 2; - if (insn->imm < 32) { - emit_shf(nfp_prog, reg_both(dst + 1), - reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), - SHF_SC_R_DSHF, 32 - insn->imm); - emit_shf(nfp_prog, reg_both(dst), - reg_none(), SHF_OP_NONE, reg_b(dst), - SHF_SC_L_SHF, insn->imm); - } else if (insn->imm == 32) { - wrp_reg_mov(nfp_prog, dst + 1, dst); - wrp_immed(nfp_prog, reg_both(dst), 0); - } else if (insn->imm > 32) { - emit_shf(nfp_prog, reg_both(dst + 1), - reg_none(), SHF_OP_NONE, reg_b(dst), - SHF_SC_L_SHF, insn->imm - 32); - wrp_immed(nfp_prog, reg_both(dst), 0); + return __shl_imm64(nfp_prog, dst, insn->imm); +} + +static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB, + reg_b(src)); + emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0)); + emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE, + reg_b(dst), SHF_SC_R_DSHF); +} + +/* NOTE: for indirect left shift, HIGH part should be calculated first. */ +static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); + emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, + reg_b(dst), SHF_SC_L_SHF); +} + +static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + shl_reg64_lt32_high(nfp_prog, dst, src); + shl_reg64_lt32_low(nfp_prog, dst, src); +} + +static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); + emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, + reg_b(dst), SHF_SC_L_SHF); + wrp_immed(nfp_prog, reg_both(dst), 0); +} + +static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + const struct bpf_insn *insn = &meta->insn; + u64 umin, umax; + u8 dst, src; + + dst = insn->dst_reg * 2; + umin = meta->umin; + umax = meta->umax; + if (umin == umax) + return __shl_imm64(nfp_prog, dst, umin); + + src = insn->src_reg * 2; + if (umax < 32) { + shl_reg64_lt32(nfp_prog, dst, src); + } else if (umin >= 32) { + shl_reg64_ge32(nfp_prog, dst, src); + } else { + /* Generate different instruction sequences depending on runtime + * value of shift amount. + */ + u16 label_ge32, label_end; + + label_ge32 = nfp_prog_current_offset(nfp_prog) + 7; + emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); + + shl_reg64_lt32_high(nfp_prog, dst, src); + label_end = nfp_prog_current_offset(nfp_prog) + 6; + emit_br(nfp_prog, BR_UNC, label_end, 2); + /* shl_reg64_lt32_low packed in delay slot. */ + shl_reg64_lt32_low(nfp_prog, dst, src); + + if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) + return -EINVAL; + shl_reg64_ge32(nfp_prog, dst, src); + + if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) + return -EINVAL; + } + + return 0; +} + +/* Pseudo code: + * if shift_amt >= 32 + * dst_high = 0; + * dst_low = dst_high >> shift_amt[4:0] + * else + * dst_high = dst_high >> shift_amt + * dst_low = (dst_high, dst_low) >> shift_amt + * + * The indirect shift will use the same logic at runtime. + */ +static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) +{ + if (shift_amt < 32) { + emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, + reg_b(dst), SHF_SC_R_DSHF, shift_amt); + emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, + reg_b(dst + 1), SHF_SC_R_SHF, shift_amt); + } else if (shift_amt == 32) { + wrp_reg_mov(nfp_prog, dst, dst + 1); + wrp_immed(nfp_prog, reg_both(dst + 1), 0); + } else if (shift_amt > 32) { + emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, + reg_b(dst + 1), SHF_SC_R_SHF, shift_amt - 32); + wrp_immed(nfp_prog, reg_both(dst + 1), 0); } return 0; @@ -1574,21 +1842,186 @@ static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) const struct bpf_insn *insn = &meta->insn; u8 dst = insn->dst_reg * 2; - if (insn->imm < 32) { - emit_shf(nfp_prog, reg_both(dst), - reg_a(dst + 1), SHF_OP_NONE, reg_b(dst), - SHF_SC_R_DSHF, insn->imm); - emit_shf(nfp_prog, reg_both(dst + 1), - reg_none(), SHF_OP_NONE, reg_b(dst + 1), - SHF_SC_R_SHF, insn->imm); - } else if (insn->imm == 32) { + return __shr_imm64(nfp_prog, dst, insn->imm); +} + +/* NOTE: for indirect right shift, LOW part should be calculated first. */ +static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); + emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE, + reg_b(dst + 1), SHF_SC_R_SHF); +} + +static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); + emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, + reg_b(dst), SHF_SC_R_DSHF); +} + +static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + shr_reg64_lt32_low(nfp_prog, dst, src); + shr_reg64_lt32_high(nfp_prog, dst, src); +} + +static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0)); + emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE, + reg_b(dst + 1), SHF_SC_R_SHF); + wrp_immed(nfp_prog, reg_both(dst + 1), 0); +} + +static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + const struct bpf_insn *insn = &meta->insn; + u64 umin, umax; + u8 dst, src; + + dst = insn->dst_reg * 2; + umin = meta->umin; + umax = meta->umax; + if (umin == umax) + return __shr_imm64(nfp_prog, dst, umin); + + src = insn->src_reg * 2; + if (umax < 32) { + shr_reg64_lt32(nfp_prog, dst, src); + } else if (umin >= 32) { + shr_reg64_ge32(nfp_prog, dst, src); + } else { + /* Generate different instruction sequences depending on runtime + * value of shift amount. + */ + u16 label_ge32, label_end; + + label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; + emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); + shr_reg64_lt32_low(nfp_prog, dst, src); + label_end = nfp_prog_current_offset(nfp_prog) + 6; + emit_br(nfp_prog, BR_UNC, label_end, 2); + /* shr_reg64_lt32_high packed in delay slot. */ + shr_reg64_lt32_high(nfp_prog, dst, src); + + if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) + return -EINVAL; + shr_reg64_ge32(nfp_prog, dst, src); + + if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) + return -EINVAL; + } + + return 0; +} + +/* Code logic is the same as __shr_imm64 except ashr requires signedness bit + * told through PREV_ALU result. + */ +static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt) +{ + if (shift_amt < 32) { + emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE, + reg_b(dst), SHF_SC_R_DSHF, shift_amt); + /* Set signedness bit. */ + emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, + reg_imm(0)); + emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF, shift_amt); + } else if (shift_amt == 32) { + /* NOTE: this also helps setting signedness bit. */ wrp_reg_mov(nfp_prog, dst, dst + 1); - wrp_immed(nfp_prog, reg_both(dst + 1), 0); - } else if (insn->imm > 32) { - emit_shf(nfp_prog, reg_both(dst), - reg_none(), SHF_OP_NONE, reg_b(dst + 1), - SHF_SC_R_SHF, insn->imm - 32); - wrp_immed(nfp_prog, reg_both(dst + 1), 0); + emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF, 31); + } else if (shift_amt > 32) { + emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR, + reg_imm(0)); + emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF, shift_amt - 32); + emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF, 31); + } + + return 0; +} + +static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + const struct bpf_insn *insn = &meta->insn; + u8 dst = insn->dst_reg * 2; + + return __ashr_imm64(nfp_prog, dst, insn->imm); +} + +static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + /* NOTE: the first insn will set both indirect shift amount (source A) + * and signedness bit (MSB of result). + */ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); + emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF); +} + +static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + /* NOTE: it is the same as logic shift because we don't need to shift in + * signedness bit when the shift amount is less than 32. + */ + return shr_reg64_lt32_low(nfp_prog, dst, src); +} + +static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + ashr_reg64_lt32_low(nfp_prog, dst, src); + ashr_reg64_lt32_high(nfp_prog, dst, src); +} + +static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src) +{ + emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1)); + emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF); + emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR, + reg_b(dst + 1), SHF_SC_R_SHF, 31); +} + +/* Like ashr_imm64, but need to use indirect shift. */ +static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + const struct bpf_insn *insn = &meta->insn; + u64 umin, umax; + u8 dst, src; + + dst = insn->dst_reg * 2; + umin = meta->umin; + umax = meta->umax; + if (umin == umax) + return __ashr_imm64(nfp_prog, dst, umin); + + src = insn->src_reg * 2; + if (umax < 32) { + ashr_reg64_lt32(nfp_prog, dst, src); + } else if (umin >= 32) { + ashr_reg64_ge32(nfp_prog, dst, src); + } else { + u16 label_ge32, label_end; + + label_ge32 = nfp_prog_current_offset(nfp_prog) + 6; + emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0); + ashr_reg64_lt32_low(nfp_prog, dst, src); + label_end = nfp_prog_current_offset(nfp_prog) + 6; + emit_br(nfp_prog, BR_UNC, label_end, 2); + /* ashr_reg64_lt32_high packed in delay slot. */ + ashr_reg64_lt32_high(nfp_prog, dst, src); + + if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32)) + return -EINVAL; + ashr_reg64_ge32(nfp_prog, dst, src); + + if (!nfp_prog_confirm_current_offset(nfp_prog, label_end)) + return -EINVAL; } return 0; @@ -2108,6 +2541,17 @@ mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, false, wrp_lmem_store); } +static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) +{ + switch (meta->insn.off) { + case offsetof(struct xdp_md, rx_queue_index): + return nfp_queue_select(nfp_prog, meta); + } + + WARN_ON_ONCE(1); /* verifier should have rejected bad accesses */ + return -EOPNOTSUPP; +} + static int mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, unsigned int size) @@ -2134,6 +2578,9 @@ static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { + if (meta->ptr.type == PTR_TO_CTX) + if (nfp_prog->type == BPF_PROG_TYPE_XDP) + return mem_stx_xdp(nfp_prog, meta); return mem_stx(nfp_prog, meta, 4); } @@ -2283,46 +2730,6 @@ static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return 0; } -static int jgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BLO, true); -} - -static int jge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BHS, false); -} - -static int jlt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BLO, false); -} - -static int jle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BHS, true); -} - -static int jsgt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BLT, true); -} - -static int jsge_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BGE, false); -} - -static int jslt_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BLT, false); -} - -static int jsle_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_imm(nfp_prog, meta, BR_BGE, true); -} - static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { const struct bpf_insn *insn = &meta->insn; @@ -2392,46 +2799,6 @@ static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return 0; } -static int jgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BLO, true); -} - -static int jge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BHS, false); -} - -static int jlt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BLO, false); -} - -static int jle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BHS, true); -} - -static int jsgt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BLT, true); -} - -static int jsge_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BGE, false); -} - -static int jslt_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BLT, false); -} - -static int jsle_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) -{ - return wrp_cmp_reg(nfp_prog, meta, BR_BGE, true); -} - static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) { return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE); @@ -2453,6 +2820,8 @@ static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta) return map_call_stack_common(nfp_prog, meta); case BPF_FUNC_get_prandom_u32: return nfp_get_prandom_u32(nfp_prog, meta); + case BPF_FUNC_perf_event_output: + return nfp_perf_event_output(nfp_prog, meta); default: WARN_ONCE(1, "verifier allowed unsupported function\n"); return -EOPNOTSUPP; @@ -2480,8 +2849,12 @@ static const instr_cb_t instr_cb[256] = { [BPF_ALU64 | BPF_SUB | BPF_X] = sub_reg64, [BPF_ALU64 | BPF_SUB | BPF_K] = sub_imm64, [BPF_ALU64 | BPF_NEG] = neg_reg64, + [BPF_ALU64 | BPF_LSH | BPF_X] = shl_reg64, [BPF_ALU64 | BPF_LSH | BPF_K] = shl_imm64, + [BPF_ALU64 | BPF_RSH | BPF_X] = shr_reg64, [BPF_ALU64 | BPF_RSH | BPF_K] = shr_imm64, + [BPF_ALU64 | BPF_ARSH | BPF_X] = ashr_reg64, + [BPF_ALU64 | BPF_ARSH | BPF_K] = ashr_imm64, [BPF_ALU | BPF_MOV | BPF_X] = mov_reg, [BPF_ALU | BPF_MOV | BPF_K] = mov_imm, [BPF_ALU | BPF_XOR | BPF_X] = xor_reg, @@ -2520,25 +2893,25 @@ static const instr_cb_t instr_cb[256] = { [BPF_ST | BPF_MEM | BPF_DW] = mem_st8, [BPF_JMP | BPF_JA | BPF_K] = jump, [BPF_JMP | BPF_JEQ | BPF_K] = jeq_imm, - [BPF_JMP | BPF_JGT | BPF_K] = jgt_imm, - [BPF_JMP | BPF_JGE | BPF_K] = jge_imm, - [BPF_JMP | BPF_JLT | BPF_K] = jlt_imm, - [BPF_JMP | BPF_JLE | BPF_K] = jle_imm, - [BPF_JMP | BPF_JSGT | BPF_K] = jsgt_imm, - [BPF_JMP | BPF_JSGE | BPF_K] = jsge_imm, - [BPF_JMP | BPF_JSLT | BPF_K] = jslt_imm, - [BPF_JMP | BPF_JSLE | BPF_K] = jsle_imm, + [BPF_JMP | BPF_JGT | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JGE | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JLT | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JLE | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JSGT | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JSGE | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JSLT | BPF_K] = cmp_imm, + [BPF_JMP | BPF_JSLE | BPF_K] = cmp_imm, [BPF_JMP | BPF_JSET | BPF_K] = jset_imm, [BPF_JMP | BPF_JNE | BPF_K] = jne_imm, [BPF_JMP | BPF_JEQ | BPF_X] = jeq_reg, - [BPF_JMP | BPF_JGT | BPF_X] = jgt_reg, - [BPF_JMP | BPF_JGE | BPF_X] = jge_reg, - [BPF_JMP | BPF_JLT | BPF_X] = jlt_reg, - [BPF_JMP | BPF_JLE | BPF_X] = jle_reg, - [BPF_JMP | BPF_JSGT | BPF_X] = jsgt_reg, - [BPF_JMP | BPF_JSGE | BPF_X] = jsge_reg, - [BPF_JMP | BPF_JSLT | BPF_X] = jslt_reg, - [BPF_JMP | BPF_JSLE | BPF_X] = jsle_reg, + [BPF_JMP | BPF_JGT | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JGE | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JLT | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JLE | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JSGT | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JSGE | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JSLT | BPF_X] = cmp_reg, + [BPF_JMP | BPF_JSLE | BPF_X] = cmp_reg, [BPF_JMP | BPF_JSET | BPF_X] = jset_reg, [BPF_JMP | BPF_JNE | BPF_X] = jne_reg, [BPF_JMP | BPF_CALL] = call, @@ -2777,6 +3150,54 @@ static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog) } } +/* abs(insn.imm) will fit better into unrestricted reg immediate - + * convert add/sub of a negative number into a sub/add of a positive one. + */ +static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog) +{ + struct nfp_insn_meta *meta; + + list_for_each_entry(meta, &nfp_prog->insns, l) { + struct bpf_insn insn = meta->insn; + + if (meta->skip) + continue; + + if (BPF_CLASS(insn.code) != BPF_ALU && + BPF_CLASS(insn.code) != BPF_ALU64 && + BPF_CLASS(insn.code) != BPF_JMP) + continue; + if (BPF_SRC(insn.code) != BPF_K) + continue; + if (insn.imm >= 0) + continue; + + if (BPF_CLASS(insn.code) == BPF_JMP) { + switch (BPF_OP(insn.code)) { + case BPF_JGE: + case BPF_JSGE: + case BPF_JLT: + case BPF_JSLT: + meta->jump_neg_op = true; + break; + default: + continue; + } + } else { + if (BPF_OP(insn.code) == BPF_ADD) + insn.code = BPF_CLASS(insn.code) | BPF_SUB; + else if (BPF_OP(insn.code) == BPF_SUB) + insn.code = BPF_CLASS(insn.code) | BPF_ADD; + else + continue; + + meta->insn.code = insn.code | BPF_K; + } + + meta->insn.imm = -insn.imm; + } +} + /* Remove masking after load since our load guarantees this is not needed */ static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog) { @@ -3212,6 +3633,7 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) { nfp_bpf_opt_reg_init(nfp_prog); + nfp_bpf_opt_neg_add_sub(nfp_prog); nfp_bpf_opt_ld_mask(nfp_prog); nfp_bpf_opt_ld_shift(nfp_prog); nfp_bpf_opt_ldst_gather(nfp_prog); @@ -3220,6 +3642,33 @@ static int nfp_bpf_optimize(struct nfp_prog *nfp_prog) return 0; } +static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog) +{ + struct nfp_insn_meta *meta1, *meta2; + struct nfp_bpf_map *nfp_map; + struct bpf_map *map; + + nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) { + if (meta1->skip || meta2->skip) + continue; + + if (meta1->insn.code != (BPF_LD | BPF_IMM | BPF_DW) || + meta1->insn.src_reg != BPF_PSEUDO_MAP_FD) + continue; + + map = (void *)(unsigned long)((u32)meta1->insn.imm | + (u64)meta2->insn.imm << 32); + if (bpf_map_offload_neutral(map)) + continue; + nfp_map = map_to_offmap(map)->dev_priv; + + meta1->insn.imm = nfp_map->tid; + meta2->insn.imm = 0; + } + + return 0; +} + static int nfp_bpf_ustore_calc(u64 *prog, unsigned int len) { __le64 *ustore = (__force __le64 *)prog; @@ -3256,6 +3705,10 @@ int nfp_bpf_jit(struct nfp_prog *nfp_prog) { int ret; + ret = nfp_bpf_replace_map_ptrs(nfp_prog); + if (ret) + return ret; + ret = nfp_bpf_optimize(nfp_prog); if (ret) return ret; @@ -3346,6 +3799,9 @@ void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv) case BPF_FUNC_map_delete_elem: val = nfp_prog->bpf->helpers.map_delete; break; + case BPF_FUNC_perf_event_output: + val = nfp_prog->bpf->helpers.perf_event_output; + break; default: pr_err("relocation of unknown helper %d\n", val); diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.c b/drivers/net/ethernet/netronome/nfp/bpf/main.c index 35fb31f682af4..fcdfb8e7fdeab 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Netronome Systems, Inc. + * Copyright (C) 2017-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -43,6 +43,14 @@ #include "fw.h" #include "main.h" +const struct rhashtable_params nfp_bpf_maps_neutral_params = { + .nelem_hint = 4, + .key_len = FIELD_SIZEOF(struct nfp_bpf_neutral_map, ptr), + .key_offset = offsetof(struct nfp_bpf_neutral_map, ptr), + .head_offset = offsetof(struct nfp_bpf_neutral_map, l), + .automatic_shrinking = true, +}; + static bool nfp_net_ebpf_capable(struct nfp_net *nn) { #ifdef __LITTLE_ENDIAN @@ -290,6 +298,9 @@ nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) case BPF_FUNC_map_delete_elem: bpf->helpers.map_delete = readl(&cap->func_addr); break; + case BPF_FUNC_perf_event_output: + bpf->helpers.perf_event_output = readl(&cap->func_addr); + break; } return 0; @@ -323,6 +334,13 @@ nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value, return 0; } +static int +nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) +{ + bpf->queue_select = true; + return 0; +} + static int nfp_bpf_parse_capabilities(struct nfp_app *app) { struct nfp_cpp *cpp = app->pf->cpp; @@ -365,6 +383,10 @@ static int nfp_bpf_parse_capabilities(struct nfp_app *app) if (nfp_bpf_parse_cap_random(app->priv, value, length)) goto err_release_free; break; + case NFP_BPF_CAP_TYPE_QUEUE_SELECT: + if (nfp_bpf_parse_cap_qsel(app->priv, value, length)) + goto err_release_free; + break; default: nfp_dbg(cpp, "unknown BPF capability: %d\n", type); break; @@ -401,17 +423,28 @@ static int nfp_bpf_init(struct nfp_app *app) init_waitqueue_head(&bpf->cmsg_wq); INIT_LIST_HEAD(&bpf->map_list); - err = nfp_bpf_parse_capabilities(app); + err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params); if (err) goto err_free_bpf; + err = nfp_bpf_parse_capabilities(app); + if (err) + goto err_free_neutral_maps; + return 0; +err_free_neutral_maps: + rhashtable_destroy(&bpf->maps_neutral); err_free_bpf: kfree(bpf); return err; } +static void nfp_check_rhashtable_empty(void *ptr, void *arg) +{ + WARN_ON_ONCE(1); +} + static void nfp_bpf_clean(struct nfp_app *app) { struct nfp_app_bpf *bpf = app->priv; @@ -419,6 +452,8 @@ static void nfp_bpf_clean(struct nfp_app *app) WARN_ON(!skb_queue_empty(&bpf->cmsg_replies)); WARN_ON(!list_empty(&bpf->map_list)); WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); + rhashtable_free_and_destroy(&bpf->maps_neutral, + nfp_check_rhashtable_empty, NULL); kfree(bpf); } diff --git a/drivers/net/ethernet/netronome/nfp/bpf/main.h b/drivers/net/ethernet/netronome/nfp/bpf/main.h index 4981c8944ca35..654fe7823e5ed 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/main.h +++ b/drivers/net/ethernet/netronome/nfp/bpf/main.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2017 Netronome Systems, Inc. + * Copyright (C) 2016-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -39,6 +39,7 @@ #include <linux/bpf_verifier.h> #include <linux/kernel.h> #include <linux/list.h> +#include <linux/rhashtable.h> #include <linux/skbuff.h> #include <linux/types.h> #include <linux/wait.h> @@ -81,10 +82,16 @@ enum static_regs { enum pkt_vec { PKT_VEC_PKT_LEN = 0, PKT_VEC_PKT_PTR = 2, + PKT_VEC_QSEL_SET = 4, + PKT_VEC_QSEL_VAL = 6, }; +#define PKT_VEL_QSEL_SET_BIT 4 + #define pv_len(np) reg_lm(1, PKT_VEC_PKT_LEN) #define pv_ctm_ptr(np) reg_lm(1, PKT_VEC_PKT_PTR) +#define pv_qsel_set(np) reg_lm(1, PKT_VEC_QSEL_SET) +#define pv_qsel_val(np) reg_lm(1, PKT_VEC_QSEL_VAL) #define stack_reg(np) reg_a(STATIC_REG_STACK) #define stack_imm(np) imm_b(np) @@ -114,6 +121,8 @@ enum pkt_vec { * @maps_in_use: number of currently offloaded maps * @map_elems_in_use: number of elements allocated to offloaded maps * + * @maps_neutral: hash table of offload-neutral maps (on pointer) + * * @adjust_head: adjust head capability * @adjust_head.flags: extra flags for adjust head * @adjust_head.off_min: minimal packet offset within buffer required @@ -133,8 +142,10 @@ enum pkt_vec { * @helpers.map_lookup: map lookup helper address * @helpers.map_update: map update helper address * @helpers.map_delete: map delete helper address + * @helpers.perf_event_output: output perf event to a ring buffer * * @pseudo_random: FW initialized the pseudo-random machinery (CSRs) + * @queue_select: BPF can set the RX queue ID in packet vector */ struct nfp_app_bpf { struct nfp_app *app; @@ -150,6 +161,8 @@ struct nfp_app_bpf { unsigned int maps_in_use; unsigned int map_elems_in_use; + struct rhashtable maps_neutral; + struct nfp_bpf_cap_adjust_head { u32 flags; int off_min; @@ -171,9 +184,11 @@ struct nfp_app_bpf { u32 map_lookup; u32 map_update; u32 map_delete; + u32 perf_event_output; } helpers; bool pseudo_random; + bool queue_select; }; enum nfp_bpf_map_use { @@ -199,6 +214,14 @@ struct nfp_bpf_map { enum nfp_bpf_map_use use_map[]; }; +struct nfp_bpf_neutral_map { + struct rhash_head l; + struct bpf_map *ptr; + u32 count; +}; + +extern const struct rhashtable_params nfp_bpf_maps_neutral_params; + struct nfp_prog; struct nfp_insn_meta; typedef int (*instr_cb_t)(struct nfp_prog *, struct nfp_insn_meta *); @@ -236,9 +259,12 @@ struct nfp_bpf_reg_state { * @xadd_over_16bit: 16bit immediate is not guaranteed * @xadd_maybe_16bit: 16bit immediate is possible * @jmp_dst: destination info for jump instructions + * @jump_neg_op: jump instruction has inverted immediate, use ADD instead of SUB * @func_id: function id for call instructions * @arg1: arg1 for call instructions * @arg2: arg2 for call instructions + * @umin: copy of core verifier umin_value. + * @umax: copy of core verifier umax_value. * @off: index of first generated machine instruction (in nfp_prog.prog) * @n: eBPF instruction number * @flags: eBPF instruction extra optimization flags @@ -264,13 +290,23 @@ struct nfp_insn_meta { bool xadd_maybe_16bit; }; /* jump */ - struct nfp_insn_meta *jmp_dst; + struct { + struct nfp_insn_meta *jmp_dst; + bool jump_neg_op; + }; /* function calls */ struct { u32 func_id; struct bpf_reg_state arg1; struct nfp_bpf_reg_state arg2; }; + /* We are interested in range info for some operands, + * for example, the shift amount. + */ + struct { + u64 umin; + u64 umax; + }; }; unsigned int off; unsigned short n; @@ -348,6 +384,25 @@ static inline bool is_mbpf_xadd(const struct nfp_insn_meta *meta) return (meta->insn.code & ~BPF_SIZE_MASK) == (BPF_STX | BPF_XADD); } +static inline bool is_mbpf_indir_shift(const struct nfp_insn_meta *meta) +{ + u8 code = meta->insn.code; + bool is_alu, is_shift; + u8 opclass, opcode; + + opclass = BPF_CLASS(code); + is_alu = opclass == BPF_ALU64 || opclass == BPF_ALU; + if (!is_alu) + return false; + + opcode = BPF_OP(code); + is_shift = opcode == BPF_LSH || opcode == BPF_RSH || opcode == BPF_ARSH; + if (!is_shift) + return false; + + return BPF_SRC(code) == BPF_X; +} + /** * struct nfp_prog - nfp BPF program * @bpf: backpointer to the bpf app priv structure @@ -363,6 +418,8 @@ static inline bool is_mbpf_xadd(const struct nfp_insn_meta *meta) * @error: error code if something went wrong * @stack_depth: max stack depth from the verifier * @adjust_head_location: if program has single adjust head call - the insn no. + * @map_records_cnt: the number of map pointers recorded for this prog + * @map_records: the map record pointers from bpf->maps_neutral * @insns: list of BPF instruction wrappers (struct nfp_insn_meta) */ struct nfp_prog { @@ -386,6 +443,9 @@ struct nfp_prog { unsigned int stack_depth; unsigned int adjust_head_location; + unsigned int map_records_cnt; + struct nfp_bpf_neutral_map **map_records; + struct list_head insns; }; @@ -436,5 +496,7 @@ int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap, int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap, void *key, void *next_key); +int nfp_bpf_event_output(struct nfp_app_bpf *bpf, struct sk_buff *skb); + void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb); #endif diff --git a/drivers/net/ethernet/netronome/nfp/bpf/offload.c b/drivers/net/ethernet/netronome/nfp/bpf/offload.c index 42d98792bd255..7eae4c0266f81 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/offload.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/offload.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2017 Netronome Systems, Inc. + * Copyright (C) 2016-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -57,6 +57,126 @@ #include "../nfp_net.h" static int +nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, + struct bpf_map *map) +{ + struct nfp_bpf_neutral_map *record; + int err; + + /* Map record paths are entered via ndo, update side is protected. */ + ASSERT_RTNL(); + + /* Reuse path - other offloaded program is already tracking this map. */ + record = rhashtable_lookup_fast(&bpf->maps_neutral, &map, + nfp_bpf_maps_neutral_params); + if (record) { + nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; + record->count++; + return 0; + } + + /* Grab a single ref to the map for our record. The prog destroy ndo + * happens after free_used_maps(). + */ + map = bpf_map_inc(map, false); + if (IS_ERR(map)) + return PTR_ERR(map); + + record = kmalloc(sizeof(*record), GFP_KERNEL); + if (!record) { + err = -ENOMEM; + goto err_map_put; + } + + record->ptr = map; + record->count = 1; + + err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l, + nfp_bpf_maps_neutral_params); + if (err) + goto err_free_rec; + + nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; + + return 0; + +err_free_rec: + kfree(record); +err_map_put: + bpf_map_put(map); + return err; +} + +static void +nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) +{ + bool freed = false; + int i; + + ASSERT_RTNL(); + + for (i = 0; i < nfp_prog->map_records_cnt; i++) { + if (--nfp_prog->map_records[i]->count) { + nfp_prog->map_records[i] = NULL; + continue; + } + + WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral, + &nfp_prog->map_records[i]->l, + nfp_bpf_maps_neutral_params)); + freed = true; + } + + if (freed) { + synchronize_rcu(); + + for (i = 0; i < nfp_prog->map_records_cnt; i++) + if (nfp_prog->map_records[i]) { + bpf_map_put(nfp_prog->map_records[i]->ptr); + kfree(nfp_prog->map_records[i]); + } + } + + kfree(nfp_prog->map_records); + nfp_prog->map_records = NULL; + nfp_prog->map_records_cnt = 0; +} + +static int +nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, + struct bpf_prog *prog) +{ + int i, cnt, err; + + /* Quickly count the maps we will have to remember */ + cnt = 0; + for (i = 0; i < prog->aux->used_map_cnt; i++) + if (bpf_map_offload_neutral(prog->aux->used_maps[i])) + cnt++; + if (!cnt) + return 0; + + nfp_prog->map_records = kmalloc_array(cnt, + sizeof(nfp_prog->map_records[0]), + GFP_KERNEL); + if (!nfp_prog->map_records) + return -ENOMEM; + + for (i = 0; i < prog->aux->used_map_cnt; i++) + if (bpf_map_offload_neutral(prog->aux->used_maps[i])) { + err = nfp_map_ptr_record(bpf, nfp_prog, + prog->aux->used_maps[i]); + if (err) { + nfp_map_ptrs_forget(bpf, nfp_prog); + return err; + } + } + WARN_ON(cnt != nfp_prog->map_records_cnt); + + return 0; +} + +static int nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, unsigned int cnt) { @@ -70,6 +190,8 @@ nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, meta->insn = prog[i]; meta->n = i; + if (is_mbpf_indir_shift(meta)) + meta->umin = U64_MAX; list_add_tail(&meta->l, &nfp_prog->insns); } @@ -151,7 +273,7 @@ static int nfp_bpf_translate(struct nfp_net *nn, struct bpf_prog *prog) prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); prog->aux->offload->jited_image = nfp_prog->prog; - return 0; + return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog); } static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog) @@ -159,6 +281,7 @@ static int nfp_bpf_destroy(struct nfp_net *nn, struct bpf_prog *prog) struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; kvfree(nfp_prog->prog); + nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog); nfp_prog_free(nfp_prog); return 0; @@ -320,6 +443,53 @@ int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf) } } +static unsigned long +nfp_bpf_perf_event_copy(void *dst, const void *src, + unsigned long off, unsigned long len) +{ + memcpy(dst, src + off, len); + return 0; +} + +int nfp_bpf_event_output(struct nfp_app_bpf *bpf, struct sk_buff *skb) +{ + struct cmsg_bpf_event *cbe = (void *)skb->data; + u32 pkt_size, data_size; + struct bpf_map *map; + + if (skb->len < sizeof(struct cmsg_bpf_event)) + goto err_drop; + + pkt_size = be32_to_cpu(cbe->pkt_size); + data_size = be32_to_cpu(cbe->data_size); + map = (void *)(unsigned long)be64_to_cpu(cbe->map_ptr); + + if (skb->len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size) + goto err_drop; + if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION) + goto err_drop; + + rcu_read_lock(); + if (!rhashtable_lookup_fast(&bpf->maps_neutral, &map, + nfp_bpf_maps_neutral_params)) { + rcu_read_unlock(); + pr_warn("perf event: dest map pointer %px not recognized, dropping event\n", + map); + goto err_drop; + } + + bpf_event_output(map, be32_to_cpu(cbe->cpu_id), + &cbe->data[round_up(pkt_size, 4)], data_size, + cbe->data, pkt_size, nfp_bpf_perf_event_copy); + rcu_read_unlock(); + + dev_consume_skb_any(skb); + return 0; +err_drop: + dev_kfree_skb_any(skb); + return -EINVAL; +} + static int nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog, struct netlink_ext_ack *extack) diff --git a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c index 06ad53ce4ad93..4bfeba7b21b22 100644 --- a/drivers/net/ethernet/netronome/nfp/bpf/verifier.c +++ b/drivers/net/ethernet/netronome/nfp/bpf/verifier.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016-2017 Netronome Systems, Inc. + * Copyright (C) 2016-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -36,6 +36,8 @@ #include <linux/kernel.h> #include <linux/pkt_cls.h> +#include "../nfp_app.h" +#include "../nfp_main.h" #include "fw.h" #include "main.h" @@ -149,15 +151,6 @@ nfp_bpf_map_call_ok(const char *fname, struct bpf_verifier_env *env, return false; } - /* Rest of the checks is only if we re-parse the same insn */ - if (!meta->func_id) - return true; - - if (meta->arg1.map_ptr != reg1->map_ptr) { - pr_vlog(env, "%s: called for different map\n", fname); - return false; - } - return true; } @@ -216,6 +209,71 @@ nfp_bpf_check_call(struct nfp_prog *nfp_prog, struct bpf_verifier_env *env, pr_vlog(env, "bpf_get_prandom_u32(): FW doesn't support random number generation\n"); return -EOPNOTSUPP; + case BPF_FUNC_perf_event_output: + BUILD_BUG_ON(NFP_BPF_SCALAR_VALUE != SCALAR_VALUE || + NFP_BPF_MAP_VALUE != PTR_TO_MAP_VALUE || + NFP_BPF_STACK != PTR_TO_STACK || + NFP_BPF_PACKET_DATA != PTR_TO_PACKET); + + if (!bpf->helpers.perf_event_output) { + pr_vlog(env, "event_output: not supported by FW\n"); + return -EOPNOTSUPP; + } + + /* Force current CPU to make sure we can report the event + * wherever we get the control message from FW. + */ + if (reg3->var_off.mask & BPF_F_INDEX_MASK || + (reg3->var_off.value & BPF_F_INDEX_MASK) != + BPF_F_CURRENT_CPU) { + char tn_buf[48]; + + tnum_strn(tn_buf, sizeof(tn_buf), reg3->var_off); + pr_vlog(env, "event_output: must use BPF_F_CURRENT_CPU, var_off: %s\n", + tn_buf); + return -EOPNOTSUPP; + } + + /* Save space in meta, we don't care about arguments other + * than 4th meta, shove it into arg1. + */ + reg1 = cur_regs(env) + BPF_REG_4; + + if (reg1->type != SCALAR_VALUE /* NULL ptr */ && + reg1->type != PTR_TO_STACK && + reg1->type != PTR_TO_MAP_VALUE && + reg1->type != PTR_TO_PACKET) { + pr_vlog(env, "event_output: unsupported ptr type: %d\n", + reg1->type); + return -EOPNOTSUPP; + } + + if (reg1->type == PTR_TO_STACK && + !nfp_bpf_stack_arg_ok("event_output", env, reg1, NULL)) + return -EOPNOTSUPP; + + /* Warn user that on offload NFP may return success even if map + * is not going to accept the event, since the event output is + * fully async and device won't know the state of the map. + * There is also FW limitation on the event length. + * + * Lost events will not show up on the perf ring, driver + * won't see them at all. Events may also get reordered. + */ + dev_warn_once(&nfp_prog->bpf->app->pf->pdev->dev, + "bpf: note: return codes and behavior of bpf_event_output() helper differs for offloaded programs!\n"); + pr_vlog(env, "warning: return codes and behavior of event_output helper differ for offload!\n"); + + if (!meta->func_id) + break; + + if (reg1->type != meta->arg1.type) { + pr_vlog(env, "event_output: ptr type changed: %d %d\n", + meta->arg1.type, reg1->type); + return -EINVAL; + } + break; + default: pr_vlog(env, "unsupported function id: %d\n", func_id); return -EOPNOTSUPP; @@ -410,6 +468,30 @@ nfp_bpf_check_ptr(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, } static int +nfp_bpf_check_store(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, + struct bpf_verifier_env *env) +{ + const struct bpf_reg_state *reg = cur_regs(env) + meta->insn.dst_reg; + + if (reg->type == PTR_TO_CTX) { + if (nfp_prog->type == BPF_PROG_TYPE_XDP) { + /* XDP ctx accesses must be 4B in size */ + switch (meta->insn.off) { + case offsetof(struct xdp_md, rx_queue_index): + if (nfp_prog->bpf->queue_select) + goto exit_check_ptr; + pr_vlog(env, "queue selection not supported by FW\n"); + return -EOPNOTSUPP; + } + } + pr_vlog(env, "unsupported store to context field\n"); + return -EOPNOTSUPP; + } +exit_check_ptr: + return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.dst_reg); +} + +static int nfp_bpf_check_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, struct bpf_verifier_env *env) { @@ -464,11 +546,19 @@ nfp_verify_insn(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx) return nfp_bpf_check_ptr(nfp_prog, meta, env, meta->insn.src_reg); if (is_mbpf_store(meta)) - return nfp_bpf_check_ptr(nfp_prog, meta, env, - meta->insn.dst_reg); + return nfp_bpf_check_store(nfp_prog, meta, env); + if (is_mbpf_xadd(meta)) return nfp_bpf_check_xadd(nfp_prog, meta, env); + if (is_mbpf_indir_shift(meta)) { + const struct bpf_reg_state *sreg = + cur_regs(env) + meta->insn.src_reg; + + meta->umin = min(meta->umin, sreg->umin_value); + meta->umax = max(meta->umax, sreg->umax_value); + } + return 0; } diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c index 80df9a5d4217d..4a6d2db750719 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/action.c +++ b/drivers/net/ethernet/netronome/nfp/flower/action.c @@ -72,6 +72,42 @@ nfp_fl_push_vlan(struct nfp_fl_push_vlan *push_vlan, push_vlan->vlan_tci = cpu_to_be16(tmp_push_vlan_tci); } +static int +nfp_fl_pre_lag(struct nfp_app *app, const struct tc_action *action, + struct nfp_fl_payload *nfp_flow, int act_len) +{ + size_t act_size = sizeof(struct nfp_fl_pre_lag); + struct nfp_fl_pre_lag *pre_lag; + struct net_device *out_dev; + int err; + + out_dev = tcf_mirred_dev(action); + if (!out_dev || !netif_is_lag_master(out_dev)) + return 0; + + if (act_len + act_size > NFP_FL_MAX_A_SIZ) + return -EOPNOTSUPP; + + /* Pre_lag action must be first on action list. + * If other actions already exist they need pushed forward. + */ + if (act_len) + memmove(nfp_flow->action_data + act_size, + nfp_flow->action_data, act_len); + + pre_lag = (struct nfp_fl_pre_lag *)nfp_flow->action_data; + err = nfp_flower_lag_populate_pre_action(app, out_dev, pre_lag); + if (err) + return err; + + pre_lag->head.jump_id = NFP_FL_ACTION_OPCODE_PRE_LAG; + pre_lag->head.len_lw = act_size >> NFP_FL_LW_SIZ; + + nfp_flow->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_NULL); + + return act_size; +} + static bool nfp_fl_netdev_is_tunnel_type(struct net_device *out_dev, enum nfp_flower_tun_type tun_type) { @@ -88,12 +124,13 @@ static bool nfp_fl_netdev_is_tunnel_type(struct net_device *out_dev, } static int -nfp_fl_output(struct nfp_fl_output *output, const struct tc_action *action, - struct nfp_fl_payload *nfp_flow, bool last, - struct net_device *in_dev, enum nfp_flower_tun_type tun_type, - int *tun_out_cnt) +nfp_fl_output(struct nfp_app *app, struct nfp_fl_output *output, + const struct tc_action *action, struct nfp_fl_payload *nfp_flow, + bool last, struct net_device *in_dev, + enum nfp_flower_tun_type tun_type, int *tun_out_cnt) { size_t act_size = sizeof(struct nfp_fl_output); + struct nfp_flower_priv *priv = app->priv; struct net_device *out_dev; u16 tmp_flags; @@ -118,6 +155,15 @@ nfp_fl_output(struct nfp_fl_output *output, const struct tc_action *action, output->flags = cpu_to_be16(tmp_flags | NFP_FL_OUT_FLAGS_USE_TUN); output->port = cpu_to_be32(NFP_FL_PORT_TYPE_TUN | tun_type); + } else if (netif_is_lag_master(out_dev) && + priv->flower_ext_feats & NFP_FL_FEATS_LAG) { + int gid; + + output->flags = cpu_to_be16(tmp_flags); + gid = nfp_flower_lag_get_output_id(app, out_dev); + if (gid < 0) + return gid; + output->port = cpu_to_be32(NFP_FL_LAG_OUT | gid); } else { /* Set action output parameters. */ output->flags = cpu_to_be16(tmp_flags); @@ -164,7 +210,7 @@ static struct nfp_fl_pre_tunnel *nfp_fl_pre_tunnel(char *act_data, int act_len) struct nfp_fl_pre_tunnel *pre_tun_act; /* Pre_tunnel action must be first on action list. - * If other actions already exist they need pushed forward. + * If other actions already exist they need to be pushed forward. */ if (act_len) memmove(act_data + act_size, act_data, act_len); @@ -443,42 +489,73 @@ nfp_fl_pedit(const struct tc_action *action, char *nfp_action, int *a_len) } static int -nfp_flower_loop_action(const struct tc_action *a, +nfp_flower_output_action(struct nfp_app *app, const struct tc_action *a, + struct nfp_fl_payload *nfp_fl, int *a_len, + struct net_device *netdev, bool last, + enum nfp_flower_tun_type *tun_type, int *tun_out_cnt, + int *out_cnt) +{ + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_output *output; + int err, prelag_size; + + if (*a_len + sizeof(struct nfp_fl_output) > NFP_FL_MAX_A_SIZ) + return -EOPNOTSUPP; + + output = (struct nfp_fl_output *)&nfp_fl->action_data[*a_len]; + err = nfp_fl_output(app, output, a, nfp_fl, last, netdev, *tun_type, + tun_out_cnt); + if (err) + return err; + + *a_len += sizeof(struct nfp_fl_output); + + if (priv->flower_ext_feats & NFP_FL_FEATS_LAG) { + /* nfp_fl_pre_lag returns -err or size of prelag action added. + * This will be 0 if it is not egressing to a lag dev. + */ + prelag_size = nfp_fl_pre_lag(app, a, nfp_fl, *a_len); + if (prelag_size < 0) + return prelag_size; + else if (prelag_size > 0 && (!last || *out_cnt)) + return -EOPNOTSUPP; + + *a_len += prelag_size; + } + (*out_cnt)++; + + return 0; +} + +static int +nfp_flower_loop_action(struct nfp_app *app, const struct tc_action *a, struct nfp_fl_payload *nfp_fl, int *a_len, struct net_device *netdev, - enum nfp_flower_tun_type *tun_type, int *tun_out_cnt) + enum nfp_flower_tun_type *tun_type, int *tun_out_cnt, + int *out_cnt) { struct nfp_fl_set_ipv4_udp_tun *set_tun; struct nfp_fl_pre_tunnel *pre_tun; struct nfp_fl_push_vlan *psh_v; struct nfp_fl_pop_vlan *pop_v; - struct nfp_fl_output *output; int err; if (is_tcf_gact_shot(a)) { nfp_fl->meta.shortcut = cpu_to_be32(NFP_FL_SC_ACT_DROP); } else if (is_tcf_mirred_egress_redirect(a)) { - if (*a_len + sizeof(struct nfp_fl_output) > NFP_FL_MAX_A_SIZ) - return -EOPNOTSUPP; - - output = (struct nfp_fl_output *)&nfp_fl->action_data[*a_len]; - err = nfp_fl_output(output, a, nfp_fl, true, netdev, *tun_type, - tun_out_cnt); + err = nfp_flower_output_action(app, a, nfp_fl, a_len, netdev, + true, tun_type, tun_out_cnt, + out_cnt); if (err) return err; - *a_len += sizeof(struct nfp_fl_output); } else if (is_tcf_mirred_egress_mirror(a)) { - if (*a_len + sizeof(struct nfp_fl_output) > NFP_FL_MAX_A_SIZ) - return -EOPNOTSUPP; - - output = (struct nfp_fl_output *)&nfp_fl->action_data[*a_len]; - err = nfp_fl_output(output, a, nfp_fl, false, netdev, *tun_type, - tun_out_cnt); + err = nfp_flower_output_action(app, a, nfp_fl, a_len, netdev, + false, tun_type, tun_out_cnt, + out_cnt); if (err) return err; - *a_len += sizeof(struct nfp_fl_output); } else if (is_tcf_vlan(a) && tcf_vlan_action(a) == TCA_VLAN_ACT_POP) { if (*a_len + sizeof(struct nfp_fl_pop_vlan) > NFP_FL_MAX_A_SIZ) return -EOPNOTSUPP; @@ -535,11 +612,12 @@ nfp_flower_loop_action(const struct tc_action *a, return 0; } -int nfp_flower_compile_action(struct tc_cls_flower_offload *flow, +int nfp_flower_compile_action(struct nfp_app *app, + struct tc_cls_flower_offload *flow, struct net_device *netdev, struct nfp_fl_payload *nfp_flow) { - int act_len, act_cnt, err, tun_out_cnt; + int act_len, act_cnt, err, tun_out_cnt, out_cnt; enum nfp_flower_tun_type tun_type; const struct tc_action *a; LIST_HEAD(actions); @@ -550,11 +628,12 @@ int nfp_flower_compile_action(struct tc_cls_flower_offload *flow, act_len = 0; act_cnt = 0; tun_out_cnt = 0; + out_cnt = 0; tcf_exts_to_list(flow->exts, &actions); list_for_each_entry(a, &actions, list) { - err = nfp_flower_loop_action(a, nfp_flow, &act_len, netdev, - &tun_type, &tun_out_cnt); + err = nfp_flower_loop_action(app, a, nfp_flow, &act_len, netdev, + &tun_type, &tun_out_cnt, &out_cnt); if (err) return err; act_cnt++; diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c index 577659f332e41..cb8565222621b 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.c +++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.c @@ -239,8 +239,10 @@ nfp_flower_cmsg_portreify_rx(struct nfp_app *app, struct sk_buff *skb) static void nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb) { + struct nfp_flower_priv *app_priv = app->priv; struct nfp_flower_cmsg_hdr *cmsg_hdr; enum nfp_flower_cmsg_type_port type; + bool skb_stored = false; cmsg_hdr = nfp_flower_cmsg_get_hdr(skb); @@ -258,13 +260,20 @@ nfp_flower_cmsg_process_one_rx(struct nfp_app *app, struct sk_buff *skb) case NFP_FLOWER_CMSG_TYPE_ACTIVE_TUNS: nfp_tunnel_keep_alive(app, skb); break; + case NFP_FLOWER_CMSG_TYPE_LAG_CONFIG: + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) { + skb_stored = nfp_flower_lag_unprocessed_msg(app, skb); + break; + } + /* fall through */ default: nfp_flower_cmsg_warn(app, "Cannot handle invalid repr control type %u\n", type); goto out; } - dev_consume_skb_any(skb); + if (!skb_stored) + dev_consume_skb_any(skb); return; out: dev_kfree_skb_any(skb); diff --git a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h index bee4367a2c388..4a7f3510a2968 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/cmsg.h +++ b/drivers/net/ethernet/netronome/nfp/flower/cmsg.h @@ -92,6 +92,7 @@ #define NFP_FL_ACTION_OPCODE_SET_IPV6_DST 12 #define NFP_FL_ACTION_OPCODE_SET_UDP 14 #define NFP_FL_ACTION_OPCODE_SET_TCP 15 +#define NFP_FL_ACTION_OPCODE_PRE_LAG 16 #define NFP_FL_ACTION_OPCODE_PRE_TUNNEL 17 #define NFP_FL_ACTION_OPCODE_NUM 32 @@ -103,6 +104,9 @@ #define NFP_FL_PUSH_VLAN_CFI BIT(12) #define NFP_FL_PUSH_VLAN_VID GENMASK(11, 0) +/* LAG ports */ +#define NFP_FL_LAG_OUT 0xC0DE0000 + /* Tunnel ports */ #define NFP_FL_PORT_TYPE_TUN 0x50000000 #define NFP_FL_IPV4_TUNNEL_TYPE GENMASK(7, 4) @@ -177,6 +181,15 @@ struct nfp_fl_pop_vlan { __be16 reserved; }; +struct nfp_fl_pre_lag { + struct nfp_fl_act_head head; + __be16 group_id; + u8 lag_version[3]; + u8 instance; +}; + +#define NFP_FL_PRE_LAG_VER_OFF 8 + struct nfp_fl_pre_tunnel { struct nfp_fl_act_head head; __be16 reserved; @@ -366,6 +379,7 @@ struct nfp_flower_cmsg_hdr { enum nfp_flower_cmsg_type_port { NFP_FLOWER_CMSG_TYPE_FLOW_ADD = 0, NFP_FLOWER_CMSG_TYPE_FLOW_DEL = 2, + NFP_FLOWER_CMSG_TYPE_LAG_CONFIG = 4, NFP_FLOWER_CMSG_TYPE_PORT_REIFY = 6, NFP_FLOWER_CMSG_TYPE_MAC_REPR = 7, NFP_FLOWER_CMSG_TYPE_PORT_MOD = 8, diff --git a/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c b/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c new file mode 100644 index 0000000000000..0c4c957717ea4 --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/flower/lag_conf.c @@ -0,0 +1,726 @@ +/* + * Copyright (C) 2018 Netronome Systems, Inc. + * + * This software is dual licensed under the GNU General License Version 2, + * June 1991 as shown in the file COPYING in the top-level directory of this + * source tree or the BSD 2-Clause License provided below. You have the + * option to license this software under the complete terms of either license. + * + * The BSD 2-Clause License: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "main.h" + +/* LAG group config flags. */ +#define NFP_FL_LAG_LAST BIT(1) +#define NFP_FL_LAG_FIRST BIT(2) +#define NFP_FL_LAG_DATA BIT(3) +#define NFP_FL_LAG_XON BIT(4) +#define NFP_FL_LAG_SYNC BIT(5) +#define NFP_FL_LAG_SWITCH BIT(6) +#define NFP_FL_LAG_RESET BIT(7) + +/* LAG port state flags. */ +#define NFP_PORT_LAG_LINK_UP BIT(0) +#define NFP_PORT_LAG_TX_ENABLED BIT(1) +#define NFP_PORT_LAG_CHANGED BIT(2) + +enum nfp_fl_lag_batch { + NFP_FL_LAG_BATCH_FIRST, + NFP_FL_LAG_BATCH_MEMBER, + NFP_FL_LAG_BATCH_FINISHED +}; + +/** + * struct nfp_flower_cmsg_lag_config - control message payload for LAG config + * @ctrl_flags: Configuration flags + * @reserved: Reserved for future use + * @ttl: Time to live of packet - host always sets to 0xff + * @pkt_number: Config message packet number - increment for each message + * @batch_ver: Batch version of messages - increment for each batch of messages + * @group_id: Group ID applicable + * @group_inst: Group instance number - increment when group is reused + * @members: Array of 32-bit words listing all active group members + */ +struct nfp_flower_cmsg_lag_config { + u8 ctrl_flags; + u8 reserved[2]; + u8 ttl; + __be32 pkt_number; + __be32 batch_ver; + __be32 group_id; + __be32 group_inst; + __be32 members[]; +}; + +/** + * struct nfp_fl_lag_group - list entry for each LAG group + * @group_id: Assigned group ID for host/kernel sync + * @group_inst: Group instance in case of ID reuse + * @list: List entry + * @master_ndev: Group master Netdev + * @dirty: Marked if the group needs synced to HW + * @offloaded: Marked if the group is currently offloaded to NIC + * @to_remove: Marked if the group should be removed from NIC + * @to_destroy: Marked if the group should be removed from driver + * @slave_cnt: Number of slaves in group + */ +struct nfp_fl_lag_group { + unsigned int group_id; + u8 group_inst; + struct list_head list; + struct net_device *master_ndev; + bool dirty; + bool offloaded; + bool to_remove; + bool to_destroy; + unsigned int slave_cnt; +}; + +#define NFP_FL_LAG_PKT_NUMBER_MASK GENMASK(30, 0) +#define NFP_FL_LAG_VERSION_MASK GENMASK(22, 0) +#define NFP_FL_LAG_HOST_TTL 0xff + +/* Use this ID with zero members to ack a batch config */ +#define NFP_FL_LAG_SYNC_ID 0 +#define NFP_FL_LAG_GROUP_MIN 1 /* ID 0 reserved */ +#define NFP_FL_LAG_GROUP_MAX 32 /* IDs 1 to 31 are valid */ + +/* wait for more config */ +#define NFP_FL_LAG_DELAY (msecs_to_jiffies(2)) + +#define NFP_FL_LAG_RETRANS_LIMIT 100 /* max retrans cmsgs to store */ + +static unsigned int nfp_fl_get_next_pkt_number(struct nfp_fl_lag *lag) +{ + lag->pkt_num++; + lag->pkt_num &= NFP_FL_LAG_PKT_NUMBER_MASK; + + return lag->pkt_num; +} + +static void nfp_fl_increment_version(struct nfp_fl_lag *lag) +{ + /* LSB is not considered by firmware so add 2 for each increment. */ + lag->batch_ver += 2; + lag->batch_ver &= NFP_FL_LAG_VERSION_MASK; + + /* Zero is reserved by firmware. */ + if (!lag->batch_ver) + lag->batch_ver += 2; +} + +static struct nfp_fl_lag_group * +nfp_fl_lag_group_create(struct nfp_fl_lag *lag, struct net_device *master) +{ + struct nfp_fl_lag_group *group; + struct nfp_flower_priv *priv; + int id; + + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + + id = ida_simple_get(&lag->ida_handle, NFP_FL_LAG_GROUP_MIN, + NFP_FL_LAG_GROUP_MAX, GFP_KERNEL); + if (id < 0) { + nfp_flower_cmsg_warn(priv->app, + "No more bonding groups available\n"); + return ERR_PTR(id); + } + + group = kmalloc(sizeof(*group), GFP_KERNEL); + if (!group) { + ida_simple_remove(&lag->ida_handle, id); + return ERR_PTR(-ENOMEM); + } + + group->group_id = id; + group->master_ndev = master; + group->dirty = true; + group->offloaded = false; + group->to_remove = false; + group->to_destroy = false; + group->slave_cnt = 0; + group->group_inst = ++lag->global_inst; + list_add_tail(&group->list, &lag->group_list); + + return group; +} + +static struct nfp_fl_lag_group * +nfp_fl_lag_find_group_for_master_with_lag(struct nfp_fl_lag *lag, + struct net_device *master) +{ + struct nfp_fl_lag_group *entry; + + if (!master) + return NULL; + + list_for_each_entry(entry, &lag->group_list, list) + if (entry->master_ndev == master) + return entry; + + return NULL; +} + +int nfp_flower_lag_populate_pre_action(struct nfp_app *app, + struct net_device *master, + struct nfp_fl_pre_lag *pre_act) +{ + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_lag_group *group = NULL; + __be32 temp_vers; + + mutex_lock(&priv->nfp_lag.lock); + group = nfp_fl_lag_find_group_for_master_with_lag(&priv->nfp_lag, + master); + if (!group) { + mutex_unlock(&priv->nfp_lag.lock); + return -ENOENT; + } + + pre_act->group_id = cpu_to_be16(group->group_id); + temp_vers = cpu_to_be32(priv->nfp_lag.batch_ver << + NFP_FL_PRE_LAG_VER_OFF); + memcpy(pre_act->lag_version, &temp_vers, 3); + pre_act->instance = group->group_inst; + mutex_unlock(&priv->nfp_lag.lock); + + return 0; +} + +int nfp_flower_lag_get_output_id(struct nfp_app *app, struct net_device *master) +{ + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_lag_group *group = NULL; + int group_id = -ENOENT; + + mutex_lock(&priv->nfp_lag.lock); + group = nfp_fl_lag_find_group_for_master_with_lag(&priv->nfp_lag, + master); + if (group) + group_id = group->group_id; + mutex_unlock(&priv->nfp_lag.lock); + + return group_id; +} + +static int +nfp_fl_lag_config_group(struct nfp_fl_lag *lag, struct nfp_fl_lag_group *group, + struct net_device **active_members, + unsigned int member_cnt, enum nfp_fl_lag_batch *batch) +{ + struct nfp_flower_cmsg_lag_config *cmsg_payload; + struct nfp_flower_priv *priv; + unsigned long int flags; + unsigned int size, i; + struct sk_buff *skb; + + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + size = sizeof(*cmsg_payload) + sizeof(__be32) * member_cnt; + skb = nfp_flower_cmsg_alloc(priv->app, size, + NFP_FLOWER_CMSG_TYPE_LAG_CONFIG, + GFP_KERNEL); + if (!skb) + return -ENOMEM; + + cmsg_payload = nfp_flower_cmsg_get_data(skb); + flags = 0; + + /* Increment batch version for each new batch of config messages. */ + if (*batch == NFP_FL_LAG_BATCH_FIRST) { + flags |= NFP_FL_LAG_FIRST; + nfp_fl_increment_version(lag); + *batch = NFP_FL_LAG_BATCH_MEMBER; + } + + /* If it is a reset msg then it is also the end of the batch. */ + if (lag->rst_cfg) { + flags |= NFP_FL_LAG_RESET; + *batch = NFP_FL_LAG_BATCH_FINISHED; + } + + /* To signal the end of a batch, both the switch and last flags are set + * and the the reserved SYNC group ID is used. + */ + if (*batch == NFP_FL_LAG_BATCH_FINISHED) { + flags |= NFP_FL_LAG_SWITCH | NFP_FL_LAG_LAST; + lag->rst_cfg = false; + cmsg_payload->group_id = cpu_to_be32(NFP_FL_LAG_SYNC_ID); + cmsg_payload->group_inst = 0; + } else { + cmsg_payload->group_id = cpu_to_be32(group->group_id); + cmsg_payload->group_inst = cpu_to_be32(group->group_inst); + } + + cmsg_payload->reserved[0] = 0; + cmsg_payload->reserved[1] = 0; + cmsg_payload->ttl = NFP_FL_LAG_HOST_TTL; + cmsg_payload->ctrl_flags = flags; + cmsg_payload->batch_ver = cpu_to_be32(lag->batch_ver); + cmsg_payload->pkt_number = cpu_to_be32(nfp_fl_get_next_pkt_number(lag)); + + for (i = 0; i < member_cnt; i++) + cmsg_payload->members[i] = + cpu_to_be32(nfp_repr_get_port_id(active_members[i])); + + nfp_ctrl_tx(priv->app->ctrl, skb); + return 0; +} + +static void nfp_fl_lag_do_work(struct work_struct *work) +{ + enum nfp_fl_lag_batch batch = NFP_FL_LAG_BATCH_FIRST; + struct nfp_fl_lag_group *entry, *storage; + struct delayed_work *delayed_work; + struct nfp_flower_priv *priv; + struct nfp_fl_lag *lag; + int err; + + delayed_work = to_delayed_work(work); + lag = container_of(delayed_work, struct nfp_fl_lag, work); + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + + mutex_lock(&lag->lock); + list_for_each_entry_safe(entry, storage, &lag->group_list, list) { + struct net_device *iter_netdev, **acti_netdevs; + struct nfp_flower_repr_priv *repr_priv; + int active_count = 0, slaves = 0; + struct nfp_repr *repr; + unsigned long *flags; + + if (entry->to_remove) { + /* Active count of 0 deletes group on hw. */ + err = nfp_fl_lag_config_group(lag, entry, NULL, 0, + &batch); + if (!err) { + entry->to_remove = false; + entry->offloaded = false; + } else { + nfp_flower_cmsg_warn(priv->app, + "group delete failed\n"); + schedule_delayed_work(&lag->work, + NFP_FL_LAG_DELAY); + continue; + } + + if (entry->to_destroy) { + ida_simple_remove(&lag->ida_handle, + entry->group_id); + list_del(&entry->list); + kfree(entry); + } + continue; + } + + acti_netdevs = kmalloc_array(entry->slave_cnt, + sizeof(*acti_netdevs), GFP_KERNEL); + + /* Include sanity check in the loop. It may be that a bond has + * changed between processing the last notification and the + * work queue triggering. If the number of slaves has changed + * or it now contains netdevs that cannot be offloaded, ignore + * the group until pending notifications are processed. + */ + rcu_read_lock(); + for_each_netdev_in_bond_rcu(entry->master_ndev, iter_netdev) { + if (!nfp_netdev_is_nfp_repr(iter_netdev)) { + slaves = 0; + break; + } + + repr = netdev_priv(iter_netdev); + + if (repr->app != priv->app) { + slaves = 0; + break; + } + + slaves++; + if (slaves > entry->slave_cnt) + break; + + /* Check the ports for state changes. */ + repr_priv = repr->app_priv; + flags = &repr_priv->lag_port_flags; + + if (*flags & NFP_PORT_LAG_CHANGED) { + *flags &= ~NFP_PORT_LAG_CHANGED; + entry->dirty = true; + } + + if ((*flags & NFP_PORT_LAG_TX_ENABLED) && + (*flags & NFP_PORT_LAG_LINK_UP)) + acti_netdevs[active_count++] = iter_netdev; + } + rcu_read_unlock(); + + if (slaves != entry->slave_cnt || !entry->dirty) { + kfree(acti_netdevs); + continue; + } + + err = nfp_fl_lag_config_group(lag, entry, acti_netdevs, + active_count, &batch); + if (!err) { + entry->offloaded = true; + entry->dirty = false; + } else { + nfp_flower_cmsg_warn(priv->app, + "group offload failed\n"); + schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); + } + + kfree(acti_netdevs); + } + + /* End the config batch if at least one packet has been batched. */ + if (batch == NFP_FL_LAG_BATCH_MEMBER) { + batch = NFP_FL_LAG_BATCH_FINISHED; + err = nfp_fl_lag_config_group(lag, NULL, NULL, 0, &batch); + if (err) + nfp_flower_cmsg_warn(priv->app, + "group batch end cmsg failed\n"); + } + + mutex_unlock(&lag->lock); +} + +static int +nfp_fl_lag_put_unprocessed(struct nfp_fl_lag *lag, struct sk_buff *skb) +{ + struct nfp_flower_cmsg_lag_config *cmsg_payload; + + cmsg_payload = nfp_flower_cmsg_get_data(skb); + if (be32_to_cpu(cmsg_payload->group_id) >= NFP_FL_LAG_GROUP_MAX) + return -EINVAL; + + /* Drop cmsg retrans if storage limit is exceeded to prevent + * overloading. If the fw notices that expected messages have not been + * received in a given time block, it will request a full resync. + */ + if (skb_queue_len(&lag->retrans_skbs) >= NFP_FL_LAG_RETRANS_LIMIT) + return -ENOSPC; + + __skb_queue_tail(&lag->retrans_skbs, skb); + + return 0; +} + +static void nfp_fl_send_unprocessed(struct nfp_fl_lag *lag) +{ + struct nfp_flower_priv *priv; + struct sk_buff *skb; + + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + + while ((skb = __skb_dequeue(&lag->retrans_skbs))) + nfp_ctrl_tx(priv->app->ctrl, skb); +} + +bool nfp_flower_lag_unprocessed_msg(struct nfp_app *app, struct sk_buff *skb) +{ + struct nfp_flower_cmsg_lag_config *cmsg_payload; + struct nfp_flower_priv *priv = app->priv; + struct nfp_fl_lag_group *group_entry; + unsigned long int flags; + bool store_skb = false; + int err; + + cmsg_payload = nfp_flower_cmsg_get_data(skb); + flags = cmsg_payload->ctrl_flags; + + /* Note the intentional fall through below. If DATA and XON are both + * set, the message will stored and sent again with the rest of the + * unprocessed messages list. + */ + + /* Store */ + if (flags & NFP_FL_LAG_DATA) + if (!nfp_fl_lag_put_unprocessed(&priv->nfp_lag, skb)) + store_skb = true; + + /* Send stored */ + if (flags & NFP_FL_LAG_XON) + nfp_fl_send_unprocessed(&priv->nfp_lag); + + /* Resend all */ + if (flags & NFP_FL_LAG_SYNC) { + /* To resend all config: + * 1) Clear all unprocessed messages + * 2) Mark all groups dirty + * 3) Reset NFP group config + * 4) Schedule a LAG config update + */ + + __skb_queue_purge(&priv->nfp_lag.retrans_skbs); + + mutex_lock(&priv->nfp_lag.lock); + list_for_each_entry(group_entry, &priv->nfp_lag.group_list, + list) + group_entry->dirty = true; + + err = nfp_flower_lag_reset(&priv->nfp_lag); + if (err) + nfp_flower_cmsg_warn(priv->app, + "mem err in group reset msg\n"); + mutex_unlock(&priv->nfp_lag.lock); + + schedule_delayed_work(&priv->nfp_lag.work, 0); + } + + return store_skb; +} + +static void +nfp_fl_lag_schedule_group_remove(struct nfp_fl_lag *lag, + struct nfp_fl_lag_group *group) +{ + group->to_remove = true; + + schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); +} + +static int +nfp_fl_lag_schedule_group_delete(struct nfp_fl_lag *lag, + struct net_device *master) +{ + struct nfp_fl_lag_group *group; + + mutex_lock(&lag->lock); + group = nfp_fl_lag_find_group_for_master_with_lag(lag, master); + if (!group) { + mutex_unlock(&lag->lock); + return -ENOENT; + } + + group->to_remove = true; + group->to_destroy = true; + mutex_unlock(&lag->lock); + + schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); + return 0; +} + +static int +nfp_fl_lag_changeupper_event(struct nfp_fl_lag *lag, + struct netdev_notifier_changeupper_info *info) +{ + struct net_device *upper = info->upper_dev, *iter_netdev; + struct netdev_lag_upper_info *lag_upper_info; + struct nfp_fl_lag_group *group; + struct nfp_flower_priv *priv; + unsigned int slave_count = 0; + bool can_offload = true; + struct nfp_repr *repr; + + if (!netif_is_lag_master(upper)) + return 0; + + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + + rcu_read_lock(); + for_each_netdev_in_bond_rcu(upper, iter_netdev) { + if (!nfp_netdev_is_nfp_repr(iter_netdev)) { + can_offload = false; + break; + } + repr = netdev_priv(iter_netdev); + + /* Ensure all ports are created by the same app/on same card. */ + if (repr->app != priv->app) { + can_offload = false; + break; + } + + slave_count++; + } + rcu_read_unlock(); + + lag_upper_info = info->upper_info; + + /* Firmware supports active/backup and L3/L4 hash bonds. */ + if (lag_upper_info && + lag_upper_info->tx_type != NETDEV_LAG_TX_TYPE_ACTIVEBACKUP && + (lag_upper_info->tx_type != NETDEV_LAG_TX_TYPE_HASH || + (lag_upper_info->hash_type != NETDEV_LAG_HASH_L34 && + lag_upper_info->hash_type != NETDEV_LAG_HASH_E34))) { + can_offload = false; + nfp_flower_cmsg_warn(priv->app, + "Unable to offload tx_type %u hash %u\n", + lag_upper_info->tx_type, + lag_upper_info->hash_type); + } + + mutex_lock(&lag->lock); + group = nfp_fl_lag_find_group_for_master_with_lag(lag, upper); + + if (slave_count == 0 || !can_offload) { + /* Cannot offload the group - remove if previously offloaded. */ + if (group && group->offloaded) + nfp_fl_lag_schedule_group_remove(lag, group); + + mutex_unlock(&lag->lock); + return 0; + } + + if (!group) { + group = nfp_fl_lag_group_create(lag, upper); + if (IS_ERR(group)) { + mutex_unlock(&lag->lock); + return PTR_ERR(group); + } + } + + group->dirty = true; + group->slave_cnt = slave_count; + + /* Group may have been on queue for removal but is now offfloable. */ + group->to_remove = false; + mutex_unlock(&lag->lock); + + schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); + return 0; +} + +static int +nfp_fl_lag_changels_event(struct nfp_fl_lag *lag, struct net_device *netdev, + struct netdev_notifier_changelowerstate_info *info) +{ + struct netdev_lag_lower_state_info *lag_lower_info; + struct nfp_flower_repr_priv *repr_priv; + struct nfp_flower_priv *priv; + struct nfp_repr *repr; + unsigned long *flags; + + if (!netif_is_lag_port(netdev) || !nfp_netdev_is_nfp_repr(netdev)) + return 0; + + lag_lower_info = info->lower_state_info; + if (!lag_lower_info) + return 0; + + priv = container_of(lag, struct nfp_flower_priv, nfp_lag); + repr = netdev_priv(netdev); + + /* Verify that the repr is associated with this app. */ + if (repr->app != priv->app) + return 0; + + repr_priv = repr->app_priv; + flags = &repr_priv->lag_port_flags; + + mutex_lock(&lag->lock); + if (lag_lower_info->link_up) + *flags |= NFP_PORT_LAG_LINK_UP; + else + *flags &= ~NFP_PORT_LAG_LINK_UP; + + if (lag_lower_info->tx_enabled) + *flags |= NFP_PORT_LAG_TX_ENABLED; + else + *flags &= ~NFP_PORT_LAG_TX_ENABLED; + + *flags |= NFP_PORT_LAG_CHANGED; + mutex_unlock(&lag->lock); + + schedule_delayed_work(&lag->work, NFP_FL_LAG_DELAY); + return 0; +} + +static int +nfp_fl_lag_netdev_event(struct notifier_block *nb, unsigned long event, + void *ptr) +{ + struct net_device *netdev; + struct nfp_fl_lag *lag; + int err; + + netdev = netdev_notifier_info_to_dev(ptr); + lag = container_of(nb, struct nfp_fl_lag, lag_nb); + + switch (event) { + case NETDEV_CHANGEUPPER: + err = nfp_fl_lag_changeupper_event(lag, ptr); + if (err) + return NOTIFY_BAD; + return NOTIFY_OK; + case NETDEV_CHANGELOWERSTATE: + err = nfp_fl_lag_changels_event(lag, netdev, ptr); + if (err) + return NOTIFY_BAD; + return NOTIFY_OK; + case NETDEV_UNREGISTER: + if (netif_is_bond_master(netdev)) { + err = nfp_fl_lag_schedule_group_delete(lag, netdev); + if (err) + return NOTIFY_BAD; + return NOTIFY_OK; + } + } + + return NOTIFY_DONE; +} + +int nfp_flower_lag_reset(struct nfp_fl_lag *lag) +{ + enum nfp_fl_lag_batch batch = NFP_FL_LAG_BATCH_FIRST; + + lag->rst_cfg = true; + return nfp_fl_lag_config_group(lag, NULL, NULL, 0, &batch); +} + +void nfp_flower_lag_init(struct nfp_fl_lag *lag) +{ + INIT_DELAYED_WORK(&lag->work, nfp_fl_lag_do_work); + INIT_LIST_HEAD(&lag->group_list); + mutex_init(&lag->lock); + ida_init(&lag->ida_handle); + + __skb_queue_head_init(&lag->retrans_skbs); + + /* 0 is a reserved batch version so increment to first valid value. */ + nfp_fl_increment_version(lag); + + lag->lag_nb.notifier_call = nfp_fl_lag_netdev_event; +} + +void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag) +{ + struct nfp_fl_lag_group *entry, *storage; + + cancel_delayed_work_sync(&lag->work); + + __skb_queue_purge(&lag->retrans_skbs); + + /* Remove all groups. */ + mutex_lock(&lag->lock); + list_for_each_entry_safe(entry, storage, &lag->group_list, list) { + list_del(&entry->list); + kfree(entry); + } + mutex_unlock(&lag->lock); + mutex_destroy(&lag->lock); + ida_destroy(&lag->ida_handle); +} diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.c b/drivers/net/ethernet/netronome/nfp/flower/main.c index 84e3b9f5abb14..19cfa162ac655 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.c +++ b/drivers/net/ethernet/netronome/nfp/flower/main.c @@ -185,6 +185,10 @@ nfp_flower_repr_netdev_init(struct nfp_app *app, struct net_device *netdev) static void nfp_flower_repr_netdev_clean(struct nfp_app *app, struct net_device *netdev) { + struct nfp_repr *repr = netdev_priv(netdev); + + kfree(repr->app_priv); + tc_setup_cb_egdev_unregister(netdev, nfp_flower_setup_tc_egress_cb, netdev_priv(netdev)); } @@ -225,7 +229,9 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app, u8 nfp_pcie = nfp_cppcore_pcie_unit(app->pf->cpp); struct nfp_flower_priv *priv = app->priv; atomic_t *replies = &priv->reify_replies; + struct nfp_flower_repr_priv *repr_priv; enum nfp_port_type port_type; + struct nfp_repr *nfp_repr; struct nfp_reprs *reprs; int i, err, reify_cnt; const u8 queue = 0; @@ -247,12 +253,25 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app, err = -ENOMEM; goto err_reprs_clean; } - RCU_INIT_POINTER(reprs->reprs[i], repr); + + repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL); + if (!repr_priv) { + err = -ENOMEM; + goto err_reprs_clean; + } + + nfp_repr = netdev_priv(repr); + nfp_repr->app_priv = repr_priv; /* For now we only support 1 PF */ WARN_ON(repr_type == NFP_REPR_TYPE_PF && i); port = nfp_port_alloc(app, port_type, repr); + if (IS_ERR(port)) { + err = PTR_ERR(port); + nfp_repr_free(repr); + goto err_reprs_clean; + } if (repr_type == NFP_REPR_TYPE_PF) { port->pf_id = i; port->vnic = priv->nn->dp.ctrl_bar; @@ -271,9 +290,11 @@ nfp_flower_spawn_vnic_reprs(struct nfp_app *app, port_id, port, priv->nn->dp.netdev); if (err) { nfp_port_free(port); + nfp_repr_free(repr); goto err_reprs_clean; } + RCU_INIT_POINTER(reprs->reprs[i], repr); nfp_info(app->cpp, "%s%d Representor(%s) created\n", repr_type == NFP_REPR_TYPE_PF ? "PF" : "VF", i, repr->name); @@ -318,6 +339,8 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv) { struct nfp_eth_table *eth_tbl = app->pf->eth_tbl; atomic_t *replies = &priv->reify_replies; + struct nfp_flower_repr_priv *repr_priv; + struct nfp_repr *nfp_repr; struct sk_buff *ctrl_skb; struct nfp_reprs *reprs; int err, reify_cnt; @@ -344,16 +367,26 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv) err = -ENOMEM; goto err_reprs_clean; } - RCU_INIT_POINTER(reprs->reprs[phys_port], repr); + + repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL); + if (!repr_priv) { + err = -ENOMEM; + goto err_reprs_clean; + } + + nfp_repr = netdev_priv(repr); + nfp_repr->app_priv = repr_priv; port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT, repr); if (IS_ERR(port)) { err = PTR_ERR(port); + nfp_repr_free(repr); goto err_reprs_clean; } err = nfp_port_init_phy_port(app->pf, app, port, i); if (err) { nfp_port_free(port); + nfp_repr_free(repr); goto err_reprs_clean; } @@ -365,6 +398,7 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv) cmsg_port_id, port, priv->nn->dp.netdev); if (err) { nfp_port_free(port); + nfp_repr_free(repr); goto err_reprs_clean; } @@ -373,6 +407,7 @@ nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv) eth_tbl->ports[i].base, phys_port); + RCU_INIT_POINTER(reprs->reprs[phys_port], repr); nfp_info(app->cpp, "Phys Port %d Representor(%s) created\n", phys_port, repr->name); } @@ -537,8 +572,22 @@ static int nfp_flower_init(struct nfp_app *app) else app_priv->flower_ext_feats = features; + /* Tell the firmware that the driver supports lag. */ + err = nfp_rtsym_write_le(app->pf->rtbl, + "_abi_flower_balance_sync_enable", 1); + if (!err) { + app_priv->flower_ext_feats |= NFP_FL_FEATS_LAG; + nfp_flower_lag_init(&app_priv->nfp_lag); + } else if (err == -ENOENT) { + nfp_warn(app->cpp, "LAG not supported by FW.\n"); + } else { + goto err_cleanup_metadata; + } + return 0; +err_cleanup_metadata: + nfp_flower_metadata_cleanup(app); err_free_app_priv: vfree(app->priv); return err; @@ -552,6 +601,9 @@ static void nfp_flower_clean(struct nfp_app *app) skb_queue_purge(&app_priv->cmsg_skbs_low); flush_work(&app_priv->cmsg_work); + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) + nfp_flower_lag_cleanup(&app_priv->nfp_lag); + nfp_flower_metadata_cleanup(app); vfree(app->priv); app->priv = NULL; @@ -618,11 +670,29 @@ nfp_flower_repr_change_mtu(struct nfp_app *app, struct net_device *netdev, static int nfp_flower_start(struct nfp_app *app) { + struct nfp_flower_priv *app_priv = app->priv; + int err; + + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) { + err = nfp_flower_lag_reset(&app_priv->nfp_lag); + if (err) + return err; + + err = register_netdevice_notifier(&app_priv->nfp_lag.lag_nb); + if (err) + return err; + } + return nfp_tunnel_config_start(app); } static void nfp_flower_stop(struct nfp_app *app) { + struct nfp_flower_priv *app_priv = app->priv; + + if (app_priv->flower_ext_feats & NFP_FL_FEATS_LAG) + unregister_netdevice_notifier(&app_priv->nfp_lag.lag_nb); + nfp_tunnel_config_stop(app); } diff --git a/drivers/net/ethernet/netronome/nfp/flower/main.h b/drivers/net/ethernet/netronome/nfp/flower/main.h index c67e1b54c6141..bbe5764d26cb7 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/main.h +++ b/drivers/net/ethernet/netronome/nfp/flower/main.h @@ -43,10 +43,13 @@ #include <net/pkt_cls.h> #include <net/tcp.h> #include <linux/workqueue.h> +#include <linux/idr.h> +struct nfp_fl_pre_lag; struct net_device; struct nfp_app; +#define NFP_FL_STATS_CTX_DONT_CARE cpu_to_be32(0xffffffff) #define NFP_FL_STATS_ENTRY_RS BIT(20) #define NFP_FL_STATS_ELEM_RS 4 #define NFP_FL_REPEATED_HASH_MAX BIT(17) @@ -66,6 +69,7 @@ struct nfp_app; /* Extra features bitmap. */ #define NFP_FL_FEATS_GENEVE BIT(0) #define NFP_FL_NBI_MTU_SETTING BIT(1) +#define NFP_FL_FEATS_LAG BIT(31) struct nfp_fl_mask_id { struct circ_buf mask_id_free_list; @@ -96,6 +100,33 @@ struct nfp_mtu_conf { }; /** + * struct nfp_fl_lag - Flower APP priv data for link aggregation + * @lag_nb: Notifier to track master/slave events + * @work: Work queue for writing configs to the HW + * @lock: Lock to protect lag_group_list + * @group_list: List of all master/slave groups offloaded + * @ida_handle: IDA to handle group ids + * @pkt_num: Incremented for each config packet sent + * @batch_ver: Incremented for each batch of config packets + * @global_inst: Instance allocator for groups + * @rst_cfg: Marker to reset HW LAG config + * @retrans_skbs: Cmsgs that could not be processed by HW and require + * retransmission + */ +struct nfp_fl_lag { + struct notifier_block lag_nb; + struct delayed_work work; + struct mutex lock; + struct list_head group_list; + struct ida ida_handle; + unsigned int pkt_num; + unsigned int batch_ver; + u8 global_inst; + bool rst_cfg; + struct sk_buff_head retrans_skbs; +}; + +/** * struct nfp_flower_priv - Flower APP per-vNIC priv data * @app: Back pointer to app * @nn: Pointer to vNIC @@ -127,6 +158,7 @@ struct nfp_mtu_conf { * from firmware for repr reify * @reify_wait_queue: wait queue for repr reify response counting * @mtu_conf: Configuration of repr MTU value + * @nfp_lag: Link aggregation data block */ struct nfp_flower_priv { struct nfp_app *app; @@ -156,6 +188,15 @@ struct nfp_flower_priv { atomic_t reify_replies; wait_queue_head_t reify_wait_queue; struct nfp_mtu_conf mtu_conf; + struct nfp_fl_lag nfp_lag; +}; + +/** + * struct nfp_flower_repr_priv - Flower APP per-repr priv data + * @lag_port_flags: Extended port flags to record lag state of repr + */ +struct nfp_flower_repr_priv { + unsigned long lag_port_flags; }; struct nfp_fl_key_ls { @@ -189,9 +230,11 @@ struct nfp_fl_payload { spinlock_t lock; /* lock stats */ struct nfp_fl_stats stats; __be32 nfp_tun_ipv4_addr; + struct net_device *ingress_dev; char *unmasked_data; char *mask_data; char *action_data; + bool ingress_offload; }; struct nfp_fl_stats_frame { @@ -211,17 +254,20 @@ int nfp_flower_compile_flow_match(struct tc_cls_flower_offload *flow, struct net_device *netdev, struct nfp_fl_payload *nfp_flow, enum nfp_flower_tun_type tun_type); -int nfp_flower_compile_action(struct tc_cls_flower_offload *flow, +int nfp_flower_compile_action(struct nfp_app *app, + struct tc_cls_flower_offload *flow, struct net_device *netdev, struct nfp_fl_payload *nfp_flow); int nfp_compile_flow_metadata(struct nfp_app *app, struct tc_cls_flower_offload *flow, - struct nfp_fl_payload *nfp_flow); + struct nfp_fl_payload *nfp_flow, + struct net_device *netdev); int nfp_modify_flow_metadata(struct nfp_app *app, struct nfp_fl_payload *nfp_flow); struct nfp_fl_payload * -nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie); +nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, + struct net_device *netdev, __be32 host_ctx); struct nfp_fl_payload * nfp_flower_remove_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie); @@ -236,5 +282,14 @@ void nfp_tunnel_request_route(struct nfp_app *app, struct sk_buff *skb); void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb); int nfp_flower_setup_tc_egress_cb(enum tc_setup_type type, void *type_data, void *cb_priv); +void nfp_flower_lag_init(struct nfp_fl_lag *lag); +void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag); +int nfp_flower_lag_reset(struct nfp_fl_lag *lag); +bool nfp_flower_lag_unprocessed_msg(struct nfp_app *app, struct sk_buff *skb); +int nfp_flower_lag_populate_pre_action(struct nfp_app *app, + struct net_device *master, + struct nfp_fl_pre_lag *pre_act); +int nfp_flower_lag_get_output_id(struct nfp_app *app, + struct net_device *master); #endif diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c index db977cf8e9332..21668aa435e81 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c +++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c @@ -99,14 +99,18 @@ static int nfp_get_stats_entry(struct nfp_app *app, u32 *stats_context_id) /* Must be called with either RTNL or rcu_read_lock */ struct nfp_fl_payload * -nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie) +nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie, + struct net_device *netdev, __be32 host_ctx) { struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *flower_entry; hash_for_each_possible_rcu(priv->flow_table, flower_entry, link, tc_flower_cookie) - if (flower_entry->tc_flower_cookie == tc_flower_cookie) + if (flower_entry->tc_flower_cookie == tc_flower_cookie && + (!netdev || flower_entry->ingress_dev == netdev) && + (host_ctx == NFP_FL_STATS_CTX_DONT_CARE || + flower_entry->meta.host_ctx_id == host_ctx)) return flower_entry; return NULL; @@ -121,13 +125,11 @@ nfp_flower_update_stats(struct nfp_app *app, struct nfp_fl_stats_frame *stats) flower_cookie = be64_to_cpu(stats->stats_cookie); rcu_read_lock(); - nfp_flow = nfp_flower_search_fl_table(app, flower_cookie); + nfp_flow = nfp_flower_search_fl_table(app, flower_cookie, NULL, + stats->stats_con_id); if (!nfp_flow) goto exit_rcu_unlock; - if (nfp_flow->meta.host_ctx_id != stats->stats_con_id) - goto exit_rcu_unlock; - spin_lock(&nfp_flow->lock); nfp_flow->stats.pkts += be32_to_cpu(stats->pkt_count); nfp_flow->stats.bytes += be64_to_cpu(stats->byte_count); @@ -317,7 +319,8 @@ nfp_check_mask_remove(struct nfp_app *app, char *mask_data, u32 mask_len, int nfp_compile_flow_metadata(struct nfp_app *app, struct tc_cls_flower_offload *flow, - struct nfp_fl_payload *nfp_flow) + struct nfp_fl_payload *nfp_flow, + struct net_device *netdev) { struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *check_entry; @@ -348,7 +351,8 @@ int nfp_compile_flow_metadata(struct nfp_app *app, nfp_flow->stats.bytes = 0; nfp_flow->stats.used = jiffies; - check_entry = nfp_flower_search_fl_table(app, flow->cookie); + check_entry = nfp_flower_search_fl_table(app, flow->cookie, netdev, + NFP_FL_STATS_CTX_DONT_CARE); if (check_entry) { if (nfp_release_stats_entry(app, stats_cxt)) return -EINVAL; diff --git a/drivers/net/ethernet/netronome/nfp/flower/offload.c b/drivers/net/ethernet/netronome/nfp/flower/offload.c index 114d2ab02a389..c42e64f32333f 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/offload.c +++ b/drivers/net/ethernet/netronome/nfp/flower/offload.c @@ -345,7 +345,7 @@ nfp_flower_calculate_key_layers(struct nfp_app *app, } static struct nfp_fl_payload * -nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer) +nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer, bool egress) { struct nfp_fl_payload *flow_pay; @@ -371,6 +371,8 @@ nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer) flow_pay->meta.flags = 0; spin_lock_init(&flow_pay->lock); + flow_pay->ingress_offload = !egress; + return flow_pay; err_free_mask: @@ -402,8 +404,20 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, struct nfp_flower_priv *priv = app->priv; struct nfp_fl_payload *flow_pay; struct nfp_fl_key_ls *key_layer; + struct net_device *ingr_dev; int err; + ingr_dev = egress ? NULL : netdev; + flow_pay = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev, + NFP_FL_STATS_CTX_DONT_CARE); + if (flow_pay) { + /* Ignore as duplicate if it has been added by different cb. */ + if (flow_pay->ingress_offload && egress) + return 0; + else + return -EOPNOTSUPP; + } + key_layer = kmalloc(sizeof(*key_layer), GFP_KERNEL); if (!key_layer) return -ENOMEM; @@ -413,22 +427,25 @@ nfp_flower_add_offload(struct nfp_app *app, struct net_device *netdev, if (err) goto err_free_key_ls; - flow_pay = nfp_flower_allocate_new(key_layer); + flow_pay = nfp_flower_allocate_new(key_layer, egress); if (!flow_pay) { err = -ENOMEM; goto err_free_key_ls; } + flow_pay->ingress_dev = egress ? NULL : netdev; + err = nfp_flower_compile_flow_match(flow, key_layer, netdev, flow_pay, tun_type); if (err) goto err_destroy_flow; - err = nfp_flower_compile_action(flow, netdev, flow_pay); + err = nfp_flower_compile_action(app, flow, netdev, flow_pay); if (err) goto err_destroy_flow; - err = nfp_compile_flow_metadata(app, flow, flow_pay); + err = nfp_compile_flow_metadata(app, flow, flow_pay, + flow_pay->ingress_dev); if (err) goto err_destroy_flow; @@ -462,6 +479,7 @@ err_free_key_ls: * @app: Pointer to the APP handle * @netdev: netdev structure. * @flow: TC flower classifier offload structure + * @egress: Netdev is the egress dev. * * Removes a flow from the repeated hash structure and clears the * action payload. @@ -470,15 +488,18 @@ err_free_key_ls: */ static int nfp_flower_del_offload(struct nfp_app *app, struct net_device *netdev, - struct tc_cls_flower_offload *flow) + struct tc_cls_flower_offload *flow, bool egress) { struct nfp_port *port = nfp_port_from_netdev(netdev); struct nfp_fl_payload *nfp_flow; + struct net_device *ingr_dev; int err; - nfp_flow = nfp_flower_search_fl_table(app, flow->cookie); + ingr_dev = egress ? NULL : netdev; + nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev, + NFP_FL_STATS_CTX_DONT_CARE); if (!nfp_flow) - return -ENOENT; + return egress ? 0 : -ENOENT; err = nfp_modify_flow_metadata(app, nfp_flow); if (err) @@ -505,7 +526,9 @@ err_free_flow: /** * nfp_flower_get_stats() - Populates flow stats obtained from hardware. * @app: Pointer to the APP handle + * @netdev: Netdev structure. * @flow: TC flower classifier offload structure + * @egress: Netdev is the egress dev. * * Populates a flow statistics structure which which corresponds to a * specific flow. @@ -513,14 +536,21 @@ err_free_flow: * Return: negative value on error, 0 if stats populated successfully. */ static int -nfp_flower_get_stats(struct nfp_app *app, struct tc_cls_flower_offload *flow) +nfp_flower_get_stats(struct nfp_app *app, struct net_device *netdev, + struct tc_cls_flower_offload *flow, bool egress) { struct nfp_fl_payload *nfp_flow; + struct net_device *ingr_dev; - nfp_flow = nfp_flower_search_fl_table(app, flow->cookie); + ingr_dev = egress ? NULL : netdev; + nfp_flow = nfp_flower_search_fl_table(app, flow->cookie, ingr_dev, + NFP_FL_STATS_CTX_DONT_CARE); if (!nfp_flow) return -EINVAL; + if (nfp_flow->ingress_offload && egress) + return 0; + spin_lock_bh(&nfp_flow->lock); tcf_exts_stats_update(flow->exts, nfp_flow->stats.bytes, nfp_flow->stats.pkts, nfp_flow->stats.used); @@ -543,9 +573,9 @@ nfp_flower_repr_offload(struct nfp_app *app, struct net_device *netdev, case TC_CLSFLOWER_REPLACE: return nfp_flower_add_offload(app, netdev, flower, egress); case TC_CLSFLOWER_DESTROY: - return nfp_flower_del_offload(app, netdev, flower); + return nfp_flower_del_offload(app, netdev, flower, egress); case TC_CLSFLOWER_STATS: - return nfp_flower_get_stats(app, flower); + return nfp_flower_get_stats(app, netdev, flower, egress); } return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_abi.h b/drivers/net/ethernet/netronome/nfp/nfp_abi.h new file mode 100644 index 0000000000000..8b56c27931bf9 --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/nfp_abi.h @@ -0,0 +1,143 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) */ +/* + * Copyright (C) 2018 Netronome Systems, Inc. + * + * This software is dual licensed under the GNU General License Version 2, + * June 1991 as shown in the file COPYING in the top-level directory of this + * source tree or the BSD 2-Clause License provided below. You have the + * option to license this software under the complete terms of either license. + * + * The BSD 2-Clause License: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef __NFP_ABI__ +#define __NFP_ABI__ 1 + +#include <linux/types.h> + +#define NFP_MBOX_SYM_NAME "_abi_nfd_pf%u_mbox" +#define NFP_MBOX_SYM_MIN_SIZE 16 /* When no data needed */ + +#define NFP_MBOX_CMD 0x00 +#define NFP_MBOX_RET 0x04 +#define NFP_MBOX_DATA_LEN 0x08 +#define NFP_MBOX_RESERVED 0x0c +#define NFP_MBOX_DATA 0x10 + +/** + * enum nfp_mbox_cmd - PF mailbox commands + * + * @NFP_MBOX_NO_CMD: null command + * Used to indicate previous command has finished. + * + * @NFP_MBOX_POOL_GET: get shared buffer pool info/config + * Input - struct nfp_shared_buf_pool_id + * Output - struct nfp_shared_buf_pool_info_get + * + * @NFP_MBOX_POOL_SET: set shared buffer pool info/config + * Input - struct nfp_shared_buf_pool_info_set + * Output - None + * + * @NFP_MBOX_PCIE_ABM_ENABLE: enable PCIe-side advanced buffer management + * Enable advanced buffer management of the PCIe block. If ABM is disabled + * PCIe block maintains a very short queue of buffers and does tail drop. + * ABM allows more advanced buffering and priority control. + * Input - None + * Output - None + * + * @NFP_MBOX_PCIE_ABM_DISABLE: disable PCIe-side advanced buffer management + * Input - None + * Output - None + */ +enum nfp_mbox_cmd { + NFP_MBOX_NO_CMD = 0x00, + + NFP_MBOX_POOL_GET = 0x01, + NFP_MBOX_POOL_SET = 0x02, + + NFP_MBOX_PCIE_ABM_ENABLE = 0x03, + NFP_MBOX_PCIE_ABM_DISABLE = 0x04, +}; + +#define NFP_SHARED_BUF_COUNT_SYM_NAME "_abi_nfd_pf%u_sb_cnt" +#define NFP_SHARED_BUF_TABLE_SYM_NAME "_abi_nfd_pf%u_sb_tbl" + +/** + * struct nfp_shared_buf - NFP shared buffer description + * @id: numerical user-visible id of the shared buffer + * @size: size in bytes of the buffer + * @ingress_pools_count: number of ingress pools + * @egress_pools_count: number of egress pools + * @ingress_tc_count: number of ingress trafic classes + * @egress_tc_count: number of egress trafic classes + * @pool_size_unit: pool size may be in credits, each credit is + * @pool_size_unit bytes + */ +struct nfp_shared_buf { + __le32 id; + __le32 size; + __le16 ingress_pools_count; + __le16 egress_pools_count; + __le16 ingress_tc_count; + __le16 egress_tc_count; + + __le32 pool_size_unit; +}; + +/** + * struct nfp_shared_buf_pool_id - shared buffer pool identification + * @shared_buf: shared buffer id + * @pool: pool index + */ +struct nfp_shared_buf_pool_id { + __le32 shared_buf; + __le32 pool; +}; + +/** + * struct nfp_shared_buf_pool_info_get - struct devlink_sb_pool_info mirror + * @pool_type: one of enum devlink_sb_pool_type + * @size: pool size in units of SB's @pool_size_unit + * @threshold_type: one of enum devlink_sb_threshold_type + */ +struct nfp_shared_buf_pool_info_get { + __le32 pool_type; + __le32 size; + __le32 threshold_type; +}; + +/** + * struct nfp_shared_buf_pool_info_set - packed args of sb_pool_set + * @id: pool identification info + * @size: pool size in units of SB's @pool_size_unit + * @threshold_type: one of enum devlink_sb_threshold_type + */ +struct nfp_shared_buf_pool_info_set { + struct nfp_shared_buf_pool_id id; + __le32 size; + __le32 threshold_type; +}; + +#endif diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.c b/drivers/net/ethernet/netronome/nfp/nfp_app.c index 6aedef0ad433a..f28b244f4ee7e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2017 Netronome Systems, Inc. + * Copyright (C) 2017-2018 Netronome Systems, Inc. * * This software is dual licensed under the GNU General License Version 2, * June 1991 as shown in the file COPYING in the top-level directory of this @@ -43,6 +43,7 @@ #include "nfp_main.h" #include "nfp_net.h" #include "nfp_net_repr.h" +#include "nfp_port.h" static const struct nfp_app_type *apps[] = { [NFP_APP_CORE_NIC] = &app_nic, @@ -54,6 +55,9 @@ static const struct nfp_app_type *apps[] = { #ifdef CONFIG_NFP_APP_FLOWER [NFP_APP_FLOWER_NIC] = &app_flower, #endif +#ifdef CONFIG_NFP_APP_ABM_NIC + [NFP_APP_ACTIVE_BUFFER_MGMT_NIC] = &app_abm, +#endif }; struct nfp_app *nfp_app_from_netdev(struct net_device *netdev) @@ -82,6 +86,27 @@ const char *nfp_app_mip_name(struct nfp_app *app) return nfp_mip_name(app->pf->mip); } +u64 *nfp_app_port_get_stats(struct nfp_port *port, u64 *data) +{ + if (!port || !port->app || !port->app->type->port_get_stats) + return data; + return port->app->type->port_get_stats(port->app, port, data); +} + +int nfp_app_port_get_stats_count(struct nfp_port *port) +{ + if (!port || !port->app || !port->app->type->port_get_stats_count) + return 0; + return port->app->type->port_get_stats_count(port->app, port); +} + +u8 *nfp_app_port_get_stats_strings(struct nfp_port *port, u8 *data) +{ + if (!port || !port->app || !port->app->type->port_get_stats_strings) + return data; + return port->app->type->port_get_stats_strings(port->app, port, data); +} + struct sk_buff * nfp_app_ctrl_msg_alloc(struct nfp_app *app, unsigned int size, gfp_t priority) { diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app.h b/drivers/net/ethernet/netronome/nfp/nfp_app.h index 2d9cb2528fc7f..ee74caacb0153 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_app.h @@ -57,11 +57,13 @@ enum nfp_app_id { NFP_APP_CORE_NIC = 0x1, NFP_APP_BPF_NIC = 0x2, NFP_APP_FLOWER_NIC = 0x3, + NFP_APP_ACTIVE_BUFFER_MGMT_NIC = 0x4, }; extern const struct nfp_app_type app_nic; extern const struct nfp_app_type app_bpf; extern const struct nfp_app_type app_flower; +extern const struct nfp_app_type app_abm; /** * struct nfp_app_type - application definition @@ -88,6 +90,9 @@ extern const struct nfp_app_type app_flower; * @repr_stop: representor netdev stop callback * @check_mtu: MTU change request on a netdev (verify it is valid) * @repr_change_mtu: MTU change request on repr (make and verify change) + * @port_get_stats: get extra ethtool statistics for a port + * @port_get_stats_count: get count of extra statistics for a port + * @port_get_stats_strings: get strings for extra statistics * @start: start application logic * @stop: stop application logic * @ctrl_msg_rx: control message handler @@ -95,6 +100,7 @@ extern const struct nfp_app_type app_flower; * @bpf: BPF ndo offload-related calls * @xdp_offload: offload an XDP program * @eswitch_mode_get: get SR-IOV eswitch mode + * @eswitch_mode_set: set SR-IOV eswitch mode (under pf->lock) * @sriov_enable: app-specific sriov initialisation * @sriov_disable: app-specific sriov clean-up * @repr_get: get representor netdev @@ -129,6 +135,12 @@ struct nfp_app_type { int (*repr_change_mtu)(struct nfp_app *app, struct net_device *netdev, int new_mtu); + u64 *(*port_get_stats)(struct nfp_app *app, + struct nfp_port *port, u64 *data); + int (*port_get_stats_count)(struct nfp_app *app, struct nfp_port *port); + u8 *(*port_get_stats_strings)(struct nfp_app *app, + struct nfp_port *port, u8 *data); + int (*start)(struct nfp_app *app); void (*stop)(struct nfp_app *app); @@ -146,6 +158,7 @@ struct nfp_app_type { void (*sriov_disable)(struct nfp_app *app); enum devlink_eswitch_mode (*eswitch_mode_get)(struct nfp_app *app); + int (*eswitch_mode_set)(struct nfp_app *app, u16 mode); struct net_device *(*repr_get)(struct nfp_app *app, u32 id); }; @@ -370,6 +383,13 @@ static inline int nfp_app_eswitch_mode_get(struct nfp_app *app, u16 *mode) return 0; } +static inline int nfp_app_eswitch_mode_set(struct nfp_app *app, u16 mode) +{ + if (!app->type->eswitch_mode_set) + return -EOPNOTSUPP; + return app->type->eswitch_mode_set(app, mode); +} + static inline int nfp_app_sriov_enable(struct nfp_app *app, int num_vfs) { if (!app || !app->type->sriov_enable) @@ -393,6 +413,10 @@ static inline struct net_device *nfp_app_repr_get(struct nfp_app *app, u32 id) struct nfp_app *nfp_app_from_netdev(struct net_device *netdev); +u64 *nfp_app_port_get_stats(struct nfp_port *port, u64 *data); +int nfp_app_port_get_stats_count(struct nfp_port *port); +u8 *nfp_app_port_get_stats_strings(struct nfp_port *port, u8 *data); + struct nfp_reprs * nfp_reprs_get_locked(struct nfp_app *app, enum nfp_repr_type type); struct nfp_reprs * @@ -410,5 +434,7 @@ void nfp_app_free(struct nfp_app *app); int nfp_app_nic_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id); +int nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app, + struct nfp_net *nn, unsigned int id); #endif diff --git a/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c b/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c index b9618c37403f9..e2dfe4f168bba 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_app_nic.c @@ -38,9 +38,8 @@ #include "nfp_net.h" #include "nfp_port.h" -static int -nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app, - struct nfp_net *nn, unsigned int id) +int nfp_app_nic_vnic_init_phy_port(struct nfp_pf *pf, struct nfp_app *app, + struct nfp_net *nn, unsigned int id) { int err; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_asm.h b/drivers/net/ethernet/netronome/nfp/nfp_asm.h index 5f2b2f24f4fae..f6677bc9875ad 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_asm.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_asm.h @@ -72,8 +72,21 @@ #define OP_BR_ADDR_LO 0x007ffc00000ULL #define OP_BR_ADDR_HI 0x10000000000ULL -#define nfp_is_br(_insn) \ - (((_insn) & OP_BR_BASE_MASK) == OP_BR_BASE) +#define OP_BR_BIT_BASE 0x0d000000000ULL +#define OP_BR_BIT_BASE_MASK 0x0f800080300ULL +#define OP_BR_BIT_A_SRC 0x000000000ffULL +#define OP_BR_BIT_B_SRC 0x0000003fc00ULL +#define OP_BR_BIT_BV 0x00000040000ULL +#define OP_BR_BIT_SRC_LMEXTN 0x40000000000ULL +#define OP_BR_BIT_DEFBR OP_BR_DEFBR +#define OP_BR_BIT_ADDR_LO OP_BR_ADDR_LO +#define OP_BR_BIT_ADDR_HI OP_BR_ADDR_HI + +static inline bool nfp_is_br(u64 insn) +{ + return (insn & OP_BR_BASE_MASK) == OP_BR_BASE || + (insn & OP_BR_BIT_BASE_MASK) == OP_BR_BIT_BASE; +} enum br_mask { BR_BEQ = 0x00, @@ -161,6 +174,7 @@ enum shf_op { SHF_OP_NONE = 0, SHF_OP_AND = 2, SHF_OP_OR = 5, + SHF_OP_ASHR = 6, }; enum shf_sc { @@ -183,16 +197,18 @@ enum shf_sc { #define OP_ALU_DST_LMEXTN 0x80000000000ULL enum alu_op { - ALU_OP_NONE = 0x00, - ALU_OP_ADD = 0x01, - ALU_OP_NOT = 0x04, - ALU_OP_ADD_2B = 0x05, - ALU_OP_AND = 0x08, - ALU_OP_SUB_C = 0x0d, - ALU_OP_ADD_C = 0x11, - ALU_OP_OR = 0x14, - ALU_OP_SUB = 0x15, - ALU_OP_XOR = 0x18, + ALU_OP_NONE = 0x00, + ALU_OP_ADD = 0x01, + ALU_OP_NOT = 0x04, + ALU_OP_ADD_2B = 0x05, + ALU_OP_AND = 0x08, + ALU_OP_AND_NOT_A = 0x0c, + ALU_OP_SUB_C = 0x0d, + ALU_OP_AND_NOT_B = 0x10, + ALU_OP_ADD_C = 0x11, + ALU_OP_OR = 0x14, + ALU_OP_SUB = 0x15, + ALU_OP_XOR = 0x18, }; enum alu_dst_ab { diff --git a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c index eb0fc614673dd..db463e20a876c 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_devlink.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_devlink.c @@ -92,7 +92,7 @@ nfp_devlink_set_lanes(struct nfp_pf *pf, unsigned int idx, unsigned int lanes) static int nfp_devlink_port_split(struct devlink *devlink, unsigned int port_index, - unsigned int count) + unsigned int count, struct netlink_ext_ack *extack) { struct nfp_pf *pf = devlink_priv(devlink); struct nfp_eth_table_port eth_port; @@ -123,7 +123,8 @@ out: } static int -nfp_devlink_port_unsplit(struct devlink *devlink, unsigned int port_index) +nfp_devlink_port_unsplit(struct devlink *devlink, unsigned int port_index, + struct netlink_ext_ack *extack) { struct nfp_pf *pf = devlink_priv(devlink); struct nfp_eth_table_port eth_port; @@ -149,6 +150,26 @@ out: return ret; } +static int +nfp_devlink_sb_pool_get(struct devlink *devlink, unsigned int sb_index, + u16 pool_index, struct devlink_sb_pool_info *pool_info) +{ + struct nfp_pf *pf = devlink_priv(devlink); + + return nfp_shared_buf_pool_get(pf, sb_index, pool_index, pool_info); +} + +static int +nfp_devlink_sb_pool_set(struct devlink *devlink, unsigned int sb_index, + u16 pool_index, + u32 size, enum devlink_sb_threshold_type threshold_type) +{ + struct nfp_pf *pf = devlink_priv(devlink); + + return nfp_shared_buf_pool_set(pf, sb_index, pool_index, + size, threshold_type); +} + static int nfp_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) { struct nfp_pf *pf = devlink_priv(devlink); @@ -156,10 +177,25 @@ static int nfp_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) return nfp_app_eswitch_mode_get(pf->app, mode); } +static int nfp_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode) +{ + struct nfp_pf *pf = devlink_priv(devlink); + int ret; + + mutex_lock(&pf->lock); + ret = nfp_app_eswitch_mode_set(pf->app, mode); + mutex_unlock(&pf->lock); + + return ret; +} + const struct devlink_ops nfp_devlink_ops = { .port_split = nfp_devlink_port_split, .port_unsplit = nfp_devlink_port_unsplit, + .sb_pool_get = nfp_devlink_sb_pool_get, + .sb_pool_set = nfp_devlink_sb_pool_set, .eswitch_mode_get = nfp_devlink_eswitch_mode_get, + .eswitch_mode_set = nfp_devlink_eswitch_mode_set, }; int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port) @@ -175,8 +211,9 @@ int nfp_devlink_port_register(struct nfp_app *app, struct nfp_port *port) return ret; devlink_port_type_eth_set(&port->dl_port, port->netdev); - if (eth_port.is_split) - devlink_port_split_set(&port->dl_port, eth_port.label_port); + devlink_port_attrs_set(&port->dl_port, DEVLINK_PORT_FLAVOUR_PHYSICAL, + eth_port.label_port, eth_port.is_split, + eth_port.label_subport); devlink = priv_to_devlink(app->pf); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.c b/drivers/net/ethernet/netronome/nfp/nfp_main.c index c4b1f344b4da2..46b76d5a726c6 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_main.c @@ -55,6 +55,7 @@ #include "nfpcore/nfp6000_pcie.h" +#include "nfp_abi.h" #include "nfp_app.h" #include "nfp_main.h" #include "nfp_net.h" @@ -75,6 +76,122 @@ static const struct pci_device_id nfp_pci_device_ids[] = { }; MODULE_DEVICE_TABLE(pci, nfp_pci_device_ids); +int nfp_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format, + unsigned int default_val) +{ + char name[256]; + int err = 0; + u64 val; + + snprintf(name, sizeof(name), format, nfp_cppcore_pcie_unit(pf->cpp)); + + val = nfp_rtsym_read_le(pf->rtbl, name, &err); + if (err) { + if (err == -ENOENT) + return default_val; + nfp_err(pf->cpp, "Unable to read symbol %s\n", name); + return err; + } + + return val; +} + +u8 __iomem * +nfp_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt, + unsigned int min_size, struct nfp_cpp_area **area) +{ + char pf_symbol[256]; + + snprintf(pf_symbol, sizeof(pf_symbol), sym_fmt, + nfp_cppcore_pcie_unit(pf->cpp)); + + return nfp_rtsym_map(pf->rtbl, pf_symbol, name, min_size, area); +} + +/* Callers should hold the devlink instance lock */ +int nfp_mbox_cmd(struct nfp_pf *pf, u32 cmd, void *in_data, u64 in_length, + void *out_data, u64 out_length) +{ + unsigned long long addr; + unsigned long err_at; + u64 max_data_sz; + u32 val = 0; + u32 cpp_id; + int n, err; + + if (!pf->mbox) + return -EOPNOTSUPP; + + cpp_id = NFP_CPP_ISLAND_ID(pf->mbox->target, NFP_CPP_ACTION_RW, 0, + pf->mbox->domain); + addr = pf->mbox->addr; + max_data_sz = pf->mbox->size - NFP_MBOX_SYM_MIN_SIZE; + + /* Check if cmd field is clear */ + err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_CMD, &val); + if (err || val) { + nfp_warn(pf->cpp, "failed to issue command (%u): %u, err: %d\n", + cmd, val, err); + return err ?: -EBUSY; + } + + in_length = min(in_length, max_data_sz); + n = nfp_cpp_write(pf->cpp, cpp_id, addr + NFP_MBOX_DATA, + in_data, in_length); + if (n != in_length) + return -EIO; + /* Write data_len and wipe reserved */ + err = nfp_cpp_writeq(pf->cpp, cpp_id, addr + NFP_MBOX_DATA_LEN, + in_length); + if (err) + return err; + + /* Read back for ordering */ + err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_DATA_LEN, &val); + if (err) + return err; + + /* Write cmd and wipe return value */ + err = nfp_cpp_writeq(pf->cpp, cpp_id, addr + NFP_MBOX_CMD, cmd); + if (err) + return err; + + err_at = jiffies + 5 * HZ; + while (true) { + /* Wait for command to go to 0 (NFP_MBOX_NO_CMD) */ + err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_CMD, &val); + if (err) + return err; + if (!val) + break; + + if (time_is_before_eq_jiffies(err_at)) + return -ETIMEDOUT; + + msleep(5); + } + + /* Copy output if any (could be error info, do it before reading ret) */ + err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_DATA_LEN, &val); + if (err) + return err; + + out_length = min_t(u32, val, min(out_length, max_data_sz)); + n = nfp_cpp_read(pf->cpp, cpp_id, addr + NFP_MBOX_DATA, + out_data, out_length); + if (n != out_length) + return -EIO; + + /* Check if there is an error */ + err = nfp_cpp_readl(pf->cpp, cpp_id, addr + NFP_MBOX_RET, &val); + if (err) + return err; + if (val) + return -val; + + return out_length; +} + static bool nfp_board_ready(struct nfp_pf *pf) { const char *cp; @@ -436,6 +553,25 @@ static void nfp_fw_unload(struct nfp_pf *pf) nfp_nsp_close(nsp); } +static int nfp_pf_find_rtsyms(struct nfp_pf *pf) +{ + char pf_symbol[256]; + unsigned int pf_id; + + pf_id = nfp_cppcore_pcie_unit(pf->cpp); + + /* Optional per-PCI PF mailbox */ + snprintf(pf_symbol, sizeof(pf_symbol), NFP_MBOX_SYM_NAME, pf_id); + pf->mbox = nfp_rtsym_lookup(pf->rtbl, pf_symbol); + if (pf->mbox && pf->mbox->size < NFP_MBOX_SYM_MIN_SIZE) { + nfp_err(pf->cpp, "PF mailbox symbol too small: %llu < %d\n", + pf->mbox->size, NFP_MBOX_SYM_MIN_SIZE); + return -EINVAL; + } + + return 0; +} + static int nfp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *pci_id) { @@ -486,6 +622,10 @@ static int nfp_pci_probe(struct pci_dev *pdev, goto err_disable_msix; } + err = nfp_resource_table_init(pf->cpp); + if (err) + goto err_cpp_free; + pf->hwinfo = nfp_hwinfo_read(pf->cpp); dev_info(&pdev->dev, "Assembly: %s%s%s-%s CPLD: %s\n", @@ -506,6 +646,10 @@ static int nfp_pci_probe(struct pci_dev *pdev, pf->mip = nfp_mip_open(pf->cpp); pf->rtbl = __nfp_rtsym_table_read(pf->cpp, pf->mip); + err = nfp_pf_find_rtsyms(pf); + if (err) + goto err_fw_unload; + pf->dump_flag = NFP_DUMP_NSP_DIAG; pf->dumpspec = nfp_net_dump_load_dumpspec(pf->cpp, pf->rtbl); @@ -548,6 +692,7 @@ err_fw_unload: vfree(pf->dumpspec); err_hwinfo_free: kfree(pf->hwinfo); +err_cpp_free: nfp_cpp_free(pf->cpp); err_disable_msix: destroy_workqueue(pf->wq); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_main.h b/drivers/net/ethernet/netronome/nfp/nfp_main.h index 42211083b51f4..595b3dc280e3b 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_main.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_main.h @@ -46,10 +46,10 @@ #include <linux/mutex.h> #include <linux/pci.h> #include <linux/workqueue.h> +#include <net/devlink.h> struct dentry; struct device; -struct devlink_ops; struct pci_dev; struct nfp_cpp; @@ -60,7 +60,9 @@ struct nfp_mip; struct nfp_net; struct nfp_nsp_identify; struct nfp_port; +struct nfp_rtsym; struct nfp_rtsym_table; +struct nfp_shared_buf; /** * struct nfp_dumpspec - NFP FW dump specification structure @@ -87,6 +89,7 @@ struct nfp_dumpspec { * @vf_cfg_mem: Pointer to mapped VF configuration area * @vfcfg_tbl2_area: Pointer to the CPP area for the VF config table * @vfcfg_tbl2: Pointer to mapped VF config table + * @mbox: RTSym of per-PCI PF mailbox (under devlink lock) * @irq_entries: Array of MSI-X entries for all vNICs * @limit_vfs: Number of VFs supported by firmware (~0 for PCI limit) * @num_vfs: Number of SR-IOV VFs enabled @@ -108,6 +111,8 @@ struct nfp_dumpspec { * @ports: Linked list of port structures (struct nfp_port) * @wq: Workqueue for running works which need to grab @lock * @port_refresh_work: Work entry for taking netdevs out + * @shared_bufs: Array of shared buffer structures if FW has any SBs + * @num_shared_bufs: Number of elements in @shared_bufs * @lock: Protects all fields which may change after probe */ struct nfp_pf { @@ -127,6 +132,8 @@ struct nfp_pf { struct nfp_cpp_area *vfcfg_tbl2_area; u8 __iomem *vfcfg_tbl2; + const struct nfp_rtsym *mbox; + struct msix_entry *irq_entries; unsigned int limit_vfs; @@ -158,6 +165,9 @@ struct nfp_pf { struct workqueue_struct *wq; struct work_struct port_refresh_work; + struct nfp_shared_buf *shared_bufs; + unsigned int num_shared_bufs; + struct mutex lock; }; @@ -177,6 +187,14 @@ nfp_net_get_mac_addr(struct nfp_pf *pf, struct net_device *netdev, bool nfp_ctrl_tx(struct nfp_net *nn, struct sk_buff *skb); +int nfp_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format, + unsigned int default_val); +u8 __iomem * +nfp_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt, + unsigned int min_size, struct nfp_cpp_area **area); +int nfp_mbox_cmd(struct nfp_pf *pf, u32 cmd, void *in_data, u64 in_length, + void *out_data, u64 out_length); + enum nfp_dump_diag { NFP_DUMP_NSP_DIAG = 0, }; @@ -188,4 +206,11 @@ s64 nfp_net_dump_calculate_size(struct nfp_pf *pf, struct nfp_dumpspec *spec, int nfp_net_dump_populate_buffer(struct nfp_pf *pf, struct nfp_dumpspec *spec, struct ethtool_dump *dump_param, void *dest); +int nfp_shared_buf_register(struct nfp_pf *pf); +void nfp_shared_buf_unregister(struct nfp_pf *pf); +int nfp_shared_buf_pool_get(struct nfp_pf *pf, unsigned int sb, u16 pool_index, + struct devlink_sb_pool_info *pool_info); +int nfp_shared_buf_pool_set(struct nfp_pf *pf, unsigned int sb, + u16 pool_index, u32 size, + enum devlink_sb_threshold_type threshold_type); #endif /* NFP_MAIN_H */ diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net.h b/drivers/net/ethernet/netronome/nfp/nfp_net.h index bd7d8ae31e173..57cb035dcc6dc 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net.h @@ -545,6 +545,7 @@ struct nfp_net_dp { /** * struct nfp_net - NFP network device structure * @dp: Datapath structure + * @id: vNIC id within the PF (0 for VFs) * @fw_ver: Firmware version * @cap: Capabilities advertised by the Firmware * @max_mtu: Maximum support MTU advertised by the Firmware @@ -597,6 +598,8 @@ struct nfp_net { struct nfp_net_fw_version fw_ver; + u32 id; + u32 cap; u32 max_mtu; @@ -909,7 +912,7 @@ int nfp_net_ring_reconfig(struct nfp_net *nn, struct nfp_net_dp *new, void nfp_net_debugfs_create(void); void nfp_net_debugfs_destroy(void); struct dentry *nfp_net_debugfs_device_add(struct pci_dev *pdev); -void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id); +void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir); void nfp_net_debugfs_dir_clean(struct dentry **dir); #else static inline void nfp_net_debugfs_create(void) @@ -926,7 +929,7 @@ static inline struct dentry *nfp_net_debugfs_device_add(struct pci_dev *pdev) } static inline void -nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id) +nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir) { } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 1eb6549f2a54e..75110c8d6a903 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -1722,7 +1722,7 @@ static int nfp_net_rx(struct nfp_net_rx_ring *rx_ring, int budget) act = bpf_prog_run_xdp(xdp_prog, &xdp); - pkt_len -= xdp.data - orig_data; + pkt_len = xdp.data_end - xdp.data; pkt_off += xdp.data - orig_data; switch (act) { @@ -3277,6 +3277,25 @@ nfp_net_features_check(struct sk_buff *skb, struct net_device *dev, return features; } +static int +nfp_net_get_phys_port_name(struct net_device *netdev, char *name, size_t len) +{ + struct nfp_net *nn = netdev_priv(netdev); + int n; + + if (nn->port) + return nfp_port_get_phys_port_name(netdev, name, len); + + if (nn->dp.is_vf) + return -EOPNOTSUPP; + + n = snprintf(name, len, "n%d", nn->id); + if (n >= len) + return -EINVAL; + + return 0; +} + /** * nfp_net_set_vxlan_port() - set vxlan port in SW and reconfigure HW * @nn: NFP Net device to reconfigure @@ -3475,7 +3494,7 @@ const struct net_device_ops nfp_net_netdev_ops = { .ndo_set_mac_address = nfp_net_set_mac_address, .ndo_set_features = nfp_net_set_features, .ndo_features_check = nfp_net_features_check, - .ndo_get_phys_port_name = nfp_port_get_phys_port_name, + .ndo_get_phys_port_name = nfp_net_get_phys_port_name, .ndo_udp_tunnel_add = nfp_net_add_vxlan_port, .ndo_udp_tunnel_del = nfp_net_del_vxlan_port, .ndo_bpf = nfp_net_xdp, diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c b/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c index 67cdd8330c59b..099b63d674514 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_debugfs.c @@ -201,7 +201,7 @@ static const struct file_operations nfp_xdp_q_fops = { .llseek = seq_lseek }; -void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id) +void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir) { struct dentry *queues, *tx, *rx, *xdp; char name[20]; @@ -211,7 +211,7 @@ void nfp_net_debugfs_vnic_add(struct nfp_net *nn, struct dentry *ddir, int id) return; if (nfp_net_is_data_vnic(nn)) - sprintf(name, "vnic%d", id); + sprintf(name, "vnic%d", nn->id); else strcpy(name, "ctrl-vnic"); nn->debugfs_dir = debugfs_create_dir(name, ddir); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index c9016419bfa0e..26d1cc4e29061 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -437,7 +437,7 @@ static int nfp_net_set_ringparam(struct net_device *netdev, return nfp_net_set_ring_size(nn, rxd_cnt, txd_cnt); } -static __printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...) +__printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...) { va_list args; @@ -637,6 +637,7 @@ static void nfp_net_get_strings(struct net_device *netdev, nn->dp.num_tx_rings, false); data = nfp_mac_get_stats_strings(netdev, data); + data = nfp_app_port_get_stats_strings(nn->port, data); break; } } @@ -651,6 +652,7 @@ nfp_net_get_stats(struct net_device *netdev, struct ethtool_stats *stats, data = nfp_vnic_get_hw_stats(data, nn->dp.ctrl_bar, nn->dp.num_rx_rings, nn->dp.num_tx_rings); data = nfp_mac_get_stats(netdev, data); + data = nfp_app_port_get_stats(nn->port, data); } static int nfp_net_get_sset_count(struct net_device *netdev, int sset) @@ -662,7 +664,8 @@ static int nfp_net_get_sset_count(struct net_device *netdev, int sset) return nfp_vnic_get_sw_stats_count(netdev) + nfp_vnic_get_hw_stats_count(nn->dp.num_rx_rings, nn->dp.num_tx_rings) + - nfp_mac_get_stats_count(netdev); + nfp_mac_get_stats_count(netdev) + + nfp_app_port_get_stats_count(nn->port); default: return -EOPNOTSUPP; } @@ -679,6 +682,7 @@ static void nfp_port_get_strings(struct net_device *netdev, data = nfp_vnic_get_hw_stats_strings(data, 0, 0, true); else data = nfp_mac_get_stats_strings(netdev, data); + data = nfp_app_port_get_stats_strings(port, data); break; } } @@ -693,6 +697,7 @@ nfp_port_get_stats(struct net_device *netdev, struct ethtool_stats *stats, data = nfp_vnic_get_hw_stats(data, port->vnic, 0, 0); else data = nfp_mac_get_stats(netdev, data); + data = nfp_app_port_get_stats(port, data); } static int nfp_port_get_sset_count(struct net_device *netdev, int sset) @@ -706,6 +711,7 @@ static int nfp_port_get_sset_count(struct net_device *netdev, int sset) count = nfp_vnic_get_hw_stats_count(0, 0); else count = nfp_mac_get_stats_count(netdev); + count += nfp_app_port_get_stats_count(port); return count; default: return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c index 45cd2092e498a..28516eecccc8d 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_main.c @@ -101,48 +101,15 @@ nfp_net_find_port(struct nfp_eth_table *eth_tbl, unsigned int index) return NULL; } -static int -nfp_net_pf_rtsym_read_optional(struct nfp_pf *pf, const char *format, - unsigned int default_val) -{ - char name[256]; - int err = 0; - u64 val; - - snprintf(name, sizeof(name), format, nfp_cppcore_pcie_unit(pf->cpp)); - - val = nfp_rtsym_read_le(pf->rtbl, name, &err); - if (err) { - if (err == -ENOENT) - return default_val; - nfp_err(pf->cpp, "Unable to read symbol %s\n", name); - return err; - } - - return val; -} - static int nfp_net_pf_get_num_ports(struct nfp_pf *pf) { - return nfp_net_pf_rtsym_read_optional(pf, "nfd_cfg_pf%u_num_ports", 1); + return nfp_pf_rtsym_read_optional(pf, "nfd_cfg_pf%u_num_ports", 1); } static int nfp_net_pf_get_app_id(struct nfp_pf *pf) { - return nfp_net_pf_rtsym_read_optional(pf, "_pf%u_net_app_id", - NFP_APP_CORE_NIC); -} - -static u8 __iomem * -nfp_net_pf_map_rtsym(struct nfp_pf *pf, const char *name, const char *sym_fmt, - unsigned int min_size, struct nfp_cpp_area **area) -{ - char pf_symbol[256]; - - snprintf(pf_symbol, sizeof(pf_symbol), sym_fmt, - nfp_cppcore_pcie_unit(pf->cpp)); - - return nfp_rtsym_map(pf->rtbl, pf_symbol, name, min_size, area); + return nfp_pf_rtsym_read_optional(pf, "_pf%u_net_app_id", + NFP_APP_CORE_NIC); } static void nfp_net_pf_free_vnic(struct nfp_pf *pf, struct nfp_net *nn) @@ -211,11 +178,13 @@ nfp_net_pf_init_vnic(struct nfp_pf *pf, struct nfp_net *nn, unsigned int id) { int err; + nn->id = id; + err = nfp_net_init(nn); if (err) return err; - nfp_net_debugfs_vnic_add(nn, pf->ddir, id); + nfp_net_debugfs_vnic_add(nn, pf->ddir); if (nn->port) { err = nfp_devlink_port_register(pf->app, nn->port); @@ -379,9 +348,8 @@ nfp_net_pf_app_init(struct nfp_pf *pf, u8 __iomem *qc_bar, unsigned int stride) if (!nfp_app_needs_ctrl_vnic(pf->app)) return 0; - ctrl_bar = nfp_net_pf_map_rtsym(pf, "net.ctrl", "_pf%u_net_ctrl_bar", - NFP_PF_CSR_SLICE_SIZE, - &pf->ctrl_vnic_bar); + ctrl_bar = nfp_pf_map_rtsym(pf, "net.ctrl", "_pf%u_net_ctrl_bar", + NFP_PF_CSR_SLICE_SIZE, &pf->ctrl_vnic_bar); if (IS_ERR(ctrl_bar)) { nfp_err(pf->cpp, "Failed to find ctrl vNIC memory symbol\n"); err = PTR_ERR(ctrl_bar); @@ -507,8 +475,8 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf) int err; min_size = pf->max_data_vnics * NFP_PF_CSR_SLICE_SIZE; - mem = nfp_net_pf_map_rtsym(pf, "net.bar0", "_pf%d_net_bar0", - min_size, &pf->data_vnic_bar); + mem = nfp_pf_map_rtsym(pf, "net.bar0", "_pf%d_net_bar0", + min_size, &pf->data_vnic_bar); if (IS_ERR(mem)) { nfp_err(pf->cpp, "Failed to find data vNIC memory symbol\n"); return PTR_ERR(mem); @@ -528,10 +496,9 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf) } } - pf->vf_cfg_mem = nfp_net_pf_map_rtsym(pf, "net.vfcfg", - "_pf%d_net_vf_bar", - NFP_NET_CFG_BAR_SZ * - pf->limit_vfs, &pf->vf_cfg_bar); + pf->vf_cfg_mem = nfp_pf_map_rtsym(pf, "net.vfcfg", "_pf%d_net_vf_bar", + NFP_NET_CFG_BAR_SZ * pf->limit_vfs, + &pf->vf_cfg_bar); if (IS_ERR(pf->vf_cfg_mem)) { if (PTR_ERR(pf->vf_cfg_mem) != -ENOENT) { err = PTR_ERR(pf->vf_cfg_mem); @@ -541,9 +508,9 @@ static int nfp_net_pci_map_mem(struct nfp_pf *pf) } min_size = NFP_NET_VF_CFG_SZ * pf->limit_vfs + NFP_NET_VF_CFG_MB_SZ; - pf->vfcfg_tbl2 = nfp_net_pf_map_rtsym(pf, "net.vfcfg_tbl2", - "_pf%d_net_vf_cfg2", - min_size, &pf->vfcfg_tbl2_area); + pf->vfcfg_tbl2 = nfp_pf_map_rtsym(pf, "net.vfcfg_tbl2", + "_pf%d_net_vf_cfg2", + min_size, &pf->vfcfg_tbl2_area); if (IS_ERR(pf->vfcfg_tbl2)) { if (PTR_ERR(pf->vfcfg_tbl2) != -ENOENT) { err = PTR_ERR(pf->vfcfg_tbl2); @@ -763,6 +730,10 @@ int nfp_net_pci_probe(struct nfp_pf *pf) if (err) goto err_app_clean; + err = nfp_shared_buf_register(pf); + if (err) + goto err_devlink_unreg; + mutex_lock(&pf->lock); pf->ddir = nfp_net_debugfs_device_add(pf->pdev); @@ -796,6 +767,8 @@ err_free_vnics: err_clean_ddir: nfp_net_debugfs_dir_clean(&pf->ddir); mutex_unlock(&pf->lock); + nfp_shared_buf_unregister(pf); +err_devlink_unreg: cancel_work_sync(&pf->port_refresh_work); devlink_unregister(devlink); err_app_clean: @@ -823,6 +796,7 @@ void nfp_net_pci_remove(struct nfp_pf *pf) mutex_unlock(&pf->lock); + nfp_shared_buf_unregister(pf); devlink_unregister(priv_to_devlink(pf)); nfp_net_pf_free_irqs(pf); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index 0cd077addb26a..d7b712f6362fa 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -277,6 +277,7 @@ const struct net_device_ops nfp_repr_netdev_ops = { .ndo_get_vf_config = nfp_app_get_vf_config, .ndo_set_vf_link_state = nfp_app_set_vf_link_state, .ndo_set_features = nfp_port_set_features, + .ndo_set_mac_address = eth_mac_addr, }; static void nfp_repr_clean(struct nfp_repr *repr) @@ -348,18 +349,24 @@ err_clean: return err; } -static void nfp_repr_free(struct nfp_repr *repr) +static void __nfp_repr_free(struct nfp_repr *repr) { free_percpu(repr->stats); free_netdev(repr->netdev); } -struct net_device *nfp_repr_alloc(struct nfp_app *app) +void nfp_repr_free(struct net_device *netdev) +{ + __nfp_repr_free(netdev_priv(netdev)); +} + +struct net_device * +nfp_repr_alloc_mqs(struct nfp_app *app, unsigned int txqs, unsigned int rxqs) { struct net_device *netdev; struct nfp_repr *repr; - netdev = alloc_etherdev(sizeof(*repr)); + netdev = alloc_etherdev_mqs(sizeof(*repr), txqs, rxqs); if (!netdev) return NULL; @@ -380,12 +387,12 @@ err_free_netdev: return NULL; } -static void nfp_repr_clean_and_free(struct nfp_repr *repr) +void nfp_repr_clean_and_free(struct nfp_repr *repr) { nfp_info(repr->app->cpp, "Destroying Representor(%s)\n", repr->netdev->name); nfp_repr_clean(repr); - nfp_repr_free(repr); + __nfp_repr_free(repr); } void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs) diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h index a621e8ff528e3..1bf2b18109ab0 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.h @@ -76,6 +76,7 @@ struct nfp_repr_pcpu_stats { * @port: Port of representor * @app: APP handle * @stats: Statistic of packets hitting CPU + * @app_priv: Pointer for APP data */ struct nfp_repr { struct net_device *netdev; @@ -83,6 +84,7 @@ struct nfp_repr { struct nfp_port *port; struct nfp_app *app; struct nfp_repr_pcpu_stats __percpu *stats; + void *app_priv; }; /** @@ -123,11 +125,18 @@ void nfp_repr_inc_rx_stats(struct net_device *netdev, unsigned int len); int nfp_repr_init(struct nfp_app *app, struct net_device *netdev, u32 cmsg_port_id, struct nfp_port *port, struct net_device *pf_netdev); -struct net_device *nfp_repr_alloc(struct nfp_app *app); +void nfp_repr_free(struct net_device *netdev); +struct net_device * +nfp_repr_alloc_mqs(struct nfp_app *app, unsigned int txqs, unsigned int rxqs); +void nfp_repr_clean_and_free(struct nfp_repr *repr); void nfp_reprs_clean_and_free(struct nfp_app *app, struct nfp_reprs *reprs); void nfp_reprs_clean_and_free_by_type(struct nfp_app *app, enum nfp_repr_type type); struct nfp_reprs *nfp_reprs_alloc(unsigned int num_reprs); int nfp_reprs_resync_phys_ports(struct nfp_app *app); +static inline struct net_device *nfp_repr_alloc(struct nfp_app *app) +{ + return nfp_repr_alloc_mqs(app, 1, 1); +} #endif /* NFP_NET_REPR_H */ diff --git a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c index b802a1d554493..68928c86b698e 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_netvf_main.c @@ -283,7 +283,7 @@ static int nfp_netvf_pci_probe(struct pci_dev *pdev, nfp_net_info(nn); vf->ddir = nfp_net_debugfs_device_add(pdev); - nfp_net_debugfs_vnic_add(nn, vf->ddir, 0); + nfp_net_debugfs_vnic_add(nn, vf->ddir); return 0; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.c b/drivers/net/ethernet/netronome/nfp/nfp_port.c index 7bd8be5c833b0..9c1298114c70f 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_port.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_port.c @@ -181,7 +181,11 @@ nfp_port_get_phys_port_name(struct net_device *netdev, char *name, size_t len) eth_port->label_subport); break; case NFP_PORT_PF_PORT: - n = snprintf(name, len, "pf%d", port->pf_id); + if (!port->pf_split) + n = snprintf(name, len, "pf%d", port->pf_id); + else + n = snprintf(name, len, "pf%ds%d", port->pf_id, + port->pf_split_id); break; case NFP_PORT_VF_PORT: n = snprintf(name, len, "pf%dvf%d", port->pf_id, port->vf_id); @@ -218,6 +222,8 @@ int nfp_port_configure(struct net_device *netdev, bool configed) eth_port = __nfp_port_get_eth_port(port); if (!eth_port) return 0; + if (port->eth_forced) + return 0; err = nfp_eth_set_configured(port->app->cpp, eth_port->index, configed); return err < 0 && err != -EOPNOTSUPP ? err : 0; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_port.h b/drivers/net/ethernet/netronome/nfp/nfp_port.h index fa7e669a969c6..51f10ae2d53e4 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_port.h +++ b/drivers/net/ethernet/netronome/nfp/nfp_port.h @@ -77,10 +77,13 @@ enum nfp_port_flags { * @app: backpointer to the app structure * @dl_port: devlink port structure * @eth_id: for %NFP_PORT_PHYS_PORT port ID in NFP enumeration scheme + * @eth_forced: for %NFP_PORT_PHYS_PORT port is forced UP or DOWN, don't change * @eth_port: for %NFP_PORT_PHYS_PORT translated ETH Table port entry * @eth_stats: for %NFP_PORT_PHYS_PORT MAC stats if available * @pf_id: for %NFP_PORT_PF_PORT, %NFP_PORT_VF_PORT ID of the PCI PF (0-3) * @vf_id: for %NFP_PORT_VF_PORT ID of the PCI VF within @pf_id + * @pf_split: for %NFP_PORT_PF_PORT %true if PCI PF has more than one vNIC + * @pf_split_id:for %NFP_PORT_PF_PORT ID of PCI PF vNIC (valid if @pf_split) * @vnic: for %NFP_PORT_PF_PORT, %NFP_PORT_VF_PORT vNIC ctrl memory * @port_list: entry on pf's list of ports */ @@ -99,6 +102,7 @@ struct nfp_port { /* NFP_PORT_PHYS_PORT */ struct { unsigned int eth_id; + bool eth_forced; struct nfp_eth_table_port *eth_port; u8 __iomem *eth_stats; }; @@ -106,6 +110,8 @@ struct nfp_port { struct { unsigned int pf_id; unsigned int vf_id; + bool pf_split; + unsigned int pf_split_id; u8 __iomem *vnic; }; }; @@ -116,6 +122,8 @@ struct nfp_port { extern const struct ethtool_ops nfp_port_ethtool_ops; extern const struct switchdev_ops nfp_port_switchdev_ops; +__printf(2, 3) u8 *nfp_pr_et(u8 *data, const char *fmt, ...); + int nfp_port_setup_tc(struct net_device *netdev, enum tc_setup_type type, void *type_data); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c b/drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c new file mode 100644 index 0000000000000..0ecd837053682 --- /dev/null +++ b/drivers/net/ethernet/netronome/nfp/nfp_shared_buf.c @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +/* + * Copyright (C) 2018 Netronome Systems, Inc. + * + * This software is dual licensed under the GNU General License Version 2, + * June 1991 as shown in the file COPYING in the top-level directory of this + * source tree or the BSD 2-Clause License provided below. You have the + * option to license this software under the complete terms of either license. + * + * The BSD 2-Clause License: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <linux/kernel.h> +#include <net/devlink.h> + +#include "nfpcore/nfp_cpp.h" +#include "nfpcore/nfp_nffw.h" +#include "nfp_abi.h" +#include "nfp_app.h" +#include "nfp_main.h" + +static u32 nfp_shared_buf_pool_unit(struct nfp_pf *pf, unsigned int sb) +{ + __le32 sb_id = cpu_to_le32(sb); + unsigned int i; + + for (i = 0; i < pf->num_shared_bufs; i++) + if (pf->shared_bufs[i].id == sb_id) + return le32_to_cpu(pf->shared_bufs[i].pool_size_unit); + + WARN_ON_ONCE(1); + return 0; +} + +int nfp_shared_buf_pool_get(struct nfp_pf *pf, unsigned int sb, u16 pool_index, + struct devlink_sb_pool_info *pool_info) +{ + struct nfp_shared_buf_pool_info_get get_data; + struct nfp_shared_buf_pool_id id = { + .shared_buf = cpu_to_le32(sb), + .pool = cpu_to_le32(pool_index), + }; + unsigned int unit_size; + int n; + + unit_size = nfp_shared_buf_pool_unit(pf, sb); + if (!unit_size) + return -EINVAL; + + n = nfp_mbox_cmd(pf, NFP_MBOX_POOL_GET, &id, sizeof(id), + &get_data, sizeof(get_data)); + if (n < 0) + return n; + if (n < sizeof(get_data)) + return -EIO; + + pool_info->pool_type = le32_to_cpu(get_data.pool_type); + pool_info->threshold_type = le32_to_cpu(get_data.threshold_type); + pool_info->size = le32_to_cpu(get_data.size) * unit_size; + + return 0; +} + +int nfp_shared_buf_pool_set(struct nfp_pf *pf, unsigned int sb, + u16 pool_index, u32 size, + enum devlink_sb_threshold_type threshold_type) +{ + struct nfp_shared_buf_pool_info_set set_data = { + .id = { + .shared_buf = cpu_to_le32(sb), + .pool = cpu_to_le32(pool_index), + }, + .threshold_type = cpu_to_le32(threshold_type), + }; + unsigned int unit_size; + + unit_size = nfp_shared_buf_pool_unit(pf, sb); + if (!unit_size || size % unit_size) + return -EINVAL; + set_data.size = cpu_to_le32(size / unit_size); + + return nfp_mbox_cmd(pf, NFP_MBOX_POOL_SET, &set_data, sizeof(set_data), + NULL, 0); +} + +int nfp_shared_buf_register(struct nfp_pf *pf) +{ + struct devlink *devlink = priv_to_devlink(pf); + unsigned int i, num_entries, entry_sz; + struct nfp_cpp_area *sb_desc_area; + u8 __iomem *sb_desc; + int n, err; + + if (!pf->mbox) + return 0; + + n = nfp_pf_rtsym_read_optional(pf, NFP_SHARED_BUF_COUNT_SYM_NAME, 0); + if (n <= 0) + return n; + num_entries = n; + + sb_desc = nfp_pf_map_rtsym(pf, "sb_tbl", NFP_SHARED_BUF_TABLE_SYM_NAME, + num_entries * sizeof(pf->shared_bufs[0]), + &sb_desc_area); + if (IS_ERR(sb_desc)) + return PTR_ERR(sb_desc); + + entry_sz = nfp_cpp_area_size(sb_desc_area) / num_entries; + + pf->shared_bufs = kmalloc_array(num_entries, sizeof(pf->shared_bufs[0]), + GFP_KERNEL); + if (!pf->shared_bufs) { + err = -ENOMEM; + goto err_release_area; + } + + for (i = 0; i < num_entries; i++) { + struct nfp_shared_buf *sb = &pf->shared_bufs[i]; + + /* Entries may be larger in future FW */ + memcpy_fromio(sb, sb_desc + i * entry_sz, sizeof(*sb)); + + err = devlink_sb_register(devlink, + le32_to_cpu(sb->id), + le32_to_cpu(sb->size), + le16_to_cpu(sb->ingress_pools_count), + le16_to_cpu(sb->egress_pools_count), + le16_to_cpu(sb->ingress_tc_count), + le16_to_cpu(sb->egress_tc_count)); + if (err) + goto err_unreg_prev; + } + pf->num_shared_bufs = num_entries; + + nfp_cpp_area_release_free(sb_desc_area); + + return 0; + +err_unreg_prev: + while (i--) + devlink_sb_unregister(devlink, + le32_to_cpu(pf->shared_bufs[i].id)); + kfree(pf->shared_bufs); +err_release_area: + nfp_cpp_area_release_free(sb_desc_area); + return err; +} + +void nfp_shared_buf_unregister(struct nfp_pf *pf) +{ + struct devlink *devlink = priv_to_devlink(pf); + unsigned int i; + + for (i = 0; i < pf->num_shared_bufs; i++) + devlink_sb_unregister(devlink, + le32_to_cpu(pf->shared_bufs[i].id)); + kfree(pf->shared_bufs); +} diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h index ced62d112aa24..f44d0a8573149 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp.h @@ -94,6 +94,8 @@ int nfp_nsp_read_sensors(struct nfp_nsp *state, unsigned int sensor_mask, /* MAC Statistics Accumulator */ #define NFP_RESOURCE_MAC_STATISTICS "mac.stat" +int nfp_resource_table_init(struct nfp_cpp *cpp); + struct nfp_resource * nfp_resource_acquire(struct nfp_cpp *cpp, const char *name); diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c index cd678323bacb4..749655c329b24 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp6000_pcie.c @@ -933,7 +933,6 @@ static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, u32 *wrptr32 = kernel_vaddr; const u32 __iomem *rdptr32; int n, width; - bool is_64; priv = nfp_cpp_area_priv(area); rdptr64 = priv->iomem + offset; @@ -943,10 +942,15 @@ static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, return -EFAULT; width = priv->width.read; - if (width <= 0) return -EINVAL; + /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */ + if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) && + priv->action == NFP_CPP_ACTION_RW && + (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) + width = TARGET_WIDTH_32; + /* Unaligned? Translate to an explicit access */ if ((priv->offset + offset) & (width - 1)) return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area), @@ -956,36 +960,29 @@ static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr, priv->offset + offset, kernel_vaddr, length, width); - is_64 = width == TARGET_WIDTH_64; - - /* MU reads via a PCIe2CPP BAR supports 32bit (and other) lengths */ - if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && - priv->action == NFP_CPP_ACTION_RW) - is_64 = false; + if (WARN_ON(!priv->bar)) + return -EFAULT; - if (is_64) { - if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) - return -EINVAL; - } else { + switch (width) { + case TARGET_WIDTH_32: if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) return -EINVAL; - } - if (WARN_ON(!priv->bar)) - return -EFAULT; + for (n = 0; n < length; n += sizeof(u32)) + *wrptr32++ = __raw_readl(rdptr32++); + return n; +#ifdef __raw_readq + case TARGET_WIDTH_64: + if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) + return -EINVAL; - if (is_64) -#ifndef __raw_readq - return -EINVAL; -#else for (n = 0; n < length; n += sizeof(u64)) *wrptr64++ = __raw_readq(rdptr64++); + return n; #endif - else - for (n = 0; n < length; n += sizeof(u32)) - *wrptr32++ = __raw_readl(rdptr32++); - - return n; + default: + return -EINVAL; + } } static int @@ -999,7 +996,6 @@ nfp6000_area_write(struct nfp_cpp_area *area, struct nfp6000_area_priv *priv; u32 __iomem *wrptr32; int n, width; - bool is_64; priv = nfp_cpp_area_priv(area); wrptr64 = priv->iomem + offset; @@ -1009,10 +1005,15 @@ nfp6000_area_write(struct nfp_cpp_area *area, return -EFAULT; width = priv->width.write; - if (width <= 0) return -EINVAL; + /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */ + if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && + priv->action == NFP_CPP_ACTION_RW && + (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4)) + width = TARGET_WIDTH_32; + /* Unaligned? Translate to an explicit access */ if ((priv->offset + offset) & (width - 1)) return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area), @@ -1022,40 +1023,33 @@ nfp6000_area_write(struct nfp_cpp_area *area, priv->offset + offset, kernel_vaddr, length, width); - is_64 = width == TARGET_WIDTH_64; - - /* MU writes via a PCIe2CPP BAR supports 32bit (and other) lengths */ - if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) && - priv->action == NFP_CPP_ACTION_RW) - is_64 = false; + if (WARN_ON(!priv->bar)) + return -EFAULT; - if (is_64) { - if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) - return -EINVAL; - } else { + switch (width) { + case TARGET_WIDTH_32: if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0) return -EINVAL; - } - if (WARN_ON(!priv->bar)) - return -EFAULT; + for (n = 0; n < length; n += sizeof(u32)) { + __raw_writel(*rdptr32++, wrptr32++); + wmb(); + } + return n; +#ifdef __raw_writeq + case TARGET_WIDTH_64: + if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0) + return -EINVAL; - if (is_64) -#ifndef __raw_writeq - return -EINVAL; -#else for (n = 0; n < length; n += sizeof(u64)) { __raw_writeq(*rdptr64++, wrptr64++); wmb(); } + return n; #endif - else - for (n = 0; n < length; n += sizeof(u32)) { - __raw_writel(*rdptr32++, wrptr32++); - wmb(); - } - - return n; + default: + return -EINVAL; + } } struct nfp6000_explicit_priv { @@ -1330,6 +1324,7 @@ struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev) /* Finished with card initialization. */ dev_info(&pdev->dev, "Netronome Flow Processor NFP4000/NFP6000 PCIe Card Probe\n"); + pcie_print_link_status(pdev); nfp = kzalloc(sizeof(*nfp), GFP_KERNEL); if (!nfp) { diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h index c8f2c064cce37..b0da3d4368505 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_cpp.h @@ -87,6 +87,11 @@ struct resource; #define NFP_CPP_TARGET_ID_MASK 0x1f +#define NFP_CPP_ATOMIC_RD(target, island) \ + NFP_CPP_ISLAND_ID((target), 3, 0, (island)) +#define NFP_CPP_ATOMIC_WR(target, island) \ + NFP_CPP_ISLAND_ID((target), 4, 0, (island)) + /** * NFP_CPP_ID() - pack target, token, and action into a CPP ID. * @target: NFP CPP target id @@ -295,6 +300,8 @@ void nfp_cpp_mutex_free(struct nfp_cpp_mutex *mutex); int nfp_cpp_mutex_lock(struct nfp_cpp_mutex *mutex); int nfp_cpp_mutex_unlock(struct nfp_cpp_mutex *mutex); int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex); +int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target, + unsigned long long address); /** * nfp_cppcore_pcie_unit() - Get PCI Unit of a CPP handle diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c index cb28ac03e4cac..c88bf673cb767 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_mutex.c @@ -59,6 +59,11 @@ static u32 nfp_mutex_unlocked(u16 interface) return (u32)interface << 16 | 0x0000; } +static u32 nfp_mutex_owner(u32 val) +{ + return val >> 16; +} + static bool nfp_mutex_is_locked(u32 val) { return (val & 0xffff) == 0x000f; @@ -351,3 +356,43 @@ int nfp_cpp_mutex_trylock(struct nfp_cpp_mutex *mutex) return nfp_mutex_is_locked(tmp) ? -EBUSY : -EINVAL; } + +/** + * nfp_cpp_mutex_reclaim() - Unlock mutex if held by local endpoint + * @cpp: NFP CPP handle + * @target: NFP CPP target ID (ie NFP_CPP_TARGET_CLS or NFP_CPP_TARGET_MU) + * @address: Offset into the address space of the NFP CPP target ID + * + * Release lock if held by local system. Extreme care is advised, call only + * when no local lock users can exist. + * + * Return: 0 if the lock was OK, 1 if locked by us, -errno on invalid mutex + */ +int nfp_cpp_mutex_reclaim(struct nfp_cpp *cpp, int target, + unsigned long long address) +{ + const u32 mur = NFP_CPP_ID(target, 3, 0); /* atomic_read */ + const u32 muw = NFP_CPP_ID(target, 4, 0); /* atomic_write */ + u16 interface = nfp_cpp_interface(cpp); + int err; + u32 tmp; + + err = nfp_cpp_mutex_validate(interface, &target, address); + if (err) + return err; + + /* Check lock */ + err = nfp_cpp_readl(cpp, mur, address, &tmp); + if (err < 0) + return err; + + if (nfp_mutex_is_unlocked(tmp) || nfp_mutex_owner(tmp) != interface) + return 0; + + /* Bust the lock */ + err = nfp_cpp_writel(cpp, muw, address, nfp_mutex_unlocked(interface)); + if (err < 0) + return err; + + return 1; +} diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h index c9724fb7ea4b2..df599d5b6bb35 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_nffw.h @@ -100,6 +100,8 @@ nfp_rtsym_lookup(struct nfp_rtsym_table *rtbl, const char *name); u64 nfp_rtsym_read_le(struct nfp_rtsym_table *rtbl, const char *name, int *error); +int nfp_rtsym_write_le(struct nfp_rtsym_table *rtbl, const char *name, + u64 value); u8 __iomem * nfp_rtsym_map(struct nfp_rtsym_table *rtbl, const char *name, const char *id, unsigned int min_size, struct nfp_cpp_area **area); diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c index 7e14725055c7b..2dd89dba9311a 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_resource.c @@ -338,3 +338,62 @@ u64 nfp_resource_size(struct nfp_resource *res) { return res->size; } + +/** + * nfp_resource_table_init() - Run initial checks on the resource table + * @cpp: NFP CPP handle + * + * Start-of-day init procedure for resource table. Must be called before + * any local resource table users may exist. + * + * Return: 0 on success, -errno on failure + */ +int nfp_resource_table_init(struct nfp_cpp *cpp) +{ + struct nfp_cpp_mutex *dev_mutex; + int i, err; + + err = nfp_cpp_mutex_reclaim(cpp, NFP_RESOURCE_TBL_TARGET, + NFP_RESOURCE_TBL_BASE); + if (err < 0) { + nfp_err(cpp, "Error: failed to reclaim resource table mutex\n"); + return err; + } + if (err) + nfp_warn(cpp, "Warning: busted main resource table mutex\n"); + + dev_mutex = nfp_cpp_mutex_alloc(cpp, NFP_RESOURCE_TBL_TARGET, + NFP_RESOURCE_TBL_BASE, + NFP_RESOURCE_TBL_KEY); + if (!dev_mutex) + return -ENOMEM; + + if (nfp_cpp_mutex_lock(dev_mutex)) { + nfp_err(cpp, "Error: failed to claim resource table mutex\n"); + nfp_cpp_mutex_free(dev_mutex); + return -EINVAL; + } + + /* Resource 0 is the dev_mutex, start from 1 */ + for (i = 1; i < NFP_RESOURCE_TBL_ENTRIES; i++) { + u64 addr = NFP_RESOURCE_TBL_BASE + + sizeof(struct nfp_resource_entry) * i; + + err = nfp_cpp_mutex_reclaim(cpp, NFP_RESOURCE_TBL_TARGET, addr); + if (err < 0) { + nfp_err(cpp, + "Error: failed to reclaim resource %d mutex\n", + i); + goto err_unlock; + } + if (err) + nfp_warn(cpp, "Warning: busted resource %d mutex\n", i); + } + + err = 0; +err_unlock: + nfp_cpp_mutex_unlock(dev_mutex); + nfp_cpp_mutex_free(dev_mutex); + + return err; +} diff --git a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c index 46107aefad1cc..9e34216578daf 100644 --- a/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c +++ b/drivers/net/ethernet/netronome/nfp/nfpcore/nfp_rtsym.c @@ -286,6 +286,49 @@ exit: return val; } +/** + * nfp_rtsym_write_le() - Write an unsigned scalar value to a symbol + * @rtbl: NFP RTsym table + * @name: Symbol name + * @value: Value to write + * + * Lookup a symbol and write a value to it. Symbol can be 4 or 8 bytes in size. + * If 4 bytes then the lower 32-bits of 'value' are used. Value will be + * written as simple little-endian unsigned value. + * + * Return: 0 on success or error code. + */ +int nfp_rtsym_write_le(struct nfp_rtsym_table *rtbl, const char *name, + u64 value) +{ + const struct nfp_rtsym *sym; + int err; + u32 id; + + sym = nfp_rtsym_lookup(rtbl, name); + if (!sym) + return -ENOENT; + + id = NFP_CPP_ISLAND_ID(sym->target, NFP_CPP_ACTION_RW, 0, sym->domain); + + switch (sym->size) { + case 4: + err = nfp_cpp_writel(rtbl->cpp, id, sym->addr, value); + break; + case 8: + err = nfp_cpp_writeq(rtbl->cpp, id, sym->addr, value); + break; + default: + nfp_err(rtbl->cpp, + "rtsym '%s' unsupported or non-scalar size: %lld\n", + name, sym->size); + err = -EINVAL; + break; + } + + return err; +} + u8 __iomem * nfp_rtsym_map(struct nfp_rtsym_table *rtbl, const char *name, const char *id, unsigned int min_size, struct nfp_cpp_area **area) |