archives/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-03-08 05:04:51 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

RDMA v6.17 merge window pull request

Browse source 
- Various minor code cleanups and fixes for hns, iser, cxgb4, hfi1, rxe,
   erdma, mana_ib
 
 - Prefetch supprot for rxe ODP
 
 - Remove memory window support from hns as new device FW is no longer
   support it
 
 - Remove qib, it is very old and obsolete now, Cornelis wishes to
   restructure the hfi1/qib shared layer
 
 - Fix a race in destroying CQs where we can still end up with work running
   because the work is cancled before the driver stops triggering it
 
 - Improve interaction with namespaces.
    * Follow the devlink namespace for newly spawned RDMA devices
    * Create iopoib net devces in the parent IB device's namespace
    * Allow CAP_NET_RAW checks to pass in user namespaces
 
 - A new flow control scheme for IB MADs to try and avoid queue overflows
   in the network
 
 - Fix 2G message sizes in bnxt_re
 
 - Optimize mkey layout for mlx5 DMABUF
 
 - New "DMA Handle" concept to allow controlling PCI TPH and steering tags
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQRRRCHOFoQz/8F5bUaFwuHvBreFYQUCaIpYoAAKCRCFwuHvBreF
 YUBaAP9Av4O3tX+xV9lpwXqOS6fE34h5KlvULoF+RMtBpkbW6QEAh+e34i3ay3lY
 gQPI3WZV0Vr1lwLv+g8Pyuxt/1JdXQ8=
 =LCBi
 -----END PGP SIGNATURE-----

Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma

Pull rdma updates from Jason Gunthorpe:

 - Various minor code cleanups and fixes for hns, iser, cxgb4, hfi1,
   rxe, erdma, mana_ib

 - Prefetch supprot for rxe ODP

 - Remove memory window support from hns as new device FW is no longer
   support it

 - Remove qib, it is very old and obsolete now, Cornelis wishes to
   restructure the hfi1/qib shared layer

 - Fix a race in destroying CQs where we can still end up with work
   running because the work is cancled before the driver stops
   triggering it

 - Improve interaction with namespaces:
     * Follow the devlink namespace for newly spawned RDMA devices
     * Create iopoib net devces in the parent IB device's namespace
     * Allow CAP_NET_RAW checks to pass in user namespaces

 - A new flow control scheme for IB MADs to try and avoid queue
   overflows in the network

 - Fix 2G message sizes in bnxt_re

 - Optimize mkey layout for mlx5 DMABUF

 - New "DMA Handle" concept to allow controlling PCI TPH and steering
   tags

* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (71 commits)
  RDMA/siw: Change maintainer email address
  RDMA/mana_ib: add support of multiple ports
  RDMA/mlx5: Refactor optional counters steering code
  RDMA/mlx5: Add DMAH support for reg_user_mr/reg_user_dmabuf_mr
  IB: Extend UVERBS_METHOD_REG_MR to get DMAH
  RDMA/mlx5: Add DMAH object support
  RDMA/core: Introduce a DMAH object and its alloc/free APIs
  IB/core: Add UVERBS_METHOD_REG_MR on the MR object
  net/mlx5: Add support for device steering tag
  net/mlx5: Expose IFC bits for TPH
  PCI/TPH: Expose pcie_tph_get_st_table_size()
  RDMA/mlx5: Fix incorrect MKEY masking
  RDMA/mlx5: Fix returned type from _mlx5r_umr_zap_mkey()
  RDMA/mlx5: remove redundant check on err on return expression
  RDMA/mana_ib: add additional port counters
  RDMA/mana_ib: Fix DSCP value in modify QP
  RDMA/efa: Add CQ with external memory support
  RDMA/core: Add umem "is_contiguous" and "start_dma_addr" helpers
  RDMA/uverbs: Add a common way to create CQ with umem
  RDMA/mlx5: Optimize DMABUF mkey page size
  ...
This commit is contained in:
Linus Torvalds 2025-07-31 12:19:55 -07:00
commit 7ce4de1cda
153 changed files with 2867 additions and 49165 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.mailmap
View file

@ -138,6 +138,7 @@ Benjamin Poirier <benjamin.poirier@gmail.com> <bpoirier@suse.de>
Benjamin Tissoires <bentiss@kernel.org> <benjamin.tissoires@gmail.com>
Benjamin Tissoires <bentiss@kernel.org> <benjamin.tissoires@redhat.com>
Benno Lossin <lossin@kernel.org> <benno.lossin@proton.me>
Bernard Metzler <bernard.metzler@linux.dev> <bmt@zurich.ibm.com>
Bingwu Zhang <xtex@aosc.io> <xtexchooser@duck.com>
Bingwu Zhang <xtex@aosc.io> <xtex@xtexx.eu.org>
Bjorn Andersson <andersson@kernel.org> <bjorn@kryo.se>

8
MAINTAINERS
View file

@ -20370,12 +20370,6 @@ S: Maintained
F: drivers/firmware/qemu_fw_cfg.c
F: include/uapi/linux/qemu_fw_cfg.h
QIB DRIVER
M: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
L: linux-rdma@vger.kernel.org
S: Supported
F: drivers/infiniband/hw/qib/
QLOGIC QL41xxx FCOE DRIVER
M: Saurav Kashyap <skashyap@marvell.com>
M: Javed Hasan <jhasan@marvell.com>
@ -23222,7 +23216,7 @@ S: Maintained
F: drivers/leds/leds-net48xx.c
SOFT-IWARP DRIVER (siw)
M: Bernard Metzler <bmt@zurich.ibm.com>
M: Bernard Metzler <bernard.metzler@linux.dev>
L: linux-rdma@vger.kernel.org
S: Supported
F: drivers/infiniband/sw/siw/

1
drivers/infiniband/Kconfig
View file

@ -92,7 +92,6 @@ source "drivers/infiniband/hw/mlx5/Kconfig"
source "drivers/infiniband/hw/mthca/Kconfig"
source "drivers/infiniband/hw/ocrdma/Kconfig"
source "drivers/infiniband/hw/qedr/Kconfig"
source "drivers/infiniband/hw/qib/Kconfig"
source "drivers/infiniband/hw/usnic/Kconfig"
source "drivers/infiniband/hw/vmw_pvrdma/Kconfig"
source "drivers/infiniband/sw/rdmavt/Kconfig"

1
drivers/infiniband/core/Makefile
View file

@ -33,6 +33,7 @@ ib_umad-y := user_mad.o
ib_uverbs-y := uverbs_main.o uverbs_cmd.o uverbs_marshall.o \
rdma_core.o uverbs_std_types.o uverbs_ioctl.o \
uverbs_std_types_cq.o \
uverbs_std_types_dmah.o \
uverbs_std_types_flow_action.o uverbs_std_types_dm.o \
uverbs_std_types_mr.o uverbs_std_types_counters.o \
uverbs_uapi.o uverbs_std_types_device.o \

47
drivers/infiniband/core/cm.c
View file

@ -161,6 +161,7 @@ struct cm_counter_attribute {
struct cm_port {
struct cm_device *cm_dev;
struct ib_mad_agent *mad_agent;
struct ib_mad_agent *rep_agent;
u32 port_num;
atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
};
@ -274,7 +275,8 @@ static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
complete(&cm_id_priv->comp);
}
static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
static struct ib_mad_send_buf *
cm_alloc_msg_agent(struct cm_id_private *cm_id_priv, bool rep_agent)
{
struct ib_mad_agent *mad_agent;
struct ib_mad_send_buf *m;
@ -286,7 +288,8 @@ static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
return ERR_PTR(-EINVAL);
read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
mad_agent = cm_id_priv->av.port->mad_agent;
mad_agent = rep_agent ? cm_id_priv->av.port->rep_agent :
cm_id_priv->av.port->mad_agent;
if (!mad_agent) {
m = ERR_PTR(-EINVAL);
goto out;
@ -315,6 +318,11 @@ out:
return m;
}
static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
{
return cm_alloc_msg_agent(cm_id_priv, false);
}
static void cm_free_msg(struct ib_mad_send_buf *msg)
{
if (msg->ah)
@ -323,13 +331,14 @@ static void cm_free_msg(struct ib_mad_send_buf *msg)
}
static struct ib_mad_send_buf *
cm_alloc_priv_msg(struct cm_id_private *cm_id_priv, enum ib_cm_state state)
cm_alloc_priv_msg_rep(struct cm_id_private *cm_id_priv, enum ib_cm_state state,
bool rep_agent)
{
struct ib_mad_send_buf *msg;
lockdep_assert_held(&cm_id_priv->lock);
msg = cm_alloc_msg(cm_id_priv);
msg = cm_alloc_msg_agent(cm_id_priv, rep_agent);
if (IS_ERR(msg))
return msg;
@ -344,6 +353,12 @@ cm_alloc_priv_msg(struct cm_id_private *cm_id_priv, enum ib_cm_state state)
return msg;
}
static struct ib_mad_send_buf *
cm_alloc_priv_msg(struct cm_id_private *cm_id_priv, enum ib_cm_state state)
{
return cm_alloc_priv_msg_rep(cm_id_priv, state, false);
}
static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
{
struct cm_id_private *cm_id_priv = msg->context[0];
@ -2295,7 +2310,7 @@ int ib_send_cm_rep(struct ib_cm_id *cm_id,
goto out;
}
msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_REP_SENT);
msg = cm_alloc_priv_msg_rep(cm_id_priv, IB_CM_REP_SENT, true);
if (IS_ERR(msg)) {
ret = PTR_ERR(msg);
goto out;
@ -4380,9 +4395,22 @@ static int cm_add_one(struct ib_device *ib_device)
goto error2;
}
port->rep_agent = ib_register_mad_agent(ib_device, i,
IB_QPT_GSI,
NULL,
0,
cm_send_handler,
NULL,
port,
0);
if (IS_ERR(port->rep_agent)) {
ret = PTR_ERR(port->rep_agent);
goto error3;
}
ret = ib_modify_port(ib_device, i, 0, &port_modify);
if (ret)
goto error3;
goto error4;
count++;
}
@ -4397,6 +4425,8 @@ static int cm_add_one(struct ib_device *ib_device)
write_unlock_irqrestore(&cm.device_lock, flags);
return 0;
error4:
ib_unregister_mad_agent(port->rep_agent);
error3:
ib_unregister_mad_agent(port->mad_agent);
error2:
@ -4410,6 +4440,7 @@ error1:
port = cm_dev->port[i-1];
ib_modify_port(ib_device, port->port_num, 0, &port_modify);
ib_unregister_mad_agent(port->rep_agent);
ib_unregister_mad_agent(port->mad_agent);
ib_port_unregister_client_groups(ib_device, i,
cm_counter_groups);
@ -4439,12 +4470,14 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
rdma_for_each_port (ib_device, i) {
struct ib_mad_agent *mad_agent;
struct ib_mad_agent *rep_agent;
if (!rdma_cap_ib_cm(ib_device, i))
continue;
port = cm_dev->port[i-1];
mad_agent = port->mad_agent;
rep_agent = port->rep_agent;
ib_modify_port(ib_device, port->port_num, 0, &port_modify);
/*
* We flush the queue here after the going_down set, this
@ -4458,8 +4491,10 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
*/
write_lock(&cm_dev->mad_agent_lock);
port->mad_agent = NULL;
port->rep_agent = NULL;
write_unlock(&cm_dev->mad_agent_lock);
ib_unregister_mad_agent(mad_agent);
ib_unregister_mad_agent(rep_agent);
ib_port_unregister_client_groups(ib_device, i,
cm_counter_groups);
}

2
drivers/infiniband/core/counters.c
View file

@ -461,7 +461,7 @@ static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num)
return NULL;
qp = container_of(res, struct ib_qp, res);
if (qp->qp_type == IB_QPT_RAW_PACKET && !capable(CAP_NET_RAW))
if (qp->qp_type == IB_QPT_RAW_PACKET && !rdma_dev_has_raw_cap(dev))
goto err;
return qp;

12
drivers/infiniband/core/cq.c
View file

@ -317,13 +317,18 @@ EXPORT_SYMBOL(__ib_alloc_cq_any);
*/
void ib_free_cq(struct ib_cq *cq)
{
int ret;
int ret = 0;
if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
return;
if (WARN_ON_ONCE(cq->cqe_used))
return;
if (cq->device->ops.pre_destroy_cq) {
ret = cq->device->ops.pre_destroy_cq(cq);
WARN_ONCE(ret, "Disable of kernel CQ shouldn't fail");
}
switch (cq->poll_ctx) {
case IB_POLL_DIRECT:
break;
@ -340,7 +345,10 @@ void ib_free_cq(struct ib_cq *cq)
rdma_dim_destroy(cq);
trace_cq_free(cq);
ret = cq->device->ops.destroy_cq(cq, NULL);
if (cq->device->ops.post_destroy_cq)
cq->device->ops.post_destroy_cq(cq);
else
ret = cq->device->ops.destroy_cq(cq, NULL);
WARN_ONCE(ret, "Destroy of kernel CQ shouldn't fail");
rdma_restrack_del(&cq->res);
kfree(cq->wc);

47
drivers/infiniband/core/device.c
View file

@ -145,6 +145,33 @@ bool rdma_dev_access_netns(const struct ib_device *dev, const struct net *net)
}
EXPORT_SYMBOL(rdma_dev_access_netns);
/**
* rdma_dev_has_raw_cap() - Returns whether a specified rdma device has
* CAP_NET_RAW capability or not.
*
* @dev: Pointer to rdma device whose capability to be checked
*
* Returns true if a rdma device's owning user namespace has CAP_NET_RAW
* capability, otherwise false. When rdma subsystem is in legacy shared network,
* namespace mode, the default net namespace is considered.
*/
bool rdma_dev_has_raw_cap(const struct ib_device *dev)
{
const struct net *net;
/* Network namespace is the resource whose user namespace
* to be considered. When in shared mode, there is no reliable
* network namespace resource, so consider the default net namespace.
*/
if (ib_devices_shared_netns)
net = &init_net;
else
net = read_pnet(&dev->coredev.rdma_net);
return ns_capable(net->user_ns, CAP_NET_RAW);
}
EXPORT_SYMBOL(rdma_dev_has_raw_cap);
/*
* xarray has this behavior where it won't iterate over NULL values stored in
* allocated arrays. So we need our own iterator to see all values stored in
@ -557,6 +584,8 @@ static void rdma_init_coredev(struct ib_core_device *coredev,
/**
* _ib_alloc_device - allocate an IB device struct
* @size:size of structure to allocate
* @net: network namespace device should be located in, namespace
* must stay valid until ib_register_device() is completed.
*
* Low-level drivers should use ib_alloc_device() to allocate &struct
* ib_device. @size is the size of the structure to be allocated,
@ -564,7 +593,7 @@ static void rdma_init_coredev(struct ib_core_device *coredev,
* ib_dealloc_device() must be used to free structures allocated with
* ib_alloc_device().
*/
struct ib_device *_ib_alloc_device(size_t size)
struct ib_device *_ib_alloc_device(size_t size, struct net *net)
{
struct ib_device *device;
unsigned int i;
@ -581,7 +610,15 @@ struct ib_device *_ib_alloc_device(size_t size)
return NULL;
}
rdma_init_coredev(&device->coredev, device, &init_net);
/* ib_devices_shared_netns can't change while we have active namespaces
* in the system which means either init_net is passed or the user has
* no idea what they are doing.
*
* To avoid breaking backward compatibility, when in shared mode,
* force to init the device in the init_net.
*/
net = ib_devices_shared_netns ? &init_net : net;
rdma_init_coredev(&device->coredev, device, net);
INIT_LIST_HEAD(&device->event_handler_list);
spin_lock_init(&device->qp_open_list_lock);
@ -2671,6 +2708,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_DEVICE_OP(dev_ops, add_sub_dev);
SET_DEVICE_OP(dev_ops, advise_mr);
SET_DEVICE_OP(dev_ops, alloc_dm);
SET_DEVICE_OP(dev_ops, alloc_dmah);
SET_DEVICE_OP(dev_ops, alloc_hw_device_stats);
SET_DEVICE_OP(dev_ops, alloc_hw_port_stats);
SET_DEVICE_OP(dev_ops, alloc_mr);
@ -2691,6 +2729,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_DEVICE_OP(dev_ops, create_ah);
SET_DEVICE_OP(dev_ops, create_counters);
SET_DEVICE_OP(dev_ops, create_cq);
SET_DEVICE_OP(dev_ops, create_cq_umem);
SET_DEVICE_OP(dev_ops, create_flow);
SET_DEVICE_OP(dev_ops, create_qp);
SET_DEVICE_OP(dev_ops, create_rwq_ind_table);
@ -2698,6 +2737,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_DEVICE_OP(dev_ops, create_user_ah);
SET_DEVICE_OP(dev_ops, create_wq);
SET_DEVICE_OP(dev_ops, dealloc_dm);
SET_DEVICE_OP(dev_ops, dealloc_dmah);
SET_DEVICE_OP(dev_ops, dealloc_driver);
SET_DEVICE_OP(dev_ops, dealloc_mw);
SET_DEVICE_OP(dev_ops, dealloc_pd);
@ -2763,8 +2803,10 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_DEVICE_OP(dev_ops, modify_srq);
SET_DEVICE_OP(dev_ops, modify_wq);
SET_DEVICE_OP(dev_ops, peek_cq);
SET_DEVICE_OP(dev_ops, pre_destroy_cq);
SET_DEVICE_OP(dev_ops, poll_cq);
SET_DEVICE_OP(dev_ops, port_groups);
SET_DEVICE_OP(dev_ops, post_destroy_cq);
SET_DEVICE_OP(dev_ops, post_recv);
SET_DEVICE_OP(dev_ops, post_send);
SET_DEVICE_OP(dev_ops, post_srq_recv);
@ -2793,6 +2835,7 @@ void ib_set_device_ops(struct ib_device *dev, const struct ib_device_ops *ops)
SET_OBJ_SIZE(dev_ops, ib_ah);
SET_OBJ_SIZE(dev_ops, ib_counters);
SET_OBJ_SIZE(dev_ops, ib_cq);
SET_OBJ_SIZE(dev_ops, ib_dmah);
SET_OBJ_SIZE(dev_ops, ib_mw);
SET_OBJ_SIZE(dev_ops, ib_pd);
SET_OBJ_SIZE(dev_ops, ib_qp);

468
drivers/infiniband/core/mad.c
View file

@ -210,6 +210,29 @@ int ib_response_mad(const struct ib_mad_hdr *hdr)
}
EXPORT_SYMBOL(ib_response_mad);
#define SOL_FC_MAX_DEFAULT_FRAC 4
#define SOL_FC_MAX_SA_FRAC 32
static int get_sol_fc_max_outstanding(struct ib_mad_reg_req *mad_reg_req)
{
if (!mad_reg_req)
/* Send only agent */
return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
switch (mad_reg_req->mgmt_class) {
case IB_MGMT_CLASS_CM:
return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
case IB_MGMT_CLASS_SUBN_ADM:
return mad_recvq_size / SOL_FC_MAX_SA_FRAC;
case IB_MGMT_CLASS_SUBN_LID_ROUTED:
case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
return min(mad_recvq_size, IB_MAD_QP_RECV_SIZE) /
SOL_FC_MAX_DEFAULT_FRAC;
default:
return 0;
}
}
/*
* ib_register_mad_agent - Register to send/receive MADs
*
@ -391,13 +414,17 @@ struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
spin_lock_init(&mad_agent_priv->lock);
INIT_LIST_HEAD(&mad_agent_priv->send_list);
INIT_LIST_HEAD(&mad_agent_priv->wait_list);
INIT_LIST_HEAD(&mad_agent_priv->done_list);
INIT_LIST_HEAD(&mad_agent_priv->rmpp_list);
INIT_LIST_HEAD(&mad_agent_priv->backlog_list);
INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
INIT_LIST_HEAD(&mad_agent_priv->local_list);
INIT_WORK(&mad_agent_priv->local_work, local_completions);
refcount_set(&mad_agent_priv->refcount, 1);
init_completion(&mad_agent_priv->comp);
mad_agent_priv->sol_fc_send_count = 0;
mad_agent_priv->sol_fc_wait_count = 0;
mad_agent_priv->sol_fc_max =
recv_handler ? get_sol_fc_max_outstanding(mad_reg_req) : 0;
ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
if (ret2) {
@ -1055,6 +1082,180 @@ int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
return ret;
}
static void handle_queued_state(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_agent_private *mad_agent_priv)
{
if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) {
mad_agent_priv->sol_fc_wait_count--;
list_move_tail(&mad_send_wr->agent_list,
&mad_agent_priv->backlog_list);
} else {
expect_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
list_add_tail(&mad_send_wr->agent_list,
&mad_agent_priv->backlog_list);
}
}
static void handle_send_state(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_agent_private *mad_agent_priv)
{
if (mad_send_wr->state == IB_MAD_STATE_INIT) {
list_add_tail(&mad_send_wr->agent_list,
&mad_agent_priv->send_list);
} else {
expect_mad_state2(mad_send_wr, IB_MAD_STATE_WAIT_RESP,
IB_MAD_STATE_QUEUED);
list_move_tail(&mad_send_wr->agent_list,
&mad_agent_priv->send_list);
}
if (mad_send_wr->is_solicited_fc) {
if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
mad_agent_priv->sol_fc_wait_count--;
mad_agent_priv->sol_fc_send_count++;
}
}
static void handle_wait_state(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_agent_private *mad_agent_priv)
{
struct ib_mad_send_wr_private *temp_mad_send_wr;
struct list_head *list_item;
unsigned long delay;
expect_mad_state3(mad_send_wr, IB_MAD_STATE_SEND_START,
IB_MAD_STATE_WAIT_RESP, IB_MAD_STATE_CANCELED);
if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
mad_send_wr->is_solicited_fc) {
mad_agent_priv->sol_fc_send_count--;
mad_agent_priv->sol_fc_wait_count++;
}
list_del_init(&mad_send_wr->agent_list);
delay = mad_send_wr->timeout;
mad_send_wr->timeout += jiffies;
if (delay) {
list_for_each_prev(list_item,
&mad_agent_priv->wait_list) {
temp_mad_send_wr = list_entry(
list_item,
struct ib_mad_send_wr_private,
agent_list);
if (time_after(mad_send_wr->timeout,
temp_mad_send_wr->timeout))
break;
}
} else {
list_item = &mad_agent_priv->wait_list;
}
list_add(&mad_send_wr->agent_list, list_item);
}
static void handle_early_resp_state(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_agent_private *mad_agent_priv)
{
expect_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
mad_agent_priv->sol_fc_send_count -= mad_send_wr->is_solicited_fc;
}
static void handle_canceled_state(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_agent_private *mad_agent_priv)
{
not_expect_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
if (mad_send_wr->is_solicited_fc) {
if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
mad_agent_priv->sol_fc_send_count--;
else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
mad_agent_priv->sol_fc_wait_count--;
}
}
static void handle_done_state(struct ib_mad_send_wr_private *mad_send_wr,
struct ib_mad_agent_private *mad_agent_priv)
{
if (mad_send_wr->is_solicited_fc) {
if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
mad_agent_priv->sol_fc_send_count--;
else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
mad_agent_priv->sol_fc_wait_count--;
}
list_del_init(&mad_send_wr->agent_list);
}
void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
enum ib_mad_state new_state)
{
struct ib_mad_agent_private *mad_agent_priv =
mad_send_wr->mad_agent_priv;
switch (new_state) {
case IB_MAD_STATE_INIT:
break;
case IB_MAD_STATE_QUEUED:
handle_queued_state(mad_send_wr, mad_agent_priv);
break;
case IB_MAD_STATE_SEND_START:
handle_send_state(mad_send_wr, mad_agent_priv);
break;
case IB_MAD_STATE_WAIT_RESP:
handle_wait_state(mad_send_wr, mad_agent_priv);
if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
return;
break;
case IB_MAD_STATE_EARLY_RESP:
handle_early_resp_state(mad_send_wr, mad_agent_priv);
break;
case IB_MAD_STATE_CANCELED:
handle_canceled_state(mad_send_wr, mad_agent_priv);
break;
case IB_MAD_STATE_DONE:
handle_done_state(mad_send_wr, mad_agent_priv);
break;
}
mad_send_wr->state = new_state;
}
static bool is_solicited_fc_mad(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_rmpp_mad *rmpp_mad;
u8 mgmt_class;
if (!mad_send_wr->timeout)
return 0;
rmpp_mad = mad_send_wr->send_buf.mad;
if (mad_send_wr->mad_agent_priv->agent.rmpp_version &&
(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE))
return 0;
mgmt_class =
((struct ib_mad_hdr *)mad_send_wr->send_buf.mad)->mgmt_class;
return mgmt_class == IB_MGMT_CLASS_CM ||
mgmt_class == IB_MGMT_CLASS_SUBN_ADM ||
mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE;
}
static bool mad_is_for_backlog(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_mad_agent_private *mad_agent_priv =
mad_send_wr->mad_agent_priv;
if (!mad_send_wr->is_solicited_fc || !mad_agent_priv->sol_fc_max)
return false;
if (!list_empty(&mad_agent_priv->backlog_list))
return true;
return mad_agent_priv->sol_fc_send_count +
mad_agent_priv->sol_fc_wait_count >=
mad_agent_priv->sol_fc_max;
}
/*
* ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated
* with the registered client
@ -1080,9 +1281,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
if (ret)
goto error;
if (!send_buf->mad_agent->send_handler ||
(send_buf->timeout_ms &&
!send_buf->mad_agent->recv_handler)) {
if (!send_buf->mad_agent->send_handler) {
ret = -EINVAL;
goto error;
}
@ -1118,15 +1317,19 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
mad_send_wr->max_retries = send_buf->retries;
mad_send_wr->retries_left = send_buf->retries;
send_buf->retries = 0;
/* Reference for work request to QP + response */
mad_send_wr->refcount = 1 + (mad_send_wr->timeout > 0);
mad_send_wr->status = IB_WC_SUCCESS;
change_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
/* Reference MAD agent until send completes */
refcount_inc(&mad_agent_priv->refcount);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
list_add_tail(&mad_send_wr->agent_list,
&mad_agent_priv->send_list);
mad_send_wr->is_solicited_fc = is_solicited_fc_mad(mad_send_wr);
if (mad_is_for_backlog(mad_send_wr)) {
change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
return 0;
}
change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
@ -1138,7 +1341,7 @@ int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
if (ret < 0) {
/* Fail send request */
spin_lock_irqsave(&mad_agent_priv->lock, flags);
list_del(&mad_send_wr->agent_list);
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
deref_mad_agent(mad_agent_priv);
goto error;
@ -1746,7 +1949,19 @@ ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
*/
(is_direct(mad_hdr->mgmt_class) ||
rcv_has_same_gid(mad_agent_priv, wr, wc)))
return (wr->status == IB_WC_SUCCESS) ? wr : NULL;
return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
}
list_for_each_entry(wr, &mad_agent_priv->backlog_list, agent_list) {
if ((wr->tid == mad_hdr->tid) &&
rcv_has_same_class(wr, wc) &&
/*
* Don't check GID for direct routed MADs.
* These might have permissive LIDs.
*/
(is_direct(mad_hdr->mgmt_class) ||
rcv_has_same_gid(mad_agent_priv, wr, wc)))
return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
}
/*
@ -1765,17 +1980,55 @@ ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
(is_direct(mad_hdr->mgmt_class) ||
rcv_has_same_gid(mad_agent_priv, wr, wc)))
/* Verify request has not been canceled */
return (wr->status == IB_WC_SUCCESS) ? wr : NULL;
return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
}
return NULL;
}
static void
process_backlog_mads(struct ib_mad_agent_private *mad_agent_priv)
{
struct ib_mad_send_wr_private *mad_send_wr;
struct ib_mad_send_wc mad_send_wc = {};
unsigned long flags;
int ret;
spin_lock_irqsave(&mad_agent_priv->lock, flags);
while (!list_empty(&mad_agent_priv->backlog_list) &&
(mad_agent_priv->sol_fc_send_count +
mad_agent_priv->sol_fc_wait_count <
mad_agent_priv->sol_fc_max)) {
mad_send_wr = list_entry(mad_agent_priv->backlog_list.next,
struct ib_mad_send_wr_private,
agent_list);
change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
ret = ib_send_mad(mad_send_wr);
if (ret) {
spin_lock_irqsave(&mad_agent_priv->lock, flags);
deref_mad_agent(mad_agent_priv);
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
mad_send_wc.send_buf = &mad_send_wr->send_buf;
mad_send_wc.status = IB_WC_LOC_QP_OP_ERR;
mad_agent_priv->agent.send_handler(
&mad_agent_priv->agent, &mad_send_wc);
}
spin_lock_irqsave(&mad_agent_priv->lock, flags);
}
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
}
void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
{
mad_send_wr->timeout = 0;
if (mad_send_wr->refcount == 1)
list_move_tail(&mad_send_wr->agent_list,
&mad_send_wr->mad_agent_priv->done_list);
if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP ||
mad_send_wr->state == IB_MAD_STATE_QUEUED)
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
else
change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP);
}
static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
@ -1784,6 +2037,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
struct ib_mad_send_wr_private *mad_send_wr;
struct ib_mad_send_wc mad_send_wc;
unsigned long flags;
bool is_mad_done;
int ret;
INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
@ -1832,6 +2086,7 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
}
} else {
ib_mark_mad_done(mad_send_wr);
is_mad_done = (mad_send_wr->state == IB_MAD_STATE_DONE);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
/* Defined behavior is to complete response before request */
@ -1841,10 +2096,13 @@ static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
mad_recv_wc);
deref_mad_agent(mad_agent_priv);
mad_send_wc.status = IB_WC_SUCCESS;
mad_send_wc.vendor_err = 0;
mad_send_wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
if (is_mad_done) {
mad_send_wc.status = IB_WC_SUCCESS;
mad_send_wc.vendor_err = 0;
mad_send_wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr,
&mad_send_wc);
}
}
} else {
mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL,
@ -2172,30 +2430,11 @@ static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)
{
struct ib_mad_agent_private *mad_agent_priv;
struct ib_mad_send_wr_private *temp_mad_send_wr;
struct list_head *list_item;
unsigned long delay;
mad_agent_priv = mad_send_wr->mad_agent_priv;
list_del(&mad_send_wr->agent_list);
delay = mad_send_wr->timeout;
mad_send_wr->timeout += jiffies;
if (delay) {
list_for_each_prev(list_item, &mad_agent_priv->wait_list) {
temp_mad_send_wr = list_entry(list_item,
struct ib_mad_send_wr_private,
agent_list);
if (time_after(mad_send_wr->timeout,
temp_mad_send_wr->timeout))
break;
}
} else {
list_item = &mad_agent_priv->wait_list;
}
list_add(&mad_send_wr->agent_list, list_item);
change_mad_state(mad_send_wr, IB_MAD_STATE_WAIT_RESP);
/* Reschedule a work item if we have a shorter timeout */
if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list)
@ -2229,32 +2468,28 @@ void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
} else
ret = IB_RMPP_RESULT_UNHANDLED;
if (mad_send_wc->status != IB_WC_SUCCESS &&
mad_send_wr->status == IB_WC_SUCCESS) {
mad_send_wr->status = mad_send_wc->status;
mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
}
if (--mad_send_wr->refcount > 0) {
if (mad_send_wr->refcount == 1 && mad_send_wr->timeout &&
mad_send_wr->status == IB_WC_SUCCESS) {
wait_for_response(mad_send_wr);
}
if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
mad_send_wc->status = IB_WC_WR_FLUSH_ERR;
else if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
mad_send_wr->timeout) {
wait_for_response(mad_send_wr);
goto done;
}
/* Remove send from MAD agent and notify client of completion */
list_del(&mad_send_wr->agent_list);
if (mad_send_wr->state != IB_MAD_STATE_DONE)
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
adjust_timeout(mad_agent_priv);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
if (mad_send_wr->status != IB_WC_SUCCESS)
mad_send_wc->status = mad_send_wr->status;
if (ret == IB_RMPP_RESULT_INTERNAL)
if (ret == IB_RMPP_RESULT_INTERNAL) {
ib_rmpp_send_handler(mad_send_wc);
else
} else {
if (mad_send_wr->is_solicited_fc)
process_backlog_mads(mad_agent_priv);
mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
mad_send_wc);
}
/* Release reference on agent taken when sending */
deref_mad_agent(mad_agent_priv);
@ -2396,40 +2631,53 @@ static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
return true;
}
static void clear_mad_error_list(struct list_head *list,
enum ib_wc_status wc_status,
struct ib_mad_agent_private *mad_agent_priv)
{
struct ib_mad_send_wr_private *mad_send_wr, *n;
struct ib_mad_send_wc mad_send_wc;
mad_send_wc.status = wc_status;
mad_send_wc.vendor_err = 0;
list_for_each_entry_safe(mad_send_wr, n, list, agent_list) {
mad_send_wc.send_buf = &mad_send_wr->send_buf;
mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
&mad_send_wc);
deref_mad_agent(mad_agent_priv);
}
}
static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
{
unsigned long flags;
struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;
struct ib_mad_send_wc mad_send_wc;
struct list_head cancel_list;
INIT_LIST_HEAD(&cancel_list);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
&mad_agent_priv->send_list, agent_list) {
if (mad_send_wr->status == IB_WC_SUCCESS) {
mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
}
&mad_agent_priv->send_list, agent_list)
change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
/* Empty wait & backlog list to prevent receives from finding request */
list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
&mad_agent_priv->wait_list, agent_list) {
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
list_add_tail(&mad_send_wr->agent_list, &cancel_list);
}
/* Empty wait list to prevent receives from finding a request */
list_splice_init(&mad_agent_priv->wait_list, &cancel_list);
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
/* Report all cancelled requests */
mad_send_wc.status = IB_WC_WR_FLUSH_ERR;
mad_send_wc.vendor_err = 0;
list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
&cancel_list, agent_list) {
mad_send_wc.send_buf = &mad_send_wr->send_buf;
list_del(&mad_send_wr->agent_list);
mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
&mad_send_wc);
deref_mad_agent(mad_agent_priv);
&mad_agent_priv->backlog_list, agent_list) {
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
list_add_tail(&mad_send_wr->agent_list, &cancel_list);
}
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
/* Report all cancelled requests */
clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
}
static struct ib_mad_send_wr_private*
@ -2451,6 +2699,13 @@ find_send_wr(struct ib_mad_agent_private *mad_agent_priv,
&mad_send_wr->send_buf == send_buf)
return mad_send_wr;
}
list_for_each_entry(mad_send_wr, &mad_agent_priv->backlog_list,
agent_list) {
if (&mad_send_wr->send_buf == send_buf)
return mad_send_wr;
}
return NULL;
}
@ -2468,16 +2723,16 @@ int ib_modify_mad(struct ib_mad_send_buf *send_buf, u32 timeout_ms)
struct ib_mad_agent_private, agent);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
mad_send_wr = find_send_wr(mad_agent_priv, send_buf);
if (!mad_send_wr || mad_send_wr->status != IB_WC_SUCCESS) {
if (!mad_send_wr || mad_send_wr->state == IB_MAD_STATE_CANCELED) {
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
return -EINVAL;
}
active = (!mad_send_wr->timeout || mad_send_wr->refcount > 1);
if (!timeout_ms) {
mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
}
active = ((mad_send_wr->state == IB_MAD_STATE_SEND_START) ||
(mad_send_wr->state == IB_MAD_STATE_EARLY_RESP) ||
(mad_send_wr->state == IB_MAD_STATE_QUEUED && timeout_ms));
if (!timeout_ms)
change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
mad_send_wr->send_buf.timeout_ms = timeout_ms;
if (active)
@ -2589,6 +2844,11 @@ static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
mad_send_wr->send_buf.retries++;
mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);
if (mad_send_wr->is_solicited_fc &&
!list_empty(&mad_send_wr->mad_agent_priv->backlog_list)) {
change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
return 0;
}
if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) {
ret = ib_retry_rmpp(mad_send_wr);
@ -2606,26 +2866,25 @@ static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
} else
ret = ib_send_mad(mad_send_wr);
if (!ret) {
mad_send_wr->refcount++;
list_add_tail(&mad_send_wr->agent_list,
&mad_send_wr->mad_agent_priv->send_list);
}
if (!ret)
change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
return ret;
}
static void timeout_sends(struct work_struct *work)
{
struct ib_mad_send_wr_private *mad_send_wr, *n;
struct ib_mad_send_wr_private *mad_send_wr;
struct ib_mad_agent_private *mad_agent_priv;
struct ib_mad_send_wc mad_send_wc;
struct list_head local_list;
struct list_head timeout_list;
struct list_head cancel_list;
struct list_head *list_item;
unsigned long flags, delay;
mad_agent_priv = container_of(work, struct ib_mad_agent_private,
timed_work.work);
mad_send_wc.vendor_err = 0;
INIT_LIST_HEAD(&local_list);
INIT_LIST_HEAD(&timeout_list);
INIT_LIST_HEAD(&cancel_list);
spin_lock_irqsave(&mad_agent_priv->lock, flags);
while (!list_empty(&mad_agent_priv->wait_list)) {
@ -2643,25 +2902,22 @@ static void timeout_sends(struct work_struct *work)
break;
}
list_del_init(&mad_send_wr->agent_list);
if (mad_send_wr->status == IB_WC_SUCCESS &&
!retry_send(mad_send_wr))
if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
list_item = &cancel_list;
else if (retry_send(mad_send_wr))
list_item = &timeout_list;
else
continue;
list_add_tail(&mad_send_wr->agent_list, &local_list);
change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
list_add_tail(&mad_send_wr->agent_list, list_item);
}
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
list_for_each_entry_safe(mad_send_wr, n, &local_list, agent_list) {
if (mad_send_wr->status == IB_WC_SUCCESS)
mad_send_wc.status = IB_WC_RESP_TIMEOUT_ERR;
else
mad_send_wc.status = mad_send_wr->status;
mad_send_wc.send_buf = &mad_send_wr->send_buf;
mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
&mad_send_wc);
deref_mad_agent(mad_agent_priv);
}
spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
process_backlog_mads(mad_agent_priv);
clear_mad_error_list(&timeout_list, IB_WC_RESP_TIMEOUT_ERR,
mad_agent_priv);
clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
}
/*

76
drivers/infiniband/core/mad_priv.h
View file

@ -95,13 +95,16 @@ struct ib_mad_agent_private {
spinlock_t lock;
struct list_head send_list;
unsigned int sol_fc_send_count;
struct list_head wait_list;
struct list_head done_list;
unsigned int sol_fc_wait_count;
struct delayed_work timed_work;
unsigned long timeout;
struct list_head local_list;
struct work_struct local_work;
struct list_head rmpp_list;
unsigned int sol_fc_max;
struct list_head backlog_list;
refcount_t refcount;
union {
@ -118,6 +121,32 @@ struct ib_mad_snoop_private {
struct completion comp;
};
enum ib_mad_state {
/* MAD is in the making and is not yet in any list */
IB_MAD_STATE_INIT,
/* MAD is in backlog list */
IB_MAD_STATE_QUEUED,
/*
* MAD was sent to the QP and is waiting for completion
* notification in send list.
*/
IB_MAD_STATE_SEND_START,
/*
* MAD send completed successfully, waiting for a response
* in wait list.
*/
IB_MAD_STATE_WAIT_RESP,
/*
* Response came early, before send completion notification,
* in send list.
*/
IB_MAD_STATE_EARLY_RESP,
/* MAD was canceled while in wait or send list */
IB_MAD_STATE_CANCELED,
/* MAD processing completed, MAD in no list */
IB_MAD_STATE_DONE
};
struct ib_mad_send_wr_private {
struct ib_mad_list_head mad_list;
struct list_head agent_list;
@ -132,8 +161,6 @@ struct ib_mad_send_wr_private {
int max_retries;
int retries_left;
int retry;
int refcount;
enum ib_wc_status status;
/* RMPP control */
struct list_head rmpp_list;
@ -143,8 +170,48 @@ struct ib_mad_send_wr_private {
int seg_num;
int newwin;
int pad;
enum ib_mad_state state;
/* Solicited MAD flow control */
bool is_solicited_fc;
};
static inline void expect_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
enum ib_mad_state expected_state)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
WARN_ON(mad_send_wr->state != expected_state);
}
static inline void expect_mad_state2(struct ib_mad_send_wr_private *mad_send_wr,
enum ib_mad_state expected_state1,
enum ib_mad_state expected_state2)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
WARN_ON(mad_send_wr->state != expected_state1 &&
mad_send_wr->state != expected_state2);
}
static inline void expect_mad_state3(struct ib_mad_send_wr_private *mad_send_wr,
enum ib_mad_state expected_state1,
enum ib_mad_state expected_state2,
enum ib_mad_state expected_state3)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
WARN_ON(mad_send_wr->state != expected_state1 &&
mad_send_wr->state != expected_state2 &&
mad_send_wr->state != expected_state3);
}
static inline void
not_expect_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
enum ib_mad_state wrong_state)
{
if (IS_ENABLED(CONFIG_LOCKDEP))
WARN_ON(mad_send_wr->state == wrong_state);
}
struct ib_mad_local_private {
struct list_head completion_list;
struct ib_mad_private *mad_priv;
@ -222,4 +289,7 @@ void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr);
void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
unsigned long timeout_ms);
void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
enum ib_mad_state new_state);
#endif /* __IB_MAD_PRIV_H__ */

41
drivers/infiniband/core/mad_rmpp.c
View file

@ -608,16 +608,20 @@ static void abort_send(struct ib_mad_agent_private *agent,
goto out; /* Unmatched send */
if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) ||
(!mad_send_wr->timeout) || (mad_send_wr->status != IB_WC_SUCCESS))
(!mad_send_wr->timeout) ||
(mad_send_wr->state == IB_MAD_STATE_CANCELED))
goto out; /* Send is already done */
ib_mark_mad_done(mad_send_wr);
if (mad_send_wr->state == IB_MAD_STATE_DONE) {
spin_unlock_irqrestore(&agent->lock, flags);
wc.status = IB_WC_REM_ABORT_ERR;
wc.vendor_err = rmpp_status;
wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &wc);
return;
}
spin_unlock_irqrestore(&agent->lock, flags);
wc.status = IB_WC_REM_ABORT_ERR;
wc.vendor_err = rmpp_status;
wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &wc);
return;
out:
spin_unlock_irqrestore(&agent->lock, flags);
@ -684,7 +688,8 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,
}
if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) ||
(!mad_send_wr->timeout) || (mad_send_wr->status != IB_WC_SUCCESS))
(!mad_send_wr->timeout) ||
(mad_send_wr->state == IB_MAD_STATE_CANCELED))
goto out; /* Send is already done */
if (seg_num > mad_send_wr->send_buf.seg_count ||
@ -709,21 +714,24 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,
struct ib_mad_send_wc wc;
ib_mark_mad_done(mad_send_wr);
if (mad_send_wr->state == IB_MAD_STATE_DONE) {
spin_unlock_irqrestore(&agent->lock, flags);
wc.status = IB_WC_SUCCESS;
wc.vendor_err = 0;
wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &wc);
return;
}
spin_unlock_irqrestore(&agent->lock, flags);
wc.status = IB_WC_SUCCESS;
wc.vendor_err = 0;
wc.send_buf = &mad_send_wr->send_buf;
ib_mad_complete_send_wr(mad_send_wr, &wc);
return;
}
if (mad_send_wr->refcount == 1)
if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
ib_reset_mad_timeout(mad_send_wr,
mad_send_wr->send_buf.timeout_ms);
spin_unlock_irqrestore(&agent->lock, flags);
ack_ds_ack(agent, mad_recv_wc);
return;
} else if (mad_send_wr->refcount == 1 &&
} else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP &&
mad_send_wr->seg_num < mad_send_wr->newwin &&
mad_send_wr->seg_num < mad_send_wr->send_buf.seg_count) {
/* Send failure will just result in a timeout/retry */
@ -731,7 +739,7 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,
if (ret)
goto out;
mad_send_wr->refcount++;
change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
list_move_tail(&mad_send_wr->agent_list,
&mad_send_wr->mad_agent_priv->send_list);
}
@ -890,7 +898,6 @@ int ib_send_rmpp_mad(struct ib_mad_send_wr_private *mad_send_wr)
mad_send_wr->newwin = init_newwin(mad_send_wr);
/* We need to wait for the final ACK even if there isn't a response */
mad_send_wr->refcount += (mad_send_wr->timeout == 0);
ret = send_next_seg(mad_send_wr);
if (!ret)
return IB_RMPP_RESULT_CONSUMED;
@ -912,7 +919,7 @@ int ib_process_rmpp_send_wc(struct ib_mad_send_wr_private *mad_send_wr,
return IB_RMPP_RESULT_INTERNAL; /* ACK, STOP, or ABORT */
if (mad_send_wc->status != IB_WC_SUCCESS ||
mad_send_wr->status != IB_WC_SUCCESS)
mad_send_wr->state == IB_MAD_STATE_CANCELED)
return IB_RMPP_RESULT_PROCESSED; /* Canceled or send error */
if (!mad_send_wr->timeout)

24
drivers/infiniband/core/nldev.c
View file

@ -255,7 +255,7 @@ EXPORT_SYMBOL(rdma_nl_put_driver_u64_hex);
bool rdma_nl_get_privileged_qkey(void)
{
return privileged_qkey || capable(CAP_NET_RAW);
return privileged_qkey;
}
EXPORT_SYMBOL(rdma_nl_get_privileged_qkey);
@ -1469,10 +1469,11 @@ static const struct nldev_fill_res_entry fill_entries[RDMA_RESTRACK_MAX] = {
};
static int res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack,
enum rdma_restrack_type res_type,
res_fill_func_t fill_func)
static noinline_for_stack int
res_get_common_doit(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack,
enum rdma_restrack_type res_type,
res_fill_func_t fill_func)
{
const struct nldev_fill_res_entry *fe = &fill_entries[res_type];
struct nlattr *tb[RDMA_NLDEV_ATTR_MAX];
@ -2263,10 +2264,10 @@ err:
return ret;
}
static int stat_get_doit_default_counter(struct sk_buff *skb,
struct nlmsghdr *nlh,
struct netlink_ext_ack *extack,
struct nlattr *tb[])
static noinline_for_stack int
stat_get_doit_default_counter(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack,
struct nlattr *tb[])
{
struct rdma_hw_stats *stats;
struct nlattr *table_attr;
@ -2356,8 +2357,9 @@ err:
return ret;
}
static int stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack, struct nlattr *tb[])
static noinline_for_stack int
stat_get_doit_qp(struct sk_buff *skb, struct nlmsghdr *nlh,
struct netlink_ext_ack *extack, struct nlattr *tb[])
{
static enum rdma_nl_counter_mode mode;

29
drivers/infiniband/core/rdma_core.c
View file

@ -1019,3 +1019,32 @@ void uverbs_finalize_object(struct ib_uobject *uobj,
WARN_ON(true);
}
}
/**
* rdma_uattrs_has_raw_cap() - Returns whether a rdma device linked to the
* uverbs attributes file has CAP_NET_RAW
* capability or not.
*
* @attrs: Pointer to uverbs attributes
*
* Returns true if a rdma device's owning user namespace has CAP_NET_RAW
* capability, otherwise false.
*/
bool rdma_uattrs_has_raw_cap(const struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_file *ufile = attrs->ufile;
struct ib_ucontext *ucontext;
bool has_cap = false;
int srcu_key;
srcu_key = srcu_read_lock(&ufile->device->disassociate_srcu);
ucontext = ib_uverbs_get_ucontext_file(ufile);
if (IS_ERR(ucontext))
goto out;
has_cap = rdma_dev_has_raw_cap(ucontext->device);
out:
srcu_read_unlock(&ufile->device->disassociate_srcu, srcu_key);
return has_cap;
}
EXPORT_SYMBOL(rdma_uattrs_has_raw_cap);

1
drivers/infiniband/core/rdma_core.h
View file

@ -156,6 +156,7 @@ extern const struct uapi_definition uverbs_def_obj_counters[];
extern const struct uapi_definition uverbs_def_obj_cq[];
extern const struct uapi_definition uverbs_def_obj_device[];
extern const struct uapi_definition uverbs_def_obj_dm[];
extern const struct uapi_definition uverbs_def_obj_dmah[];
extern const struct uapi_definition uverbs_def_obj_flow_action[];
extern const struct uapi_definition uverbs_def_obj_intf[];
extern const struct uapi_definition uverbs_def_obj_mr[];

2
drivers/infiniband/core/restrack.c
View file

@ -100,6 +100,8 @@ static struct ib_device *res_to_dev(struct rdma_restrack_entry *res)
return container_of(res, struct rdma_counter, res)->device;
case RDMA_RESTRACK_SRQ:
return container_of(res, struct ib_srq, res)->device;
case RDMA_RESTRACK_DMAH:
return container_of(res, struct ib_dmah, res)->device;
default:
WARN_ONCE(true, "Wrong resource tracking type %u\n", res->type);
return NULL;

13
drivers/infiniband/core/uverbs_cmd.c
View file

@ -741,7 +741,7 @@ static int ib_uverbs_reg_mr(struct uverbs_attr_bundle *attrs)
}
mr = pd->device->ops.reg_user_mr(pd, cmd.start, cmd.length, cmd.hca_va,
cmd.access_flags,
cmd.access_flags, NULL,
&attrs->driver_udata);
if (IS_ERR(mr)) {
ret = PTR_ERR(mr);
@ -1312,9 +1312,9 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
switch (cmd->qp_type) {
case IB_QPT_RAW_PACKET:
if (!capable(CAP_NET_RAW))
if (!rdma_uattrs_has_raw_cap(attrs))
return -EPERM;
break;
fallthrough;
case IB_QPT_RC:
case IB_QPT_UC:
case IB_QPT_UD:
@ -1451,7 +1451,7 @@ static int create_qp(struct uverbs_attr_bundle *attrs,
}
if (attr.create_flags & IB_QP_CREATE_SOURCE_QPN) {
if (!capable(CAP_NET_RAW)) {
if (!rdma_uattrs_has_raw_cap(attrs)) {
ret = -EPERM;
goto err_put;
}
@ -1877,7 +1877,8 @@ static int modify_qp(struct uverbs_attr_bundle *attrs,
attr->path_mig_state = cmd->base.path_mig_state;
if (cmd->base.attr_mask & IB_QP_QKEY) {
if (cmd->base.qkey & IB_QP_SET_QKEY &&
!rdma_nl_get_privileged_qkey()) {
!(rdma_nl_get_privileged_qkey() ||
rdma_uattrs_has_raw_cap(attrs))) {
ret = -EPERM;
goto release_qp;
}
@ -3225,7 +3226,7 @@ static int ib_uverbs_ex_create_flow(struct uverbs_attr_bundle *attrs)
if (cmd.comp_mask)
return -EINVAL;
if (!capable(CAP_NET_RAW))
if (!rdma_uattrs_has_raw_cap(attrs))
return -EPERM;
if (cmd.flow_attr.flags >= IB_FLOW_ATTR_FLAGS_RESERVED)

87
drivers/infiniband/core/uverbs_std_types_cq.c
View file

@ -64,15 +64,21 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
struct ib_ucq_object *obj = container_of(
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE),
typeof(*obj), uevent.uobject);
struct ib_uverbs_completion_event_file *ev_file = NULL;
struct ib_device *ib_dev = attrs->context->device;
int ret;
u64 user_handle;
struct ib_umem_dmabuf *umem_dmabuf;
struct ib_cq_init_attr attr = {};
struct ib_cq *cq;
struct ib_uverbs_completion_event_file *ev_file = NULL;
struct ib_uobject *ev_file_uobj;
struct ib_umem *umem = NULL;
u64 buffer_length;
u64 buffer_offset;
struct ib_cq *cq;
u64 user_handle;
u64 buffer_va;
int buffer_fd;
int ret;
if (!ib_dev->ops.create_cq || !ib_dev->ops.destroy_cq)
if ((!ib_dev->ops.create_cq && !ib_dev->ops.create_cq_umem) || !ib_dev->ops.destroy_cq)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.comp_vector, attrs,
@ -112,9 +118,66 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
INIT_LIST_HEAD(&obj->comp_list);
INIT_LIST_HEAD(&obj->uevent.event_list);
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA)) {
ret = uverbs_copy_from(&buffer_va, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA);
if (ret)
goto err_event_file;
ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);
if (ret)
goto err_event_file;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD) ||
uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
!ib_dev->ops.create_cq_umem) {
ret = -EINVAL;
goto err_event_file;
}
umem = ib_umem_get(ib_dev, buffer_va, buffer_length, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem)) {
ret = PTR_ERR(umem);
goto err_event_file;
}
} else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD)) {
ret = uverbs_get_raw_fd(&buffer_fd, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_FD);
if (ret)
goto err_event_file;
ret = uverbs_copy_from(&buffer_offset, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET);
if (ret)
goto err_event_file;
ret = uverbs_copy_from(&buffer_length, attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH);
if (ret)
goto err_event_file;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_VA) ||
!ib_dev->ops.create_cq_umem) {
ret = -EINVAL;
goto err_event_file;
}
umem_dmabuf = ib_umem_dmabuf_get_pinned(ib_dev, buffer_offset, buffer_length,
buffer_fd, IB_ACCESS_LOCAL_WRITE);
if (IS_ERR(umem_dmabuf)) {
ret = PTR_ERR(umem_dmabuf);
goto err_event_file;
}
umem = &umem_dmabuf->umem;
} else if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET) ||
uverbs_attr_is_valid(attrs, UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH) ||
!ib_dev->ops.create_cq) {
ret = -EINVAL;
goto err_event_file;
}
cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
if (!cq) {
ret = -ENOMEM;
ib_umem_release(umem);
goto err_event_file;
}
@ -128,7 +191,8 @@ static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
rdma_restrack_set_name(&cq->res, NULL);
ret = ib_dev->ops.create_cq(cq, &attr, attrs);
ret = umem ? ib_dev->ops.create_cq_umem(cq, &attr, umem, attrs) :
ib_dev->ops.create_cq(cq, &attr, attrs);
if (ret)
goto err_free;
@ -180,6 +244,17 @@ DECLARE_UVERBS_NAMED_METHOD(
UVERBS_OBJECT_ASYNC_EVENT,
UVERBS_ACCESS_READ,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_VA,
UVERBS_ATTR_TYPE(u64),
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_LENGTH,
UVERBS_ATTR_TYPE(u64),
UA_OPTIONAL),
UVERBS_ATTR_RAW_FD(UVERBS_ATTR_CREATE_CQ_BUFFER_FD,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_BUFFER_OFFSET,
UVERBS_ATTR_TYPE(u64),
UA_OPTIONAL),
UVERBS_ATTR_UHW());
static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(

145
drivers/infiniband/core/uverbs_std_types_dmah.c Normal file
View file

@ -0,0 +1,145 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved
*/
#include "rdma_core.h"
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
#include "restrack.h"
static int uverbs_free_dmah(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_dmah *dmah = uobject->object;
int ret;
if (atomic_read(&dmah->usecnt))
return -EBUSY;
ret = dmah->device->ops.dealloc_dmah(dmah, attrs);
if (ret)
return ret;
rdma_restrack_del(&dmah->res);
kfree(dmah);
return 0;
}
static int UVERBS_HANDLER(UVERBS_METHOD_DMAH_ALLOC)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj =
uverbs_attr_get(attrs, UVERBS_ATTR_ALLOC_DMAH_HANDLE)
->obj_attr.uobject;
struct ib_device *ib_dev = attrs->context->device;
struct ib_dmah *dmah;
int ret;
dmah = rdma_zalloc_drv_obj(ib_dev, ib_dmah);
if (!dmah)
return -ENOMEM;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_ALLOC_DMAH_CPU_ID)) {
ret = uverbs_copy_from(&dmah->cpu_id, attrs,
UVERBS_ATTR_ALLOC_DMAH_CPU_ID);
if (ret)
goto err;
if (!cpumask_test_cpu(dmah->cpu_id, current->cpus_ptr)) {
ret = -EPERM;
goto err;
}
dmah->valid_fields |= BIT(IB_DMAH_CPU_ID_EXISTS);
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_ALLOC_DMAH_TPH_MEM_TYPE)) {
dmah->mem_type = uverbs_attr_get_enum_id(attrs,
UVERBS_ATTR_ALLOC_DMAH_TPH_MEM_TYPE);
dmah->valid_fields |= BIT(IB_DMAH_MEM_TYPE_EXISTS);
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_ALLOC_DMAH_PH)) {
ret = uverbs_copy_from(&dmah->ph, attrs,
UVERBS_ATTR_ALLOC_DMAH_PH);
if (ret)
goto err;
/* Per PCIe spec 6.2-1.0, only the lowest two bits are applicable */
if (dmah->ph & 0xFC) {
ret = -EINVAL;
goto err;
}
dmah->valid_fields |= BIT(IB_DMAH_PH_EXISTS);
}
dmah->device = ib_dev;
dmah->uobject = uobj;
atomic_set(&dmah->usecnt, 0);
rdma_restrack_new(&dmah->res, RDMA_RESTRACK_DMAH);
rdma_restrack_set_name(&dmah->res, NULL);
ret = ib_dev->ops.alloc_dmah(dmah, attrs);
if (ret) {
rdma_restrack_put(&dmah->res);
goto err;
}
uobj->object = dmah;
rdma_restrack_add(&dmah->res);
uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_ALLOC_DMAH_HANDLE);
return 0;
err:
kfree(dmah);
return ret;
}
static const struct uverbs_attr_spec uverbs_dmah_mem_type[] = {
[TPH_MEM_TYPE_VM] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_NO_DATA(),
},
[TPH_MEM_TYPE_PM] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_NO_DATA(),
},
};
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_DMAH_ALLOC,
UVERBS_ATTR_IDR(UVERBS_ATTR_ALLOC_DMAH_HANDLE,
UVERBS_OBJECT_DMAH,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DMAH_CPU_ID,
UVERBS_ATTR_TYPE(u32),
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_ALLOC_DMAH_TPH_MEM_TYPE,
uverbs_dmah_mem_type,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DMAH_PH,
UVERBS_ATTR_TYPE(u8),
UA_OPTIONAL));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_DMAH_FREE,
UVERBS_ATTR_IDR(UVERBS_ATTR_FREE_DMA_HANDLE,
UVERBS_OBJECT_DMAH,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_DMAH,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_dmah),
&UVERBS_METHOD(UVERBS_METHOD_DMAH_ALLOC),
&UVERBS_METHOD(UVERBS_METHOD_DMAH_FREE));
const struct uapi_definition uverbs_def_obj_dmah[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DMAH,
UAPI_DEF_OBJ_NEEDS_FN(dealloc_dmah),
UAPI_DEF_OBJ_NEEDS_FN(alloc_dmah)),
{}
};

172
drivers/infiniband/core/uverbs_std_types_mr.c
View file

@ -238,7 +238,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_DMABUF_MR)(
return ret;
mr = pd->device->ops.reg_user_mr_dmabuf(pd, offset, length, iova, fd,
access_flags,
access_flags, NULL,
attrs);
if (IS_ERR(mr))
return PTR_ERR(mr);
@ -266,6 +266,135 @@ static int UVERBS_HANDLER(UVERBS_METHOD_REG_DMABUF_MR)(
return ret;
}
static int UVERBS_HANDLER(UVERBS_METHOD_REG_MR)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj =
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_REG_MR_HANDLE);
struct ib_pd *pd =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_MR_PD_HANDLE);
u32 valid_access_flags = IB_ACCESS_SUPPORTED;
u64 length, iova, fd_offset = 0, addr = 0;
struct ib_device *ib_dev = pd->device;
struct ib_dmah *dmah = NULL;
bool has_fd_offset = false;
bool has_addr = false;
bool has_fd = false;
u32 access_flags;
struct ib_mr *mr;
int fd;
int ret;
ret = uverbs_copy_from(&iova, attrs, UVERBS_ATTR_REG_MR_IOVA);
if (ret)
return ret;
ret = uverbs_copy_from(&length, attrs, UVERBS_ATTR_REG_MR_LENGTH);
if (ret)
return ret;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_ADDR)) {
ret = uverbs_copy_from(&addr, attrs,
UVERBS_ATTR_REG_MR_ADDR);
if (ret)
return ret;
has_addr = true;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_FD_OFFSET)) {
ret = uverbs_copy_from(&fd_offset, attrs,
UVERBS_ATTR_REG_MR_FD_OFFSET);
if (ret)
return ret;
has_fd_offset = true;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_FD)) {
ret = uverbs_get_raw_fd(&fd, attrs,
UVERBS_ATTR_REG_MR_FD);
if (ret)
return ret;
has_fd = true;
}
if (has_fd) {
if (!ib_dev->ops.reg_user_mr_dmabuf)
return -EOPNOTSUPP;
/* FD requires offset and can't come with addr */
if (!has_fd_offset || has_addr)
return -EINVAL;
if ((fd_offset & ~PAGE_MASK) != (iova & ~PAGE_MASK))
return -EINVAL;
valid_access_flags = IB_ACCESS_LOCAL_WRITE |
IB_ACCESS_REMOTE_READ |
IB_ACCESS_REMOTE_WRITE |
IB_ACCESS_REMOTE_ATOMIC |
IB_ACCESS_RELAXED_ORDERING;
} else {
if (!has_addr || has_fd_offset)
return -EINVAL;
if ((addr & ~PAGE_MASK) != (iova & ~PAGE_MASK))
return -EINVAL;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_REG_MR_DMA_HANDLE)) {
dmah = uverbs_attr_get_obj(attrs,
UVERBS_ATTR_REG_MR_DMA_HANDLE);
if (IS_ERR(dmah))
return PTR_ERR(dmah);
}
ret = uverbs_get_flags32(&access_flags, attrs,
UVERBS_ATTR_REG_MR_ACCESS_FLAGS,
valid_access_flags);
if (ret)
return ret;
ret = ib_check_mr_access(ib_dev, access_flags);
if (ret)
return ret;
if (has_fd)
mr = pd->device->ops.reg_user_mr_dmabuf(pd, fd_offset, length,
iova, fd, access_flags,
dmah, attrs);
else
mr = pd->device->ops.reg_user_mr(pd, addr, length, iova,
access_flags, dmah, NULL);
if (IS_ERR(mr))
return PTR_ERR(mr);
mr->device = pd->device;
mr->pd = pd;
mr->type = IB_MR_TYPE_USER;
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
if (dmah) {
mr->dmah = dmah;
atomic_inc(&dmah->usecnt);
}
rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
rdma_restrack_set_name(&mr->res, NULL);
rdma_restrack_add(&mr->res);
uobj->object = mr;
uverbs_finalize_uobj_create(attrs, UVERBS_ATTR_REG_MR_HANDLE);
ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_MR_RESP_LKEY,
&mr->lkey, sizeof(mr->lkey));
if (ret)
return ret;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_MR_RESP_RKEY,
&mr->rkey, sizeof(mr->rkey));
return ret;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_ADVISE_MR,
UVERBS_ATTR_IDR(UVERBS_ATTR_ADVISE_MR_PD_HANDLE,
@ -362,6 +491,44 @@ DECLARE_UVERBS_NAMED_METHOD(
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_REG_MR,
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_MR_HANDLE,
UVERBS_OBJECT_MR,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_MR_PD_HANDLE,
UVERBS_OBJECT_PD,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_MR_DMA_HANDLE,
UVERBS_OBJECT_DMAH,
UVERBS_ACCESS_READ,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_IOVA,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_LENGTH,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_REG_MR_ACCESS_FLAGS,
enum ib_access_flags,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_ADDR,
UVERBS_ATTR_TYPE(u64),
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_MR_FD_OFFSET,
UVERBS_ATTR_TYPE(u64),
UA_OPTIONAL),
UVERBS_ATTR_RAW_FD(UVERBS_ATTR_REG_MR_FD,
UA_OPTIONAL),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_LKEY,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_MR_RESP_RKEY,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_MR_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_MR_HANDLE,
@ -376,7 +543,8 @@ DECLARE_UVERBS_NAMED_OBJECT(
&UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG),
&UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY),
&UVERBS_METHOD(UVERBS_METHOD_QUERY_MR),
&UVERBS_METHOD(UVERBS_METHOD_REG_DMABUF_MR));
&UVERBS_METHOD(UVERBS_METHOD_REG_DMABUF_MR),
&UVERBS_METHOD(UVERBS_METHOD_REG_MR));
const struct uapi_definition uverbs_def_obj_mr[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR,

2
drivers/infiniband/core/uverbs_std_types_qp.c
View file

@ -133,7 +133,7 @@ static int UVERBS_HANDLER(UVERBS_METHOD_QP_CREATE)(
device = xrcd->device;
break;
case IB_UVERBS_QPT_RAW_PACKET:
if (!capable(CAP_NET_RAW))
if (!rdma_uattrs_has_raw_cap(attrs))
return -EPERM;
fallthrough;
case IB_UVERBS_QPT_RC:

1
drivers/infiniband/core/uverbs_uapi.c
View file

@ -631,6 +631,7 @@ static const struct uapi_definition uverbs_core_api[] = {
UAPI_DEF_CHAIN(uverbs_def_obj_cq),
UAPI_DEF_CHAIN(uverbs_def_obj_device),
UAPI_DEF_CHAIN(uverbs_def_obj_dm),
UAPI_DEF_CHAIN(uverbs_def_obj_dmah),
UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
UAPI_DEF_CHAIN(uverbs_def_obj_intf),
UAPI_DEF_CHAIN(uverbs_def_obj_mr),

5
drivers/infiniband/core/verbs.c
View file

@ -2223,7 +2223,7 @@ struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
}
mr = pd->device->ops.reg_user_mr(pd, start, length, virt_addr,
access_flags, NULL);
access_flags, NULL, NULL);
if (IS_ERR(mr))
return mr;
@ -2262,6 +2262,7 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
{
struct ib_pd *pd = mr->pd;
struct ib_dm *dm = mr->dm;
struct ib_dmah *dmah = mr->dmah;
struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
int ret;
@ -2272,6 +2273,8 @@ int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
atomic_dec(&pd->usecnt);
if (dm)
atomic_dec(&dm->usecnt);
if (dmah)
atomic_dec(&dmah->usecnt);
kfree(sig_attrs);
}

1
drivers/infiniband/hw/Makefile
View file

@ -1,6 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_INFINIBAND_MTHCA) += mthca/
obj-$(CONFIG_INFINIBAND_QIB) += qib/
obj-$(CONFIG_INFINIBAND_CXGB4) += cxgb4/
obj-$(CONFIG_INFINIBAND_EFA) += efa/
obj-$(CONFIG_INFINIBAND_IRDMA) += irdma/

10
drivers/infiniband/hw/bnxt_re/ib_verbs.c
View file

@ -4235,6 +4235,7 @@ free_mr:
struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
u64 virt_addr, int mr_access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
@ -4242,6 +4243,9 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
struct ib_umem *umem;
struct ib_mr *ib_mr;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
umem = ib_umem_get(&rdev->ibdev, start, length, mr_access_flags);
if (IS_ERR(umem))
return ERR_CAST(umem);
@ -4255,6 +4259,7 @@ struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *ib_pd, u64 start, u64 length,
struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
u64 length, u64 virt_addr, int fd,
int mr_access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs)
{
struct bnxt_re_pd *pd = container_of(ib_pd, struct bnxt_re_pd, ib_pd);
@ -4263,6 +4268,9 @@ struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
struct ib_umem *umem;
struct ib_mr *ib_mr;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
umem_dmabuf = ib_umem_dmabuf_get_pinned(&rdev->ibdev, start, length,
fd, mr_access_flags);
if (IS_ERR(umem_dmabuf))
@ -4738,7 +4746,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
return err;
err = uverbs_copy_to(attrs, BNXT_RE_TOGGLE_MEM_MMAP_OFFSET,
&offset, sizeof(length));
&offset, sizeof(offset));
if (err)
return err;

2
drivers/infiniband/hw/bnxt_re/ib_verbs.h
View file

@ -258,10 +258,12 @@ struct ib_mw *bnxt_re_alloc_mw(struct ib_pd *ib_pd, enum ib_mw_type type,
int bnxt_re_dealloc_mw(struct ib_mw *mw);
struct ib_mr *bnxt_re_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int mr_access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata);
struct ib_mr *bnxt_re_reg_user_mr_dmabuf(struct ib_pd *ib_pd, u64 start,
u64 length, u64 virt_addr,
int fd, int mr_access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs);
int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata);
void bnxt_re_dealloc_ucontext(struct ib_ucontext *context);

28
drivers/infiniband/hw/bnxt_re/qplib_fp.c
View file

@ -1750,9 +1750,9 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp,
}
}
static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
struct bnxt_qplib_swqe *wqe,
u16 *idx)
static unsigned int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
struct bnxt_qplib_swqe *wqe,
u32 *idx)
{
struct bnxt_qplib_hwq *hwq;
int len, t_len, offt;
@ -1769,7 +1769,7 @@ static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
il_src = (void *)wqe->sg_list[indx].addr;
t_len += len;
if (t_len > qp->max_inline_data)
return -ENOMEM;
return BNXT_RE_INVAL_MSG_SIZE;
while (len) {
if (pull_dst) {
pull_dst = false;
@ -1795,9 +1795,9 @@ static int bnxt_qplib_put_inline(struct bnxt_qplib_qp *qp,
return t_len;
}
static u32 bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_sge *ssge,
u16 nsge, u16 *idx)
static unsigned int bnxt_qplib_put_sges(struct bnxt_qplib_hwq *hwq,
struct bnxt_qplib_sge *ssge,
u32 nsge, u32 *idx)
{
struct sq_sge *dsge;
int indx, len = 0;
@ -1878,14 +1878,12 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
struct bnxt_qplib_hwq *hwq;
struct bnxt_qplib_swq *swq;
bool sch_handler = false;
u32 wqe_idx, slots, idx;
u16 wqe_sz, qdf = 0;
bool msn_update;
void *base_hdr;
void *ext_hdr;
__le32 temp32;
u32 wqe_idx;
u32 slots;
u16 idx;
hwq = &sq->hwq;
if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS &&
@ -1937,8 +1935,10 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp,
else
data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge,
&idx);
if (data_len < 0)
goto queue_err;
if (data_len > BNXT_RE_MAX_MSG_SIZE) {
rc = -EINVAL;
goto done;
}
/* Make sure we update MSN table only for wired wqes */
msn_update = true;
/* Specifics */
@ -2139,8 +2139,8 @@ int bnxt_qplib_post_recv(struct bnxt_qplib_qp *qp,
struct bnxt_qplib_hwq *hwq;
struct bnxt_qplib_swq *swq;
bool sch_handler = false;
u16 wqe_sz, idx;
u32 wqe_idx;
u32 wqe_idx, idx;
u16 wqe_sz;
int rc = 0;
hwq = &rq->hwq;

3
drivers/infiniband/hw/bnxt_re/qplib_fp.h
View file

@ -346,6 +346,9 @@ struct bnxt_qplib_qp {
u8 tos_dscp;
};
#define BNXT_RE_MAX_MSG_SIZE 0x80000000
#define BNXT_RE_INVAL_MSG_SIZE 0xFFFFFFFF
#define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base)
#define CQE_CNT_PER_PG (PAGE_SIZE / BNXT_QPLIB_MAX_CQE_ENTRY_SIZE)

2
drivers/infiniband/hw/bnxt_re/qplib_sp.c
View file

@ -674,7 +674,7 @@ int bnxt_qplib_reg_mr(struct bnxt_qplib_res *res, struct bnxt_qplib_mrw *mr,
req.log2_pbl_pg_size = cpu_to_le16(((ilog2(PAGE_SIZE) <<
CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_SFT) &
CMDQ_REGISTER_MR_LOG2_PBL_PG_SIZE_MASK));
req.access = (mr->access_flags & 0xFFFF);
req.access = (mr->access_flags & BNXT_QPLIB_MR_ACCESS_MASK);
req.va = cpu_to_le64(mr->va);
req.key = cpu_to_le32(mr->lkey);
if (_is_alloc_mr_unified(res->dattr->dev_cap_flags))

1
drivers/infiniband/hw/bnxt_re/qplib_sp.h
View file

@ -111,6 +111,7 @@ struct bnxt_qplib_mrw {
struct bnxt_qplib_pd *pd;
int type;
u32 access_flags;
#define BNXT_QPLIB_MR_ACCESS_MASK 0xFF
#define BNXT_QPLIB_FR_PMR 0x80000000
u32 lkey;
u32 rkey;

3
drivers/infiniband/hw/cxgb4/device.c
View file

@ -905,8 +905,7 @@ static int c4iw_rdev_open(struct c4iw_rdev *rdev)
return 0;
err_free_status_page_and_wr_log:
if (c4iw_wr_log && rdev->wr_log)
kfree(rdev->wr_log);
kfree(rdev->wr_log);
free_page((unsigned long)rdev->status_page);
destroy_ocqp_pool:
c4iw_ocqp_pool_destroy(rdev);

1
drivers/infiniband/hw/cxgb4/iw_cxgb4.h
View file

@ -1006,6 +1006,7 @@ int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
void c4iw_dealloc(struct uld_ctx *ctx);
struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start,
u64 length, u64 virt, int acc,
struct ib_dmah *dmah,
struct ib_udata *udata);
struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);

6
drivers/infiniband/hw/cxgb4/mem.c
View file

@ -489,7 +489,8 @@ err_free_mhp:
}
struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *udata)
u64 virt, int acc, struct ib_dmah *dmah,
struct ib_udata *udata)
{
__be64 *pages;
int shift, n, i;
@ -501,6 +502,9 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
pr_debug("ib_pd %p\n", pd);
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
if (length == ~0ULL)
return ERR_PTR(-EINVAL);

5
drivers/infiniband/hw/efa/efa.h
View file

@ -107,6 +107,7 @@ struct efa_cq {
u16 cq_idx;
/* NULL when no interrupts requested */
struct efa_eq *eq;
struct ib_umem *umem;
};
struct efa_qp {
@ -162,12 +163,16 @@ int efa_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init_attr,
int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct uverbs_attr_bundle *attrs);
int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_umem *umem, struct uverbs_attr_bundle *attrs);
struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata);
struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
u64 length, u64 virt_addr,
int fd, int access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs);
int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
int efa_get_port_immutable(struct ib_device *ibdev, u32 port_num,

17
drivers/infiniband/hw/efa/efa_admin_cmds_defs.h
View file

@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_ADMIN_CMDS_H_
@ -68,6 +68,7 @@ enum efa_admin_get_stats_type {
EFA_ADMIN_GET_STATS_TYPE_MESSAGES = 1,
EFA_ADMIN_GET_STATS_TYPE_RDMA_READ = 2,
EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE = 3,
EFA_ADMIN_GET_STATS_TYPE_NETWORK = 4,
};
enum efa_admin_get_stats_scope {
@ -651,6 +652,18 @@ struct efa_admin_rdma_write_stats {
u64 write_recv_bytes;
};
struct efa_admin_network_stats {
u64 retrans_bytes;
u64 retrans_pkts;
u64 retrans_timeout_events;
u64 unresponsive_remote_events;
u64 impaired_remote_conn_events;
};
struct efa_admin_acq_get_stats_resp {
struct efa_admin_acq_common_desc acq_common_desc;
@ -662,6 +675,8 @@ struct efa_admin_acq_get_stats_resp {
struct efa_admin_rdma_read_stats rdma_read_stats;
struct efa_admin_rdma_write_stats rdma_write_stats;
struct efa_admin_network_stats network_stats;
} u;
};

53
drivers/infiniband/hw/efa/efa_com_cmd.c
View file

@ -1,6 +1,6 @@
// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
/*
* Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#include "efa_com.h"
@ -769,6 +769,11 @@ int efa_com_get_stats(struct efa_com_dev *edev,
struct efa_com_admin_queue *aq = &edev->aq;
struct efa_admin_aq_get_stats_cmd cmd = {};
struct efa_admin_acq_get_stats_resp resp;
struct efa_admin_rdma_write_stats *rws;
struct efa_admin_rdma_read_stats *rrs;
struct efa_admin_messages_stats *ms;
struct efa_admin_network_stats *ns;
struct efa_admin_basic_stats *bs;
int err;
cmd.aq_common_descriptor.opcode = EFA_ADMIN_GET_STATS;
@ -791,29 +796,41 @@ int efa_com_get_stats(struct efa_com_dev *edev,
switch (cmd.type) {
case EFA_ADMIN_GET_STATS_TYPE_BASIC:
result->basic_stats.tx_bytes = resp.u.basic_stats.tx_bytes;
result->basic_stats.tx_pkts = resp.u.basic_stats.tx_pkts;
result->basic_stats.rx_bytes = resp.u.basic_stats.rx_bytes;
result->basic_stats.rx_pkts = resp.u.basic_stats.rx_pkts;
result->basic_stats.rx_drops = resp.u.basic_stats.rx_drops;
bs = &resp.u.basic_stats;
result->basic_stats.tx_bytes = bs->tx_bytes;
result->basic_stats.tx_pkts = bs->tx_pkts;
result->basic_stats.rx_bytes = bs->rx_bytes;
result->basic_stats.rx_pkts = bs->rx_pkts;
result->basic_stats.rx_drops = bs->rx_drops;
break;
case EFA_ADMIN_GET_STATS_TYPE_MESSAGES:
result->messages_stats.send_bytes = resp.u.messages_stats.send_bytes;
result->messages_stats.send_wrs = resp.u.messages_stats.send_wrs;
result->messages_stats.recv_bytes = resp.u.messages_stats.recv_bytes;
result->messages_stats.recv_wrs = resp.u.messages_stats.recv_wrs;
ms = &resp.u.messages_stats;
result->messages_stats.send_bytes = ms->send_bytes;
result->messages_stats.send_wrs = ms->send_wrs;
result->messages_stats.recv_bytes = ms->recv_bytes;
result->messages_stats.recv_wrs = ms->recv_wrs;
break;
case EFA_ADMIN_GET_STATS_TYPE_RDMA_READ:
result->rdma_read_stats.read_wrs = resp.u.rdma_read_stats.read_wrs;
result->rdma_read_stats.read_bytes = resp.u.rdma_read_stats.read_bytes;
result->rdma_read_stats.read_wr_err = resp.u.rdma_read_stats.read_wr_err;
result->rdma_read_stats.read_resp_bytes = resp.u.rdma_read_stats.read_resp_bytes;
rrs = &resp.u.rdma_read_stats;
result->rdma_read_stats.read_wrs = rrs->read_wrs;
result->rdma_read_stats.read_bytes = rrs->read_bytes;
result->rdma_read_stats.read_wr_err = rrs->read_wr_err;
result->rdma_read_stats.read_resp_bytes = rrs->read_resp_bytes;
break;
case EFA_ADMIN_GET_STATS_TYPE_RDMA_WRITE:
result->rdma_write_stats.write_wrs = resp.u.rdma_write_stats.write_wrs;
result->rdma_write_stats.write_bytes = resp.u.rdma_write_stats.write_bytes;
result->rdma_write_stats.write_wr_err = resp.u.rdma_write_stats.write_wr_err;
result->rdma_write_stats.write_recv_bytes = resp.u.rdma_write_stats.write_recv_bytes;
rws = &resp.u.rdma_write_stats;
result->rdma_write_stats.write_wrs = rws->write_wrs;
result->rdma_write_stats.write_bytes = rws->write_bytes;
result->rdma_write_stats.write_wr_err = rws->write_wr_err;
result->rdma_write_stats.write_recv_bytes = rws->write_recv_bytes;
break;
case EFA_ADMIN_GET_STATS_TYPE_NETWORK:
ns = &resp.u.network_stats;
result->network_stats.retrans_bytes = ns->retrans_bytes;
result->network_stats.retrans_pkts = ns->retrans_pkts;
result->network_stats.retrans_timeout_events = ns->retrans_timeout_events;
result->network_stats.unresponsive_remote_events = ns->unresponsive_remote_events;
result->network_stats.impaired_remote_conn_events = ns->impaired_remote_conn_events;
break;
}

11
drivers/infiniband/hw/efa/efa_com_cmd.h
View file

@ -1,6 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
/*
* Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2018-2025 Amazon.com, Inc. or its affiliates. All rights reserved.
*/
#ifndef _EFA_COM_CMD_H_
@ -283,11 +283,20 @@ struct efa_com_rdma_write_stats {
u64 write_recv_bytes;
};
struct efa_com_network_stats {
u64 retrans_bytes;
u64 retrans_pkts;
u64 retrans_timeout_events;
u64 unresponsive_remote_events;
u64 impaired_remote_conn_events;
};
union efa_com_get_stats_result {
struct efa_com_basic_stats basic_stats;
struct efa_com_messages_stats messages_stats;
struct efa_com_rdma_read_stats rdma_read_stats;
struct efa_com_rdma_write_stats rdma_write_stats;
struct efa_com_network_stats network_stats;
};
int efa_com_create_qp(struct efa_com_dev *edev,

1
drivers/infiniband/hw/efa/efa_main.c
View file

@ -372,6 +372,7 @@ static const struct ib_device_ops efa_dev_ops = {
.alloc_pd = efa_alloc_pd,
.alloc_ucontext = efa_alloc_ucontext,
.create_cq = efa_create_cq,
.create_cq_umem = efa_create_cq_umem,
.create_qp = efa_create_qp,
.create_user_ah = efa_create_ah,
.dealloc_pd = efa_dealloc_pd,

91
drivers/infiniband/hw/efa/efa_verbs.c
View file

@ -64,6 +64,11 @@ struct efa_user_mmap_entry {
op(EFA_RDMA_WRITE_BYTES, "rdma_write_bytes") \
op(EFA_RDMA_WRITE_WR_ERR, "rdma_write_wr_err") \
op(EFA_RDMA_WRITE_RECV_BYTES, "rdma_write_recv_bytes") \
op(EFA_RETRANS_BYTES, "retrans_bytes") \
op(EFA_RETRANS_PKTS, "retrans_pkts") \
op(EFA_RETRANS_TIMEOUT_EVENS, "retrans_timeout_events") \
op(EFA_UNRESPONSIVE_REMOTE_EVENTS, "unresponsive_remote_events") \
op(EFA_IMPAIRED_REMOTE_CONN_EVENTS, "impaired_remote_conn_events") \
#define EFA_STATS_ENUM(ename, name) ename,
#define EFA_STATS_STR(ename, nam) \
@ -249,6 +254,7 @@ int efa_query_device(struct ib_device *ibdev,
resp.max_rdma_size = dev_attr->max_rdma_size;
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_SGID;
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_CQ_WITH_EXT_MEM;
if (EFA_DEV_CAP(dev, RDMA_READ))
resp.device_caps |= EFA_QUERY_DEVICE_CAPS_RDMA_READ;
@ -1082,8 +1088,11 @@ int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
xa_erase(&dev->cqs_xa, cq->cq_idx);
synchronize_irq(cq->eq->irq.irqn);
}
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
if (cq->umem)
ib_umem_release(cq->umem);
else
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE);
return 0;
}
@ -1122,8 +1131,8 @@ static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq,
return 0;
}
int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct uverbs_attr_bundle *attrs)
int efa_create_cq_umem(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_umem *umem, struct uverbs_attr_bundle *attrs)
{
struct ib_udata *udata = &attrs->driver_udata;
struct efa_ucontext *ucontext = rdma_udata_to_drv_context(
@ -1202,11 +1211,30 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
cq->ucontext = ucontext;
cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs);
cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
if (!cq->cpu_addr) {
err = -ENOMEM;
goto err_out;
if (umem) {
if (umem->length < cq->size) {
ibdev_dbg(&dev->ibdev, "External memory too small\n");
err = -EINVAL;
goto err_free_mem;
}
if (!ib_umem_is_contiguous(umem)) {
ibdev_dbg(&dev->ibdev, "Non contiguous CQ unsupported\n");
err = -EINVAL;
goto err_free_mem;
}
cq->cpu_addr = NULL;
cq->dma_addr = ib_umem_start_dma_addr(umem);
cq->umem = umem;
} else {
cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
if (!cq->cpu_addr) {
err = -ENOMEM;
goto err_out;
}
}
params.uarn = cq->ucontext->uarn;
@ -1223,7 +1251,7 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
err = efa_com_create_cq(&dev->edev, &params, &result);
if (err)
goto err_free_mapped;
goto err_free_mem;
resp.db_off = result.db_off;
resp.cq_idx = result.cq_idx;
@ -1231,7 +1259,9 @@ int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
cq->ibcq.cqe = result.actual_depth;
WARN_ON_ONCE(entries != result.actual_depth);
err = cq_mmap_entries_setup(dev, cq, &resp, result.db_valid);
if (!umem)
err = cq_mmap_entries_setup(dev, cq, &resp, result.db_valid);
if (err) {
ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n",
cq->cq_idx);
@ -1269,15 +1299,23 @@ err_remove_mmap:
efa_cq_user_mmap_entries_remove(cq);
err_destroy_cq:
efa_destroy_cq_idx(dev, cq->cq_idx);
err_free_mapped:
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size,
DMA_FROM_DEVICE);
err_free_mem:
if (umem)
ib_umem_release(umem);
else
efa_free_mapped(dev, cq->cpu_addr, cq->dma_addr, cq->size, DMA_FROM_DEVICE);
err_out:
atomic64_inc(&dev->stats.create_cq_err);
return err;
}
int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct uverbs_attr_bundle *attrs)
{
return efa_create_cq_umem(ibcq, attr, NULL, attrs);
}
static int umem_to_page_list(struct efa_dev *dev,
struct ib_umem *umem,
u64 *page_list,
@ -1727,6 +1765,7 @@ static int efa_register_mr(struct ib_pd *ibpd, struct efa_mr *mr, u64 start,
struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
u64 length, u64 virt_addr,
int fd, int access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs)
{
struct efa_dev *dev = to_edev(ibpd->device);
@ -1734,6 +1773,11 @@ struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
struct efa_mr *mr;
int err;
if (dmah) {
err = -EOPNOTSUPP;
goto err_out;
}
mr = efa_alloc_mr(ibpd, access_flags, &attrs->driver_udata);
if (IS_ERR(mr)) {
err = PTR_ERR(mr);
@ -1766,12 +1810,18 @@ err_out:
struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct efa_dev *dev = to_edev(ibpd->device);
struct efa_mr *mr;
int err;
if (dmah) {
err = -EOPNOTSUPP;
goto err_out;
}
mr = efa_alloc_mr(ibpd, access_flags, udata);
if (IS_ERR(mr)) {
err = PTR_ERR(mr);
@ -2186,6 +2236,7 @@ static int efa_fill_port_stats(struct efa_dev *dev, struct rdma_hw_stats *stats,
struct efa_com_rdma_write_stats *rws;
struct efa_com_rdma_read_stats *rrs;
struct efa_com_messages_stats *ms;
struct efa_com_network_stats *ns;
struct efa_com_basic_stats *bs;
int err;
@ -2238,6 +2289,18 @@ static int efa_fill_port_stats(struct efa_dev *dev, struct rdma_hw_stats *stats,
stats->value[EFA_RDMA_WRITE_RECV_BYTES] = rws->write_recv_bytes;
}
params.type = EFA_ADMIN_GET_STATS_TYPE_NETWORK;
err = efa_com_get_stats(&dev->edev, &params, &result);
if (err)
return err;
ns = &result.network_stats;
stats->value[EFA_RETRANS_BYTES] = ns->retrans_bytes;
stats->value[EFA_RETRANS_PKTS] = ns->retrans_pkts;
stats->value[EFA_RETRANS_TIMEOUT_EVENS] = ns->retrans_timeout_events;
stats->value[EFA_UNRESPONSIVE_REMOTE_EVENTS] = ns->unresponsive_remote_events;
stats->value[EFA_IMPAIRED_REMOTE_CONN_EVENTS] = ns->impaired_remote_conn_events;
return ARRAY_SIZE(efa_port_stats_descs);
}

9
drivers/infiniband/hw/erdma/erdma_verbs.c
View file

@ -629,7 +629,8 @@ err_free_mtt:
static void erdma_destroy_mtt_buf_sg(struct erdma_dev *dev,
struct erdma_mtt *mtt)
{
dma_unmap_sg(&dev->pdev->dev, mtt->sglist, mtt->nsg, DMA_TO_DEVICE);
dma_unmap_sg(&dev->pdev->dev, mtt->sglist,
DIV_ROUND_UP(mtt->size, PAGE_SIZE), DMA_TO_DEVICE);
vfree(mtt->sglist);
}
@ -1199,13 +1200,17 @@ int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
}
struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
u64 virt, int access, struct ib_udata *udata)
u64 virt, int access, struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct erdma_mr *mr = NULL;
struct erdma_dev *dev = to_edev(ibpd->device);
u32 stag;
int ret;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
if (!len || len > dev->attrs.max_mr_size)
return ERR_PTR(-EINVAL);

3
drivers/infiniband/hw/erdma/erdma_verbs.h
View file

@ -452,7 +452,8 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
void erdma_disassociate_ucontext(struct ib_ucontext *ibcontext);
int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
u64 virt, int access, struct ib_udata *udata);
u64 virt, int access, struct ib_dmah *dmah,
struct ib_udata *udata);
struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);

94
drivers/infiniband/hw/hfi1/affinity.c
View file

@ -92,9 +92,7 @@ static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
/* Initialize non-HT cpu cores mask */
void init_real_cpu_mask(void)
{
int possible, curr_cpu, i, ht;
cpumask_clear(&node_affinity.real_cpu_mask);
int possible, curr_cpu, ht;
/* Start with cpu online mask as the real cpu mask */
cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
@ -110,17 +108,10 @@ void init_real_cpu_mask(void)
* "real" cores. Assumes that HT cores are not enumerated in
* succession (except in the single core case).
*/
curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
for (i = 0; i < possible / ht; i++)
curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
/*
* Step 2. Remove the remaining HT siblings. Use cpumask_next() to
* skip any gaps.
*/
for (; i < possible; i++) {
cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
}
curr_cpu = cpumask_nth(possible / ht, &node_affinity.real_cpu_mask) + 1;
/* Step 2. Remove the remaining HT siblings. */
cpumask_clear_cpus(&node_affinity.real_cpu_mask, curr_cpu, nr_cpu_ids - curr_cpu);
}
int node_affinity_init(void)
@ -346,9 +337,10 @@ static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
&entry->def_intr.used);
/* If there are non-interrupt CPUs available, use them first */
if (!cpumask_empty(non_intr_cpus))
cpu = cpumask_first(non_intr_cpus);
else /* Otherwise, use interrupt CPUs */
cpu = cpumask_first(non_intr_cpus);
/* Otherwise, use interrupt CPUs */
if (cpu >= nr_cpu_ids)
cpu = cpumask_first(available_cpus);
if (cpu >= nr_cpu_ids) { /* empty */
@ -963,32 +955,23 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
struct hfi1_affinity_node_list *affinity)
{
int possible, curr_cpu, i;
uint num_cores_per_socket = node_affinity.num_online_cpus /
affinity->num_core_siblings /
node_affinity.num_online_nodes;
int curr_cpu;
uint num_cores;
cpumask_copy(hw_thread_mask, &affinity->proc.mask);
if (affinity->num_core_siblings > 0) {
/* Removing other siblings not needed for now */
possible = cpumask_weight(hw_thread_mask);
curr_cpu = cpumask_first(hw_thread_mask);
for (i = 0;
i < num_cores_per_socket * node_affinity.num_online_nodes;
i++)
curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
for (; i < possible; i++) {
cpumask_clear_cpu(curr_cpu, hw_thread_mask);
curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
}
if (affinity->num_core_siblings == 0)
return;
/* Identifying correct HW threads within physical cores */
cpumask_shift_left(hw_thread_mask, hw_thread_mask,
num_cores_per_socket *
node_affinity.num_online_nodes *
hw_thread_no);
}
num_cores = rounddown(node_affinity.num_online_cpus / affinity->num_core_siblings,
node_affinity.num_online_nodes);
/* Removing other siblings not needed for now */
curr_cpu = cpumask_nth(num_cores * node_affinity.num_online_nodes, hw_thread_mask) + 1;
cpumask_clear_cpus(hw_thread_mask, curr_cpu, nr_cpu_ids - curr_cpu);
/* Identifying correct HW threads within physical cores */
cpumask_shift_left(hw_thread_mask, hw_thread_mask, num_cores * hw_thread_no);
}
int hfi1_get_proc_affinity(int node)
@ -1087,22 +1070,19 @@ int hfi1_get_proc_affinity(int node)
* If HT cores are enabled, identify which HW threads within the
* physical cores should be used.
*/
if (affinity->num_core_siblings > 0) {
for (i = 0; i < affinity->num_core_siblings; i++) {
find_hw_thread_mask(i, hw_thread_mask, affinity);
for (i = 0; i < affinity->num_core_siblings; i++) {
find_hw_thread_mask(i, hw_thread_mask, affinity);
/*
* If there's at least one available core for this HW
* thread number, stop looking for a core.
*
* diff will always be not empty at least once in this
* loop as the used mask gets reset when
* (set->mask == set->used) before this loop.
*/
cpumask_andnot(diff, hw_thread_mask, &set->used);
if (!cpumask_empty(diff))
break;
}
/*
* If there's at least one available core for this HW
* thread number, stop looking for a core.
*
* diff will always be not empty at least once in this
* loop as the used mask gets reset when
* (set->mask == set->used) before this loop.
*/
if (cpumask_andnot(diff, hw_thread_mask, &set->used))
break;
}
hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
cpumask_pr_args(hw_thread_mask));
@ -1133,8 +1113,7 @@ int hfi1_get_proc_affinity(int node)
* used for process assignments using the same method as
* the preferred NUMA node.
*/
cpumask_andnot(diff, available_mask, intrs_mask);
if (!cpumask_empty(diff))
if (cpumask_andnot(diff, available_mask, intrs_mask))
cpumask_copy(available_mask, diff);
/* If we don't have CPUs on the preferred node, use other NUMA nodes */
@ -1150,8 +1129,7 @@ int hfi1_get_proc_affinity(int node)
* At first, we don't want to place processes on the same
* CPUs as interrupt handlers.
*/
cpumask_andnot(diff, available_mask, intrs_mask);
if (!cpumask_empty(diff))
if (cpumask_andnot(diff, available_mask, intrs_mask))
cpumask_copy(available_mask, diff);
}
hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",

21
drivers/infiniband/hw/hns/hns_roce_device.h
View file

@ -316,16 +316,6 @@ struct hns_roce_mtr {
struct hns_roce_hem_cfg hem_cfg; /* config for hardware addressing */
};
struct hns_roce_mw {
struct ib_mw ibmw;
u32 pdn;
u32 rkey;
int enabled; /* MW's active status */
u32 pbl_hop_num;
u32 pbl_ba_pg_sz;
u32 pbl_buf_pg_sz;
};
struct hns_roce_mr {
struct ib_mr ibmr;
u64 iova; /* MR's virtual original addr */
@ -856,6 +846,7 @@ struct hns_roce_caps {
u16 default_ceq_arm_st;
u8 cong_cap;
enum hns_roce_cong_type default_cong_type;
u32 max_ack_req_msg_len;
};
enum hns_roce_device_state {
@ -933,7 +924,6 @@ struct hns_roce_hw {
struct hns_roce_mr *mr, int flags,
void *mb_buf);
int (*frmr_write_mtpt)(void *mb_buf, struct hns_roce_mr *mr);
int (*mw_write_mtpt)(void *mb_buf, struct hns_roce_mw *mw);
void (*write_cqc)(struct hns_roce_dev *hr_dev,
struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
dma_addr_t dma_handle);
@ -1078,11 +1068,6 @@ static inline struct hns_roce_mr *to_hr_mr(struct ib_mr *ibmr)
return container_of(ibmr, struct hns_roce_mr, ibmr);
}
static inline struct hns_roce_mw *to_hr_mw(struct ib_mw *ibmw)
{
return container_of(ibmw, struct hns_roce_mw, ibmw);
}
static inline struct hns_roce_qp *to_hr_qp(struct ib_qp *ibqp)
{
return container_of(ibqp, struct hns_roce_qp, ibqp);
@ -1234,6 +1219,7 @@ int hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc);
struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata);
struct ib_mr *hns_roce_rereg_user_mr(struct ib_mr *mr, int flags, u64 start,
u64 length, u64 virt_addr,
@ -1246,9 +1232,6 @@ int hns_roce_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
unsigned long key_to_hw_index(u32 key);
int hns_roce_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);
int hns_roce_dealloc_mw(struct ib_mw *ibmw);
void hns_roce_buf_free(struct hns_roce_dev *hr_dev, struct hns_roce_buf *buf);
struct hns_roce_buf *hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size,
u32 page_shift, u32 flags);

18
drivers/infiniband/hw/hns/hns_roce_hem.c
View file

@ -249,15 +249,12 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
}
static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
unsigned long hem_alloc_size,
gfp_t gfp_mask)
unsigned long hem_alloc_size)
{
struct hns_roce_hem *hem;
int order;
void *buf;
WARN_ON(gfp_mask & __GFP_HIGHMEM);
order = get_order(hem_alloc_size);
if (PAGE_SIZE << order != hem_alloc_size) {
dev_err(hr_dev->dev, "invalid hem_alloc_size: %lu!\n",
@ -265,13 +262,12 @@ static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
return NULL;
}
hem = kmalloc(sizeof(*hem),
gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
hem = kmalloc(sizeof(*hem), GFP_KERNEL);
if (!hem)
return NULL;
buf = dma_alloc_coherent(hr_dev->dev, hem_alloc_size,
&hem->dma, gfp_mask);
&hem->dma, GFP_KERNEL);
if (!buf)
goto fail;
@ -378,7 +374,6 @@ static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
{
u32 bt_size = mhop->bt_chunk_size;
struct device *dev = hr_dev->dev;
gfp_t flag;
u64 bt_ba;
u32 size;
int ret;
@ -417,8 +412,7 @@ static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
* alloc bt space chunk for MTT/CQE.
*/
size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
flag = GFP_KERNEL | __GFP_NOWARN;
table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size, flag);
table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size);
if (!table->hem[index->buf]) {
ret = -ENOMEM;
goto err_alloc_hem;
@ -546,9 +540,7 @@ int hns_roce_table_get(struct hns_roce_dev *hr_dev,
goto out;
}
table->hem[i] = hns_roce_alloc_hem(hr_dev,
table->table_chunk_size,
GFP_KERNEL | __GFP_NOWARN);
table->hem[i] = hns_roce_alloc_hem(hr_dev, table->table_chunk_size);
if (!table->hem[i]) {
ret = -ENOMEM;
goto out;

134
drivers/infiniband/hw/hns/hns_roce_hw_v2.c
View file

@ -144,7 +144,7 @@ static void set_frmr_seg(struct hns_roce_v2_rc_send_wqe *rc_sq_wqe,
u64 pbl_ba;
/* use ib_access_flags */
hr_reg_write_bool(fseg, FRMR_BIND_EN, wr->access & IB_ACCESS_MW_BIND);
hr_reg_write_bool(fseg, FRMR_BIND_EN, 0);
hr_reg_write_bool(fseg, FRMR_ATOMIC,
wr->access & IB_ACCESS_REMOTE_ATOMIC);
hr_reg_write_bool(fseg, FRMR_RR, wr->access & IB_ACCESS_REMOTE_READ);
@ -2196,31 +2196,36 @@ static void apply_func_caps(struct hns_roce_dev *hr_dev)
static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
{
struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM];
struct hns_roce_cmq_desc desc[HNS_ROCE_QUERY_PF_CAPS_CMD_NUM] = {};
struct hns_roce_caps *caps = &hr_dev->caps;
struct hns_roce_query_pf_caps_a *resp_a;
struct hns_roce_query_pf_caps_b *resp_b;
struct hns_roce_query_pf_caps_c *resp_c;
struct hns_roce_query_pf_caps_d *resp_d;
struct hns_roce_query_pf_caps_e *resp_e;
struct hns_roce_query_pf_caps_f *resp_f;
enum hns_roce_opcode_type cmd;
int ctx_hop_num;
int pbl_hop_num;
int cmd_num;
int ret;
int i;
cmd = hr_dev->is_vf ? HNS_ROCE_OPC_QUERY_VF_CAPS_NUM :
HNS_ROCE_OPC_QUERY_PF_CAPS_NUM;
cmd_num = hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 ?
HNS_ROCE_QUERY_PF_CAPS_CMD_NUM_HIP08 :
HNS_ROCE_QUERY_PF_CAPS_CMD_NUM;
for (i = 0; i < HNS_ROCE_QUERY_PF_CAPS_CMD_NUM; i++) {
for (i = 0; i < cmd_num - 1; i++) {
hns_roce_cmq_setup_basic_desc(&desc[i], cmd, true);
if (i < (HNS_ROCE_QUERY_PF_CAPS_CMD_NUM - 1))
desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
else
desc[i].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
desc[i].flag |= cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
}
ret = hns_roce_cmq_send(hr_dev, desc, HNS_ROCE_QUERY_PF_CAPS_CMD_NUM);
hns_roce_cmq_setup_basic_desc(&desc[cmd_num - 1], cmd, true);
desc[cmd_num - 1].flag &= ~cpu_to_le16(HNS_ROCE_CMD_FLAG_NEXT);
ret = hns_roce_cmq_send(hr_dev, desc, cmd_num);
if (ret)
return ret;
@ -2229,6 +2234,7 @@ static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
resp_c = (struct hns_roce_query_pf_caps_c *)desc[2].data;
resp_d = (struct hns_roce_query_pf_caps_d *)desc[3].data;
resp_e = (struct hns_roce_query_pf_caps_e *)desc[4].data;
resp_f = (struct hns_roce_query_pf_caps_f *)desc[5].data;
caps->local_ca_ack_delay = resp_a->local_ca_ack_delay;
caps->max_sq_sg = le16_to_cpu(resp_a->max_sq_sg);
@ -2293,6 +2299,8 @@ static int hns_roce_query_caps(struct hns_roce_dev *hr_dev)
caps->reserved_srqs = hr_reg_read(resp_e, PF_CAPS_E_RSV_SRQS);
caps->reserved_lkey = hr_reg_read(resp_e, PF_CAPS_E_RSV_LKEYS);
caps->max_ack_req_msg_len = le32_to_cpu(resp_f->max_ack_req_msg_len);
caps->qpc_hop_num = ctx_hop_num;
caps->sccc_hop_num = ctx_hop_num;
caps->srqc_hop_num = ctx_hop_num;
@ -2627,7 +2635,7 @@ static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)
struct ib_pd *pd;
hr_pd = kzalloc(sizeof(*hr_pd), GFP_KERNEL);
if (ZERO_OR_NULL_PTR(hr_pd))
if (!hr_pd)
return NULL;
pd = &hr_pd->ibpd;
pd->device = ibdev;
@ -2658,7 +2666,7 @@ static struct ib_cq *free_mr_init_cq(struct hns_roce_dev *hr_dev)
cq_init_attr.cqe = HNS_ROCE_FREE_MR_USED_CQE_NUM;
hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL);
if (ZERO_OR_NULL_PTR(hr_cq))
if (!hr_cq)
return NULL;
cq = &hr_cq->ib_cq;
@ -2691,7 +2699,7 @@ static int free_mr_init_qp(struct hns_roce_dev *hr_dev, struct ib_cq *cq,
int ret;
hr_qp = kzalloc(sizeof(*hr_qp), GFP_KERNEL);
if (ZERO_OR_NULL_PTR(hr_qp))
if (!hr_qp)
return -ENOMEM;
qp = &hr_qp->ibqp;
@ -2986,14 +2994,22 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
{
int ret;
if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
ret = free_mr_init(hr_dev);
if (ret) {
dev_err(hr_dev->dev, "failed to init free mr!\n");
return ret;
}
}
/* The hns ROCEE requires the extdb info to be cleared before using */
ret = hns_roce_clear_extdb_list_info(hr_dev);
if (ret)
return ret;
goto err_clear_extdb_failed;
ret = get_hem_table(hr_dev);
if (ret)
return ret;
goto err_get_hem_table_failed;
if (hr_dev->is_vf)
return 0;
@ -3008,6 +3024,11 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
err_llm_init_failed:
put_hem_table(hr_dev);
err_get_hem_table_failed:
hns_roce_function_clear(hr_dev);
err_clear_extdb_failed:
if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
free_mr_exit(hr_dev);
return ret;
}
@ -3313,8 +3334,6 @@ static int hns_roce_v2_write_mtpt(struct hns_roce_dev *hr_dev,
hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_VALID);
hr_reg_write(mpt_entry, MPT_PD, mr->pd);
hr_reg_write_bool(mpt_entry, MPT_BIND_EN,
mr->access & IB_ACCESS_MW_BIND);
hr_reg_write_bool(mpt_entry, MPT_ATOMIC_EN,
mr->access & IB_ACCESS_REMOTE_ATOMIC);
hr_reg_write_bool(mpt_entry, MPT_RR_EN,
@ -3358,8 +3377,6 @@ static int hns_roce_v2_rereg_write_mtpt(struct hns_roce_dev *hr_dev,
hr_reg_write(mpt_entry, MPT_PD, mr->pd);
if (flags & IB_MR_REREG_ACCESS) {
hr_reg_write(mpt_entry, MPT_BIND_EN,
(mr_access_flags & IB_ACCESS_MW_BIND ? 1 : 0));
hr_reg_write(mpt_entry, MPT_ATOMIC_EN,
mr_access_flags & IB_ACCESS_REMOTE_ATOMIC ? 1 : 0);
hr_reg_write(mpt_entry, MPT_RR_EN,
@ -3397,7 +3414,6 @@ static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
hr_reg_enable(mpt_entry, MPT_R_INV_EN);
hr_reg_enable(mpt_entry, MPT_FRE);
hr_reg_clear(mpt_entry, MPT_MR_MW);
hr_reg_enable(mpt_entry, MPT_BPD);
hr_reg_clear(mpt_entry, MPT_PA);
@ -3417,38 +3433,6 @@ static int hns_roce_v2_frmr_write_mtpt(void *mb_buf, struct hns_roce_mr *mr)
return 0;
}
static int hns_roce_v2_mw_write_mtpt(void *mb_buf, struct hns_roce_mw *mw)
{
struct hns_roce_v2_mpt_entry *mpt_entry;
mpt_entry = mb_buf;
memset(mpt_entry, 0, sizeof(*mpt_entry));
hr_reg_write(mpt_entry, MPT_ST, V2_MPT_ST_FREE);
hr_reg_write(mpt_entry, MPT_PD, mw->pdn);
hr_reg_enable(mpt_entry, MPT_R_INV_EN);
hr_reg_enable(mpt_entry, MPT_LW_EN);
hr_reg_enable(mpt_entry, MPT_MR_MW);
hr_reg_enable(mpt_entry, MPT_BPD);
hr_reg_clear(mpt_entry, MPT_PA);
hr_reg_write(mpt_entry, MPT_BQP,
mw->ibmw.type == IB_MW_TYPE_1 ? 0 : 1);
mpt_entry->lkey = cpu_to_le32(mw->rkey);
hr_reg_write(mpt_entry, MPT_PBL_HOP_NUM,
mw->pbl_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 :
mw->pbl_hop_num);
hr_reg_write(mpt_entry, MPT_PBL_BA_PG_SZ,
mw->pbl_ba_pg_sz + PG_SHIFT_OFFSET);
hr_reg_write(mpt_entry, MPT_PBL_BUF_PG_SZ,
mw->pbl_buf_pg_sz + PG_SHIFT_OFFSET);
return 0;
}
static int free_mr_post_send_lp_wqe(struct hns_roce_qp *hr_qp)
{
struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
@ -3849,7 +3833,6 @@ static const u32 wc_send_op_map[] = {
HR_WC_OP_MAP(ATOM_MSK_CMP_AND_SWAP, MASKED_COMP_SWAP),
HR_WC_OP_MAP(ATOM_MSK_FETCH_AND_ADD, MASKED_FETCH_ADD),
HR_WC_OP_MAP(FAST_REG_PMR, REG_MR),
HR_WC_OP_MAP(BIND_MW, REG_MR),
};
static int to_ib_wc_send_op(u32 hr_opcode)
@ -4560,7 +4543,9 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
dma_addr_t trrl_ba;
dma_addr_t irrl_ba;
enum ib_mtu ib_mtu;
u8 ack_req_freq;
const u8 *smac;
int lp_msg_len;
u8 lp_pktn_ini;
u64 *mtts;
u8 *dmac;
@ -4643,7 +4628,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
return -EINVAL;
#define MIN_LP_MSG_LEN 1024
/* mtu * (2 ^ lp_pktn_ini) should be in the range of 1024 to mtu */
lp_pktn_ini = ilog2(max(mtu, MIN_LP_MSG_LEN) / mtu);
lp_msg_len = max(mtu, MIN_LP_MSG_LEN);
lp_pktn_ini = ilog2(lp_msg_len / mtu);
if (attr_mask & IB_QP_PATH_MTU) {
hr_reg_write(context, QPC_MTU, ib_mtu);
@ -4653,8 +4639,22 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp,
hr_reg_write(context, QPC_LP_PKTN_INI, lp_pktn_ini);
hr_reg_clear(qpc_mask, QPC_LP_PKTN_INI);
/* ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI */
hr_reg_write(context, QPC_ACK_REQ_FREQ, lp_pktn_ini);
/*
* There are several constraints for ACK_REQ_FREQ:
* 1. mtu * (2 ^ ACK_REQ_FREQ) should not be too large, otherwise
* it may cause some unexpected retries when sending large
* payload.
* 2. ACK_REQ_FREQ should be larger than or equal to LP_PKTN_INI.
* 3. ACK_REQ_FREQ must be equal to LP_PKTN_INI when using LDCP
* or HC3 congestion control algorithm.
*/
if (hr_qp->cong_type == CONG_TYPE_LDCP ||
hr_qp->cong_type == CONG_TYPE_HC3 ||
hr_dev->caps.max_ack_req_msg_len < lp_msg_len)
ack_req_freq = lp_pktn_ini;
else
ack_req_freq = ilog2(hr_dev->caps.max_ack_req_msg_len / mtu);
hr_reg_write(context, QPC_ACK_REQ_FREQ, ack_req_freq);
hr_reg_clear(qpc_mask, QPC_ACK_REQ_FREQ);
hr_reg_clear(qpc_mask, QPC_RX_REQ_PSN_ERR);
@ -5349,11 +5349,10 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
{
struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
struct hns_roce_v2_qp_context ctx[2];
struct hns_roce_v2_qp_context *context = ctx;
struct hns_roce_v2_qp_context *qpc_mask = ctx + 1;
struct hns_roce_v2_qp_context *context;
struct hns_roce_v2_qp_context *qpc_mask;
struct ib_device *ibdev = &hr_dev->ib_dev;
int ret;
int ret = -ENOMEM;
if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
return -EOPNOTSUPP;
@ -5364,7 +5363,11 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
* we should set all bits of the relevant fields in context mask to
* 0 at the same time, else set them to 0x1.
*/
memset(context, 0, hr_dev->caps.qpc_sz);
context = kvzalloc(sizeof(*context), GFP_KERNEL);
qpc_mask = kvzalloc(sizeof(*qpc_mask), GFP_KERNEL);
if (!context || !qpc_mask)
goto out;
memset(qpc_mask, 0xff, hr_dev->caps.qpc_sz);
ret = hns_roce_v2_set_abs_fields(ibqp, attr, attr_mask, cur_state,
@ -5406,6 +5409,8 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp,
clear_qp(hr_qp);
out:
kvfree(qpc_mask);
kvfree(context);
return ret;
}
@ -6948,7 +6953,6 @@ static const struct hns_roce_hw hns_roce_hw_v2 = {
.write_mtpt = hns_roce_v2_write_mtpt,
.rereg_write_mtpt = hns_roce_v2_rereg_write_mtpt,
.frmr_write_mtpt = hns_roce_v2_frmr_write_mtpt,
.mw_write_mtpt = hns_roce_v2_mw_write_mtpt,
.write_cqc = hns_roce_v2_write_cqc,
.set_hem = hns_roce_v2_set_hem,
.clear_hem = hns_roce_v2_clear_hem,
@ -7044,21 +7048,11 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
goto error_failed_roce_init;
}
if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08) {
ret = free_mr_init(hr_dev);
if (ret) {
dev_err(hr_dev->dev, "failed to init free mr!\n");
goto error_failed_free_mr_init;
}
}
handle->priv = hr_dev;
return 0;
error_failed_free_mr_init:
hns_roce_exit(hr_dev);
error_failed_roce_init:
kfree(hr_dev->priv);

16
drivers/infiniband/hw/hns/hns_roce_hw_v2.h
View file

@ -814,24 +814,16 @@ struct hns_roce_v2_mpt_entry {
#define V2_MPT_BYTE_8_LW_EN_S 7
#define V2_MPT_BYTE_8_MW_CNT_S 8
#define V2_MPT_BYTE_8_MW_CNT_M GENMASK(31, 8)
#define V2_MPT_BYTE_12_FRE_S 0
#define V2_MPT_BYTE_12_PA_S 1
#define V2_MPT_BYTE_12_MR_MW_S 4
#define V2_MPT_BYTE_12_BPD_S 5
#define V2_MPT_BYTE_12_BQP_S 6
#define V2_MPT_BYTE_12_INNER_PA_VLD_S 7
#define V2_MPT_BYTE_12_MW_BIND_QPN_S 8
#define V2_MPT_BYTE_12_MW_BIND_QPN_M GENMASK(31, 8)
#define V2_MPT_BYTE_48_PBL_BA_H_S 0
#define V2_MPT_BYTE_48_PBL_BA_H_M GENMASK(28, 0)
@ -1168,7 +1160,8 @@ struct hns_roce_cfg_gmv_tb_b {
#define GMV_TB_B_SMAC_H GMV_TB_B_FIELD_LOC(47, 32)
#define GMV_TB_B_SGID_IDX GMV_TB_B_FIELD_LOC(71, 64)
#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 5
#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM_HIP08 5
#define HNS_ROCE_QUERY_PF_CAPS_CMD_NUM 6
struct hns_roce_query_pf_caps_a {
u8 number_ports;
u8 local_ca_ack_delay;
@ -1280,6 +1273,11 @@ struct hns_roce_query_pf_caps_e {
__le16 aeq_period;
};
struct hns_roce_query_pf_caps_f {
__le32 max_ack_req_msg_len;
__le32 rsv[5];
};
#define PF_CAPS_E_FIELD_LOC(h, l) \
FIELD_LOC(struct hns_roce_query_pf_caps_e, h, l)

32
drivers/infiniband/hw/hns/hns_roce_main.c
View file

@ -672,13 +672,6 @@ static const struct ib_device_ops hns_roce_dev_mr_ops = {
.rereg_user_mr = hns_roce_rereg_user_mr,
};
static const struct ib_device_ops hns_roce_dev_mw_ops = {
.alloc_mw = hns_roce_alloc_mw,
.dealloc_mw = hns_roce_dealloc_mw,
INIT_RDMA_OBJ_SIZE(ib_mw, hns_roce_mw, ibmw),
};
static const struct ib_device_ops hns_roce_dev_frmr_ops = {
.alloc_mr = hns_roce_alloc_mr,
.map_mr_sg = hns_roce_map_mr_sg,
@ -732,9 +725,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_REREG_MR)
ib_set_device_ops(ib_dev, &hns_roce_dev_mr_ops);
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_MW)
ib_set_device_ops(ib_dev, &hns_roce_dev_mw_ops);
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_FRMR)
ib_set_device_ops(ib_dev, &hns_roce_dev_frmr_ops);
@ -947,10 +937,7 @@ err_unmap_dmpt:
static void hns_roce_teardown_hca(struct hns_roce_dev *hr_dev)
{
hns_roce_cleanup_bitmap(hr_dev);
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||
hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB)
mutex_destroy(&hr_dev->pgdir_mutex);
mutex_destroy(&hr_dev->pgdir_mutex);
}
/**
@ -965,11 +952,11 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
spin_lock_init(&hr_dev->sm_lock);
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||
hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) {
INIT_LIST_HEAD(&hr_dev->pgdir_list);
mutex_init(&hr_dev->pgdir_mutex);
}
INIT_LIST_HEAD(&hr_dev->qp_list);
spin_lock_init(&hr_dev->qp_list_lock);
INIT_LIST_HEAD(&hr_dev->pgdir_list);
mutex_init(&hr_dev->pgdir_mutex);
hns_roce_init_uar_table(hr_dev);
@ -1001,9 +988,7 @@ static int hns_roce_setup_hca(struct hns_roce_dev *hr_dev)
err_uar_table_free:
ida_destroy(&hr_dev->uar_ida.ida);
if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQ_RECORD_DB ||
hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB)
mutex_destroy(&hr_dev->pgdir_mutex);
mutex_destroy(&hr_dev->pgdir_mutex);
return ret;
}
@ -1132,9 +1117,6 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)
}
}
INIT_LIST_HEAD(&hr_dev->qp_list);
spin_lock_init(&hr_dev->qp_list_lock);
ret = hns_roce_register_device(hr_dev);
if (ret)
goto error_failed_register_device;

120
drivers/infiniband/hw/hns/hns_roce_mr.c
View file

@ -231,12 +231,18 @@ err_free:
struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
struct hns_roce_mr *mr;
int ret;
if (dmah) {
ret = -EOPNOTSUPP;
goto err_out;
}
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
if (!mr) {
ret = -ENOMEM;
@ -483,120 +489,6 @@ err_page_list:
return sg_num;
}
static void hns_roce_mw_free(struct hns_roce_dev *hr_dev,
struct hns_roce_mw *mw)
{
struct device *dev = hr_dev->dev;
int ret;
if (mw->enabled) {
ret = hns_roce_destroy_hw_ctx(hr_dev, HNS_ROCE_CMD_DESTROY_MPT,
key_to_hw_index(mw->rkey) &
(hr_dev->caps.num_mtpts - 1));
if (ret)
dev_warn(dev, "MW DESTROY_MPT failed (%d)\n", ret);
hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
key_to_hw_index(mw->rkey));
}
ida_free(&hr_dev->mr_table.mtpt_ida.ida,
(int)key_to_hw_index(mw->rkey));
}
static int hns_roce_mw_enable(struct hns_roce_dev *hr_dev,
struct hns_roce_mw *mw)
{
struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
struct hns_roce_cmd_mailbox *mailbox;
struct device *dev = hr_dev->dev;
unsigned long mtpt_idx = key_to_hw_index(mw->rkey);
int ret;
/* prepare HEM entry memory */
ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
if (ret)
return ret;
mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
if (IS_ERR(mailbox)) {
ret = PTR_ERR(mailbox);
goto err_table;
}
ret = hr_dev->hw->mw_write_mtpt(mailbox->buf, mw);
if (ret) {
dev_err(dev, "MW write mtpt fail!\n");
goto err_page;
}
ret = hns_roce_create_hw_ctx(hr_dev, mailbox, HNS_ROCE_CMD_CREATE_MPT,
mtpt_idx & (hr_dev->caps.num_mtpts - 1));
if (ret) {
dev_err(dev, "MW CREATE_MPT failed (%d)\n", ret);
goto err_page;
}
mw->enabled = 1;
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
return 0;
err_page:
hns_roce_free_cmd_mailbox(hr_dev, mailbox);
err_table:
hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
return ret;
}
int hns_roce_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device);
struct hns_roce_ida *mtpt_ida = &hr_dev->mr_table.mtpt_ida;
struct ib_device *ibdev = &hr_dev->ib_dev;
struct hns_roce_mw *mw = to_hr_mw(ibmw);
int ret;
int id;
/* Allocate a key for mw from mr_table */
id = ida_alloc_range(&mtpt_ida->ida, mtpt_ida->min, mtpt_ida->max,
GFP_KERNEL);
if (id < 0) {
ibdev_err(ibdev, "failed to alloc id for MW key, id(%d)\n", id);
return -ENOMEM;
}
mw->rkey = hw_index_to_key(id);
ibmw->rkey = mw->rkey;
mw->pdn = to_hr_pd(ibmw->pd)->pdn;
mw->pbl_hop_num = hr_dev->caps.pbl_hop_num;
mw->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
mw->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
ret = hns_roce_mw_enable(hr_dev, mw);
if (ret)
goto err_mw;
return 0;
err_mw:
hns_roce_mw_free(hr_dev, mw);
return ret;
}
int hns_roce_dealloc_mw(struct ib_mw *ibmw)
{
struct hns_roce_dev *hr_dev = to_hr_dev(ibmw->device);
struct hns_roce_mw *mw = to_hr_mw(ibmw);
hns_roce_mw_free(hr_dev, mw);
return 0;
}
static int mtr_map_region(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr,
struct hns_roce_buf_region *region, dma_addr_t *pages,
int max_count)

4
drivers/infiniband/hw/hns/hns_roce_qp.c
View file

@ -1003,14 +1003,14 @@ static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
int ret;
sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
if (ZERO_OR_NULL_PTR(sq_wrid)) {
if (!sq_wrid) {
ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
return -ENOMEM;
}
if (hr_qp->rq.wqe_cnt) {
rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
if (ZERO_OR_NULL_PTR(rq_wrid)) {
if (!rq_wrid) {
ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
ret = -ENOMEM;
goto err_sq;

9
drivers/infiniband/hw/irdma/verbs.c
View file

@ -3013,10 +3013,12 @@ static int irdma_reg_user_mr_type_cq(struct irdma_mem_reg_req req,
* @len: length of mr
* @virt: virtual address
* @access: access of mr
* @dmah: dma handle
* @udata: user data
*/
static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
u64 virt, int access,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
#define IRDMA_MEM_REG_MIN_REQ_LEN offsetofend(struct irdma_mem_reg_req, sq_pages)
@ -3026,6 +3028,9 @@ static struct ib_mr *irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
struct irdma_mr *iwmr = NULL;
int err;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
return ERR_PTR(-EINVAL);
@ -3085,6 +3090,7 @@ error:
static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
u64 len, u64 virt,
int fd, int access,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs)
{
struct irdma_device *iwdev = to_iwdev(pd->device);
@ -3092,6 +3098,9 @@ static struct ib_mr *irdma_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
struct irdma_mr *iwmr;
int err;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
if (len > iwdev->rf->sc_dev.hw_attrs.max_mr_size)
return ERR_PTR(-EINVAL);

78
drivers/infiniband/hw/mana/counters.c
View file

@ -32,8 +32,32 @@ static const struct rdma_stat_desc mana_ib_port_stats_desc[] = {
[MANA_IB_RATE_INC_EVENTS].name = "rate_inc_events",
[MANA_IB_NUM_QPS_RECOVERED].name = "num_qps_recovered",
[MANA_IB_CURRENT_RATE].name = "current_rate",
[MANA_IB_DUP_RX_REQ].name = "dup_rx_requests",
[MANA_IB_TX_BYTES].name = "tx_bytes",
[MANA_IB_RX_BYTES].name = "rx_bytes",
[MANA_IB_RX_SEND_REQ].name = "rx_send_requests",
[MANA_IB_RX_WRITE_REQ].name = "rx_write_requests",
[MANA_IB_RX_READ_REQ].name = "rx_read_requests",
[MANA_IB_TX_PKT].name = "tx_packets",
[MANA_IB_RX_PKT].name = "rx_packets",
};
static const struct rdma_stat_desc mana_ib_device_stats_desc[] = {
[MANA_IB_SENT_CNPS].name = "sent_cnps",
[MANA_IB_RECEIVED_ECNS].name = "received_ecns",
[MANA_IB_RECEIVED_CNP_COUNT].name = "received_cnp_count",
[MANA_IB_QP_CONGESTED_EVENTS].name = "qp_congested_events",
[MANA_IB_QP_RECOVERED_EVENTS].name = "qp_recovered_events",
[MANA_IB_DEV_RATE_INC_EVENTS].name = "rate_inc_events",
};
struct rdma_hw_stats *mana_ib_alloc_hw_device_stats(struct ib_device *ibdev)
{
return rdma_alloc_hw_stats_struct(mana_ib_device_stats_desc,
ARRAY_SIZE(mana_ib_device_stats_desc),
RDMA_HW_STATS_DEFAULT_LIFESPAN);
}
struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
u32 port_num)
{
@ -42,8 +66,39 @@ struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
RDMA_HW_STATS_DEFAULT_LIFESPAN);
}
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
u32 port_num, int index)
static int mana_ib_get_hw_device_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats)
{
struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,
ib_dev);
struct mana_rnic_query_device_cntrs_resp resp = {};
struct mana_rnic_query_device_cntrs_req req = {};
int err;
mana_gd_init_req_hdr(&req.hdr, MANA_IB_QUERY_DEVICE_COUNTERS,
sizeof(req), sizeof(resp));
req.hdr.dev_id = mdev->gdma_dev->dev_id;
req.adapter = mdev->adapter_handle;
err = mana_gd_send_request(mdev_to_gc(mdev), sizeof(req), &req,
sizeof(resp), &resp);
if (err) {
ibdev_err(&mdev->ib_dev, "Failed to query device counters err %d",
err);
return err;
}
stats->value[MANA_IB_SENT_CNPS] = resp.sent_cnps;
stats->value[MANA_IB_RECEIVED_ECNS] = resp.received_ecns;
stats->value[MANA_IB_RECEIVED_CNP_COUNT] = resp.received_cnp_count;
stats->value[MANA_IB_QP_CONGESTED_EVENTS] = resp.qp_congested_events;
stats->value[MANA_IB_QP_RECOVERED_EVENTS] = resp.qp_recovered_events;
stats->value[MANA_IB_DEV_RATE_INC_EVENTS] = resp.rate_inc_events;
return ARRAY_SIZE(mana_ib_device_stats_desc);
}
static int mana_ib_get_hw_port_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
u32 port_num)
{
struct mana_ib_dev *mdev = container_of(ibdev, struct mana_ib_dev,
ib_dev);
@ -53,6 +108,7 @@ int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
mana_gd_init_req_hdr(&req.hdr, MANA_IB_QUERY_VF_COUNTERS,
sizeof(req), sizeof(resp));
req.hdr.resp.msg_version = GDMA_MESSAGE_V2;
req.hdr.dev_id = mdev->gdma_dev->dev_id;
req.adapter = mdev->adapter_handle;
@ -101,5 +157,23 @@ int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
stats->value[MANA_IB_NUM_QPS_RECOVERED] = resp.num_qps_recovered;
stats->value[MANA_IB_CURRENT_RATE] = resp.current_rate;
stats->value[MANA_IB_DUP_RX_REQ] = resp.dup_rx_req;
stats->value[MANA_IB_TX_BYTES] = resp.tx_bytes;
stats->value[MANA_IB_RX_BYTES] = resp.rx_bytes;
stats->value[MANA_IB_RX_SEND_REQ] = resp.rx_send_req;
stats->value[MANA_IB_RX_WRITE_REQ] = resp.rx_write_req;
stats->value[MANA_IB_RX_READ_REQ] = resp.rx_read_req;
stats->value[MANA_IB_TX_PKT] = resp.tx_pkt;
stats->value[MANA_IB_RX_PKT] = resp.rx_pkt;
return ARRAY_SIZE(mana_ib_port_stats_desc);
}
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
u32 port_num, int index)
{
if (!port_num)
return mana_ib_get_hw_device_stats(ibdev, stats);
else
return mana_ib_get_hw_port_stats(ibdev, stats, port_num);
}

18
drivers/infiniband/hw/mana/counters.h
View file

@ -35,10 +35,28 @@ enum mana_ib_port_counters {
MANA_IB_RATE_INC_EVENTS,
MANA_IB_NUM_QPS_RECOVERED,
MANA_IB_CURRENT_RATE,
MANA_IB_DUP_RX_REQ,
MANA_IB_TX_BYTES,
MANA_IB_RX_BYTES,
MANA_IB_RX_SEND_REQ,
MANA_IB_RX_WRITE_REQ,
MANA_IB_RX_READ_REQ,
MANA_IB_TX_PKT,
MANA_IB_RX_PKT,
};
enum mana_ib_device_counters {
MANA_IB_SENT_CNPS,
MANA_IB_RECEIVED_ECNS,
MANA_IB_RECEIVED_CNP_COUNT,
MANA_IB_QP_CONGESTED_EVENTS,
MANA_IB_QP_RECOVERED_EVENTS,
MANA_IB_DEV_RATE_INC_EVENTS,
};
struct rdma_hw_stats *mana_ib_alloc_hw_port_stats(struct ib_device *ibdev,
u32 port_num);
struct rdma_hw_stats *mana_ib_alloc_hw_device_stats(struct ib_device *ibdev);
int mana_ib_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
u32 port_num, int index);
#endif /* _COUNTERS_H_ */

116
drivers/infiniband/hw/mana/device.c
View file

@ -65,6 +65,10 @@ static const struct ib_device_ops mana_ib_stats_ops = {
.get_hw_stats = mana_ib_get_hw_stats,
};
static const struct ib_device_ops mana_ib_device_stats_ops = {
.alloc_hw_device_stats = mana_ib_alloc_hw_device_stats,
};
static int mana_ib_netdev_event(struct notifier_block *this,
unsigned long event, void *ptr)
{
@ -73,28 +77,31 @@ static int mana_ib_netdev_event(struct notifier_block *this,
struct gdma_context *gc = dev->gdma_dev->gdma_context;
struct mana_context *mc = gc->mana.driver_data;
struct net_device *ndev;
int i;
/* Only process events from our parent device */
if (event_dev != mc->ports[0])
return NOTIFY_DONE;
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
if (event_dev == mc->ports[i]) {
switch (event) {
case NETDEV_CHANGEUPPER:
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
/*
* RDMA core will setup GID based on updated netdev.
* It's not possible to race with the core as rtnl lock is being
* held.
*/
ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
switch (event) {
case NETDEV_CHANGEUPPER:
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
/*
* RDMA core will setup GID based on updated netdev.
* It's not possible to race with the core as rtnl lock is being
* held.
*/
ib_device_set_netdev(&dev->ib_dev, ndev, 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
if (ndev)
netdev_put(ndev, &dev->dev_tracker);
/* mana_get_primary_netdev() returns ndev with refcount held */
netdev_put(ndev, &dev->dev_tracker);
return NOTIFY_OK;
default:
return NOTIFY_DONE;
}
return NOTIFY_OK;
default:
return NOTIFY_DONE;
}
}
return NOTIFY_DONE;
}
static int mana_ib_probe(struct auxiliary_device *adev,
@ -107,7 +114,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
struct net_device *ndev;
struct mana_ib_dev *dev;
u8 mac_addr[ETH_ALEN];
int ret;
int ret, i;
dev = ib_alloc_device(mana_ib_dev, ib_dev);
if (!dev)
@ -122,51 +129,56 @@ static int mana_ib_probe(struct auxiliary_device *adev,
if (mana_ib_is_rnic(dev)) {
dev->ib_dev.phys_port_cnt = 1;
ndev = mana_get_primary_netdev(mc, 0, &dev->dev_tracker);
if (!ndev) {
ret = -ENODEV;
ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1");
goto free_ib_device;
}
ether_addr_copy(mac_addr, ndev->dev_addr);
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr);
ret = ib_device_set_netdev(&dev->ib_dev, ndev, 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
netdev_put(ndev, &dev->dev_tracker);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
goto free_ib_device;
}
dev->nb.notifier_call = mana_ib_netdev_event;
ret = register_netdevice_notifier(&dev->nb);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d",
ret);
goto free_ib_device;
}
addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
ret = mana_ib_gd_query_adapter_caps(dev);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
goto deregister_net_notifier;
goto free_ib_device;
}
ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT)
ib_set_device_ops(&dev->ib_dev, &mana_ib_device_stats_ops);
ret = mana_ib_create_eqs(dev);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
goto deregister_net_notifier;
goto free_ib_device;
}
ret = mana_ib_gd_create_rnic_adapter(dev);
if (ret)
goto destroy_eqs;
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
dev->ib_dev.phys_port_cnt = mc->num_ports;
for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
if (!ndev) {
ret = -ENODEV;
ibdev_err(&dev->ib_dev,
"Failed to get netdev for IB port %d", i + 1);
goto destroy_rnic;
}
ether_addr_copy(mac_addr, ndev->dev_addr);
ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
/* mana_get_primary_netdev() returns ndev with refcount held */
netdev_put(ndev, &dev->dev_tracker);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
goto destroy_rnic;
}
ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
goto destroy_rnic;
}
}
dev->nb.notifier_call = mana_ib_netdev_event;
ret = register_netdevice_notifier(&dev->nb);
if (ret) {
ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
goto destroy_rnic;
}
} else {
@ -182,7 +194,7 @@ static int mana_ib_probe(struct auxiliary_device *adev,
MANA_AV_BUFFER_SIZE, 0);
if (!dev->av_pool) {
ret = -ENOMEM;
goto destroy_rnic;
goto deregister_net_notifier;
}
ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
@ -199,15 +211,15 @@ static int mana_ib_probe(struct auxiliary_device *adev,
deallocate_pool:
dma_pool_destroy(dev->av_pool);
deregister_net_notifier:
if (mana_ib_is_rnic(dev))
unregister_netdevice_notifier(&dev->nb);
destroy_rnic:
if (mana_ib_is_rnic(dev))
mana_ib_gd_destroy_rnic_adapter(dev);
destroy_eqs:
if (mana_ib_is_rnic(dev))
mana_ib_destroy_eqs(dev);
deregister_net_notifier:
if (mana_ib_is_rnic(dev))
unregister_netdevice_notifier(&dev->nb);
free_ib_device:
xa_destroy(&dev->qp_table_wq);
ib_dealloc_device(&dev->ib_dev);
@ -221,9 +233,9 @@ static void mana_ib_remove(struct auxiliary_device *adev)
ib_unregister_device(&dev->ib_dev);
dma_pool_destroy(dev->av_pool);
if (mana_ib_is_rnic(dev)) {
unregister_netdevice_notifier(&dev->nb);
mana_ib_gd_destroy_rnic_adapter(dev);
mana_ib_destroy_eqs(dev);
unregister_netdevice_notifier(&dev->nb);
}
xa_destroy(&dev->qp_table_wq);
ib_dealloc_device(&dev->ib_dev);

13
drivers/infiniband/hw/mana/main.c
View file

@ -563,8 +563,14 @@ int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num,
immutable->gid_tbl_len = attr.gid_tbl_len;
if (mana_ib_is_rnic(dev)) {
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
if (port_num == 1) {
immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
immutable->max_mad_size = IB_MGMT_MAD_SIZE;
} else {
immutable->core_cap_flags = RDMA_CORE_CAP_PROT_ROCE_UDP_ENCAP
| RDMA_CORE_CAP_ETH_AH;
immutable->max_mad_size = 0;
}
} else {
immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
}
@ -633,8 +639,9 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 port,
props->pkey_tbl_len = 1;
if (mana_ib_is_rnic(dev)) {
props->gid_tbl_len = 16;
props->port_cap_flags = IB_PORT_CM_SUP;
props->ip_gids = true;
if (port == 1)
props->port_cap_flags = IB_PORT_CM_SUP;
}
return 0;

30
drivers/infiniband/hw/mana/mana_ib.h
View file

@ -210,6 +210,7 @@ enum mana_ib_command_code {
MANA_IB_DESTROY_RC_QP = 0x3000b,
MANA_IB_SET_QP_STATE = 0x3000d,
MANA_IB_QUERY_VF_COUNTERS = 0x30022,
MANA_IB_QUERY_DEVICE_COUNTERS = 0x30023,
};
struct mana_ib_query_adapter_caps_req {
@ -218,6 +219,8 @@ struct mana_ib_query_adapter_caps_req {
enum mana_ib_adapter_features {
MANA_IB_FEATURE_CLIENT_ERROR_CQE_SUPPORT = BIT(4),
MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT = BIT(5),
MANA_IB_FEATURE_MULTI_PORTS_SUPPORT = BIT(6),
};
struct mana_ib_query_adapter_caps_resp {
@ -514,6 +517,31 @@ struct mana_rnic_query_vf_cntrs_resp {
u64 rate_inc_events;
u64 num_qps_recovered;
u64 current_rate;
u64 dup_rx_req;
u64 tx_bytes;
u64 rx_bytes;
u64 rx_send_req;
u64 rx_write_req;
u64 rx_read_req;
u64 tx_pkt;
u64 rx_pkt;
}; /* HW Data */
struct mana_rnic_query_device_cntrs_req {
struct gdma_req_hdr hdr;
mana_handle_t adapter;
}; /* HW Data */
struct mana_rnic_query_device_cntrs_resp {
struct gdma_resp_hdr hdr;
u32 sent_cnps;
u32 received_ecns;
u32 reserved1;
u32 received_cnp_count;
u32 qp_congested_events;
u32 qp_recovered_events;
u32 rate_inc_events;
u32 reserved2;
}; /* HW Data */
static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
@ -605,6 +633,7 @@ struct ib_mr *mana_ib_get_dma_mr(struct ib_pd *ibpd, int access_flags);
struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 iova, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata);
int mana_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
@ -694,5 +723,6 @@ int mana_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
u64 iova, int fd, int mr_access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs);
#endif

8
drivers/infiniband/hw/mana/mr.c
View file

@ -106,6 +106,7 @@ static int mana_ib_gd_destroy_mr(struct mana_ib_dev *dev, u64 mr_handle)
struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
u64 iova, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
@ -116,6 +117,9 @@ struct ib_mr *mana_ib_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
u64 dma_region_handle;
int err;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
ibdev_dbg(ibdev,
@ -188,6 +192,7 @@ err_free:
struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 length,
u64 iova, int fd, int access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs)
{
struct mana_ib_pd *pd = container_of(ibpd, struct mana_ib_pd, ibpd);
@ -199,6 +204,9 @@ struct ib_mr *mana_ib_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 leng
u64 dma_region_handle;
int err;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
dev = container_of(ibdev, struct mana_ib_dev, ib_dev);
access_flags &= ~IB_ACCESS_OPTIONAL;

2
drivers/infiniband/hw/mana/qp.c
View file

@ -772,7 +772,7 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
req.ah_attr.dest_port = ROCE_V2_UDP_DPORT;
req.ah_attr.src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
ibqp->qp_num, attr->dest_qp_num);
req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class;
req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class >> 2;
req.ah_attr.hop_limit = attr->ah_attr.grh.hop_limit;
}

1
drivers/infiniband/hw/mlx4/mlx4_ib.h
View file

@ -759,6 +759,7 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
struct ib_umem *umem);
struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata);
int mlx4_ib_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
int mlx4_ib_alloc_mw(struct ib_mw *mw, struct ib_udata *udata);

4
drivers/infiniband/hw/mlx4/mr.c
View file

@ -139,6 +139,7 @@ static struct ib_umem *mlx4_get_umem_mr(struct ib_device *device, u64 start,
struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(pd->device);
@ -147,6 +148,9 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
int err;
int n;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
mr = kzalloc(sizeof(*mr), GFP_KERNEL);
if (!mr)
return ERR_PTR(-ENOMEM);

1
drivers/infiniband/hw/mlx5/Makefile
View file

@ -8,6 +8,7 @@ mlx5_ib-y := ah.o \
cq.o \
data_direct.o \
dm.o \
dmah.o \
doorbell.o \
fs.o \
gsi.o \

30
drivers/infiniband/hw/mlx5/counters.c
View file

@ -16,6 +16,18 @@ struct mlx5_ib_counter {
u32 type;
};
struct mlx5_rdma_counter {
struct rdma_counter rdma_counter;
struct mlx5_fc *fc[MLX5_IB_OPCOUNTER_MAX];
struct xarray qpn_opfc_xa;
};
static struct mlx5_rdma_counter *to_mcounter(struct rdma_counter *counter)
{
return container_of(counter, struct mlx5_rdma_counter, rdma_counter);
}
#define INIT_Q_COUNTER(_name) \
{ .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
@ -602,7 +614,7 @@ static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
return 0;
WARN_ON(!xa_empty(&mcounter->qpn_opfc_xa));
mlx5r_fs_destroy_fcs(dev, counter);
mlx5r_fs_destroy_fcs(dev, mcounter->fc);
MLX5_SET(dealloc_q_counter_in, in, opcode,
MLX5_CMD_OP_DEALLOC_Q_COUNTER);
MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
@ -612,6 +624,7 @@ static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
struct ib_qp *qp, u32 port)
{
struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
struct mlx5_ib_dev *dev = to_mdev(qp->device);
bool new = false;
int err;
@ -635,7 +648,11 @@ static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
if (err)
goto fail_set_counter;
err = mlx5r_fs_bind_op_fc(qp, counter, port);
if (!counter->mode.bind_opcnt)
return 0;
err = mlx5r_fs_bind_op_fc(qp, mcounter->fc, &mcounter->qpn_opfc_xa,
port);
if (err)
goto fail_bind_op_fc;
@ -655,9 +672,12 @@ fail_set_counter:
static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp, u32 port)
{
struct rdma_counter *counter = qp->counter;
struct mlx5_rdma_counter *mcounter;
int err;
mlx5r_fs_unbind_op_fc(qp, counter);
mcounter = to_mcounter(counter);
mlx5r_fs_unbind_op_fc(qp, &mcounter->qpn_opfc_xa);
err = mlx5_ib_qp_set_counter(qp, NULL);
if (err)
@ -666,7 +686,9 @@ static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp, u32 port)
return 0;
fail_set_counter:
mlx5r_fs_bind_op_fc(qp, counter, port);
if (counter->mode.bind_opcnt)
mlx5r_fs_bind_op_fc(qp, mcounter->fc,
&mcounter->qpn_opfc_xa, port);
return err;
}

13
drivers/infiniband/hw/mlx5/counters.h
View file

@ -8,19 +8,6 @@
#include "mlx5_ib.h"
struct mlx5_rdma_counter {
struct rdma_counter rdma_counter;
struct mlx5_fc *fc[MLX5_IB_OPCOUNTER_MAX];
struct xarray qpn_opfc_xa;
};
static inline struct mlx5_rdma_counter *
to_mcounter(struct rdma_counter *counter)
{
return container_of(counter, struct mlx5_rdma_counter, rdma_counter);
}
int mlx5_ib_counters_init(struct mlx5_ib_dev *dev);
void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev);
void mlx5_ib_counters_clear_description(struct ib_counters *counters);

19
drivers/infiniband/hw/mlx5/cq.c
View file

@ -1055,20 +1055,31 @@ err_cqb:
return err;
}
int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
int mlx5_ib_pre_destroy_cq(struct ib_cq *cq)
{
struct mlx5_ib_dev *dev = to_mdev(cq->device);
struct mlx5_ib_cq *mcq = to_mcq(cq);
return mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
}
void mlx5_ib_post_destroy_cq(struct ib_cq *cq)
{
destroy_cq_kernel(to_mdev(cq->device), to_mcq(cq));
}
int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
int ret;
ret = mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
ret = mlx5_ib_pre_destroy_cq(cq);
if (ret)
return ret;
if (udata)
destroy_cq_user(mcq, udata);
destroy_cq_user(to_mcq(cq), udata);
else
destroy_cq_kernel(dev, mcq);
mlx5_ib_post_destroy_cq(cq);
return 0;
}

6
drivers/infiniband/hw/mlx5/devx.c
View file

@ -159,7 +159,7 @@ int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user, u64 req_ucaps)
uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
if (is_user &&
(MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX) &&
capable(CAP_NET_RAW))
rdma_dev_has_raw_cap(&dev->ib_dev))
cap |= MLX5_UCTX_CAP_RAW_TX;
if (is_user &&
(MLX5_CAP_GEN(dev->mdev, uctx_cap) &
@ -1393,6 +1393,10 @@ static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
}
MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
/* TPH is not allowed to bypass the regular kernel's verbs flow */
MLX5_SET(mkc, mkc, pcie_tph_en, 0);
MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX);
return 0;
}

54
drivers/infiniband/hw/mlx5/dmah.c Normal file
View file

@ -0,0 +1,54 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved
*/
#include <rdma/uverbs_std_types.h>
#include <linux/pci-tph.h>
#include "dmah.h"
#define UVERBS_MODULE_NAME mlx5_ib
#include <rdma/uverbs_named_ioctl.h>
static int mlx5_ib_alloc_dmah(struct ib_dmah *ibdmah,
struct uverbs_attr_bundle *attrs)
{
struct mlx5_core_dev *mdev = to_mdev(ibdmah->device)->mdev;
struct mlx5_ib_dmah *dmah = to_mdmah(ibdmah);
u16 st_bits = BIT(IB_DMAH_CPU_ID_EXISTS) |
BIT(IB_DMAH_MEM_TYPE_EXISTS);
int err;
/* PH is a must for TPH following PCIe spec 6.2-1.0 */
if (!(ibdmah->valid_fields & BIT(IB_DMAH_PH_EXISTS)))
return -EINVAL;
/* ST is optional; however, partial data for it is not allowed */
if (ibdmah->valid_fields & st_bits) {
if ((ibdmah->valid_fields & st_bits) != st_bits)
return -EINVAL;
err = mlx5_st_alloc_index(mdev, ibdmah->mem_type,
ibdmah->cpu_id, &dmah->st_index);
if (err)
return err;
}
return 0;
}
static int mlx5_ib_dealloc_dmah(struct ib_dmah *ibdmah,
struct uverbs_attr_bundle *attrs)
{
struct mlx5_ib_dmah *dmah = to_mdmah(ibdmah);
struct mlx5_core_dev *mdev = to_mdev(ibdmah->device)->mdev;
if (ibdmah->valid_fields & BIT(IB_DMAH_CPU_ID_EXISTS))
return mlx5_st_dealloc_index(mdev, dmah->st_index);
return 0;
}
const struct ib_device_ops mlx5_ib_dev_dmah_ops = {
.alloc_dmah = mlx5_ib_alloc_dmah,
.dealloc_dmah = mlx5_ib_dealloc_dmah,
};

23
drivers/infiniband/hw/mlx5/dmah.h Normal file
View file

@ -0,0 +1,23 @@
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
/*
* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved
*/
#ifndef _MLX5_IB_DMAH_H
#define _MLX5_IB_DMAH_H
#include "mlx5_ib.h"
extern const struct ib_device_ops mlx5_ib_dev_dmah_ops;
struct mlx5_ib_dmah {
struct ib_dmah ibdmah;
u16 st_index;
};
static inline struct mlx5_ib_dmah *to_mdmah(struct ib_dmah *ibdmah)
{
return container_of(ibdmah, struct mlx5_ib_dmah, ibdmah);
}
#endif /* _MLX5_IB_DMAH_H */

121
drivers/infiniband/hw/mlx5/fs.c
View file

@ -1012,14 +1012,14 @@ static int get_per_qp_prio(struct mlx5_ib_dev *dev,
return 0;
}
static struct mlx5_per_qp_opfc *
get_per_qp_opfc(struct mlx5_rdma_counter *mcounter, u32 qp_num, bool *new)
static struct mlx5_per_qp_opfc *get_per_qp_opfc(struct xarray *qpn_opfc_xa,
u32 qp_num, bool *new)
{
struct mlx5_per_qp_opfc *per_qp_opfc;
*new = false;
per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp_num);
per_qp_opfc = xa_load(qpn_opfc_xa, qp_num);
if (per_qp_opfc)
return per_qp_opfc;
per_qp_opfc = kzalloc(sizeof(*per_qp_opfc), GFP_KERNEL);
@ -1032,7 +1032,8 @@ get_per_qp_opfc(struct mlx5_rdma_counter *mcounter, u32 qp_num, bool *new)
}
static int add_op_fc_rules(struct mlx5_ib_dev *dev,
struct mlx5_rdma_counter *mcounter,
struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
struct xarray *qpn_opfc_xa,
struct mlx5_per_qp_opfc *per_qp_opfc,
struct mlx5_ib_flow_prio *prio,
enum mlx5_ib_optional_counter_type type,
@ -1055,7 +1056,7 @@ static int add_op_fc_rules(struct mlx5_ib_dev *dev,
return 0;
}
opfc->fc = mcounter->fc[type];
opfc->fc = fc_arr[type];
spec = kcalloc(MAX_OPFC_RULES, sizeof(*spec), GFP_KERNEL);
if (!spec) {
@ -1148,8 +1149,7 @@ static int add_op_fc_rules(struct mlx5_ib_dev *dev,
}
prio->refcount += spec_num;
err = xa_err(xa_store(&mcounter->qpn_opfc_xa, qp_num, per_qp_opfc,
GFP_KERNEL));
err = xa_err(xa_store(qpn_opfc_xa, qp_num, per_qp_opfc, GFP_KERNEL));
if (err)
goto del_rules;
@ -1168,8 +1168,9 @@ null_fc:
return err;
}
static bool is_fc_shared_and_in_use(struct mlx5_rdma_counter *mcounter,
u32 type, struct mlx5_fc **fc)
static bool
is_fc_shared_and_in_use(struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX], u32 type,
struct mlx5_fc **fc)
{
u32 shared_fc_type;
@ -1190,7 +1191,7 @@ static bool is_fc_shared_and_in_use(struct mlx5_rdma_counter *mcounter,
return false;
}
*fc = mcounter->fc[shared_fc_type];
*fc = fc_arr[shared_fc_type];
if (!(*fc))
return false;
@ -1198,24 +1199,23 @@ static bool is_fc_shared_and_in_use(struct mlx5_rdma_counter *mcounter,
}
void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
struct rdma_counter *counter)
struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX])
{
struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
struct mlx5_fc *in_use_fc;
int i;
for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
if (!mcounter->fc[i])
if (!fc_arr[i])
continue;
if (is_fc_shared_and_in_use(mcounter, i, &in_use_fc)) {
mcounter->fc[i] = NULL;
if (is_fc_shared_and_in_use(fc_arr, i, &in_use_fc)) {
fc_arr[i] = NULL;
continue;
}
mlx5_fc_destroy(dev->mdev, mcounter->fc[i]);
mcounter->fc[i] = NULL;
mlx5_fc_destroy(dev->mdev, fc_arr[i]);
fc_arr[i] = NULL;
}
}
@ -1359,16 +1359,15 @@ void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
put_per_qp_prio(dev, type);
}
void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct rdma_counter *counter)
void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa)
{
struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
struct mlx5_ib_dev *dev = to_mdev(counter->device);
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct mlx5_per_qp_opfc *per_qp_opfc;
struct mlx5_ib_op_fc *in_use_opfc;
struct mlx5_ib_flow_prio *prio;
int i, j;
per_qp_opfc = xa_load(&mcounter->qpn_opfc_xa, qp->qp_num);
per_qp_opfc = xa_load(qpn_opfc_xa, qp->qp_num);
if (!per_qp_opfc)
return;
@ -1394,13 +1393,13 @@ void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct rdma_counter *counter)
}
kfree(per_qp_opfc);
xa_erase(&mcounter->qpn_opfc_xa, qp->qp_num);
xa_erase(qpn_opfc_xa, qp->qp_num);
}
int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter,
u32 port)
int mlx5r_fs_bind_op_fc(struct ib_qp *qp,
struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
struct xarray *qpn_opfc_xa, u32 port)
{
struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct mlx5_per_qp_opfc *per_qp_opfc;
struct mlx5_ib_flow_prio *prio;
@ -1410,9 +1409,6 @@ int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter,
int i, err, per_qp_type;
bool new;
if (!counter->mode.bind_opcnt)
return 0;
cnts = &dev->port[port - 1].cnts;
for (i = 0; i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; i++) {
@ -1424,23 +1420,22 @@ int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter,
prio = get_opfc_prio(dev, per_qp_type);
WARN_ON(!prio->flow_table);
if (is_fc_shared_and_in_use(mcounter, per_qp_type, &in_use_fc))
mcounter->fc[per_qp_type] = in_use_fc;
if (is_fc_shared_and_in_use(fc_arr, per_qp_type, &in_use_fc))
fc_arr[per_qp_type] = in_use_fc;
if (!mcounter->fc[per_qp_type]) {
mcounter->fc[per_qp_type] = mlx5_fc_create(dev->mdev,
false);
if (IS_ERR(mcounter->fc[per_qp_type]))
return PTR_ERR(mcounter->fc[per_qp_type]);
if (!fc_arr[per_qp_type]) {
fc_arr[per_qp_type] = mlx5_fc_create(dev->mdev, false);
if (IS_ERR(fc_arr[per_qp_type]))
return PTR_ERR(fc_arr[per_qp_type]);
}
per_qp_opfc = get_per_qp_opfc(mcounter, qp->qp_num, &new);
per_qp_opfc = get_per_qp_opfc(qpn_opfc_xa, qp->qp_num, &new);
if (!per_qp_opfc) {
err = -ENOMEM;
goto free_fc;
}
err = add_op_fc_rules(dev, mcounter, per_qp_opfc, prio,
per_qp_type, qp->qp_num, port);
err = add_op_fc_rules(dev, fc_arr, qpn_opfc_xa, per_qp_opfc,
prio, per_qp_type, qp->qp_num, port);
if (err)
goto del_rules;
}
@ -1448,12 +1443,12 @@ int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter,
return 0;
del_rules:
mlx5r_fs_unbind_op_fc(qp, counter);
mlx5r_fs_unbind_op_fc(qp, qpn_opfc_xa);
if (new)
kfree(per_qp_opfc);
free_fc:
if (xa_empty(&mcounter->qpn_opfc_xa))
mlx5r_fs_destroy_fcs(dev, counter);
if (xa_empty(qpn_opfc_xa))
mlx5r_fs_destroy_fcs(dev, fc_arr);
return err;
}
@ -1966,7 +1961,8 @@ _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
break;
case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
if (ib_port == 0 || user_priority > MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
if (ib_port == 0 ||
user_priority >= MLX5_RDMA_TRANSPORT_BYPASS_PRIO)
return ERR_PTR(-EINVAL);
ret = mlx5_ib_fill_transport_ns_info(dev, ns_type, &flags,
&vport_idx, &vport,
@ -2016,10 +2012,10 @@ _get_flow_table(struct mlx5_ib_dev *dev, u16 user_priority,
prio = &dev->flow_db->rdma_tx[priority];
break;
case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_RX:
prio = &dev->flow_db->rdma_transport_rx[ib_port - 1];
prio = &dev->flow_db->rdma_transport_rx[priority][ib_port - 1];
break;
case MLX5_FLOW_NAMESPACE_RDMA_TRANSPORT_TX:
prio = &dev->flow_db->rdma_transport_tx[ib_port - 1];
prio = &dev->flow_db->rdma_transport_tx[priority][ib_port - 1];
break;
default: return ERR_PTR(-EINVAL);
}
@ -2458,7 +2454,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_CREATE_FLOW)(
struct mlx5_ib_dev *dev;
u32 flags;
if (!capable(CAP_NET_RAW))
if (!rdma_uattrs_has_raw_cap(attrs))
return -EPERM;
fs_matcher = uverbs_attr_get_obj(attrs,
@ -2989,7 +2985,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_STEERING_ANCHOR_CREATE)(
u32 ft_id;
int err;
if (!capable(CAP_NET_RAW))
if (!rdma_dev_has_raw_cap(&dev->ib_dev))
return -EPERM;
err = uverbs_get_const(&ib_uapi_ft_type, attrs,
@ -3466,31 +3462,40 @@ static const struct ib_device_ops flow_ops = {
int mlx5_ib_fs_init(struct mlx5_ib_dev *dev)
{
int i, j;
dev->flow_db = kzalloc(sizeof(*dev->flow_db), GFP_KERNEL);
if (!dev->flow_db)
return -ENOMEM;
dev->flow_db->rdma_transport_rx = kcalloc(dev->num_ports,
sizeof(struct mlx5_ib_flow_prio),
GFP_KERNEL);
if (!dev->flow_db->rdma_transport_rx)
goto free_flow_db;
for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++) {
dev->flow_db->rdma_transport_rx[i] =
kcalloc(dev->num_ports,
sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
if (!dev->flow_db->rdma_transport_rx[i])
goto free_rdma_transport_rx;
}
dev->flow_db->rdma_transport_tx = kcalloc(dev->num_ports,
sizeof(struct mlx5_ib_flow_prio),
GFP_KERNEL);
if (!dev->flow_db->rdma_transport_tx)
goto free_rdma_transport_rx;
for (j = 0; j < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; j++) {
dev->flow_db->rdma_transport_tx[j] =
kcalloc(dev->num_ports,
sizeof(struct mlx5_ib_flow_prio), GFP_KERNEL);
if (!dev->flow_db->rdma_transport_tx[j])
goto free_rdma_transport_tx;
}
mutex_init(&dev->flow_db->lock);
ib_set_device_ops(&dev->ib_dev, &flow_ops);
return 0;
free_rdma_transport_tx:
while (j--)
kfree(dev->flow_db->rdma_transport_tx[j]);
free_rdma_transport_rx:
kfree(dev->flow_db->rdma_transport_rx);
free_flow_db:
while (i--)
kfree(dev->flow_db->rdma_transport_rx[i]);
kfree(dev->flow_db);
return -ENOMEM;
}

8
drivers/infiniband/hw/mlx5/fs.h
View file

@ -13,6 +13,8 @@ void mlx5_ib_fs_cleanup_anchor(struct mlx5_ib_dev *dev);
static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev)
{
int i;
/* When a steering anchor is created, a special flow table is also
* created for the user to reference. Since the user can reference it,
* the kernel cannot trust that when the user destroys the steering
@ -25,8 +27,10 @@ static inline void mlx5_ib_fs_cleanup(struct mlx5_ib_dev *dev)
* is a safe assumption that all references are gone.
*/
mlx5_ib_fs_cleanup_anchor(dev);
kfree(dev->flow_db->rdma_transport_tx);
kfree(dev->flow_db->rdma_transport_rx);
for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++)
kfree(dev->flow_db->rdma_transport_tx[i]);
for (i = 0; i < MLX5_RDMA_TRANSPORT_BYPASS_PRIO; i++)
kfree(dev->flow_db->rdma_transport_rx[i]);
kfree(dev->flow_db);
}
#endif /* _MLX5_IB_FS_H */

3
drivers/infiniband/hw/mlx5/ib_rep.c
View file

@ -88,7 +88,8 @@ mlx5_ib_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep)
else
return mlx5_ib_set_vport_rep(lag_master, rep, vport_index);
ibdev = ib_alloc_device(mlx5_ib_dev, ib_dev);
ibdev = ib_alloc_device_with_net(mlx5_ib_dev, ib_dev,
mlx5_core_net(lag_master));
if (!ibdev)
return -ENOMEM;

13
drivers/infiniband/hw/mlx5/main.c
View file

@ -50,6 +50,7 @@
#include <rdma/ib_ucaps.h>
#include "macsec.h"
#include "data_direct.h"
#include "dmah.h"
#define UVERBS_MODULE_NAME mlx5_ib
#include <rdma/uverbs_named_ioctl.h>
@ -4190,7 +4191,9 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
.modify_port = mlx5_ib_modify_port,
.modify_qp = mlx5_ib_modify_qp,
.modify_srq = mlx5_ib_modify_srq,
.pre_destroy_cq = mlx5_ib_pre_destroy_cq,
.poll_cq = mlx5_ib_poll_cq,
.post_destroy_cq = mlx5_ib_post_destroy_cq,
.post_recv = mlx5_ib_post_recv_nodrain,
.post_send = mlx5_ib_post_send_nodrain,
.post_srq_recv = mlx5_ib_post_srq_recv,
@ -4212,6 +4215,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
INIT_RDMA_OBJ_SIZE(ib_dmah, mlx5_ib_dmah, ibdmah),
INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
@ -4339,6 +4343,9 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
if (mdev->st)
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dmah_ops);
ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
@ -4824,7 +4831,8 @@ static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
!MLX5_CAP_GEN_2(mparent->mdev, multiplane_qp_ud))
return ERR_PTR(-EOPNOTSUPP);
mplane = ib_alloc_device(mlx5_ib_dev, ib_dev);
mplane = ib_alloc_device_with_net(mlx5_ib_dev, ib_dev,
mlx5_core_net(mparent->mdev));
if (!mplane)
return ERR_PTR(-ENOMEM);
@ -4938,7 +4946,8 @@ static int mlx5r_probe(struct auxiliary_device *adev,
num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
MLX5_CAP_GEN(mdev, num_vhca_ports));
dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
dev = ib_alloc_device_with_net(mlx5_ib_dev, ib_dev,
mlx5_core_net(mdev));
if (!dev)
return -ENOMEM;

99
drivers/infiniband/hw/mlx5/mlx5_ib.h
View file

@ -104,19 +104,6 @@ unsigned long __mlx5_umem_find_best_quantized_pgoff(
__mlx5_bit_sz(typ, page_offset_fld), 0, scale, \
page_offset_quantized)
static inline unsigned long
mlx5_umem_dmabuf_find_best_pgsz(struct ib_umem_dmabuf *umem_dmabuf)
{
/*
* mkeys used for dmabuf are fixed at PAGE_SIZE because we must be able
* to hold any sgl after a move operation. Ideally the mkc page size
* could be changed at runtime to be optimal, but right now the driver
* cannot do that.
*/
return ib_umem_find_best_pgsz(&umem_dmabuf->umem, PAGE_SIZE,
umem_dmabuf->umem.iova);
}
enum {
MLX5_IB_MMAP_OFFSET_START = 9,
MLX5_IB_MMAP_OFFSET_END = 255,
@ -320,8 +307,8 @@ struct mlx5_ib_flow_db {
struct mlx5_ib_flow_prio rdma_tx[MLX5_IB_NUM_FLOW_FT];
struct mlx5_ib_flow_prio opfcs[MLX5_IB_OPCOUNTER_MAX];
struct mlx5_flow_table *lag_demux_ft;
struct mlx5_ib_flow_prio *rdma_transport_rx;
struct mlx5_ib_flow_prio *rdma_transport_tx;
struct mlx5_ib_flow_prio *rdma_transport_rx[MLX5_RDMA_TRANSPORT_BYPASS_PRIO];
struct mlx5_ib_flow_prio *rdma_transport_tx[MLX5_RDMA_TRANSPORT_BYPASS_PRIO];
/* Protect flow steering bypass flow tables
* when add/del flow rules.
* only single add/removal of flow steering rule could be done
@ -352,6 +339,7 @@ struct mlx5_ib_flow_db {
#define MLX5_IB_UPD_XLT_ACCESS BIT(5)
#define MLX5_IB_UPD_XLT_INDIRECT BIT(6)
#define MLX5_IB_UPD_XLT_DOWNGRADE BIT(7)
#define MLX5_IB_UPD_XLT_KEEP_PGSZ BIT(8)
/* Private QP creation flags to be passed in ib_qp_init_attr.create_flags.
*
@ -650,8 +638,13 @@ enum mlx5_mkey_type {
MLX5_MKEY_IMPLICIT_CHILD,
};
/* Used for non-existent ph value */
#define MLX5_IB_NO_PH 0xff
struct mlx5r_cache_rb_key {
u8 ats:1;
u8 ph;
u16 st_index;
unsigned int access_mode;
unsigned int access_flags;
unsigned int ndescs;
@ -739,6 +732,8 @@ struct mlx5_ib_mr {
struct mlx5_ib_mr *dd_crossed_mr;
struct list_head dd_node;
u8 revoked :1;
/* Indicates previous dmabuf page fault occurred */
u8 dmabuf_faulted:1;
struct mlx5_ib_mkey null_mmkey;
};
};
@ -899,13 +894,14 @@ void mlx5_ib_fs_remove_op_fc(struct mlx5_ib_dev *dev,
struct mlx5_ib_op_fc *opfc,
enum mlx5_ib_optional_counter_type type);
int mlx5r_fs_bind_op_fc(struct ib_qp *qp, struct rdma_counter *counter,
u32 port);
int mlx5r_fs_bind_op_fc(struct ib_qp *qp,
struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX],
struct xarray *qpn_opfc_xa, u32 port);
void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct rdma_counter *counter);
void mlx5r_fs_unbind_op_fc(struct ib_qp *qp, struct xarray *qpn_opfc_xa);
void mlx5r_fs_destroy_fcs(struct mlx5_ib_dev *dev,
struct rdma_counter *counter);
struct mlx5_fc *fc_arr[MLX5_IB_OPCOUNTER_MAX]);
struct mlx5_ib_multiport_info;
@ -1372,16 +1368,20 @@ int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct uverbs_attr_bundle *attrs);
int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mlx5_ib_pre_destroy_cq(struct ib_cq *cq);
void mlx5_ib_post_destroy_cq(struct ib_cq *cq);
int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
int mlx5_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc);
struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata);
struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 start,
u64 length, u64 virt_addr,
int fd, int access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs);
int mlx5_ib_advise_mr(struct ib_pd *pd,
enum ib_uverbs_advise_mr_advice advice,
@ -1748,20 +1748,71 @@ static inline u32 smi_to_native_portnum(struct mlx5_ib_dev *dev, u32 port)
return (port - 1) / dev->num_ports + 1;
}
static inline unsigned int get_max_log_entity_size_cap(struct mlx5_ib_dev *dev,
int access_mode)
{
int max_log_size = 0;
if (access_mode == MLX5_MKC_ACCESS_MODE_MTT)
max_log_size =
MLX5_CAP_GEN_2(dev->mdev, max_mkey_log_entity_size_mtt);
else if (access_mode == MLX5_MKC_ACCESS_MODE_KSM)
max_log_size = MLX5_CAP_GEN_2(
dev->mdev, max_mkey_log_entity_size_fixed_buffer);
if (!max_log_size ||
(max_log_size > 31 &&
!MLX5_CAP_GEN_2(dev->mdev, umr_log_entity_size_5)))
max_log_size = 31;
return max_log_size;
}
static inline unsigned int get_min_log_entity_size_cap(struct mlx5_ib_dev *dev,
int access_mode)
{
int min_log_size = 0;
if (access_mode == MLX5_MKC_ACCESS_MODE_KSM &&
MLX5_CAP_GEN_2(dev->mdev,
min_mkey_log_entity_size_fixed_buffer_valid))
min_log_size = MLX5_CAP_GEN_2(
dev->mdev, min_mkey_log_entity_size_fixed_buffer);
else
min_log_size =
MLX5_CAP_GEN_2(dev->mdev, log_min_mkey_entity_size);
min_log_size = max(min_log_size, MLX5_ADAPTER_PAGE_SHIFT);
return min_log_size;
}
/*
* For mkc users, instead of a page_offset the command has a start_iova which
* specifies both the page_offset and the on-the-wire IOVA
*/
static __always_inline unsigned long
mlx5_umem_mkc_find_best_pgsz(struct mlx5_ib_dev *dev, struct ib_umem *umem,
u64 iova)
u64 iova, int access_mode)
{
int page_size_bits =
MLX5_CAP_GEN_2(dev->mdev, umr_log_entity_size_5) ? 6 : 5;
unsigned long bitmap =
__mlx5_log_page_size_to_bitmap(page_size_bits, 0);
unsigned int max_log_entity_size_cap, min_log_entity_size_cap;
unsigned long bitmap;
max_log_entity_size_cap = get_max_log_entity_size_cap(dev, access_mode);
min_log_entity_size_cap = get_min_log_entity_size_cap(dev, access_mode);
bitmap = GENMASK_ULL(max_log_entity_size_cap, min_log_entity_size_cap);
return ib_umem_find_best_pgsz(umem, bitmap, iova);
}
static inline unsigned long
mlx5_umem_dmabuf_find_best_pgsz(struct ib_umem_dmabuf *umem_dmabuf,
int access_mode)
{
return mlx5_umem_mkc_find_best_pgsz(to_mdev(umem_dmabuf->umem.ibdev),
&umem_dmabuf->umem,
umem_dmabuf->umem.iova,
access_mode);
}
#endif /* MLX5_IB_H */

116
drivers/infiniband/hw/mlx5/mr.c
View file

@ -44,6 +44,7 @@
#include "mlx5_ib.h"
#include "umr.h"
#include "data_direct.h"
#include "dmah.h"
enum {
MAX_PENDING_REG_MR = 8,
@ -57,7 +58,7 @@ create_mkey_callback(int status, struct mlx5_async_work *context);
static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
u64 iova, int access_flags,
unsigned long page_size, bool populate,
int access_mode);
int access_mode, u16 st_index, u8 ph);
static int __mlx5_ib_dereg_mr(struct ib_mr *ibmr);
static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr,
@ -256,6 +257,14 @@ static void set_cache_mkc(struct mlx5_cache_ent *ent, void *mkc)
get_mkc_octo_size(ent->rb_key.access_mode,
ent->rb_key.ndescs));
MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
if (ent->rb_key.ph != MLX5_IB_NO_PH) {
MLX5_SET(mkc, mkc, pcie_tph_en, 1);
MLX5_SET(mkc, mkc, pcie_tph_ph, ent->rb_key.ph);
if (ent->rb_key.st_index != MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)
MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index,
ent->rb_key.st_index);
}
}
/* Asynchronously schedule new MRs to be populated in the cache. */
@ -641,6 +650,14 @@ static int cache_ent_key_cmp(struct mlx5r_cache_rb_key key1,
if (res)
return res;
res = key1.st_index - key2.st_index;
if (res)
return res;
res = key1.ph - key2.ph;
if (res)
return res;
/*
* keep ndescs the last in the compare table since the find function
* searches for an exact match on all properties and only closest
@ -712,6 +729,8 @@ mkey_cache_ent_from_rb_key(struct mlx5_ib_dev *dev,
smallest->rb_key.access_mode == rb_key.access_mode &&
smallest->rb_key.access_flags == rb_key.access_flags &&
smallest->rb_key.ats == rb_key.ats &&
smallest->rb_key.st_index == rb_key.st_index &&
smallest->rb_key.ph == rb_key.ph &&
smallest->rb_key.ndescs <= ndescs_limit) ?
smallest :
NULL;
@ -786,7 +805,8 @@ struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev,
struct mlx5r_cache_rb_key rb_key = {
.ndescs = ndescs,
.access_mode = access_mode,
.access_flags = get_unchangeable_access_flags(dev, access_flags)
.access_flags = get_unchangeable_access_flags(dev, access_flags),
.ph = MLX5_IB_NO_PH,
};
struct mlx5_cache_ent *ent = mkey_cache_ent_from_rb_key(dev, rb_key);
@ -943,6 +963,7 @@ int mlx5_mkey_cache_init(struct mlx5_ib_dev *dev)
struct rb_root *root = &dev->cache.rb_root;
struct mlx5r_cache_rb_key rb_key = {
.access_mode = MLX5_MKC_ACCESS_MODE_MTT,
.ph = MLX5_IB_NO_PH,
};
struct mlx5_cache_ent *ent;
struct rb_node *node;
@ -1119,7 +1140,8 @@ static unsigned int mlx5_umem_dmabuf_default_pgsz(struct ib_umem *umem,
static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
struct ib_umem *umem, u64 iova,
int access_flags, int access_mode)
int access_flags, int access_mode,
u16 st_index, u8 ph)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5r_cache_rb_key rb_key = {};
@ -1130,7 +1152,8 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
if (umem->is_dmabuf)
page_size = mlx5_umem_dmabuf_default_pgsz(umem, iova);
else
page_size = mlx5_umem_mkc_find_best_pgsz(dev, umem, iova);
page_size = mlx5_umem_mkc_find_best_pgsz(dev, umem, iova,
access_mode);
if (WARN_ON(!page_size))
return ERR_PTR(-EINVAL);
@ -1138,6 +1161,8 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
rb_key.ndescs = ib_umem_num_dma_blocks(umem, page_size);
rb_key.ats = mlx5_umem_needs_ats(dev, umem, access_flags);
rb_key.access_flags = get_unchangeable_access_flags(dev, access_flags);
rb_key.st_index = st_index;
rb_key.ph = ph;
ent = mkey_cache_ent_from_rb_key(dev, rb_key);
/*
* If the MR can't come from the cache then synchronously create an uncached
@ -1145,7 +1170,8 @@ static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd,
*/
if (!ent) {
mutex_lock(&dev->slow_path_mutex);
mr = reg_create(pd, umem, iova, access_flags, page_size, false, access_mode);
mr = reg_create(pd, umem, iova, access_flags, page_size, false, access_mode,
st_index, ph);
mutex_unlock(&dev->slow_path_mutex);
if (IS_ERR(mr))
return mr;
@ -1230,7 +1256,7 @@ err_1:
static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
u64 iova, int access_flags,
unsigned long page_size, bool populate,
int access_mode)
int access_mode, u16 st_index, u8 ph)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_mr *mr;
@ -1240,7 +1266,8 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
u32 *in;
int err;
bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)) &&
(access_mode == MLX5_MKC_ACCESS_MODE_MTT);
(access_mode == MLX5_MKC_ACCESS_MODE_MTT) &&
(ph == MLX5_IB_NO_PH);
bool ksm_mode = (access_mode == MLX5_MKC_ACCESS_MODE_KSM);
if (!page_size)
@ -1304,6 +1331,13 @@ static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem,
get_octo_len(iova, umem->length, mr->page_shift));
}
if (ph != MLX5_IB_NO_PH) {
MLX5_SET(mkc, mkc, pcie_tph_en, 1);
MLX5_SET(mkc, mkc, pcie_tph_ph, ph);
if (st_index != MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX)
MLX5_SET(mkc, mkc, pcie_tph_steering_tag_index, st_index);
}
err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
if (err) {
mlx5_ib_warn(dev, "create mkey failed\n");
@ -1423,24 +1457,37 @@ struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm,
}
static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem,
u64 iova, int access_flags)
u64 iova, int access_flags,
struct ib_dmah *dmah)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_mr *mr = NULL;
bool xlt_with_umr;
u16 st_index = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX;
u8 ph = MLX5_IB_NO_PH;
int err;
if (dmah) {
struct mlx5_ib_dmah *mdmah = to_mdmah(dmah);
ph = dmah->ph;
if (dmah->valid_fields & BIT(IB_DMAH_CPU_ID_EXISTS))
st_index = mdmah->st_index;
}
xlt_with_umr = mlx5r_umr_can_load_pas(dev, umem->length);
if (xlt_with_umr) {
mr = alloc_cacheable_mr(pd, umem, iova, access_flags,
MLX5_MKC_ACCESS_MODE_MTT);
MLX5_MKC_ACCESS_MODE_MTT,
st_index, ph);
} else {
unsigned long page_size =
mlx5_umem_mkc_find_best_pgsz(dev, umem, iova);
unsigned long page_size = mlx5_umem_mkc_find_best_pgsz(
dev, umem, iova, MLX5_MKC_ACCESS_MODE_MTT);
mutex_lock(&dev->slow_path_mutex);
mr = reg_create(pd, umem, iova, access_flags, page_size,
true, MLX5_MKC_ACCESS_MODE_MTT);
true, MLX5_MKC_ACCESS_MODE_MTT,
st_index, ph);
mutex_unlock(&dev->slow_path_mutex);
}
if (IS_ERR(mr)) {
@ -1504,7 +1551,9 @@ static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length,
return ERR_CAST(odp);
mr = alloc_cacheable_mr(pd, &odp->umem, iova, access_flags,
MLX5_MKC_ACCESS_MODE_MTT);
MLX5_MKC_ACCESS_MODE_MTT,
MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX,
MLX5_IB_NO_PH);
if (IS_ERR(mr)) {
ib_umem_release(&odp->umem);
return ERR_CAST(mr);
@ -1528,13 +1577,15 @@ err_dereg_mr:
struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 iova, int access_flags,
struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct ib_umem *umem;
int err;
if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) ||
((access_flags & IB_ACCESS_ON_DEMAND) && dmah))
return ERR_PTR(-EOPNOTSUPP);
mlx5_ib_dbg(dev, "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n",
@ -1550,7 +1601,7 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
umem = ib_umem_get(&dev->ib_dev, start, length, access_flags);
if (IS_ERR(umem))
return ERR_CAST(umem);
return create_real_mr(pd, umem, iova, access_flags);
return create_real_mr(pd, umem, iova, access_flags, dmah);
}
static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach)
@ -1575,12 +1626,15 @@ static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = {
static struct ib_mr *
reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
u64 offset, u64 length, u64 virt_addr,
int fd, int access_flags, int access_mode)
int fd, int access_flags, int access_mode,
struct ib_dmah *dmah)
{
bool pinned_mode = (access_mode == MLX5_MKC_ACCESS_MODE_KSM);
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_mr *mr = NULL;
struct ib_umem_dmabuf *umem_dmabuf;
u16 st_index = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX;
u8 ph = MLX5_IB_NO_PH;
int err;
err = mlx5r_umr_resource_init(dev);
@ -1603,8 +1657,17 @@ reg_user_mr_dmabuf(struct ib_pd *pd, struct device *dma_device,
return ERR_CAST(umem_dmabuf);
}
if (dmah) {
struct mlx5_ib_dmah *mdmah = to_mdmah(dmah);
ph = dmah->ph;
if (dmah->valid_fields & BIT(IB_DMAH_CPU_ID_EXISTS))
st_index = mdmah->st_index;
}
mr = alloc_cacheable_mr(pd, &umem_dmabuf->umem, virt_addr,
access_flags, access_mode);
access_flags, access_mode,
st_index, ph);
if (IS_ERR(mr)) {
ib_umem_release(&umem_dmabuf->umem);
return ERR_CAST(mr);
@ -1661,7 +1724,8 @@ reg_user_mr_dmabuf_by_data_direct(struct ib_pd *pd, u64 offset,
access_flags &= ~IB_ACCESS_RELAXED_ORDERING;
crossed_mr = reg_user_mr_dmabuf(pd, &data_direct_dev->pdev->dev,
offset, length, virt_addr, fd,
access_flags, MLX5_MKC_ACCESS_MODE_KSM);
access_flags, MLX5_MKC_ACCESS_MODE_KSM,
NULL);
if (IS_ERR(crossed_mr)) {
ret = PTR_ERR(crossed_mr);
goto end;
@ -1688,6 +1752,7 @@ end:
struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset,
u64 length, u64 virt_addr,
int fd, int access_flags,
struct ib_dmah *dmah,
struct uverbs_attr_bundle *attrs)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
@ -1720,7 +1785,8 @@ struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset,
return reg_user_mr_dmabuf(pd, pd->device->dma_device,
offset, length, virt_addr,
fd, access_flags, MLX5_MKC_ACCESS_MODE_MTT);
fd, access_flags, MLX5_MKC_ACCESS_MODE_MTT,
dmah);
}
/*
@ -1754,7 +1820,8 @@ static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr,
if (!mlx5r_umr_can_load_pas(dev, new_umem->length))
return false;
*page_size = mlx5_umem_mkc_find_best_pgsz(dev, new_umem, iova);
*page_size = mlx5_umem_mkc_find_best_pgsz(
dev, new_umem, iova, mr->mmkey.cache_ent->rb_key.access_mode);
if (WARN_ON(!*page_size))
return false;
return (mr->mmkey.cache_ent->rb_key.ndescs) >=
@ -1817,7 +1884,8 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
struct mlx5_ib_mr *mr = to_mmr(ib_mr);
int err;
if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || mr->data_direct)
if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || mr->data_direct ||
mr->mmkey.rb_key.ph != MLX5_IB_NO_PH)
return ERR_PTR(-EOPNOTSUPP);
mlx5_ib_dbg(
@ -1861,7 +1929,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages);
return create_real_mr(new_pd, umem, mr->ibmr.iova,
new_access_flags);
new_access_flags, NULL);
}
/*
@ -1892,7 +1960,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
}
return NULL;
}
return create_real_mr(new_pd, new_umem, iova, new_access_flags);
return create_real_mr(new_pd, new_umem, iova, new_access_flags, NULL);
}
/*
@ -1901,7 +1969,7 @@ struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
*/
recreate:
return mlx5_ib_reg_user_mr(new_pd, start, length, iova,
new_access_flags, udata);
new_access_flags, NULL, udata);
}
static int

32
drivers/infiniband/hw/mlx5/odp.c
View file

@ -836,9 +836,13 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt,
u32 *bytes_mapped, u32 flags)
{
struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem);
int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM :
MLX5_MKC_ACCESS_MODE_MTT;
unsigned int old_page_shift = mr->page_shift;
unsigned int page_shift;
unsigned long page_size;
u32 xlt_flags = 0;
int err;
unsigned long page_size;
if (flags & MLX5_PF_FLAGS_ENABLE)
xlt_flags |= MLX5_IB_UPD_XLT_ENABLE;
@ -850,20 +854,33 @@ static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt,
return err;
}
page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf);
page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode);
if (!page_size) {
ib_umem_dmabuf_unmap_pages(umem_dmabuf);
err = -EINVAL;
} else {
if (mr->data_direct)
err = mlx5r_umr_update_data_direct_ksm_pas(mr, xlt_flags);
else
err = mlx5r_umr_update_mr_pas(mr, xlt_flags);
page_shift = order_base_2(page_size);
if (page_shift != mr->page_shift && mr->dmabuf_faulted) {
err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags,
page_shift);
} else {
mr->page_shift = page_shift;
if (mr->data_direct)
err = mlx5r_umr_update_data_direct_ksm_pas(
mr, xlt_flags);
else
err = mlx5r_umr_update_mr_pas(mr,
xlt_flags);
}
}
dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv);
if (err)
if (err) {
mr->page_shift = old_page_shift;
return err;
}
mr->dmabuf_faulted = 1;
if (bytes_mapped)
*bytes_mapped += bcnt;
@ -1866,6 +1883,7 @@ int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev)
struct mlx5r_cache_rb_key rb_key = {
.access_mode = MLX5_MKC_ACCESS_MODE_KSM,
.ndescs = mlx5_imr_ksm_entries,
.ph = MLX5_IB_NO_PH,
};
struct mlx5_cache_ent *ent;

303
drivers/infiniband/hw/mlx5/umr.c
View file

@ -659,6 +659,9 @@ static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev,
wqe->ctrl_seg.mkey_mask |= get_umr_update_translation_mask(dev);
if (!mr->ibmr.length)
MLX5_SET(mkc, &wqe->mkey_seg, length64, 1);
if (flags & MLX5_IB_UPD_XLT_KEEP_PGSZ)
wqe->ctrl_seg.mkey_mask &=
cpu_to_be64(~MLX5_MKEY_MASK_PAGE_SIZE);
}
wqe->ctrl_seg.xlt_octowords =
@ -666,46 +669,78 @@ static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev,
wqe->data_seg.byte_count = cpu_to_be32(sg->length);
}
static void
_mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, struct mlx5r_umr_wqe *wqe,
struct ib_sge *sg, unsigned int flags,
unsigned int page_shift, bool dd)
{
struct mlx5_ib_dev *dev = mr_to_mdev(mr);
mlx5r_umr_set_update_xlt_ctrl_seg(&wqe->ctrl_seg, flags, sg);
mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, page_shift);
if (dd) /* Use the data direct internal kernel PD */
MLX5_SET(mkc, &wqe->mkey_seg, pd, dev->ddr.pdn);
mlx5r_umr_set_update_xlt_data_seg(&wqe->data_seg, sg);
}
static int
_mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd)
_mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd,
size_t start_block, size_t nblocks)
{
size_t ent_size = dd ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt);
struct mlx5_ib_dev *dev = mr_to_mdev(mr);
struct device *ddev = &dev->mdev->pdev->dev;
struct mlx5r_umr_wqe wqe = {};
size_t processed_blocks = 0;
struct ib_block_iter biter;
size_t cur_block_idx = 0;
struct mlx5_ksm *cur_ksm;
struct mlx5_mtt *cur_mtt;
size_t orig_sg_length;
size_t total_blocks;
size_t final_size;
void *curr_entry;
struct ib_sge sg;
void *entry;
u64 offset = 0;
u64 offset;
int err = 0;
entry = mlx5r_umr_create_xlt(dev, &sg,
ib_umem_num_dma_blocks(mr->umem, 1 << mr->page_shift),
ent_size, flags);
total_blocks = ib_umem_num_dma_blocks(mr->umem, 1UL << mr->page_shift);
if (start_block > total_blocks)
return -EINVAL;
/* nblocks 0 means update all blocks starting from start_block */
if (nblocks)
total_blocks = nblocks;
entry = mlx5r_umr_create_xlt(dev, &sg, total_blocks, ent_size, flags);
if (!entry)
return -ENOMEM;
orig_sg_length = sg.length;
mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg);
mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr,
mr->page_shift);
if (dd) {
/* Use the data direct internal kernel PD */
MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn);
cur_ksm = entry;
} else {
cur_mtt = entry;
}
mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg);
_mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, mr->page_shift, dd);
/* Set initial translation offset to start_block */
offset = (u64)start_block * ent_size;
mlx5r_umr_update_offset(&wqe.ctrl_seg, offset);
if (dd)
cur_ksm = entry;
else
cur_mtt = entry;
curr_entry = entry;
rdma_umem_for_each_dma_block(mr->umem, &biter, BIT(mr->page_shift)) {
if (cur_block_idx < start_block) {
cur_block_idx++;
continue;
}
if (nblocks && processed_blocks >= nblocks)
break;
if (curr_entry == entry + sg.length) {
dma_sync_single_for_device(ddev, sg.addr, sg.length,
DMA_TO_DEVICE);
@ -727,6 +762,11 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd)
if (dd) {
cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter));
cur_ksm->key = cpu_to_be32(dev->ddr.mkey);
if (mr->umem->is_dmabuf &&
(flags & MLX5_IB_UPD_XLT_ZAP)) {
cur_ksm->va = 0;
cur_ksm->key = 0;
}
cur_ksm++;
curr_entry = cur_ksm;
} else {
@ -738,6 +778,8 @@ _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd)
cur_mtt++;
curr_entry = cur_mtt;
}
processed_blocks++;
}
final_size = curr_entry - entry;
@ -754,13 +796,32 @@ err:
return err;
}
int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int flags)
int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr,
unsigned int flags,
size_t start_block,
size_t nblocks)
{
/* No invalidation flow is expected */
if (WARN_ON(!mr->umem->is_dmabuf) || (flags & MLX5_IB_UPD_XLT_ZAP))
if (WARN_ON(!mr->umem->is_dmabuf) || ((flags & MLX5_IB_UPD_XLT_ZAP) &&
!(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ)))
return -EINVAL;
return _mlx5r_umr_update_mr_pas(mr, flags, true);
return _mlx5r_umr_update_mr_pas(mr, flags, true, start_block, nblocks);
}
int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr,
unsigned int flags)
{
return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, 0, 0);
}
int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags,
size_t start_block, size_t nblocks)
{
if (WARN_ON(mr->umem->is_odp))
return -EINVAL;
return _mlx5r_umr_update_mr_pas(mr, flags, false, start_block, nblocks);
}
/*
@ -770,10 +831,7 @@ int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int fla
*/
int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)
{
if (WARN_ON(mr->umem->is_odp))
return -EINVAL;
return _mlx5r_umr_update_mr_pas(mr, flags, false);
return mlx5r_umr_update_mr_pas_range(mr, flags, 0, 0);
}
static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
@ -866,3 +924,202 @@ int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
mlx5r_umr_unmap_free_xlt(dev, xlt, &sg);
return err;
}
/*
* Update only the page-size (log_page_size) field of an existing memory key
* using UMR. This is useful when the MR's physical layout stays the same
* but the optimal page shift has changed (e.g. dmabuf after pages are
* pinned and the HW can switch from 4K to huge-page alignment).
*/
int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr,
unsigned int page_shift,
bool dd)
{
struct mlx5_ib_dev *dev = mr_to_mdev(mr);
struct mlx5r_umr_wqe wqe = {};
int err;
/* Build UMR wqe: we touch only PAGE_SIZE, so use the dedicated mask */
wqe.ctrl_seg.mkey_mask = get_umr_update_translation_mask(dev);
/* MR must be free while page size is modified */
wqe.ctrl_seg.flags = MLX5_UMR_CHECK_FREE | MLX5_UMR_INLINE;
/* Fill mkey segment with the new page size, keep the rest unchanged */
MLX5_SET(mkc, &wqe.mkey_seg, log_page_size, page_shift);
if (dd)
MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn);
else
MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn);
MLX5_SET64(mkc, &wqe.mkey_seg, start_addr, mr->ibmr.iova);
MLX5_SET64(mkc, &wqe.mkey_seg, len, mr->ibmr.length);
MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff);
MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0,
mlx5_mkey_variant(mr->mmkey.key));
err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false);
if (!err)
mr->page_shift = page_shift;
return err;
}
static inline int
_mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags,
size_t start_block, size_t nblocks, bool dd)
{
if (dd)
return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags,
start_block,
nblocks);
else
return mlx5r_umr_update_mr_pas_range(mr, flags, start_block,
nblocks);
}
/**
* This function makes an mkey non-present by zapping the translation entries of
* the mkey by zapping (zeroing out) the first N entries, where N is determined
* by the largest page size supported by the device and the MR length.
* It then updates the mkey's page size to the largest possible value, ensuring
* the MR is completely non-present and safe for further updates.
* It is useful to update the page size of a dmabuf MR on a page fault.
*
* Return: On success, returns the number of entries that were zapped.
* On error, returns a negative error code.
*/
static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr,
unsigned int flags,
unsigned int page_shift,
size_t *nblocks,
bool dd)
{
unsigned int old_page_shift = mr->page_shift;
struct mlx5_ib_dev *dev = mr_to_mdev(mr);
unsigned int max_page_shift;
size_t page_shift_nblocks;
unsigned int max_log_size;
int access_mode;
int err;
access_mode = dd ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT;
flags |= MLX5_IB_UPD_XLT_KEEP_PGSZ | MLX5_IB_UPD_XLT_ZAP |
MLX5_IB_UPD_XLT_ATOMIC;
max_log_size = get_max_log_entity_size_cap(dev, access_mode);
max_page_shift = order_base_2(mr->ibmr.length);
max_page_shift = min(max(max_page_shift, page_shift), max_log_size);
/* Count blocks in units of max_page_shift, we will zap exactly this
* many to make the whole MR non-present.
* Block size must be aligned to MLX5_UMR_FLEX_ALIGNMENT since it may
* be used as offset into the XLT later on.
*/
*nblocks = ib_umem_num_dma_blocks(mr->umem, 1UL << max_page_shift);
if (dd)
*nblocks = ALIGN(*nblocks, MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT);
else
*nblocks = ALIGN(*nblocks, MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT);
page_shift_nblocks = ib_umem_num_dma_blocks(mr->umem,
1UL << page_shift);
/* If the number of blocks at max possible page shift is greater than
* the number of blocks at the new page size, we should just go over the
* whole mkey entries.
*/
if (*nblocks >= page_shift_nblocks)
*nblocks = 0;
/* Make the first nblocks entries non-present without changing
* page size yet.
*/
if (*nblocks)
mr->page_shift = max_page_shift;
err = _mlx5r_dmabuf_umr_update_pas(mr, flags, 0, *nblocks, dd);
if (err) {
mr->page_shift = old_page_shift;
return err;
}
/* Change page size to the max page size now that the MR is completely
* non-present.
*/
if (*nblocks) {
err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift, dd);
if (err) {
mr->page_shift = old_page_shift;
return err;
}
}
return 0;
}
/**
* mlx5r_umr_dmabuf_update_pgsz - Safely update DMABUF MR page size and its
* entries accordingly
* @mr: The memory region to update
* @xlt_flags: Translation table update flags
* @page_shift: The new (optimized) page shift to use
*
* This function updates the page size and mkey translation entries for a DMABUF
* MR in a safe, multi-step process to avoid exposing partially updated mappings
* The update is performed in 5 steps:
* 1. Make the first X entries non-present, while X is calculated to be
* minimal according to a large page shift that can be used to cover the
* MR length.
* 2. Update the page size to the large supported page size
* 3. Load the remaining N-X entries according to the (optimized) page_shift
* 4. Update the page size according to the (optimized) page_shift
* 5. Load the first X entries with the correct translations
*
* This ensures that at no point is the MR accessible with a partially updated
* translation table, maintaining correctness and preventing access to stale or
* inconsistent mappings.
*
* Returns 0 on success or a negative error code on failure.
*/
int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags,
unsigned int page_shift)
{
unsigned int old_page_shift = mr->page_shift;
size_t zapped_blocks;
size_t total_blocks;
int err;
err = _mlx5r_umr_zap_mkey(mr, xlt_flags, page_shift, &zapped_blocks,
mr->data_direct);
if (err)
return err;
/* _mlx5r_umr_zap_mkey already enables the mkey */
xlt_flags &= ~MLX5_IB_UPD_XLT_ENABLE;
mr->page_shift = page_shift;
total_blocks = ib_umem_num_dma_blocks(mr->umem, 1UL << mr->page_shift);
if (zapped_blocks && zapped_blocks < total_blocks) {
/* Update PAS according to the new page size but don't update
* the page size in the mkey yet.
*/
err = _mlx5r_dmabuf_umr_update_pas(
mr,
xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ,
zapped_blocks,
total_blocks - zapped_blocks,
mr->data_direct);
if (err)
goto err;
}
err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift,
mr->data_direct);
if (err)
goto err;
err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks,
mr->data_direct);
if (err)
goto err;
return 0;
err:
mr->page_shift = old_page_shift;
return err;
}

13
drivers/infiniband/hw/mlx5/umr.h
View file

@ -94,9 +94,20 @@ struct mlx5r_umr_wqe {
int mlx5r_umr_revoke_mr(struct mlx5_ib_mr *mr);
int mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd,
int access_flags);
int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags);
int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr,
unsigned int flags,
size_t start_block,
size_t nblocks);
int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, unsigned int flags);
int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags,
size_t start_block, size_t nblocks);
int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags);
int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
int page_shift, int flags);
int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr,
unsigned int page_shift,
bool dd);
int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags,
unsigned int page_shift);
#endif /* _MLX5_IB_UMR_H */

6
drivers/infiniband/hw/mthca/mthca_provider.c
View file

@ -825,7 +825,8 @@ static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc)
}
static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *udata)
u64 virt, int acc, struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct mthca_dev *dev = to_mdev(pd->device);
struct ib_block_iter biter;
@ -838,6 +839,9 @@ static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
int err = 0;
int write_mtt_size;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
if (udata->inlen < sizeof ucmd) {
if (!context->reg_mr_warned) {
mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n",

6
drivers/infiniband/hw/ocrdma/ocrdma_verbs.c
View file

@ -847,13 +847,17 @@ static void build_user_pbes(struct ocrdma_dev *dev, struct ocrdma_mr *mr)
}
struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
u64 usr_addr, int acc, struct ib_udata *udata)
u64 usr_addr, int acc, struct ib_dmah *dmah,
struct ib_udata *udata)
{
int status = -ENOMEM;
struct ocrdma_dev *dev = get_ocrdma_dev(ibpd->device);
struct ocrdma_mr *mr;
struct ocrdma_pd *pd;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
pd = get_ocrdma_pd(ibpd);
if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))

3
drivers/infiniband/hw/ocrdma/ocrdma_verbs.h
View file

@ -98,7 +98,8 @@ int ocrdma_post_srq_recv(struct ib_srq *, const struct ib_recv_wr *,
int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
struct ib_mr *ocrdma_get_dma_mr(struct ib_pd *, int acc);
struct ib_mr *ocrdma_reg_user_mr(struct ib_pd *, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *);
u64 virt, int acc, struct ib_dmah *dmah,
struct ib_udata *);
struct ib_mr *ocrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg);
int ocrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,

6
drivers/infiniband/hw/qedr/verbs.c
View file

@ -2953,13 +2953,17 @@ done:
}
struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
u64 usr_addr, int acc, struct ib_udata *udata)
u64 usr_addr, int acc, struct ib_dmah *dmah,
struct ib_udata *udata)
{
struct qedr_dev *dev = get_qedr_dev(ibpd->device);
struct qedr_mr *mr;
struct qedr_pd *pd;
int rc = -ENOMEM;
if (dmah)
return ERR_PTR(-EOPNOTSUPP);
pd = get_qedr_pd(ibpd);
DP_DEBUG(dev, QEDR_MSG_MR,
"qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",

3
drivers/infiniband/hw/qedr/verbs.h
View file

@ -79,7 +79,8 @@ int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
struct ib_mr *qedr_get_dma_mr(struct ib_pd *, int acc);
struct ib_mr *qedr_reg_user_mr(struct ib_pd *, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *);
u64 virt, int acc, struct ib_dmah *dmah,
struct ib_udata *);
int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);

17
drivers/infiniband/hw/qib/Kconfig
View file

@ -1,17 +0,0 @@
# SPDX-License-Identifier: GPL-2.0-only
config INFINIBAND_QIB
tristate "Intel PCIe HCA support"
depends on 64BIT && INFINIBAND_RDMAVT
depends on PCI
help
This is a low-level driver for Intel PCIe QLE InfiniBand host
channel adapters. This driver does not support the Intel
HyperTransport card (model QHT7140).
config INFINIBAND_QIB_DCA
bool "QIB DCA support"
depends on INFINIBAND_QIB && DCA && SMP && !(INFINIBAND_QIB=y && DCA=m)
default y
help
Setting this enables DCA support on some Intel chip sets
with the iba7322 HCA.

17
drivers/infiniband/hw/qib/Makefile
View file

@ -1,17 +0,0 @@
# SPDX-License-Identifier: GPL-2.0
obj-$(CONFIG_INFINIBAND_QIB) += ib_qib.o
ib_qib-y := qib_diag.o qib_driver.o qib_eeprom.o \
qib_file_ops.o qib_fs.o qib_init.o qib_intr.o \
qib_mad.o qib_pcie.o qib_pio_copy.o \
qib_qp.o qib_qsfp.o qib_rc.o qib_ruc.o qib_sdma.o \
qib_sysfs.o qib_twsi.o qib_tx.o qib_uc.o qib_ud.o \
qib_user_pages.o qib_user_sdma.o qib_iba7220.o \
qib_sd7220.o qib_iba7322.o qib_verbs.o
# 6120 has no fallback if no MSI interrupts, others can do INTx
ib_qib-$(CONFIG_PCI_MSI) += qib_iba6120.o
ib_qib-$(CONFIG_X86_64) += qib_wc_x86_64.o
ib_qib-$(CONFIG_PPC64) += qib_wc_ppc64.o
ib_qib-$(CONFIG_DEBUG_FS) += qib_debugfs.o

1492
drivers/infiniband/hw/qib/qib.h
View file

File diff suppressed because it is too large Load diff

977
drivers/infiniband/hw/qib/qib_6120_regs.h
View file

@ -1,977 +0,0 @@
/*
* Copyright (c) 2008, 2009, 2010 QLogic Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* This file is mechanically generated from RTL. Any hand-edits will be lost! */
#define QIB_6120_Revision_OFFS 0x0
#define QIB_6120_Revision_R_Simulator_LSB 0x3F
#define QIB_6120_Revision_R_Simulator_RMASK 0x1
#define QIB_6120_Revision_Reserved_LSB 0x28
#define QIB_6120_Revision_Reserved_RMASK 0x7FFFFF
#define QIB_6120_Revision_BoardID_LSB 0x20
#define QIB_6120_Revision_BoardID_RMASK 0xFF
#define QIB_6120_Revision_R_SW_LSB 0x18
#define QIB_6120_Revision_R_SW_RMASK 0xFF
#define QIB_6120_Revision_R_Arch_LSB 0x10
#define QIB_6120_Revision_R_Arch_RMASK 0xFF
#define QIB_6120_Revision_R_ChipRevMajor_LSB 0x8
#define QIB_6120_Revision_R_ChipRevMajor_RMASK 0xFF
#define QIB_6120_Revision_R_ChipRevMinor_LSB 0x0
#define QIB_6120_Revision_R_ChipRevMinor_RMASK 0xFF
#define QIB_6120_Control_OFFS 0x8
#define QIB_6120_Control_TxLatency_LSB 0x4
#define QIB_6120_Control_TxLatency_RMASK 0x1
#define QIB_6120_Control_PCIERetryBufDiagEn_LSB 0x3
#define QIB_6120_Control_PCIERetryBufDiagEn_RMASK 0x1
#define QIB_6120_Control_LinkEn_LSB 0x2
#define QIB_6120_Control_LinkEn_RMASK 0x1
#define QIB_6120_Control_FreezeMode_LSB 0x1
#define QIB_6120_Control_FreezeMode_RMASK 0x1
#define QIB_6120_Control_SyncReset_LSB 0x0
#define QIB_6120_Control_SyncReset_RMASK 0x1
#define QIB_6120_PageAlign_OFFS 0x10
#define QIB_6120_PortCnt_OFFS 0x18
#define QIB_6120_SendRegBase_OFFS 0x30
#define QIB_6120_UserRegBase_OFFS 0x38
#define QIB_6120_CntrRegBase_OFFS 0x40
#define QIB_6120_Scratch_OFFS 0x48
#define QIB_6120_Scratch_TopHalf_LSB 0x20
#define QIB_6120_Scratch_TopHalf_RMASK 0xFFFFFFFF
#define QIB_6120_Scratch_BottomHalf_LSB 0x0
#define QIB_6120_Scratch_BottomHalf_RMASK 0xFFFFFFFF
#define QIB_6120_IntBlocked_OFFS 0x60
#define QIB_6120_IntBlocked_ErrorIntBlocked_LSB 0x1F
#define QIB_6120_IntBlocked_ErrorIntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_PioSetIntBlocked_LSB 0x1E
#define QIB_6120_IntBlocked_PioSetIntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_PioBufAvailIntBlocked_LSB 0x1D
#define QIB_6120_IntBlocked_PioBufAvailIntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_assertGPIOIntBlocked_LSB 0x1C
#define QIB_6120_IntBlocked_assertGPIOIntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_Reserved_LSB 0xF
#define QIB_6120_IntBlocked_Reserved_RMASK 0x1FFF
#define QIB_6120_IntBlocked_RcvAvail4IntBlocked_LSB 0x10
#define QIB_6120_IntBlocked_RcvAvail4IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvAvail3IntBlocked_LSB 0xF
#define QIB_6120_IntBlocked_RcvAvail3IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvAvail2IntBlocked_LSB 0xE
#define QIB_6120_IntBlocked_RcvAvail2IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvAvail1IntBlocked_LSB 0xD
#define QIB_6120_IntBlocked_RcvAvail1IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvAvail0IntBlocked_LSB 0xC
#define QIB_6120_IntBlocked_RcvAvail0IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_Reserved1_LSB 0x5
#define QIB_6120_IntBlocked_Reserved1_RMASK 0x7F
#define QIB_6120_IntBlocked_RcvUrg4IntBlocked_LSB 0x4
#define QIB_6120_IntBlocked_RcvUrg4IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvUrg3IntBlocked_LSB 0x3
#define QIB_6120_IntBlocked_RcvUrg3IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvUrg2IntBlocked_LSB 0x2
#define QIB_6120_IntBlocked_RcvUrg2IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvUrg1IntBlocked_LSB 0x1
#define QIB_6120_IntBlocked_RcvUrg1IntBlocked_RMASK 0x1
#define QIB_6120_IntBlocked_RcvUrg0IntBlocked_LSB 0x0
#define QIB_6120_IntBlocked_RcvUrg0IntBlocked_RMASK 0x1
#define QIB_6120_IntMask_OFFS 0x68
#define QIB_6120_IntMask_ErrorIntMask_LSB 0x1F
#define QIB_6120_IntMask_ErrorIntMask_RMASK 0x1
#define QIB_6120_IntMask_PioSetIntMask_LSB 0x1E
#define QIB_6120_IntMask_PioSetIntMask_RMASK 0x1
#define QIB_6120_IntMask_PioBufAvailIntMask_LSB 0x1D
#define QIB_6120_IntMask_PioBufAvailIntMask_RMASK 0x1
#define QIB_6120_IntMask_assertGPIOIntMask_LSB 0x1C
#define QIB_6120_IntMask_assertGPIOIntMask_RMASK 0x1
#define QIB_6120_IntMask_Reserved_LSB 0x11
#define QIB_6120_IntMask_Reserved_RMASK 0x7FF
#define QIB_6120_IntMask_RcvAvail4IntMask_LSB 0x10
#define QIB_6120_IntMask_RcvAvail4IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvAvail3IntMask_LSB 0xF
#define QIB_6120_IntMask_RcvAvail3IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvAvail2IntMask_LSB 0xE
#define QIB_6120_IntMask_RcvAvail2IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvAvail1IntMask_LSB 0xD
#define QIB_6120_IntMask_RcvAvail1IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvAvail0IntMask_LSB 0xC
#define QIB_6120_IntMask_RcvAvail0IntMask_RMASK 0x1
#define QIB_6120_IntMask_Reserved1_LSB 0x5
#define QIB_6120_IntMask_Reserved1_RMASK 0x7F
#define QIB_6120_IntMask_RcvUrg4IntMask_LSB 0x4
#define QIB_6120_IntMask_RcvUrg4IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvUrg3IntMask_LSB 0x3
#define QIB_6120_IntMask_RcvUrg3IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvUrg2IntMask_LSB 0x2
#define QIB_6120_IntMask_RcvUrg2IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvUrg1IntMask_LSB 0x1
#define QIB_6120_IntMask_RcvUrg1IntMask_RMASK 0x1
#define QIB_6120_IntMask_RcvUrg0IntMask_LSB 0x0
#define QIB_6120_IntMask_RcvUrg0IntMask_RMASK 0x1
#define QIB_6120_IntStatus_OFFS 0x70
#define QIB_6120_IntStatus_Error_LSB 0x1F
#define QIB_6120_IntStatus_Error_RMASK 0x1
#define QIB_6120_IntStatus_PioSent_LSB 0x1E
#define QIB_6120_IntStatus_PioSent_RMASK 0x1
#define QIB_6120_IntStatus_PioBufAvail_LSB 0x1D
#define QIB_6120_IntStatus_PioBufAvail_RMASK 0x1
#define QIB_6120_IntStatus_assertGPIO_LSB 0x1C
#define QIB_6120_IntStatus_assertGPIO_RMASK 0x1
#define QIB_6120_IntStatus_Reserved_LSB 0xF
#define QIB_6120_IntStatus_Reserved_RMASK 0x1FFF
#define QIB_6120_IntStatus_RcvAvail4_LSB 0x10
#define QIB_6120_IntStatus_RcvAvail4_RMASK 0x1
#define QIB_6120_IntStatus_RcvAvail3_LSB 0xF
#define QIB_6120_IntStatus_RcvAvail3_RMASK 0x1
#define QIB_6120_IntStatus_RcvAvail2_LSB 0xE
#define QIB_6120_IntStatus_RcvAvail2_RMASK 0x1
#define QIB_6120_IntStatus_RcvAvail1_LSB 0xD
#define QIB_6120_IntStatus_RcvAvail1_RMASK 0x1
#define QIB_6120_IntStatus_RcvAvail0_LSB 0xC
#define QIB_6120_IntStatus_RcvAvail0_RMASK 0x1
#define QIB_6120_IntStatus_Reserved1_LSB 0x5
#define QIB_6120_IntStatus_Reserved1_RMASK 0x7F
#define QIB_6120_IntStatus_RcvUrg4_LSB 0x4
#define QIB_6120_IntStatus_RcvUrg4_RMASK 0x1
#define QIB_6120_IntStatus_RcvUrg3_LSB 0x3
#define QIB_6120_IntStatus_RcvUrg3_RMASK 0x1
#define QIB_6120_IntStatus_RcvUrg2_LSB 0x2
#define QIB_6120_IntStatus_RcvUrg2_RMASK 0x1
#define QIB_6120_IntStatus_RcvUrg1_LSB 0x1
#define QIB_6120_IntStatus_RcvUrg1_RMASK 0x1
#define QIB_6120_IntStatus_RcvUrg0_LSB 0x0
#define QIB_6120_IntStatus_RcvUrg0_RMASK 0x1
#define QIB_6120_IntClear_OFFS 0x78
#define QIB_6120_IntClear_ErrorIntClear_LSB 0x1F
#define QIB_6120_IntClear_ErrorIntClear_RMASK 0x1
#define QIB_6120_IntClear_PioSetIntClear_LSB 0x1E
#define QIB_6120_IntClear_PioSetIntClear_RMASK 0x1
#define QIB_6120_IntClear_PioBufAvailIntClear_LSB 0x1D
#define QIB_6120_IntClear_PioBufAvailIntClear_RMASK 0x1
#define QIB_6120_IntClear_assertGPIOIntClear_LSB 0x1C
#define QIB_6120_IntClear_assertGPIOIntClear_RMASK 0x1
#define QIB_6120_IntClear_Reserved_LSB 0xF
#define QIB_6120_IntClear_Reserved_RMASK 0x1FFF
#define QIB_6120_IntClear_RcvAvail4IntClear_LSB 0x10
#define QIB_6120_IntClear_RcvAvail4IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvAvail3IntClear_LSB 0xF
#define QIB_6120_IntClear_RcvAvail3IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvAvail2IntClear_LSB 0xE
#define QIB_6120_IntClear_RcvAvail2IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvAvail1IntClear_LSB 0xD
#define QIB_6120_IntClear_RcvAvail1IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvAvail0IntClear_LSB 0xC
#define QIB_6120_IntClear_RcvAvail0IntClear_RMASK 0x1
#define QIB_6120_IntClear_Reserved1_LSB 0x5
#define QIB_6120_IntClear_Reserved1_RMASK 0x7F
#define QIB_6120_IntClear_RcvUrg4IntClear_LSB 0x4
#define QIB_6120_IntClear_RcvUrg4IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvUrg3IntClear_LSB 0x3
#define QIB_6120_IntClear_RcvUrg3IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvUrg2IntClear_LSB 0x2
#define QIB_6120_IntClear_RcvUrg2IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvUrg1IntClear_LSB 0x1
#define QIB_6120_IntClear_RcvUrg1IntClear_RMASK 0x1
#define QIB_6120_IntClear_RcvUrg0IntClear_LSB 0x0
#define QIB_6120_IntClear_RcvUrg0IntClear_RMASK 0x1
#define QIB_6120_ErrMask_OFFS 0x80
#define QIB_6120_ErrMask_Reserved_LSB 0x34
#define QIB_6120_ErrMask_Reserved_RMASK 0xFFF
#define QIB_6120_ErrMask_HardwareErrMask_LSB 0x33
#define QIB_6120_ErrMask_HardwareErrMask_RMASK 0x1
#define QIB_6120_ErrMask_ResetNegatedMask_LSB 0x32
#define QIB_6120_ErrMask_ResetNegatedMask_RMASK 0x1
#define QIB_6120_ErrMask_InvalidAddrErrMask_LSB 0x31
#define QIB_6120_ErrMask_InvalidAddrErrMask_RMASK 0x1
#define QIB_6120_ErrMask_IBStatusChangedMask_LSB 0x30
#define QIB_6120_ErrMask_IBStatusChangedMask_RMASK 0x1
#define QIB_6120_ErrMask_Reserved1_LSB 0x26
#define QIB_6120_ErrMask_Reserved1_RMASK 0x3FF
#define QIB_6120_ErrMask_SendUnsupportedVLErrMask_LSB 0x25
#define QIB_6120_ErrMask_SendUnsupportedVLErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendUnexpectedPktNumErrMask_LSB 0x24
#define QIB_6120_ErrMask_SendUnexpectedPktNumErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendPioArmLaunchErrMask_LSB 0x23
#define QIB_6120_ErrMask_SendPioArmLaunchErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendDroppedDataPktErrMask_LSB 0x22
#define QIB_6120_ErrMask_SendDroppedDataPktErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendDroppedSmpPktErrMask_LSB 0x21
#define QIB_6120_ErrMask_SendDroppedSmpPktErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendPktLenErrMask_LSB 0x20
#define QIB_6120_ErrMask_SendPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendUnderRunErrMask_LSB 0x1F
#define QIB_6120_ErrMask_SendUnderRunErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendMaxPktLenErrMask_LSB 0x1E
#define QIB_6120_ErrMask_SendMaxPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_SendMinPktLenErrMask_LSB 0x1D
#define QIB_6120_ErrMask_SendMinPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_Reserved2_LSB 0x12
#define QIB_6120_ErrMask_Reserved2_RMASK 0x7FF
#define QIB_6120_ErrMask_RcvIBLostLinkErrMask_LSB 0x11
#define QIB_6120_ErrMask_RcvIBLostLinkErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvHdrErrMask_LSB 0x10
#define QIB_6120_ErrMask_RcvHdrErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvHdrLenErrMask_LSB 0xF
#define QIB_6120_ErrMask_RcvHdrLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvBadTidErrMask_LSB 0xE
#define QIB_6120_ErrMask_RcvBadTidErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvHdrFullErrMask_LSB 0xD
#define QIB_6120_ErrMask_RcvHdrFullErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvEgrFullErrMask_LSB 0xC
#define QIB_6120_ErrMask_RcvEgrFullErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvBadVersionErrMask_LSB 0xB
#define QIB_6120_ErrMask_RcvBadVersionErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvIBFlowErrMask_LSB 0xA
#define QIB_6120_ErrMask_RcvIBFlowErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvEBPErrMask_LSB 0x9
#define QIB_6120_ErrMask_RcvEBPErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvUnsupportedVLErrMask_LSB 0x8
#define QIB_6120_ErrMask_RcvUnsupportedVLErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvUnexpectedCharErrMask_LSB 0x7
#define QIB_6120_ErrMask_RcvUnexpectedCharErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvShortPktLenErrMask_LSB 0x6
#define QIB_6120_ErrMask_RcvShortPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvLongPktLenErrMask_LSB 0x5
#define QIB_6120_ErrMask_RcvLongPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvMaxPktLenErrMask_LSB 0x4
#define QIB_6120_ErrMask_RcvMaxPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvMinPktLenErrMask_LSB 0x3
#define QIB_6120_ErrMask_RcvMinPktLenErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvICRCErrMask_LSB 0x2
#define QIB_6120_ErrMask_RcvICRCErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvVCRCErrMask_LSB 0x1
#define QIB_6120_ErrMask_RcvVCRCErrMask_RMASK 0x1
#define QIB_6120_ErrMask_RcvFormatErrMask_LSB 0x0
#define QIB_6120_ErrMask_RcvFormatErrMask_RMASK 0x1
#define QIB_6120_ErrStatus_OFFS 0x88
#define QIB_6120_ErrStatus_Reserved_LSB 0x34
#define QIB_6120_ErrStatus_Reserved_RMASK 0xFFF
#define QIB_6120_ErrStatus_HardwareErr_LSB 0x33
#define QIB_6120_ErrStatus_HardwareErr_RMASK 0x1
#define QIB_6120_ErrStatus_ResetNegated_LSB 0x32
#define QIB_6120_ErrStatus_ResetNegated_RMASK 0x1
#define QIB_6120_ErrStatus_InvalidAddrErr_LSB 0x31
#define QIB_6120_ErrStatus_InvalidAddrErr_RMASK 0x1
#define QIB_6120_ErrStatus_IBStatusChanged_LSB 0x30
#define QIB_6120_ErrStatus_IBStatusChanged_RMASK 0x1
#define QIB_6120_ErrStatus_Reserved1_LSB 0x26
#define QIB_6120_ErrStatus_Reserved1_RMASK 0x3FF
#define QIB_6120_ErrStatus_SendUnsupportedVLErr_LSB 0x25
#define QIB_6120_ErrStatus_SendUnsupportedVLErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendUnexpectedPktNumErr_LSB 0x24
#define QIB_6120_ErrStatus_SendUnexpectedPktNumErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendPioArmLaunchErr_LSB 0x23
#define QIB_6120_ErrStatus_SendPioArmLaunchErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendDroppedDataPktErr_LSB 0x22
#define QIB_6120_ErrStatus_SendDroppedDataPktErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendDroppedSmpPktErr_LSB 0x21
#define QIB_6120_ErrStatus_SendDroppedSmpPktErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendPktLenErr_LSB 0x20
#define QIB_6120_ErrStatus_SendPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendUnderRunErr_LSB 0x1F
#define QIB_6120_ErrStatus_SendUnderRunErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendMaxPktLenErr_LSB 0x1E
#define QIB_6120_ErrStatus_SendMaxPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_SendMinPktLenErr_LSB 0x1D
#define QIB_6120_ErrStatus_SendMinPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_Reserved2_LSB 0x12
#define QIB_6120_ErrStatus_Reserved2_RMASK 0x7FF
#define QIB_6120_ErrStatus_RcvIBLostLinkErr_LSB 0x11
#define QIB_6120_ErrStatus_RcvIBLostLinkErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvHdrErr_LSB 0x10
#define QIB_6120_ErrStatus_RcvHdrErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvHdrLenErr_LSB 0xF
#define QIB_6120_ErrStatus_RcvHdrLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvBadTidErr_LSB 0xE
#define QIB_6120_ErrStatus_RcvBadTidErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvHdrFullErr_LSB 0xD
#define QIB_6120_ErrStatus_RcvHdrFullErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvEgrFullErr_LSB 0xC
#define QIB_6120_ErrStatus_RcvEgrFullErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvBadVersionErr_LSB 0xB
#define QIB_6120_ErrStatus_RcvBadVersionErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvIBFlowErr_LSB 0xA
#define QIB_6120_ErrStatus_RcvIBFlowErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvEBPErr_LSB 0x9
#define QIB_6120_ErrStatus_RcvEBPErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvUnsupportedVLErr_LSB 0x8
#define QIB_6120_ErrStatus_RcvUnsupportedVLErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvUnexpectedCharErr_LSB 0x7
#define QIB_6120_ErrStatus_RcvUnexpectedCharErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvShortPktLenErr_LSB 0x6
#define QIB_6120_ErrStatus_RcvShortPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvLongPktLenErr_LSB 0x5
#define QIB_6120_ErrStatus_RcvLongPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvMaxPktLenErr_LSB 0x4
#define QIB_6120_ErrStatus_RcvMaxPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvMinPktLenErr_LSB 0x3
#define QIB_6120_ErrStatus_RcvMinPktLenErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvICRCErr_LSB 0x2
#define QIB_6120_ErrStatus_RcvICRCErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvVCRCErr_LSB 0x1
#define QIB_6120_ErrStatus_RcvVCRCErr_RMASK 0x1
#define QIB_6120_ErrStatus_RcvFormatErr_LSB 0x0
#define QIB_6120_ErrStatus_RcvFormatErr_RMASK 0x1
#define QIB_6120_ErrClear_OFFS 0x90
#define QIB_6120_ErrClear_Reserved_LSB 0x34
#define QIB_6120_ErrClear_Reserved_RMASK 0xFFF
#define QIB_6120_ErrClear_HardwareErrClear_LSB 0x33
#define QIB_6120_ErrClear_HardwareErrClear_RMASK 0x1
#define QIB_6120_ErrClear_ResetNegatedClear_LSB 0x32
#define QIB_6120_ErrClear_ResetNegatedClear_RMASK 0x1
#define QIB_6120_ErrClear_InvalidAddrErrClear_LSB 0x31
#define QIB_6120_ErrClear_InvalidAddrErrClear_RMASK 0x1
#define QIB_6120_ErrClear_IBStatusChangedClear_LSB 0x30
#define QIB_6120_ErrClear_IBStatusChangedClear_RMASK 0x1
#define QIB_6120_ErrClear_Reserved1_LSB 0x26
#define QIB_6120_ErrClear_Reserved1_RMASK 0x3FF
#define QIB_6120_ErrClear_SendUnsupportedVLErrClear_LSB 0x25
#define QIB_6120_ErrClear_SendUnsupportedVLErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendUnexpectedPktNumErrClear_LSB 0x24
#define QIB_6120_ErrClear_SendUnexpectedPktNumErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendPioArmLaunchErrClear_LSB 0x23
#define QIB_6120_ErrClear_SendPioArmLaunchErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendDroppedDataPktErrClear_LSB 0x22
#define QIB_6120_ErrClear_SendDroppedDataPktErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendDroppedSmpPktErrClear_LSB 0x21
#define QIB_6120_ErrClear_SendDroppedSmpPktErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendPktLenErrClear_LSB 0x20
#define QIB_6120_ErrClear_SendPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendUnderRunErrClear_LSB 0x1F
#define QIB_6120_ErrClear_SendUnderRunErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendMaxPktLenErrClear_LSB 0x1E
#define QIB_6120_ErrClear_SendMaxPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_SendMinPktLenErrClear_LSB 0x1D
#define QIB_6120_ErrClear_SendMinPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_Reserved2_LSB 0x12
#define QIB_6120_ErrClear_Reserved2_RMASK 0x7FF
#define QIB_6120_ErrClear_RcvIBLostLinkErrClear_LSB 0x11
#define QIB_6120_ErrClear_RcvIBLostLinkErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvHdrErrClear_LSB 0x10
#define QIB_6120_ErrClear_RcvHdrErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvHdrLenErrClear_LSB 0xF
#define QIB_6120_ErrClear_RcvHdrLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvBadTidErrClear_LSB 0xE
#define QIB_6120_ErrClear_RcvBadTidErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvHdrFullErrClear_LSB 0xD
#define QIB_6120_ErrClear_RcvHdrFullErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvEgrFullErrClear_LSB 0xC
#define QIB_6120_ErrClear_RcvEgrFullErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvBadVersionErrClear_LSB 0xB
#define QIB_6120_ErrClear_RcvBadVersionErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvIBFlowErrClear_LSB 0xA
#define QIB_6120_ErrClear_RcvIBFlowErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvEBPErrClear_LSB 0x9
#define QIB_6120_ErrClear_RcvEBPErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvUnsupportedVLErrClear_LSB 0x8
#define QIB_6120_ErrClear_RcvUnsupportedVLErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvUnexpectedCharErrClear_LSB 0x7
#define QIB_6120_ErrClear_RcvUnexpectedCharErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvShortPktLenErrClear_LSB 0x6
#define QIB_6120_ErrClear_RcvShortPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvLongPktLenErrClear_LSB 0x5
#define QIB_6120_ErrClear_RcvLongPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvMaxPktLenErrClear_LSB 0x4
#define QIB_6120_ErrClear_RcvMaxPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvMinPktLenErrClear_LSB 0x3
#define QIB_6120_ErrClear_RcvMinPktLenErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvICRCErrClear_LSB 0x2
#define QIB_6120_ErrClear_RcvICRCErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvVCRCErrClear_LSB 0x1
#define QIB_6120_ErrClear_RcvVCRCErrClear_RMASK 0x1
#define QIB_6120_ErrClear_RcvFormatErrClear_LSB 0x0
#define QIB_6120_ErrClear_RcvFormatErrClear_RMASK 0x1
#define QIB_6120_HwErrMask_OFFS 0x98
#define QIB_6120_HwErrMask_IBCBusFromSPCParityErrMask_LSB 0x3F
#define QIB_6120_HwErrMask_IBCBusFromSPCParityErrMask_RMASK 0x1
#define QIB_6120_HwErrMask_IBCBusToSPCParityErrMask_LSB 0x3E
#define QIB_6120_HwErrMask_IBCBusToSPCParityErrMask_RMASK 0x1
#define QIB_6120_HwErrMask_Reserved_LSB 0x3D
#define QIB_6120_HwErrMask_Reserved_RMASK 0x1
#define QIB_6120_HwErrMask_IBSerdesPClkNotDetectMask_LSB 0x3C
#define QIB_6120_HwErrMask_IBSerdesPClkNotDetectMask_RMASK 0x1
#define QIB_6120_HwErrMask_PCIESerdesQ0PClkNotDetectMask_LSB 0x3B
#define QIB_6120_HwErrMask_PCIESerdesQ0PClkNotDetectMask_RMASK 0x1
#define QIB_6120_HwErrMask_PCIESerdesQ1PClkNotDetectMask_LSB 0x3A
#define QIB_6120_HwErrMask_PCIESerdesQ1PClkNotDetectMask_RMASK 0x1
#define QIB_6120_HwErrMask_Reserved1_LSB 0x39
#define QIB_6120_HwErrMask_Reserved1_RMASK 0x1
#define QIB_6120_HwErrMask_IBPLLrfSlipMask_LSB 0x38
#define QIB_6120_HwErrMask_IBPLLrfSlipMask_RMASK 0x1
#define QIB_6120_HwErrMask_IBPLLfbSlipMask_LSB 0x37
#define QIB_6120_HwErrMask_IBPLLfbSlipMask_RMASK 0x1
#define QIB_6120_HwErrMask_PowerOnBISTFailedMask_LSB 0x36
#define QIB_6120_HwErrMask_PowerOnBISTFailedMask_RMASK 0x1
#define QIB_6120_HwErrMask_Reserved2_LSB 0x33
#define QIB_6120_HwErrMask_Reserved2_RMASK 0x7
#define QIB_6120_HwErrMask_RXEMemParityErrMask_LSB 0x2C
#define QIB_6120_HwErrMask_RXEMemParityErrMask_RMASK 0x7F
#define QIB_6120_HwErrMask_TXEMemParityErrMask_LSB 0x28
#define QIB_6120_HwErrMask_TXEMemParityErrMask_RMASK 0xF
#define QIB_6120_HwErrMask_Reserved3_LSB 0x22
#define QIB_6120_HwErrMask_Reserved3_RMASK 0x3F
#define QIB_6120_HwErrMask_PCIeBusParityErrMask_LSB 0x1F
#define QIB_6120_HwErrMask_PCIeBusParityErrMask_RMASK 0x7
#define QIB_6120_HwErrMask_PcieCplTimeoutMask_LSB 0x1E
#define QIB_6120_HwErrMask_PcieCplTimeoutMask_RMASK 0x1
#define QIB_6120_HwErrMask_PoisonedTLPMask_LSB 0x1D
#define QIB_6120_HwErrMask_PoisonedTLPMask_RMASK 0x1
#define QIB_6120_HwErrMask_Reserved4_LSB 0x6
#define QIB_6120_HwErrMask_Reserved4_RMASK 0x7FFFFF
#define QIB_6120_HwErrMask_PCIeMemParityErrMask_LSB 0x0
#define QIB_6120_HwErrMask_PCIeMemParityErrMask_RMASK 0x3F
#define QIB_6120_HwErrStatus_OFFS 0xA0
#define QIB_6120_HwErrStatus_IBCBusFromSPCParityErr_LSB 0x3F
#define QIB_6120_HwErrStatus_IBCBusFromSPCParityErr_RMASK 0x1
#define QIB_6120_HwErrStatus_IBCBusToSPCParityErr_LSB 0x3E
#define QIB_6120_HwErrStatus_IBCBusToSPCParityErr_RMASK 0x1
#define QIB_6120_HwErrStatus_Reserved_LSB 0x3D
#define QIB_6120_HwErrStatus_Reserved_RMASK 0x1
#define QIB_6120_HwErrStatus_IBSerdesPClkNotDetect_LSB 0x3C
#define QIB_6120_HwErrStatus_IBSerdesPClkNotDetect_RMASK 0x1
#define QIB_6120_HwErrStatus_PCIESerdesQ0PClkNotDetect_LSB 0x3B
#define QIB_6120_HwErrStatus_PCIESerdesQ0PClkNotDetect_RMASK 0x1
#define QIB_6120_HwErrStatus_PCIESerdesQ1PClkNotDetect_LSB 0x3A
#define QIB_6120_HwErrStatus_PCIESerdesQ1PClkNotDetect_RMASK 0x1
#define QIB_6120_HwErrStatus_Reserved1_LSB 0x39
#define QIB_6120_HwErrStatus_Reserved1_RMASK 0x1
#define QIB_6120_HwErrStatus_IBPLLrfSlip_LSB 0x38
#define QIB_6120_HwErrStatus_IBPLLrfSlip_RMASK 0x1
#define QIB_6120_HwErrStatus_IBPLLfbSlip_LSB 0x37
#define QIB_6120_HwErrStatus_IBPLLfbSlip_RMASK 0x1
#define QIB_6120_HwErrStatus_PowerOnBISTFailed_LSB 0x36
#define QIB_6120_HwErrStatus_PowerOnBISTFailed_RMASK 0x1
#define QIB_6120_HwErrStatus_Reserved2_LSB 0x33
#define QIB_6120_HwErrStatus_Reserved2_RMASK 0x7
#define QIB_6120_HwErrStatus_RXEMemParity_LSB 0x2C
#define QIB_6120_HwErrStatus_RXEMemParity_RMASK 0x7F
#define QIB_6120_HwErrStatus_TXEMemParity_LSB 0x28
#define QIB_6120_HwErrStatus_TXEMemParity_RMASK 0xF
#define QIB_6120_HwErrStatus_Reserved3_LSB 0x22
#define QIB_6120_HwErrStatus_Reserved3_RMASK 0x3F
#define QIB_6120_HwErrStatus_PCIeBusParity_LSB 0x1F
#define QIB_6120_HwErrStatus_PCIeBusParity_RMASK 0x7
#define QIB_6120_HwErrStatus_PcieCplTimeout_LSB 0x1E
#define QIB_6120_HwErrStatus_PcieCplTimeout_RMASK 0x1
#define QIB_6120_HwErrStatus_PoisenedTLP_LSB 0x1D
#define QIB_6120_HwErrStatus_PoisenedTLP_RMASK 0x1
#define QIB_6120_HwErrStatus_Reserved4_LSB 0x6
#define QIB_6120_HwErrStatus_Reserved4_RMASK 0x7FFFFF
#define QIB_6120_HwErrStatus_PCIeMemParity_LSB 0x0
#define QIB_6120_HwErrStatus_PCIeMemParity_RMASK 0x3F
#define QIB_6120_HwErrClear_OFFS 0xA8
#define QIB_6120_HwErrClear_IBCBusFromSPCParityErrClear_LSB 0x3F
#define QIB_6120_HwErrClear_IBCBusFromSPCParityErrClear_RMASK 0x1
#define QIB_6120_HwErrClear_IBCBusToSPCparityErrClear_LSB 0x3E
#define QIB_6120_HwErrClear_IBCBusToSPCparityErrClear_RMASK 0x1
#define QIB_6120_HwErrClear_Reserved_LSB 0x3D
#define QIB_6120_HwErrClear_Reserved_RMASK 0x1
#define QIB_6120_HwErrClear_IBSerdesPClkNotDetectClear_LSB 0x3C
#define QIB_6120_HwErrClear_IBSerdesPClkNotDetectClear_RMASK 0x1
#define QIB_6120_HwErrClear_PCIESerdesQ0PClkNotDetectClear_LSB 0x3B
#define QIB_6120_HwErrClear_PCIESerdesQ0PClkNotDetectClear_RMASK 0x1
#define QIB_6120_HwErrClear_PCIESerdesQ1PClkNotDetectClear_LSB 0x3A
#define QIB_6120_HwErrClear_PCIESerdesQ1PClkNotDetectClear_RMASK 0x1
#define QIB_6120_HwErrClear_Reserved1_LSB 0x39
#define QIB_6120_HwErrClear_Reserved1_RMASK 0x1
#define QIB_6120_HwErrClear_IBPLLrfSlipClear_LSB 0x38
#define QIB_6120_HwErrClear_IBPLLrfSlipClear_RMASK 0x1
#define QIB_6120_HwErrClear_IBPLLfbSlipClear_LSB 0x37
#define QIB_6120_HwErrClear_IBPLLfbSlipClear_RMASK 0x1
#define QIB_6120_HwErrClear_PowerOnBISTFailedClear_LSB 0x36
#define QIB_6120_HwErrClear_PowerOnBISTFailedClear_RMASK 0x1
#define QIB_6120_HwErrClear_Reserved2_LSB 0x33
#define QIB_6120_HwErrClear_Reserved2_RMASK 0x7
#define QIB_6120_HwErrClear_RXEMemParityClear_LSB 0x2C
#define QIB_6120_HwErrClear_RXEMemParityClear_RMASK 0x7F
#define QIB_6120_HwErrClear_TXEMemParityClear_LSB 0x28
#define QIB_6120_HwErrClear_TXEMemParityClear_RMASK 0xF
#define QIB_6120_HwErrClear_Reserved3_LSB 0x22
#define QIB_6120_HwErrClear_Reserved3_RMASK 0x3F
#define QIB_6120_HwErrClear_PCIeBusParityClr_LSB 0x1F
#define QIB_6120_HwErrClear_PCIeBusParityClr_RMASK 0x7
#define QIB_6120_HwErrClear_PcieCplTimeoutClear_LSB 0x1E
#define QIB_6120_HwErrClear_PcieCplTimeoutClear_RMASK 0x1
#define QIB_6120_HwErrClear_PoisonedTLPClear_LSB 0x1D
#define QIB_6120_HwErrClear_PoisonedTLPClear_RMASK 0x1
#define QIB_6120_HwErrClear_Reserved4_LSB 0x6
#define QIB_6120_HwErrClear_Reserved4_RMASK 0x7FFFFF
#define QIB_6120_HwErrClear_PCIeMemParityClr_LSB 0x0
#define QIB_6120_HwErrClear_PCIeMemParityClr_RMASK 0x3F
#define QIB_6120_HwDiagCtrl_OFFS 0xB0
#define QIB_6120_HwDiagCtrl_ForceIBCBusFromSPCParityErr_LSB 0x3F
#define QIB_6120_HwDiagCtrl_ForceIBCBusFromSPCParityErr_RMASK 0x1
#define QIB_6120_HwDiagCtrl_ForceIBCBusToSPCParityErr_LSB 0x3E
#define QIB_6120_HwDiagCtrl_ForceIBCBusToSPCParityErr_RMASK 0x1
#define QIB_6120_HwDiagCtrl_CounterWrEnable_LSB 0x3D
#define QIB_6120_HwDiagCtrl_CounterWrEnable_RMASK 0x1
#define QIB_6120_HwDiagCtrl_CounterDisable_LSB 0x3C
#define QIB_6120_HwDiagCtrl_CounterDisable_RMASK 0x1
#define QIB_6120_HwDiagCtrl_Reserved_LSB 0x33
#define QIB_6120_HwDiagCtrl_Reserved_RMASK 0x1FF
#define QIB_6120_HwDiagCtrl_ForceRxMemParityErr_LSB 0x2C
#define QIB_6120_HwDiagCtrl_ForceRxMemParityErr_RMASK 0x7F
#define QIB_6120_HwDiagCtrl_ForceTxMemparityErr_LSB 0x28
#define QIB_6120_HwDiagCtrl_ForceTxMemparityErr_RMASK 0xF
#define QIB_6120_HwDiagCtrl_Reserved1_LSB 0x23
#define QIB_6120_HwDiagCtrl_Reserved1_RMASK 0x1F
#define QIB_6120_HwDiagCtrl_forcePCIeBusParity_LSB 0x1F
#define QIB_6120_HwDiagCtrl_forcePCIeBusParity_RMASK 0xF
#define QIB_6120_HwDiagCtrl_Reserved2_LSB 0x6
#define QIB_6120_HwDiagCtrl_Reserved2_RMASK 0x1FFFFFF
#define QIB_6120_HwDiagCtrl_forcePCIeMemParity_LSB 0x0
#define QIB_6120_HwDiagCtrl_forcePCIeMemParity_RMASK 0x3F
#define QIB_6120_IBCStatus_OFFS 0xC0
#define QIB_6120_IBCStatus_TxCreditOk_LSB 0x1F
#define QIB_6120_IBCStatus_TxCreditOk_RMASK 0x1
#define QIB_6120_IBCStatus_TxReady_LSB 0x1E
#define QIB_6120_IBCStatus_TxReady_RMASK 0x1
#define QIB_6120_IBCStatus_Reserved_LSB 0x7
#define QIB_6120_IBCStatus_Reserved_RMASK 0x7FFFFF
#define QIB_6120_IBCStatus_LinkState_LSB 0x4
#define QIB_6120_IBCStatus_LinkState_RMASK 0x7
#define QIB_6120_IBCStatus_LinkTrainingState_LSB 0x0
#define QIB_6120_IBCStatus_LinkTrainingState_RMASK 0xF
#define QIB_6120_IBCCtrl_OFFS 0xC8
#define QIB_6120_IBCCtrl_Loopback_LSB 0x3F
#define QIB_6120_IBCCtrl_Loopback_RMASK 0x1
#define QIB_6120_IBCCtrl_LinkDownDefaultState_LSB 0x3E
#define QIB_6120_IBCCtrl_LinkDownDefaultState_RMASK 0x1
#define QIB_6120_IBCCtrl_Reserved_LSB 0x2B
#define QIB_6120_IBCCtrl_Reserved_RMASK 0x7FFFF
#define QIB_6120_IBCCtrl_CreditScale_LSB 0x28
#define QIB_6120_IBCCtrl_CreditScale_RMASK 0x7
#define QIB_6120_IBCCtrl_OverrunThreshold_LSB 0x24
#define QIB_6120_IBCCtrl_OverrunThreshold_RMASK 0xF
#define QIB_6120_IBCCtrl_PhyerrThreshold_LSB 0x20
#define QIB_6120_IBCCtrl_PhyerrThreshold_RMASK 0xF
#define QIB_6120_IBCCtrl_Reserved1_LSB 0x1F
#define QIB_6120_IBCCtrl_Reserved1_RMASK 0x1
#define QIB_6120_IBCCtrl_MaxPktLen_LSB 0x14
#define QIB_6120_IBCCtrl_MaxPktLen_RMASK 0x7FF
#define QIB_6120_IBCCtrl_LinkCmd_LSB 0x12
#define QIB_6120_IBCCtrl_LinkCmd_RMASK 0x3
#define QIB_6120_IBCCtrl_LinkInitCmd_LSB 0x10
#define QIB_6120_IBCCtrl_LinkInitCmd_RMASK 0x3
#define QIB_6120_IBCCtrl_FlowCtrlWaterMark_LSB 0x8
#define QIB_6120_IBCCtrl_FlowCtrlWaterMark_RMASK 0xFF
#define QIB_6120_IBCCtrl_FlowCtrlPeriod_LSB 0x0
#define QIB_6120_IBCCtrl_FlowCtrlPeriod_RMASK 0xFF
#define QIB_6120_EXTStatus_OFFS 0xD0
#define QIB_6120_EXTStatus_GPIOIn_LSB 0x30
#define QIB_6120_EXTStatus_GPIOIn_RMASK 0xFFFF
#define QIB_6120_EXTStatus_Reserved_LSB 0x20
#define QIB_6120_EXTStatus_Reserved_RMASK 0xFFFF
#define QIB_6120_EXTStatus_Reserved1_LSB 0x10
#define QIB_6120_EXTStatus_Reserved1_RMASK 0xFFFF
#define QIB_6120_EXTStatus_MemBISTFoundErr_LSB 0xF
#define QIB_6120_EXTStatus_MemBISTFoundErr_RMASK 0x1
#define QIB_6120_EXTStatus_MemBISTEndTest_LSB 0xE
#define QIB_6120_EXTStatus_MemBISTEndTest_RMASK 0x1
#define QIB_6120_EXTStatus_Reserved2_LSB 0x0
#define QIB_6120_EXTStatus_Reserved2_RMASK 0x3FFF
#define QIB_6120_EXTCtrl_OFFS 0xD8
#define QIB_6120_EXTCtrl_GPIOOe_LSB 0x30
#define QIB_6120_EXTCtrl_GPIOOe_RMASK 0xFFFF
#define QIB_6120_EXTCtrl_GPIOInvert_LSB 0x20
#define QIB_6120_EXTCtrl_GPIOInvert_RMASK 0xFFFF
#define QIB_6120_EXTCtrl_Reserved_LSB 0x4
#define QIB_6120_EXTCtrl_Reserved_RMASK 0xFFFFFFF
#define QIB_6120_EXTCtrl_LEDPriPortGreenOn_LSB 0x3
#define QIB_6120_EXTCtrl_LEDPriPortGreenOn_RMASK 0x1
#define QIB_6120_EXTCtrl_LEDPriPortYellowOn_LSB 0x2
#define QIB_6120_EXTCtrl_LEDPriPortYellowOn_RMASK 0x1
#define QIB_6120_EXTCtrl_LEDGblOkGreenOn_LSB 0x1
#define QIB_6120_EXTCtrl_LEDGblOkGreenOn_RMASK 0x1
#define QIB_6120_EXTCtrl_LEDGblErrRedOff_LSB 0x0
#define QIB_6120_EXTCtrl_LEDGblErrRedOff_RMASK 0x1
#define QIB_6120_GPIOOut_OFFS 0xE0
#define QIB_6120_GPIOMask_OFFS 0xE8
#define QIB_6120_GPIOStatus_OFFS 0xF0
#define QIB_6120_GPIOClear_OFFS 0xF8
#define QIB_6120_RcvCtrl_OFFS 0x100
#define QIB_6120_RcvCtrl_TailUpd_LSB 0x1F
#define QIB_6120_RcvCtrl_TailUpd_RMASK 0x1
#define QIB_6120_RcvCtrl_RcvPartitionKeyDisable_LSB 0x1E
#define QIB_6120_RcvCtrl_RcvPartitionKeyDisable_RMASK 0x1
#define QIB_6120_RcvCtrl_Reserved_LSB 0x15
#define QIB_6120_RcvCtrl_Reserved_RMASK 0x1FF
#define QIB_6120_RcvCtrl_IntrAvail_LSB 0x10
#define QIB_6120_RcvCtrl_IntrAvail_RMASK 0x1F
#define QIB_6120_RcvCtrl_Reserved1_LSB 0x9
#define QIB_6120_RcvCtrl_Reserved1_RMASK 0x7F
#define QIB_6120_RcvCtrl_Reserved2_LSB 0x5
#define QIB_6120_RcvCtrl_Reserved2_RMASK 0xF
#define QIB_6120_RcvCtrl_PortEnable_LSB 0x0
#define QIB_6120_RcvCtrl_PortEnable_RMASK 0x1F
#define QIB_6120_RcvBTHQP_OFFS 0x108
#define QIB_6120_RcvBTHQP_BTHQP_Mask_LSB 0x1E
#define QIB_6120_RcvBTHQP_BTHQP_Mask_RMASK 0x3
#define QIB_6120_RcvBTHQP_Reserved_LSB 0x18
#define QIB_6120_RcvBTHQP_Reserved_RMASK 0x3F
#define QIB_6120_RcvBTHQP_RcvBTHQP_LSB 0x0
#define QIB_6120_RcvBTHQP_RcvBTHQP_RMASK 0xFFFFFF
#define QIB_6120_RcvHdrSize_OFFS 0x110
#define QIB_6120_RcvHdrCnt_OFFS 0x118
#define QIB_6120_RcvHdrEntSize_OFFS 0x120
#define QIB_6120_RcvTIDBase_OFFS 0x128
#define QIB_6120_RcvTIDCnt_OFFS 0x130
#define QIB_6120_RcvEgrBase_OFFS 0x138
#define QIB_6120_RcvEgrCnt_OFFS 0x140
#define QIB_6120_RcvBufBase_OFFS 0x148
#define QIB_6120_RcvBufSize_OFFS 0x150
#define QIB_6120_RxIntMemBase_OFFS 0x158
#define QIB_6120_RxIntMemSize_OFFS 0x160
#define QIB_6120_RcvPartitionKey_OFFS 0x168
#define QIB_6120_RcvPktLEDCnt_OFFS 0x178
#define QIB_6120_RcvPktLEDCnt_ONperiod_LSB 0x20
#define QIB_6120_RcvPktLEDCnt_ONperiod_RMASK 0xFFFFFFFF
#define QIB_6120_RcvPktLEDCnt_OFFperiod_LSB 0x0
#define QIB_6120_RcvPktLEDCnt_OFFperiod_RMASK 0xFFFFFFFF
#define QIB_6120_SendCtrl_OFFS 0x1C0
#define QIB_6120_SendCtrl_Disarm_LSB 0x1F
#define QIB_6120_SendCtrl_Disarm_RMASK 0x1
#define QIB_6120_SendCtrl_Reserved_LSB 0x17
#define QIB_6120_SendCtrl_Reserved_RMASK 0xFF
#define QIB_6120_SendCtrl_DisarmPIOBuf_LSB 0x10
#define QIB_6120_SendCtrl_DisarmPIOBuf_RMASK 0x7F
#define QIB_6120_SendCtrl_Reserved1_LSB 0x4
#define QIB_6120_SendCtrl_Reserved1_RMASK 0xFFF
#define QIB_6120_SendCtrl_PIOEnable_LSB 0x3
#define QIB_6120_SendCtrl_PIOEnable_RMASK 0x1
#define QIB_6120_SendCtrl_PIOBufAvailUpd_LSB 0x2
#define QIB_6120_SendCtrl_PIOBufAvailUpd_RMASK 0x1
#define QIB_6120_SendCtrl_PIOIntBufAvail_LSB 0x1
#define QIB_6120_SendCtrl_PIOIntBufAvail_RMASK 0x1
#define QIB_6120_SendCtrl_Abort_LSB 0x0
#define QIB_6120_SendCtrl_Abort_RMASK 0x1
#define QIB_6120_SendPIOBufBase_OFFS 0x1C8
#define QIB_6120_SendPIOBufBase_Reserved_LSB 0x35
#define QIB_6120_SendPIOBufBase_Reserved_RMASK 0x7FF
#define QIB_6120_SendPIOBufBase_BaseAddr_LargePIO_LSB 0x20
#define QIB_6120_SendPIOBufBase_BaseAddr_LargePIO_RMASK 0x1FFFFF
#define QIB_6120_SendPIOBufBase_Reserved1_LSB 0x15
#define QIB_6120_SendPIOBufBase_Reserved1_RMASK 0x7FF
#define QIB_6120_SendPIOBufBase_BaseAddr_SmallPIO_LSB 0x0
#define QIB_6120_SendPIOBufBase_BaseAddr_SmallPIO_RMASK 0x1FFFFF
#define QIB_6120_SendPIOSize_OFFS 0x1D0
#define QIB_6120_SendPIOSize_Reserved_LSB 0x2D
#define QIB_6120_SendPIOSize_Reserved_RMASK 0xFFFFF
#define QIB_6120_SendPIOSize_Size_LargePIO_LSB 0x20
#define QIB_6120_SendPIOSize_Size_LargePIO_RMASK 0x1FFF
#define QIB_6120_SendPIOSize_Reserved1_LSB 0xC
#define QIB_6120_SendPIOSize_Reserved1_RMASK 0xFFFFF
#define QIB_6120_SendPIOSize_Size_SmallPIO_LSB 0x0
#define QIB_6120_SendPIOSize_Size_SmallPIO_RMASK 0xFFF
#define QIB_6120_SendPIOBufCnt_OFFS 0x1D8
#define QIB_6120_SendPIOBufCnt_Reserved_LSB 0x24
#define QIB_6120_SendPIOBufCnt_Reserved_RMASK 0xFFFFFFF
#define QIB_6120_SendPIOBufCnt_Num_LargePIO_LSB 0x20
#define QIB_6120_SendPIOBufCnt_Num_LargePIO_RMASK 0xF
#define QIB_6120_SendPIOBufCnt_Reserved1_LSB 0x9
#define QIB_6120_SendPIOBufCnt_Reserved1_RMASK 0x7FFFFF
#define QIB_6120_SendPIOBufCnt_Num_SmallPIO_LSB 0x0
#define QIB_6120_SendPIOBufCnt_Num_SmallPIO_RMASK 0x1FF
#define QIB_6120_SendPIOAvailAddr_OFFS 0x1E0
#define QIB_6120_SendPIOAvailAddr_SendPIOAvailAddr_LSB 0x6
#define QIB_6120_SendPIOAvailAddr_SendPIOAvailAddr_RMASK 0x3FFFFFFFF
#define QIB_6120_SendPIOAvailAddr_Reserved_LSB 0x0
#define QIB_6120_SendPIOAvailAddr_Reserved_RMASK 0x3F
#define QIB_6120_SendBufErr0_OFFS 0x240
#define QIB_6120_SendBufErr0_SendBufErrPIO_63_0_LSB 0x0
#define QIB_6120_SendBufErr0_SendBufErrPIO_63_0_RMASK 0x0
#define QIB_6120_RcvHdrAddr0_OFFS 0x280
#define QIB_6120_RcvHdrAddr0_RcvHdrAddr0_LSB 0x2
#define QIB_6120_RcvHdrAddr0_RcvHdrAddr0_RMASK 0x3FFFFFFFFF
#define QIB_6120_RcvHdrAddr0_Reserved_LSB 0x0
#define QIB_6120_RcvHdrAddr0_Reserved_RMASK 0x3
#define QIB_6120_RcvHdrTailAddr0_OFFS 0x300
#define QIB_6120_RcvHdrTailAddr0_RcvHdrTailAddr0_LSB 0x2
#define QIB_6120_RcvHdrTailAddr0_RcvHdrTailAddr0_RMASK 0x3FFFFFFFFF
#define QIB_6120_RcvHdrTailAddr0_Reserved_LSB 0x0
#define QIB_6120_RcvHdrTailAddr0_Reserved_RMASK 0x3
#define QIB_6120_SerdesCfg0_OFFS 0x3C0
#define QIB_6120_SerdesCfg0_DisableIBTxIdleDetect_LSB 0x3F
#define QIB_6120_SerdesCfg0_DisableIBTxIdleDetect_RMASK 0x1
#define QIB_6120_SerdesCfg0_Reserved_LSB 0x38
#define QIB_6120_SerdesCfg0_Reserved_RMASK 0x7F
#define QIB_6120_SerdesCfg0_RxEqCtl_LSB 0x36
#define QIB_6120_SerdesCfg0_RxEqCtl_RMASK 0x3
#define QIB_6120_SerdesCfg0_TxTermAdj_LSB 0x34
#define QIB_6120_SerdesCfg0_TxTermAdj_RMASK 0x3
#define QIB_6120_SerdesCfg0_RxTermAdj_LSB 0x32
#define QIB_6120_SerdesCfg0_RxTermAdj_RMASK 0x3
#define QIB_6120_SerdesCfg0_TermAdj1_LSB 0x31
#define QIB_6120_SerdesCfg0_TermAdj1_RMASK 0x1
#define QIB_6120_SerdesCfg0_TermAdj0_LSB 0x30
#define QIB_6120_SerdesCfg0_TermAdj0_RMASK 0x1
#define QIB_6120_SerdesCfg0_LPBKA_LSB 0x2F
#define QIB_6120_SerdesCfg0_LPBKA_RMASK 0x1
#define QIB_6120_SerdesCfg0_LPBKB_LSB 0x2E
#define QIB_6120_SerdesCfg0_LPBKB_RMASK 0x1
#define QIB_6120_SerdesCfg0_LPBKC_LSB 0x2D
#define QIB_6120_SerdesCfg0_LPBKC_RMASK 0x1
#define QIB_6120_SerdesCfg0_LPBKD_LSB 0x2C
#define QIB_6120_SerdesCfg0_LPBKD_RMASK 0x1
#define QIB_6120_SerdesCfg0_PW_LSB 0x2B
#define QIB_6120_SerdesCfg0_PW_RMASK 0x1
#define QIB_6120_SerdesCfg0_RefSel_LSB 0x29
#define QIB_6120_SerdesCfg0_RefSel_RMASK 0x3
#define QIB_6120_SerdesCfg0_ParReset_LSB 0x28
#define QIB_6120_SerdesCfg0_ParReset_RMASK 0x1
#define QIB_6120_SerdesCfg0_ParLPBK_LSB 0x27
#define QIB_6120_SerdesCfg0_ParLPBK_RMASK 0x1
#define QIB_6120_SerdesCfg0_OffsetEn_LSB 0x26
#define QIB_6120_SerdesCfg0_OffsetEn_RMASK 0x1
#define QIB_6120_SerdesCfg0_Offset_LSB 0x1E
#define QIB_6120_SerdesCfg0_Offset_RMASK 0xFF
#define QIB_6120_SerdesCfg0_L2PwrDn_LSB 0x1D
#define QIB_6120_SerdesCfg0_L2PwrDn_RMASK 0x1
#define QIB_6120_SerdesCfg0_ResetPLL_LSB 0x1C
#define QIB_6120_SerdesCfg0_ResetPLL_RMASK 0x1
#define QIB_6120_SerdesCfg0_RxTermEnX_LSB 0x18
#define QIB_6120_SerdesCfg0_RxTermEnX_RMASK 0xF
#define QIB_6120_SerdesCfg0_BeaconTxEnX_LSB 0x14
#define QIB_6120_SerdesCfg0_BeaconTxEnX_RMASK 0xF
#define QIB_6120_SerdesCfg0_RxDetEnX_LSB 0x10
#define QIB_6120_SerdesCfg0_RxDetEnX_RMASK 0xF
#define QIB_6120_SerdesCfg0_TxIdeEnX_LSB 0xC
#define QIB_6120_SerdesCfg0_TxIdeEnX_RMASK 0xF
#define QIB_6120_SerdesCfg0_RxIdleEnX_LSB 0x8
#define QIB_6120_SerdesCfg0_RxIdleEnX_RMASK 0xF
#define QIB_6120_SerdesCfg0_L1PwrDnA_LSB 0x7
#define QIB_6120_SerdesCfg0_L1PwrDnA_RMASK 0x1
#define QIB_6120_SerdesCfg0_L1PwrDnB_LSB 0x6
#define QIB_6120_SerdesCfg0_L1PwrDnB_RMASK 0x1
#define QIB_6120_SerdesCfg0_L1PwrDnC_LSB 0x5
#define QIB_6120_SerdesCfg0_L1PwrDnC_RMASK 0x1
#define QIB_6120_SerdesCfg0_L1PwrDnD_LSB 0x4
#define QIB_6120_SerdesCfg0_L1PwrDnD_RMASK 0x1
#define QIB_6120_SerdesCfg0_ResetA_LSB 0x3
#define QIB_6120_SerdesCfg0_ResetA_RMASK 0x1
#define QIB_6120_SerdesCfg0_ResetB_LSB 0x2
#define QIB_6120_SerdesCfg0_ResetB_RMASK 0x1
#define QIB_6120_SerdesCfg0_ResetC_LSB 0x1
#define QIB_6120_SerdesCfg0_ResetC_RMASK 0x1
#define QIB_6120_SerdesCfg0_ResetD_LSB 0x0
#define QIB_6120_SerdesCfg0_ResetD_RMASK 0x1
#define QIB_6120_SerdesStat_OFFS 0x3D0
#define QIB_6120_SerdesStat_Reserved_LSB 0xC
#define QIB_6120_SerdesStat_Reserved_RMASK 0xFFFFFFFFFFFFF
#define QIB_6120_SerdesStat_BeaconDetA_LSB 0xB
#define QIB_6120_SerdesStat_BeaconDetA_RMASK 0x1
#define QIB_6120_SerdesStat_BeaconDetB_LSB 0xA
#define QIB_6120_SerdesStat_BeaconDetB_RMASK 0x1
#define QIB_6120_SerdesStat_BeaconDetC_LSB 0x9
#define QIB_6120_SerdesStat_BeaconDetC_RMASK 0x1
#define QIB_6120_SerdesStat_BeaconDetD_LSB 0x8
#define QIB_6120_SerdesStat_BeaconDetD_RMASK 0x1
#define QIB_6120_SerdesStat_RxDetA_LSB 0x7
#define QIB_6120_SerdesStat_RxDetA_RMASK 0x1
#define QIB_6120_SerdesStat_RxDetB_LSB 0x6
#define QIB_6120_SerdesStat_RxDetB_RMASK 0x1
#define QIB_6120_SerdesStat_RxDetC_LSB 0x5
#define QIB_6120_SerdesStat_RxDetC_RMASK 0x1
#define QIB_6120_SerdesStat_RxDetD_LSB 0x4
#define QIB_6120_SerdesStat_RxDetD_RMASK 0x1
#define QIB_6120_SerdesStat_TxIdleDetA_LSB 0x3
#define QIB_6120_SerdesStat_TxIdleDetA_RMASK 0x1
#define QIB_6120_SerdesStat_TxIdleDetB_LSB 0x2
#define QIB_6120_SerdesStat_TxIdleDetB_RMASK 0x1
#define QIB_6120_SerdesStat_TxIdleDetC_LSB 0x1
#define QIB_6120_SerdesStat_TxIdleDetC_RMASK 0x1
#define QIB_6120_SerdesStat_TxIdleDetD_LSB 0x0
#define QIB_6120_SerdesStat_TxIdleDetD_RMASK 0x1
#define QIB_6120_XGXSCfg_OFFS 0x3D8
#define QIB_6120_XGXSCfg_ArmLaunchErrorDisable_LSB 0x3F
#define QIB_6120_XGXSCfg_ArmLaunchErrorDisable_RMASK 0x1
#define QIB_6120_XGXSCfg_Reserved_LSB 0x17
#define QIB_6120_XGXSCfg_Reserved_RMASK 0xFFFFFFFFFF
#define QIB_6120_XGXSCfg_polarity_inv_LSB 0x13
#define QIB_6120_XGXSCfg_polarity_inv_RMASK 0xF
#define QIB_6120_XGXSCfg_link_sync_mask_LSB 0x9
#define QIB_6120_XGXSCfg_link_sync_mask_RMASK 0x3FF
#define QIB_6120_XGXSCfg_port_addr_LSB 0x4
#define QIB_6120_XGXSCfg_port_addr_RMASK 0x1F
#define QIB_6120_XGXSCfg_mdd_30_LSB 0x3
#define QIB_6120_XGXSCfg_mdd_30_RMASK 0x1
#define QIB_6120_XGXSCfg_xcv_resetn_LSB 0x2
#define QIB_6120_XGXSCfg_xcv_resetn_RMASK 0x1
#define QIB_6120_XGXSCfg_Reserved1_LSB 0x1
#define QIB_6120_XGXSCfg_Reserved1_RMASK 0x1
#define QIB_6120_XGXSCfg_tx_rx_resetn_LSB 0x0
#define QIB_6120_XGXSCfg_tx_rx_resetn_RMASK 0x1
#define QIB_6120_LBIntCnt_OFFS 0x12000
#define QIB_6120_LBFlowStallCnt_OFFS 0x12008
#define QIB_6120_TxUnsupVLErrCnt_OFFS 0x12018
#define QIB_6120_TxDataPktCnt_OFFS 0x12020
#define QIB_6120_TxFlowPktCnt_OFFS 0x12028
#define QIB_6120_TxDwordCnt_OFFS 0x12030
#define QIB_6120_TxLenErrCnt_OFFS 0x12038
#define QIB_6120_TxMaxMinLenErrCnt_OFFS 0x12040
#define QIB_6120_TxUnderrunCnt_OFFS 0x12048
#define QIB_6120_TxFlowStallCnt_OFFS 0x12050
#define QIB_6120_TxDroppedPktCnt_OFFS 0x12058
#define QIB_6120_RxDroppedPktCnt_OFFS 0x12060
#define QIB_6120_RxDataPktCnt_OFFS 0x12068
#define QIB_6120_RxFlowPktCnt_OFFS 0x12070
#define QIB_6120_RxDwordCnt_OFFS 0x12078
#define QIB_6120_RxLenErrCnt_OFFS 0x12080
#define QIB_6120_RxMaxMinLenErrCnt_OFFS 0x12088
#define QIB_6120_RxICRCErrCnt_OFFS 0x12090
#define QIB_6120_RxVCRCErrCnt_OFFS 0x12098
#define QIB_6120_RxFlowCtrlErrCnt_OFFS 0x120A0
#define QIB_6120_RxBadFormatCnt_OFFS 0x120A8
#define QIB_6120_RxLinkProblemCnt_OFFS 0x120B0
#define QIB_6120_RxEBPCnt_OFFS 0x120B8
#define QIB_6120_RxLPCRCErrCnt_OFFS 0x120C0
#define QIB_6120_RxBufOvflCnt_OFFS 0x120C8
#define QIB_6120_RxTIDFullErrCnt_OFFS 0x120D0
#define QIB_6120_RxTIDValidErrCnt_OFFS 0x120D8
#define QIB_6120_RxPKeyMismatchCnt_OFFS 0x120E0
#define QIB_6120_RxP0HdrEgrOvflCnt_OFFS 0x120E8
#define QIB_6120_IBStatusChangeCnt_OFFS 0x12140
#define QIB_6120_IBLinkErrRecoveryCnt_OFFS 0x12148
#define QIB_6120_IBLinkDownedCnt_OFFS 0x12150
#define QIB_6120_IBSymbolErrCnt_OFFS 0x12158
#define QIB_6120_PcieRetryBufDiagQwordCnt_OFFS 0x12170
#define QIB_6120_RcvEgrArray0_OFFS 0x14000
#define QIB_6120_RcvTIDArray0_OFFS 0x54000
#define QIB_6120_PIOLaunchFIFO_OFFS 0x64000
#define QIB_6120_SendPIOpbcCache_OFFS 0x64800
#define QIB_6120_RcvBuf1_OFFS 0x72000
#define QIB_6120_RcvBuf2_OFFS 0x75000
#define QIB_6120_RcvFlags_OFFS 0x77000
#define QIB_6120_RcvLookupBuf1_OFFS 0x79000
#define QIB_6120_RcvDMABuf_OFFS 0x7B000
#define QIB_6120_MiscRXEIntMem_OFFS 0x7C000
#define QIB_6120_PCIERcvBuf_OFFS 0x80000
#define QIB_6120_PCIERetryBuf_OFFS 0x82000
#define QIB_6120_PCIERcvBufRdToWrAddr_OFFS 0x84000
#define QIB_6120_PIOBuf0_MA_OFFS 0x100000

149
drivers/infiniband/hw/qib/qib_7220.h
View file

@ -1,149 +0,0 @@
#ifndef _QIB_7220_H
#define _QIB_7220_H
/*
* Copyright (c) 2007, 2009, 2010 QLogic Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* grab register-defs auto-generated by HW */
#include "qib_7220_regs.h"
/* The number of eager receive TIDs for context zero. */
#define IBA7220_KRCVEGRCNT 2048U
#define IB_7220_LT_STATE_CFGRCVFCFG 0x09
#define IB_7220_LT_STATE_CFGWAITRMT 0x0a
#define IB_7220_LT_STATE_TXREVLANES 0x0d
#define IB_7220_LT_STATE_CFGENH 0x10
struct qib_chip_specific {
u64 __iomem *cregbase;
u64 *cntrs;
u64 *portcntrs;
spinlock_t sdepb_lock; /* serdes EPB bus */
spinlock_t rcvmod_lock; /* protect rcvctrl shadow changes */
spinlock_t gpio_lock; /* RMW of shadows/regs for ExtCtrl and GPIO */
u64 hwerrmask;
u64 errormask;
u64 gpio_out; /* shadow of kr_gpio_out, for rmw ops */
u64 gpio_mask; /* shadow the gpio mask register */
u64 extctrl; /* shadow the gpio output enable, etc... */
u32 ncntrs;
u32 nportcntrs;
u32 cntrnamelen;
u32 portcntrnamelen;
u32 numctxts;
u32 rcvegrcnt;
u32 autoneg_tries;
u32 serdes_first_init_done;
u32 sdmabufcnt;
u32 lastbuf_for_pio;
u32 updthresh; /* current AvailUpdThld */
u32 updthresh_dflt; /* default AvailUpdThld */
u8 presets_needed;
u8 relock_timer_active;
char emsgbuf[128];
char sdmamsgbuf[192];
char bitsmsgbuf[64];
struct timer_list relock_timer;
unsigned int relock_interval; /* in jiffies */
struct qib_devdata *dd;
};
struct qib_chippport_specific {
struct qib_pportdata pportdata;
wait_queue_head_t autoneg_wait;
struct delayed_work autoneg_work;
struct timer_list chase_timer;
/*
* these 5 fields are used to establish deltas for IB symbol
* errors and linkrecovery errors. They can be reported on
* some chips during link negotiation prior to INIT, and with
* DDR when faking DDR negotiations with non-IBTA switches.
* The chip counters are adjusted at driver unload if there is
* a non-zero delta.
*/
u64 ibdeltainprog;
u64 ibsymdelta;
u64 ibsymsnap;
u64 iblnkerrdelta;
u64 iblnkerrsnap;
u64 ibcctrl; /* kr_ibcctrl shadow */
u64 ibcddrctrl; /* kr_ibcddrctrl shadow */
unsigned long chase_end;
u32 last_delay_mult;
};
/*
* This header file provides the declarations and common definitions
* for (mostly) manipulation of the SerDes blocks within the IBA7220.
* the functions declared should only be called from within other
* 7220-related files such as qib_iba7220.c or qib_sd7220.c.
*/
int qib_sd7220_presets(struct qib_devdata *dd);
int qib_sd7220_init(struct qib_devdata *dd);
void qib_sd7220_clr_ibpar(struct qib_devdata *);
/*
* Below used for sdnum parameter, selecting one of the two sections
* used for PCIe, or the single SerDes used for IB, which is the
* only one currently used
*/
#define IB_7220_SERDES 2
static inline u32 qib_read_kreg32(const struct qib_devdata *dd,
const u16 regno)
{
if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
return -1;
return readl((u32 __iomem *)&dd->kregbase[regno]);
}
static inline u64 qib_read_kreg64(const struct qib_devdata *dd,
const u16 regno)
{
if (!dd->kregbase || !(dd->flags & QIB_PRESENT))
return -1;
return readq(&dd->kregbase[regno]);
}
static inline void qib_write_kreg(const struct qib_devdata *dd,
const u16 regno, u64 value)
{
if (dd->kregbase)
writeq(value, &dd->kregbase[regno]);
}
void set_7220_relock_poll(struct qib_devdata *, int);
void shutdown_7220_relock_poll(struct qib_devdata *);
void toggle_7220_rclkrls(struct qib_devdata *);
#endif /* _QIB_7220_H */

1496
drivers/infiniband/hw/qib/qib_7220_regs.h
View file

File diff suppressed because it is too large Load diff

3163
drivers/infiniband/hw/qib/qib_7322_regs.h
View file

File diff suppressed because it is too large Load diff

798
drivers/infiniband/hw/qib/qib_common.h
View file

@ -1,798 +0,0 @@
/*
* Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
* All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _QIB_COMMON_H
#define _QIB_COMMON_H
/*
* This file contains defines, structures, etc. that are used
* to communicate between kernel and user code.
*/
/* This is the IEEE-assigned OUI for QLogic Inc. QLogic_IB */
#define QIB_SRC_OUI_1 0x00
#define QIB_SRC_OUI_2 0x11
#define QIB_SRC_OUI_3 0x75
/* version of protocol header (known to chip also). In the long run,
* we should be able to generate and accept a range of version numbers;
* for now we only accept one, and it's compiled in.
*/
#define IPS_PROTO_VERSION 2
/*
* These are compile time constants that you may want to enable or disable
* if you are trying to debug problems with code or performance.
* QIB_VERBOSE_TRACING define as 1 if you want additional tracing in
* fastpath code
* QIB_TRACE_REGWRITES define as 1 if you want register writes to be
* traced in fastpath code
* _QIB_TRACING define as 0 if you want to remove all tracing in a
* compilation unit
*/
/*
* The value in the BTH QP field that QLogic_IB uses to differentiate
* an qlogic_ib protocol IB packet vs standard IB transport
* This it needs to be even (0x656b78), because the LSB is sometimes
* used for the MSB of context. The change may cause a problem
* interoperating with older software.
*/
#define QIB_KD_QP 0x656b78
/*
* These are the status bits readable (in ascii form, 64bit value)
* from the "status" sysfs file. For binary compatibility, values
* must remain as is; removed states can be reused for different
* purposes.
*/
#define QIB_STATUS_INITTED 0x1 /* basic initialization done */
/* Chip has been found and initted */
#define QIB_STATUS_CHIP_PRESENT 0x20
/* IB link is at ACTIVE, usable for data traffic */
#define QIB_STATUS_IB_READY 0x40
/* link is configured, LID, MTU, etc. have been set */
#define QIB_STATUS_IB_CONF 0x80
/* A Fatal hardware error has occurred. */
#define QIB_STATUS_HWERROR 0x200
/*
* The list of usermode accessible registers. Also see Reg_* later in file.
*/
enum qib_ureg {
/* (RO) DMA RcvHdr to be used next. */
ur_rcvhdrtail = 0,
/* (RW) RcvHdr entry to be processed next by host. */
ur_rcvhdrhead = 1,
/* (RO) Index of next Eager index to use. */
ur_rcvegrindextail = 2,
/* (RW) Eager TID to be processed next */
ur_rcvegrindexhead = 3,
/* For internal use only; max register number. */
_QIB_UregMax
};
/* bit values for spi_runtime_flags */
#define QIB_RUNTIME_PCIE 0x0002
#define QIB_RUNTIME_FORCE_WC_ORDER 0x0004
#define QIB_RUNTIME_RCVHDR_COPY 0x0008
#define QIB_RUNTIME_MASTER 0x0010
#define QIB_RUNTIME_RCHK 0x0020
#define QIB_RUNTIME_NODMA_RTAIL 0x0080
#define QIB_RUNTIME_SPECIAL_TRIGGER 0x0100
#define QIB_RUNTIME_SDMA 0x0200
#define QIB_RUNTIME_FORCE_PIOAVAIL 0x0400
#define QIB_RUNTIME_PIO_REGSWAPPED 0x0800
#define QIB_RUNTIME_CTXT_MSB_IN_QP 0x1000
#define QIB_RUNTIME_CTXT_REDIRECT 0x2000
#define QIB_RUNTIME_HDRSUPP 0x4000
/*
* This structure is returned by qib_userinit() immediately after
* open to get implementation-specific info, and info specific to this
* instance.
*
* This struct must have explict pad fields where type sizes
* may result in different alignments between 32 and 64 bit
* programs, since the 64 bit * bit kernel requires the user code
* to have matching offsets
*/
struct qib_base_info {
/* version of hardware, for feature checking. */
__u32 spi_hw_version;
/* version of software, for feature checking. */
__u32 spi_sw_version;
/* QLogic_IB context assigned, goes into sent packets */
__u16 spi_ctxt;
__u16 spi_subctxt;
/*
* IB MTU, packets IB data must be less than this.
* The MTU is in bytes, and will be a multiple of 4 bytes.
*/
__u32 spi_mtu;
/*
* Size of a PIO buffer. Any given packet's total size must be less
* than this (in words). Included is the starting control word, so
* if 513 is returned, then total pkt size is 512 words or less.
*/
__u32 spi_piosize;
/* size of the TID cache in qlogic_ib, in entries */
__u32 spi_tidcnt;
/* size of the TID Eager list in qlogic_ib, in entries */
__u32 spi_tidegrcnt;
/* size of a single receive header queue entry in words. */
__u32 spi_rcvhdrent_size;
/*
* Count of receive header queue entries allocated.
* This may be less than the spu_rcvhdrcnt passed in!.
*/
__u32 spi_rcvhdr_cnt;
/* per-chip and other runtime features bitmap (QIB_RUNTIME_*) */
__u32 spi_runtime_flags;
/* address where hardware receive header queue is mapped */
__u64 spi_rcvhdr_base;
/* user program. */
/* base address of eager TID receive buffers used by hardware. */
__u64 spi_rcv_egrbufs;
/* Allocated by initialization code, not by protocol. */
/*
* Size of each TID buffer in host memory, starting at
* spi_rcv_egrbufs. The buffers are virtually contiguous.
*/
__u32 spi_rcv_egrbufsize;
/*
* The special QP (queue pair) value that identifies an qlogic_ib
* protocol packet from standard IB packets. More, probably much
* more, to be added.
*/
__u32 spi_qpair;
/*
* User register base for init code, not to be used directly by
* protocol or applications. Always points to chip registers,
* for normal or shared context.
*/
__u64 spi_uregbase;
/*
* Maximum buffer size in bytes that can be used in a single TID
* entry (assuming the buffer is aligned to this boundary). This is
* the minimum of what the hardware and software support Guaranteed
* to be a power of 2.
*/
__u32 spi_tid_maxsize;
/*
* alignment of each pio send buffer (byte count
* to add to spi_piobufbase to get to second buffer)
*/
__u32 spi_pioalign;
/*
* The index of the first pio buffer available to this process;
* needed to do lookup in spi_pioavailaddr; not added to
* spi_piobufbase.
*/
__u32 spi_pioindex;
/* number of buffers mapped for this process */
__u32 spi_piocnt;
/*
* Base address of writeonly pio buffers for this process.
* Each buffer has spi_piosize words, and is aligned on spi_pioalign
* boundaries. spi_piocnt buffers are mapped from this address
*/
__u64 spi_piobufbase;
/*
* Base address of readonly memory copy of the pioavail registers.
* There are 2 bits for each buffer.
*/
__u64 spi_pioavailaddr;
/*
* Address where driver updates a copy of the interface and driver
* status (QIB_STATUS_*) as a 64 bit value. It's followed by a
* link status qword (formerly combined with driver status), then a
* string indicating hardware error, if there was one.
*/
__u64 spi_status;
/* number of chip ctxts available to user processes */
__u32 spi_nctxts;
__u16 spi_unit; /* unit number of chip we are using */
__u16 spi_port; /* IB port number we are using */
/* num bufs in each contiguous set */
__u32 spi_rcv_egrperchunk;
/* size in bytes of each contiguous set */
__u32 spi_rcv_egrchunksize;
/* total size of mmap to cover full rcvegrbuffers */
__u32 spi_rcv_egrbuftotlen;
__u32 spi_rhf_offset; /* dword offset in hdrqent for rcvhdr flags */
/* address of readonly memory copy of the rcvhdrq tail register. */
__u64 spi_rcvhdr_tailaddr;
/*
* shared memory pages for subctxts if ctxt is shared; these cover
* all the processes in the group sharing a single context.
* all have enough space for the num_subcontexts value on this job.
*/
__u64 spi_subctxt_uregbase;
__u64 spi_subctxt_rcvegrbuf;
__u64 spi_subctxt_rcvhdr_base;
/* shared memory page for send buffer disarm status */
__u64 spi_sendbuf_status;
} __aligned(8);
/*
* This version number is given to the driver by the user code during
* initialization in the spu_userversion field of qib_user_info, so
* the driver can check for compatibility with user code.
*
* The major version changes when data structures
* change in an incompatible way. The driver must be the same or higher
* for initialization to succeed. In some cases, a higher version
* driver will not interoperate with older software, and initialization
* will return an error.
*/
#define QIB_USER_SWMAJOR 1
/*
* Minor version differences are always compatible
* a within a major version, however if user software is larger
* than driver software, some new features and/or structure fields
* may not be implemented; the user code must deal with this if it
* cares, or it must abort after initialization reports the difference.
*/
#define QIB_USER_SWMINOR 13
#define QIB_USER_SWVERSION ((QIB_USER_SWMAJOR << 16) | QIB_USER_SWMINOR)
#ifndef QIB_KERN_TYPE
#define QIB_KERN_TYPE 0
#endif
/*
* Similarly, this is the kernel version going back to the user. It's
* slightly different, in that we want to tell if the driver was built as
* part of a QLogic release, or from the driver from openfabrics.org,
* kernel.org, or a standard distribution, for support reasons.
* The high bit is 0 for non-QLogic and 1 for QLogic-built/supplied.
*
* It's returned by the driver to the user code during initialization in the
* spi_sw_version field of qib_base_info, so the user code can in turn
* check for compatibility with the kernel.
*/
#define QIB_KERN_SWVERSION ((QIB_KERN_TYPE << 31) | QIB_USER_SWVERSION)
/*
* Define the driver version number. This is something that refers only
* to the driver itself, not the software interfaces it supports.
*/
#define QIB_DRIVER_VERSION_BASE "1.11"
/* create the final driver version string */
#ifdef QIB_IDSTR
#define QIB_DRIVER_VERSION QIB_DRIVER_VERSION_BASE " " QIB_IDSTR
#else
#define QIB_DRIVER_VERSION QIB_DRIVER_VERSION_BASE
#endif
/*
* If the unit is specified via open, HCA choice is fixed. If port is
* specified, it's also fixed. Otherwise we try to spread contexts
* across ports and HCAs, using different algorithims. WITHIN is
* the old default, prior to this mechanism.
*/
#define QIB_PORT_ALG_ACROSS 0 /* round robin contexts across HCAs, then
* ports; this is the default */
#define QIB_PORT_ALG_WITHIN 1 /* use all contexts on an HCA (round robin
* active ports within), then next HCA */
#define QIB_PORT_ALG_COUNT 2 /* number of algorithm choices */
/*
* This structure is passed to qib_userinit() to tell the driver where
* user code buffers are, sizes, etc. The offsets and sizes of the
* fields must remain unchanged, for binary compatibility. It can
* be extended, if userversion is changed so user code can tell, if needed
*/
struct qib_user_info {
/*
* version of user software, to detect compatibility issues.
* Should be set to QIB_USER_SWVERSION.
*/
__u32 spu_userversion;
__u32 _spu_unused2;
/* size of struct base_info to write to */
__u32 spu_base_info_size;
__u32 spu_port_alg; /* which QIB_PORT_ALG_*; unused user minor < 11 */
/*
* If two or more processes wish to share a context, each process
* must set the spu_subctxt_cnt and spu_subctxt_id to the same
* values. The only restriction on the spu_subctxt_id is that
* it be unique for a given node.
*/
__u16 spu_subctxt_cnt;
__u16 spu_subctxt_id;
__u32 spu_port; /* IB port requested by user if > 0 */
/*
* address of struct base_info to write to
*/
__u64 spu_base_info;
} __aligned(8);
/* User commands. */
/* 16 available, was: old set up userspace (for old user code) */
#define QIB_CMD_CTXT_INFO 17 /* find out what resources we got */
#define QIB_CMD_RECV_CTRL 18 /* control receipt of packets */
#define QIB_CMD_TID_UPDATE 19 /* update expected TID entries */
#define QIB_CMD_TID_FREE 20 /* free expected TID entries */
#define QIB_CMD_SET_PART_KEY 21 /* add partition key */
/* 22 available, was: return info on slave processes (for old user code) */
#define QIB_CMD_ASSIGN_CTXT 23 /* allocate HCA and ctxt */
#define QIB_CMD_USER_INIT 24 /* set up userspace */
#define QIB_CMD_UNUSED_1 25
#define QIB_CMD_UNUSED_2 26
#define QIB_CMD_PIOAVAILUPD 27 /* force an update of PIOAvail reg */
#define QIB_CMD_POLL_TYPE 28 /* set the kind of polling we want */
#define QIB_CMD_ARMLAUNCH_CTRL 29 /* armlaunch detection control */
/* 30 is unused */
#define QIB_CMD_SDMA_INFLIGHT 31 /* sdma inflight counter request */
#define QIB_CMD_SDMA_COMPLETE 32 /* sdma completion counter request */
/* 33 available, was a testing feature */
#define QIB_CMD_DISARM_BUFS 34 /* disarm send buffers w/ errors */
#define QIB_CMD_ACK_EVENT 35 /* ack & clear bits */
#define QIB_CMD_CPUS_LIST 36 /* list of cpus allocated, for pinned
* processes: qib_cpus_list */
/*
* QIB_CMD_ACK_EVENT obsoletes QIB_CMD_DISARM_BUFS, but we keep it for
* compatibility with libraries from previous release. The ACK_EVENT
* will take appropriate driver action (if any, just DISARM for now),
* then clear the bits passed in as part of the mask. These bits are
* in the first 64bit word at spi_sendbuf_status, and are passed to
* the driver in the event_mask union as well.
*/
#define _QIB_EVENT_DISARM_BUFS_BIT 0
#define _QIB_EVENT_LINKDOWN_BIT 1
#define _QIB_EVENT_LID_CHANGE_BIT 2
#define _QIB_EVENT_LMC_CHANGE_BIT 3
#define _QIB_EVENT_SL2VL_CHANGE_BIT 4
#define _QIB_MAX_EVENT_BIT _QIB_EVENT_SL2VL_CHANGE_BIT
#define QIB_EVENT_DISARM_BUFS_BIT (1UL << _QIB_EVENT_DISARM_BUFS_BIT)
#define QIB_EVENT_LINKDOWN_BIT (1UL << _QIB_EVENT_LINKDOWN_BIT)
#define QIB_EVENT_LID_CHANGE_BIT (1UL << _QIB_EVENT_LID_CHANGE_BIT)
#define QIB_EVENT_LMC_CHANGE_BIT (1UL << _QIB_EVENT_LMC_CHANGE_BIT)
#define QIB_EVENT_SL2VL_CHANGE_BIT (1UL << _QIB_EVENT_SL2VL_CHANGE_BIT)
/*
* Poll types
*/
#define QIB_POLL_TYPE_ANYRCV 0x0
#define QIB_POLL_TYPE_URGENT 0x1
struct qib_ctxt_info {
__u16 num_active; /* number of active units */
__u16 unit; /* unit (chip) assigned to caller */
__u16 port; /* IB port assigned to caller (1-based) */
__u16 ctxt; /* ctxt on unit assigned to caller */
__u16 subctxt; /* subctxt on unit assigned to caller */
__u16 num_ctxts; /* number of ctxts available on unit */
__u16 num_subctxts; /* number of subctxts opened on ctxt */
__u16 rec_cpu; /* cpu # for affinity (ffff if none) */
};
struct qib_tid_info {
__u32 tidcnt;
/* make structure same size in 32 and 64 bit */
__u32 tid__unused;
/* virtual address of first page in transfer */
__u64 tidvaddr;
/* pointer (same size 32/64 bit) to __u16 tid array */
__u64 tidlist;
/*
* pointer (same size 32/64 bit) to bitmap of TIDs used
* for this call; checked for being large enough at open
*/
__u64 tidmap;
};
struct qib_cmd {
__u32 type; /* command type */
union {
struct qib_tid_info tid_info;
struct qib_user_info user_info;
/*
* address in userspace where we should put the sdma
* inflight counter
*/
__u64 sdma_inflight;
/*
* address in userspace where we should put the sdma
* completion counter
*/
__u64 sdma_complete;
/* address in userspace of struct qib_ctxt_info to
write result to */
__u64 ctxt_info;
/* enable/disable receipt of packets */
__u32 recv_ctrl;
/* enable/disable armlaunch errors (non-zero to enable) */
__u32 armlaunch_ctrl;
/* partition key to set */
__u16 part_key;
/* user address of __u32 bitmask of active slaves */
__u64 slave_mask_addr;
/* type of polling we want */
__u16 poll_type;
/* back pressure enable bit for one particular context */
__u8 ctxt_bp;
/* qib_user_event_ack(), IPATH_EVENT_* bits */
__u64 event_mask;
} cmd;
};
struct qib_iovec {
/* Pointer to data, but same size 32 and 64 bit */
__u64 iov_base;
/*
* Length of data; don't need 64 bits, but want
* qib_sendpkt to remain same size as before 32 bit changes, so...
*/
__u64 iov_len;
};
/*
* Describes a single packet for send. Each packet can have one or more
* buffers, but the total length (exclusive of IB headers) must be less
* than the MTU, and if using the PIO method, entire packet length,
* including IB headers, must be less than the qib_piosize value (words).
* Use of this necessitates including sys/uio.h
*/
struct __qib_sendpkt {
__u32 sps_flags; /* flags for packet (TBD) */
__u32 sps_cnt; /* number of entries to use in sps_iov */
/* array of iov's describing packet. TEMPORARY */
struct qib_iovec sps_iov[4];
};
/*
* Diagnostics can send a packet by "writing" the following
* structs to the diag data special file.
* This allows a custom
* pbc (+ static rate) qword, so that special modes and deliberate
* changes to CRCs can be used. The elements were also re-ordered
* for better alignment and to avoid padding issues.
*/
#define _DIAG_XPKT_VERS 3
struct qib_diag_xpkt {
__u16 version;
__u16 unit;
__u16 port;
__u16 len;
__u64 data;
__u64 pbc_wd;
};
/*
* Data layout in I2C flash (for GUID, etc.)
* All fields are little-endian binary unless otherwise stated
*/
#define QIB_FLASH_VERSION 2
struct qib_flash {
/* flash layout version (QIB_FLASH_VERSION) */
__u8 if_fversion;
/* checksum protecting if_length bytes */
__u8 if_csum;
/*
* valid length (in use, protected by if_csum), including
* if_fversion and if_csum themselves)
*/
__u8 if_length;
/* the GUID, in network order */
__u8 if_guid[8];
/* number of GUIDs to use, starting from if_guid */
__u8 if_numguid;
/* the (last 10 characters of) board serial number, in ASCII */
char if_serial[12];
/* board mfg date (YYYYMMDD ASCII) */
char if_mfgdate[8];
/* last board rework/test date (YYYYMMDD ASCII) */
char if_testdate[8];
/* logging of error counts, TBD */
__u8 if_errcntp[4];
/* powered on hours, updated at driver unload */
__u8 if_powerhour[2];
/* ASCII free-form comment field */
char if_comment[32];
/* Backwards compatible prefix for longer QLogic Serial Numbers */
char if_sprefix[4];
/* 82 bytes used, min flash size is 128 bytes */
__u8 if_future[46];
};
/*
* These are the counters implemented in the chip, and are listed in order.
* The InterCaps naming is taken straight from the chip spec.
*/
struct qlogic_ib_counters {
__u64 LBIntCnt;
__u64 LBFlowStallCnt;
__u64 TxSDmaDescCnt; /* was Reserved1 */
__u64 TxUnsupVLErrCnt;
__u64 TxDataPktCnt;
__u64 TxFlowPktCnt;
__u64 TxDwordCnt;
__u64 TxLenErrCnt;
__u64 TxMaxMinLenErrCnt;
__u64 TxUnderrunCnt;
__u64 TxFlowStallCnt;
__u64 TxDroppedPktCnt;
__u64 RxDroppedPktCnt;
__u64 RxDataPktCnt;
__u64 RxFlowPktCnt;
__u64 RxDwordCnt;
__u64 RxLenErrCnt;
__u64 RxMaxMinLenErrCnt;
__u64 RxICRCErrCnt;
__u64 RxVCRCErrCnt;
__u64 RxFlowCtrlErrCnt;
__u64 RxBadFormatCnt;
__u64 RxLinkProblemCnt;
__u64 RxEBPCnt;
__u64 RxLPCRCErrCnt;
__u64 RxBufOvflCnt;
__u64 RxTIDFullErrCnt;
__u64 RxTIDValidErrCnt;
__u64 RxPKeyMismatchCnt;
__u64 RxP0HdrEgrOvflCnt;
__u64 RxP1HdrEgrOvflCnt;
__u64 RxP2HdrEgrOvflCnt;
__u64 RxP3HdrEgrOvflCnt;
__u64 RxP4HdrEgrOvflCnt;
__u64 RxP5HdrEgrOvflCnt;
__u64 RxP6HdrEgrOvflCnt;
__u64 RxP7HdrEgrOvflCnt;
__u64 RxP8HdrEgrOvflCnt;
__u64 RxP9HdrEgrOvflCnt;
__u64 RxP10HdrEgrOvflCnt;
__u64 RxP11HdrEgrOvflCnt;
__u64 RxP12HdrEgrOvflCnt;
__u64 RxP13HdrEgrOvflCnt;
__u64 RxP14HdrEgrOvflCnt;
__u64 RxP15HdrEgrOvflCnt;
__u64 RxP16HdrEgrOvflCnt;
__u64 IBStatusChangeCnt;
__u64 IBLinkErrRecoveryCnt;
__u64 IBLinkDownedCnt;
__u64 IBSymbolErrCnt;
__u64 RxVL15DroppedPktCnt;
__u64 RxOtherLocalPhyErrCnt;
__u64 PcieRetryBufDiagQwordCnt;
__u64 ExcessBufferOvflCnt;
__u64 LocalLinkIntegrityErrCnt;
__u64 RxVlErrCnt;
__u64 RxDlidFltrCnt;
};
/*
* The next set of defines are for packet headers, and chip register
* and memory bits that are visible to and/or used by user-mode software.
*/
/* RcvHdrFlags bits */
#define QLOGIC_IB_RHF_LENGTH_MASK 0x7FF
#define QLOGIC_IB_RHF_LENGTH_SHIFT 0
#define QLOGIC_IB_RHF_RCVTYPE_MASK 0x7
#define QLOGIC_IB_RHF_RCVTYPE_SHIFT 11
#define QLOGIC_IB_RHF_EGRINDEX_MASK 0xFFF
#define QLOGIC_IB_RHF_EGRINDEX_SHIFT 16
#define QLOGIC_IB_RHF_SEQ_MASK 0xF
#define QLOGIC_IB_RHF_SEQ_SHIFT 0
#define QLOGIC_IB_RHF_HDRQ_OFFSET_MASK 0x7FF
#define QLOGIC_IB_RHF_HDRQ_OFFSET_SHIFT 4
#define QLOGIC_IB_RHF_H_ICRCERR 0x80000000
#define QLOGIC_IB_RHF_H_VCRCERR 0x40000000
#define QLOGIC_IB_RHF_H_PARITYERR 0x20000000
#define QLOGIC_IB_RHF_H_LENERR 0x10000000
#define QLOGIC_IB_RHF_H_MTUERR 0x08000000
#define QLOGIC_IB_RHF_H_IHDRERR 0x04000000
#define QLOGIC_IB_RHF_H_TIDERR 0x02000000
#define QLOGIC_IB_RHF_H_MKERR 0x01000000
#define QLOGIC_IB_RHF_H_IBERR 0x00800000
#define QLOGIC_IB_RHF_H_ERR_MASK 0xFF800000
#define QLOGIC_IB_RHF_L_USE_EGR 0x80000000
#define QLOGIC_IB_RHF_L_SWA 0x00008000
#define QLOGIC_IB_RHF_L_SWB 0x00004000
/* qlogic_ib header fields */
#define QLOGIC_IB_I_VERS_MASK 0xF
#define QLOGIC_IB_I_VERS_SHIFT 28
#define QLOGIC_IB_I_CTXT_MASK 0xF
#define QLOGIC_IB_I_CTXT_SHIFT 24
#define QLOGIC_IB_I_TID_MASK 0x7FF
#define QLOGIC_IB_I_TID_SHIFT 13
#define QLOGIC_IB_I_OFFSET_MASK 0x1FFF
#define QLOGIC_IB_I_OFFSET_SHIFT 0
/* K_PktFlags bits */
#define QLOGIC_IB_KPF_INTR 0x1
#define QLOGIC_IB_KPF_SUBCTXT_MASK 0x3
#define QLOGIC_IB_KPF_SUBCTXT_SHIFT 1
#define QLOGIC_IB_MAX_SUBCTXT 4
/* SendPIO per-buffer control */
#define QLOGIC_IB_SP_TEST 0x40
#define QLOGIC_IB_SP_TESTEBP 0x20
#define QLOGIC_IB_SP_TRIGGER_SHIFT 15
/* SendPIOAvail bits */
#define QLOGIC_IB_SENDPIOAVAIL_BUSY_SHIFT 1
#define QLOGIC_IB_SENDPIOAVAIL_CHECK_SHIFT 0
/* qlogic_ib header format */
struct qib_header {
/*
* Version - 4 bits, Context - 4 bits, TID - 10 bits and Offset -
* 14 bits before ECO change ~28 Dec 03. After that, Vers 4,
* Context 4, TID 11, offset 13.
*/
__le32 ver_ctxt_tid_offset;
__le16 chksum;
__le16 pkt_flags;
};
/*
* qlogic_ib user message header format.
* This structure contains the first 4 fields common to all protocols
* that employ qlogic_ib.
*/
struct qib_message_header {
__be16 lrh[4];
__be32 bth[3];
/* fields below this point are in host byte order */
struct qib_header iph;
/* fields below are simplified, but should match PSM */
/* some are accessed by driver when packet spliting is needed */
__u8 sub_opcode;
__u8 flags;
__u16 commidx;
__u32 ack_seq_num;
__u8 flowid;
__u8 hdr_dlen;
__u16 mqhdr;
__u32 uwords[4];
};
/* sequence number bits for message */
union qib_seqnum {
struct {
__u32 seq:11;
__u32 gen:8;
__u32 flow:5;
};
struct {
__u32 pkt:16;
__u32 msg:8;
};
__u32 val;
};
/* qib receiving-dma tid-session-member */
struct qib_tid_session_member {
__u16 tid;
__u16 offset;
__u16 length;
};
/* IB - LRH header consts */
#define QIB_LRH_GRH 0x0003 /* 1. word of IB LRH - next header: GRH */
#define QIB_LRH_BTH 0x0002 /* 1. word of IB LRH - next header: BTH */
/* misc. */
#define SIZE_OF_CRC 1
#define QIB_DEFAULT_P_KEY 0xFFFF
#define QIB_PSN_MASK 0xFFFFFF
#define QIB_EAGER_TID_ID QLOGIC_IB_I_TID_MASK
#define QIB_MULTICAST_QPN 0xFFFFFF
/* Receive Header Queue: receive type (from qlogic_ib) */
#define RCVHQ_RCV_TYPE_EXPECTED 0
#define RCVHQ_RCV_TYPE_EAGER 1
#define RCVHQ_RCV_TYPE_NON_KD 2
#define RCVHQ_RCV_TYPE_ERROR 3
#define QIB_HEADER_QUEUE_WORDS 9
/* functions for extracting fields from rcvhdrq entries for the driver.
*/
static inline __u32 qib_hdrget_err_flags(const __le32 *rbuf)
{
return __le32_to_cpu(rbuf[1]) & QLOGIC_IB_RHF_H_ERR_MASK;
}
static inline __u32 qib_hdrget_rcv_type(const __le32 *rbuf)
{
return (__le32_to_cpu(rbuf[0]) >> QLOGIC_IB_RHF_RCVTYPE_SHIFT) &
QLOGIC_IB_RHF_RCVTYPE_MASK;
}
static inline __u32 qib_hdrget_length_in_bytes(const __le32 *rbuf)
{
return ((__le32_to_cpu(rbuf[0]) >> QLOGIC_IB_RHF_LENGTH_SHIFT) &
QLOGIC_IB_RHF_LENGTH_MASK) << 2;
}
static inline __u32 qib_hdrget_index(const __le32 *rbuf)
{
return (__le32_to_cpu(rbuf[0]) >> QLOGIC_IB_RHF_EGRINDEX_SHIFT) &
QLOGIC_IB_RHF_EGRINDEX_MASK;
}
static inline __u32 qib_hdrget_seq(const __le32 *rbuf)
{
return (__le32_to_cpu(rbuf[1]) >> QLOGIC_IB_RHF_SEQ_SHIFT) &
QLOGIC_IB_RHF_SEQ_MASK;
}
static inline __u32 qib_hdrget_offset(const __le32 *rbuf)
{
return (__le32_to_cpu(rbuf[1]) >> QLOGIC_IB_RHF_HDRQ_OFFSET_SHIFT) &
QLOGIC_IB_RHF_HDRQ_OFFSET_MASK;
}
static inline __u32 qib_hdrget_use_egr_buf(const __le32 *rbuf)
{
return __le32_to_cpu(rbuf[0]) & QLOGIC_IB_RHF_L_USE_EGR;
}
#endif /* _QIB_COMMON_H */

274
drivers/infiniband/hw/qib/qib_debugfs.c
View file

@ -1,274 +0,0 @@
/*
* Copyright (c) 2013 - 2017 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/debugfs.h>
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/export.h>
#include "qib.h"
#include "qib_verbs.h"
#include "qib_debugfs.h"
static struct dentry *qib_dbg_root;
#define DEBUGFS_FILE(name) \
static const struct seq_operations _##name##_seq_ops = { \
.start = _##name##_seq_start, \
.next = _##name##_seq_next, \
.stop = _##name##_seq_stop, \
.show = _##name##_seq_show \
}; \
static int _##name##_open(struct inode *inode, struct file *s) \
{ \
struct seq_file *seq; \
int ret; \
ret = seq_open(s, &_##name##_seq_ops); \
if (ret) \
return ret; \
seq = s->private_data; \
seq->private = inode->i_private; \
return 0; \
} \
static const struct file_operations _##name##_file_ops = { \
.owner = THIS_MODULE, \
.open = _##name##_open, \
.read = seq_read, \
.llseek = seq_lseek, \
.release = seq_release \
};
static void *_opcode_stats_seq_start(struct seq_file *s, loff_t *pos)
{
struct qib_opcode_stats_perctx *opstats;
if (*pos >= ARRAY_SIZE(opstats->stats))
return NULL;
return pos;
}
static void *_opcode_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
{
struct qib_opcode_stats_perctx *opstats;
++*pos;
if (*pos >= ARRAY_SIZE(opstats->stats))
return NULL;
return pos;
}
static void _opcode_stats_seq_stop(struct seq_file *s, void *v)
{
/* nothing allocated */
}
static int _opcode_stats_seq_show(struct seq_file *s, void *v)
{
loff_t *spos = v;
loff_t i = *spos, j;
u64 n_packets = 0, n_bytes = 0;
struct qib_ibdev *ibd = (struct qib_ibdev *)s->private;
struct qib_devdata *dd = dd_from_dev(ibd);
for (j = 0; j < dd->first_user_ctxt; j++) {
if (!dd->rcd[j])
continue;
n_packets += dd->rcd[j]->opstats->stats[i].n_packets;
n_bytes += dd->rcd[j]->opstats->stats[i].n_bytes;
}
if (!n_packets && !n_bytes)
return SEQ_SKIP;
seq_printf(s, "%02llx %llu/%llu\n", i,
(unsigned long long) n_packets,
(unsigned long long) n_bytes);
return 0;
}
DEBUGFS_FILE(opcode_stats)
static void *_ctx_stats_seq_start(struct seq_file *s, loff_t *pos)
{
struct qib_ibdev *ibd = (struct qib_ibdev *)s->private;
struct qib_devdata *dd = dd_from_dev(ibd);
if (!*pos)
return SEQ_START_TOKEN;
if (*pos >= dd->first_user_ctxt)
return NULL;
return pos;
}
static void *_ctx_stats_seq_next(struct seq_file *s, void *v, loff_t *pos)
{
struct qib_ibdev *ibd = (struct qib_ibdev *)s->private;
struct qib_devdata *dd = dd_from_dev(ibd);
if (v == SEQ_START_TOKEN)
return pos;
++*pos;
if (*pos >= dd->first_user_ctxt)
return NULL;
return pos;
}
static void _ctx_stats_seq_stop(struct seq_file *s, void *v)
{
/* nothing allocated */
}
static int _ctx_stats_seq_show(struct seq_file *s, void *v)
{
loff_t *spos;
loff_t i, j;
u64 n_packets = 0;
struct qib_ibdev *ibd = (struct qib_ibdev *)s->private;
struct qib_devdata *dd = dd_from_dev(ibd);
if (v == SEQ_START_TOKEN) {
seq_puts(s, "Ctx:npkts\n");
return 0;
}
spos = v;
i = *spos;
if (!dd->rcd[i])
return SEQ_SKIP;
for (j = 0; j < ARRAY_SIZE(dd->rcd[i]->opstats->stats); j++)
n_packets += dd->rcd[i]->opstats->stats[j].n_packets;
if (!n_packets)
return SEQ_SKIP;
seq_printf(s, " %llu:%llu\n", i, n_packets);
return 0;
}
DEBUGFS_FILE(ctx_stats)
static void *_qp_stats_seq_start(struct seq_file *s, loff_t *pos)
__acquires(RCU)
{
struct rvt_qp_iter *iter;
loff_t n = *pos;
iter = rvt_qp_iter_init(s->private, 0, NULL);
/* stop calls rcu_read_unlock */
rcu_read_lock();
if (!iter)
return NULL;
do {
if (rvt_qp_iter_next(iter)) {
kfree(iter);
return NULL;
}
} while (n--);
return iter;
}
static void *_qp_stats_seq_next(struct seq_file *s, void *iter_ptr,
loff_t *pos)
__must_hold(RCU)
{
struct rvt_qp_iter *iter = iter_ptr;
(*pos)++;
if (rvt_qp_iter_next(iter)) {
kfree(iter);
return NULL;
}
return iter;
}
static void _qp_stats_seq_stop(struct seq_file *s, void *iter_ptr)
__releases(RCU)
{
rcu_read_unlock();
}
static int _qp_stats_seq_show(struct seq_file *s, void *iter_ptr)
{
struct rvt_qp_iter *iter = iter_ptr;
if (!iter)
return 0;
qib_qp_iter_print(s, iter);
return 0;
}
DEBUGFS_FILE(qp_stats)
void qib_dbg_ibdev_init(struct qib_ibdev *ibd)
{
struct dentry *root;
char name[10];
snprintf(name, sizeof(name), "qib%d", dd_from_dev(ibd)->unit);
root = debugfs_create_dir(name, qib_dbg_root);
ibd->qib_ibdev_dbg = root;
debugfs_create_file("opcode_stats", 0400, root, ibd,
&_opcode_stats_file_ops);
debugfs_create_file("ctx_stats", 0400, root, ibd, &_ctx_stats_file_ops);
debugfs_create_file("qp_stats", 0400, root, ibd, &_qp_stats_file_ops);
}
void qib_dbg_ibdev_exit(struct qib_ibdev *ibd)
{
if (!qib_dbg_root)
goto out;
debugfs_remove_recursive(ibd->qib_ibdev_dbg);
out:
ibd->qib_ibdev_dbg = NULL;
}
void qib_dbg_init(void)
{
qib_dbg_root = debugfs_create_dir(QIB_DRV_NAME, NULL);
}
void qib_dbg_exit(void)
{
debugfs_remove_recursive(qib_dbg_root);
qib_dbg_root = NULL;
}

45
drivers/infiniband/hw/qib/qib_debugfs.h
View file

@ -1,45 +0,0 @@
#ifndef _QIB_DEBUGFS_H
#define _QIB_DEBUGFS_H
#ifdef CONFIG_DEBUG_FS
/*
* Copyright (c) 2013 Intel Corporation. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
struct qib_ibdev;
void qib_dbg_ibdev_init(struct qib_ibdev *ibd);
void qib_dbg_ibdev_exit(struct qib_ibdev *ibd);
void qib_dbg_init(void);
void qib_dbg_exit(void);
#endif
#endif /* _QIB_DEBUGFS_H */

906
drivers/infiniband/hw/qib/qib_diag.c
View file

@ -1,906 +0,0 @@
/*
* Copyright (c) 2012 Intel Corporation. All rights reserved.
* Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/*
* This file contains support for diagnostic functions. It is accessed by
* opening the qib_diag device, normally minor number 129. Diagnostic use
* of the QLogic_IB chip may render the chip or board unusable until the
* driver is unloaded, or in some cases, until the system is rebooted.
*
* Accesses to the chip through this interface are not similar to going
* through the /sys/bus/pci resource mmap interface.
*/
#include <linux/io.h>
#include <linux/pci.h>
#include <linux/poll.h>
#include <linux/vmalloc.h>
#include <linux/export.h>
#include <linux/fs.h>
#include <linux/uaccess.h>
#include "qib.h"
#include "qib_common.h"
#undef pr_fmt
#define pr_fmt(fmt) QIB_DRV_NAME ": " fmt
/*
* Each client that opens the diag device must read then write
* offset 0, to prevent lossage from random cat or od. diag_state
* sequences this "handshake".
*/
enum diag_state { UNUSED = 0, OPENED, INIT, READY };
/* State for an individual client. PID so children cannot abuse handshake */
static struct qib_diag_client {
struct qib_diag_client *next;
struct qib_devdata *dd;
pid_t pid;
enum diag_state state;
} *client_pool;
/*
* Get a client struct. Recycled if possible, else kmalloc.
* Must be called with qib_mutex held
*/
static struct qib_diag_client *get_client(struct qib_devdata *dd)
{
struct qib_diag_client *dc;
dc = client_pool;
if (dc)
/* got from pool remove it and use */
client_pool = dc->next;
else
/* None in pool, alloc and init */
dc = kmalloc(sizeof(*dc), GFP_KERNEL);
if (dc) {
dc->next = NULL;
dc->dd = dd;
dc->pid = current->pid;
dc->state = OPENED;
}
return dc;
}
/*
* Return to pool. Must be called with qib_mutex held
*/
static void return_client(struct qib_diag_client *dc)
{
struct qib_devdata *dd = dc->dd;
struct qib_diag_client *tdc, *rdc;
rdc = NULL;
if (dc == dd->diag_client) {
dd->diag_client = dc->next;
rdc = dc;
} else {
tdc = dc->dd->diag_client;
while (tdc) {
if (dc == tdc->next) {
tdc->next = dc->next;
rdc = dc;
break;
}
tdc = tdc->next;
}
}
if (rdc) {
rdc->state = UNUSED;
rdc->dd = NULL;
rdc->pid = 0;
rdc->next = client_pool;
client_pool = rdc;
}
}
static int qib_diag_open(struct inode *in, struct file *fp);
static int qib_diag_release(struct inode *in, struct file *fp);
static ssize_t qib_diag_read(struct file *fp, char __user *data,
size_t count, loff_t *off);
static ssize_t qib_diag_write(struct file *fp, const char __user *data,
size_t count, loff_t *off);
static const struct file_operations diag_file_ops = {
.owner = THIS_MODULE,
.write = qib_diag_write,
.read = qib_diag_read,
.open = qib_diag_open,
.release = qib_diag_release,
.llseek = default_llseek,
};
static atomic_t diagpkt_count = ATOMIC_INIT(0);
static struct cdev *diagpkt_cdev;
static struct device *diagpkt_device;
static ssize_t qib_diagpkt_write(struct file *fp, const char __user *data,
size_t count, loff_t *off);
static const struct file_operations diagpkt_file_ops = {
.owner = THIS_MODULE,
.write = qib_diagpkt_write,
.llseek = noop_llseek,
};
int qib_diag_add(struct qib_devdata *dd)
{
char name[16];
int ret = 0;
if (atomic_inc_return(&diagpkt_count) == 1) {
ret = qib_cdev_init(QIB_DIAGPKT_MINOR, "ipath_diagpkt",
&diagpkt_file_ops, &diagpkt_cdev,
&diagpkt_device);
if (ret)
goto done;
}
snprintf(name, sizeof(name), "ipath_diag%d", dd->unit);
ret = qib_cdev_init(QIB_DIAG_MINOR_BASE + dd->unit, name,
&diag_file_ops, &dd->diag_cdev,
&dd->diag_device);
done:
return ret;
}
static void qib_unregister_observers(struct qib_devdata *dd);
void qib_diag_remove(struct qib_devdata *dd)
{
struct qib_diag_client *dc;
if (atomic_dec_and_test(&diagpkt_count))
qib_cdev_cleanup(&diagpkt_cdev, &diagpkt_device);
qib_cdev_cleanup(&dd->diag_cdev, &dd->diag_device);
/*
* Return all diag_clients of this device. There should be none,
* as we are "guaranteed" that no clients are still open
*/
while (dd->diag_client)
return_client(dd->diag_client);
/* Now clean up all unused client structs */
while (client_pool) {
dc = client_pool;
client_pool = dc->next;
kfree(dc);
}
/* Clean up observer list */
qib_unregister_observers(dd);
}
/* qib_remap_ioaddr32 - remap an offset into chip address space to __iomem *
*
* @dd: the qlogic_ib device
* @offs: the offset in chip-space
* @cntp: Pointer to max (byte) count for transfer starting at offset
* This returns a u32 __iomem * so it can be used for both 64 and 32-bit
* mapping. It is needed because with the use of PAT for control of
* write-combining, the logically contiguous address-space of the chip
* may be split into virtually non-contiguous spaces, with different
* attributes, which are them mapped to contiguous physical space
* based from the first BAR.
*
* The code below makes the same assumptions as were made in
* init_chip_wc_pat() (qib_init.c), copied here:
* Assumes chip address space looks like:
* - kregs + sregs + cregs + uregs (in any order)
* - piobufs (2K and 4K bufs in either order)
* or:
* - kregs + sregs + cregs (in any order)
* - piobufs (2K and 4K bufs in either order)
* - uregs
*
* If cntp is non-NULL, returns how many bytes from offset can be accessed
* Returns 0 if the offset is not mapped.
*/
static u32 __iomem *qib_remap_ioaddr32(struct qib_devdata *dd, u32 offset,
u32 *cntp)
{
u32 kreglen;
u32 snd_bottom, snd_lim = 0;
u32 __iomem *krb32 = (u32 __iomem *)dd->kregbase;
u32 __iomem *map = NULL;
u32 cnt = 0;
u32 tot4k, offs4k;
/* First, simplest case, offset is within the first map. */
kreglen = (dd->kregend - dd->kregbase) * sizeof(u64);
if (offset < kreglen) {
map = krb32 + (offset / sizeof(u32));
cnt = kreglen - offset;
goto mapped;
}
/*
* Next check for user regs, the next most common case,
* and a cheap check because if they are not in the first map
* they are last in chip.
*/
if (dd->userbase) {
/* If user regs mapped, they are after send, so set limit. */
u32 ulim = (dd->cfgctxts * dd->ureg_align) + dd->uregbase;
if (!dd->piovl15base)
snd_lim = dd->uregbase;
krb32 = (u32 __iomem *)dd->userbase;
if (offset >= dd->uregbase && offset < ulim) {
map = krb32 + (offset - dd->uregbase) / sizeof(u32);
cnt = ulim - offset;
goto mapped;
}
}
/*
* Lastly, check for offset within Send Buffers.
* This is gnarly because struct devdata is deliberately vague
* about things like 7322 VL15 buffers, and we are not in
* chip-specific code here, so should not make many assumptions.
* The one we _do_ make is that the only chip that has more sndbufs
* than we admit is the 7322, and it has userregs above that, so
* we know the snd_lim.
*/
/* Assume 2K buffers are first. */
snd_bottom = dd->pio2k_bufbase;
if (snd_lim == 0) {
u32 tot2k = dd->piobcnt2k * ALIGN(dd->piosize2k, dd->palign);
snd_lim = snd_bottom + tot2k;
}
/* If 4k buffers exist, account for them by bumping
* appropriate limit.
*/
tot4k = dd->piobcnt4k * dd->align4k;
offs4k = dd->piobufbase >> 32;
if (dd->piobcnt4k) {
if (snd_bottom > offs4k)
snd_bottom = offs4k;
else {
/* 4k above 2k. Bump snd_lim, if needed*/
if (!dd->userbase || dd->piovl15base)
snd_lim = offs4k + tot4k;
}
}
/*
* Judgement call: can we ignore the space between SendBuffs and
* UserRegs, where we would like to see vl15 buffs, but not more?
*/
if (offset >= snd_bottom && offset < snd_lim) {
offset -= snd_bottom;
map = (u32 __iomem *)dd->piobase + (offset / sizeof(u32));
cnt = snd_lim - offset;
}
if (!map && offs4k && dd->piovl15base) {
snd_lim = offs4k + tot4k + 2 * dd->align4k;
if (offset >= (offs4k + tot4k) && offset < snd_lim) {
map = (u32 __iomem *)dd->piovl15base +
((offset - (offs4k + tot4k)) / sizeof(u32));
cnt = snd_lim - offset;
}
}
mapped:
if (cntp)
*cntp = cnt;
return map;
}
/*
* qib_read_umem64 - read a 64-bit quantity from the chip into user space
* @dd: the qlogic_ib device
* @uaddr: the location to store the data in user memory
* @regoffs: the offset from BAR0 (_NOT_ full pointer, anymore)
* @count: number of bytes to copy (multiple of 32 bits)
*
* This function also localizes all chip memory accesses.
* The copy should be written such that we read full cacheline packets
* from the chip. This is usually used for a single qword
*
* NOTE: This assumes the chip address is 64-bit aligned.
*/
static int qib_read_umem64(struct qib_devdata *dd, void __user *uaddr,
u32 regoffs, size_t count)
{
const u64 __iomem *reg_addr;
const u64 __iomem *reg_end;
u32 limit;
int ret;
reg_addr = (const u64 __iomem *)qib_remap_ioaddr32(dd, regoffs, &limit);
if (reg_addr == NULL || limit == 0 || !(dd->flags & QIB_PRESENT)) {
ret = -EINVAL;
goto bail;
}
if (count >= limit)
count = limit;
reg_end = reg_addr + (count / sizeof(u64));
/* not very efficient, but it works for now */
while (reg_addr < reg_end) {
u64 data = readq(reg_addr);
if (copy_to_user(uaddr, &data, sizeof(u64))) {
ret = -EFAULT;
goto bail;
}
reg_addr++;
uaddr += sizeof(u64);
}
ret = 0;
bail:
return ret;
}
/*
* qib_write_umem64 - write a 64-bit quantity to the chip from user space
* @dd: the qlogic_ib device
* @regoffs: the offset from BAR0 (_NOT_ full pointer, anymore)
* @uaddr: the source of the data in user memory
* @count: the number of bytes to copy (multiple of 32 bits)
*
* This is usually used for a single qword
* NOTE: This assumes the chip address is 64-bit aligned.
*/
static int qib_write_umem64(struct qib_devdata *dd, u32 regoffs,
const void __user *uaddr, size_t count)
{
u64 __iomem *reg_addr;
const u64 __iomem *reg_end;
u32 limit;
int ret;
reg_addr = (u64 __iomem *)qib_remap_ioaddr32(dd, regoffs, &limit);
if (reg_addr == NULL || limit == 0 || !(dd->flags & QIB_PRESENT)) {
ret = -EINVAL;
goto bail;
}
if (count >= limit)
count = limit;
reg_end = reg_addr + (count / sizeof(u64));
/* not very efficient, but it works for now */
while (reg_addr < reg_end) {
u64 data;
if (copy_from_user(&data, uaddr, sizeof(data))) {
ret = -EFAULT;
goto bail;
}
writeq(data, reg_addr);
reg_addr++;
uaddr += sizeof(u64);
}
ret = 0;
bail:
return ret;
}
/*
* qib_read_umem32 - read a 32-bit quantity from the chip into user space
* @dd: the qlogic_ib device
* @uaddr: the location to store the data in user memory
* @regoffs: the offset from BAR0 (_NOT_ full pointer, anymore)
* @count: number of bytes to copy
*
* read 32 bit values, not 64 bit; for memories that only
* support 32 bit reads; usually a single dword.
*/
static int qib_read_umem32(struct qib_devdata *dd, void __user *uaddr,
u32 regoffs, size_t count)
{
const u32 __iomem *reg_addr;
const u32 __iomem *reg_end;
u32 limit;
int ret;
reg_addr = qib_remap_ioaddr32(dd, regoffs, &limit);
if (reg_addr == NULL || limit == 0 || !(dd->flags & QIB_PRESENT)) {
ret = -EINVAL;
goto bail;
}
if (count >= limit)
count = limit;
reg_end = reg_addr + (count / sizeof(u32));
/* not very efficient, but it works for now */
while (reg_addr < reg_end) {
u32 data = readl(reg_addr);
if (copy_to_user(uaddr, &data, sizeof(data))) {
ret = -EFAULT;
goto bail;
}
reg_addr++;
uaddr += sizeof(u32);
}
ret = 0;
bail:
return ret;
}
/*
* qib_write_umem32 - write a 32-bit quantity to the chip from user space
* @dd: the qlogic_ib device
* @regoffs: the offset from BAR0 (_NOT_ full pointer, anymore)
* @uaddr: the source of the data in user memory
* @count: number of bytes to copy
*
* write 32 bit values, not 64 bit; for memories that only
* support 32 bit write; usually a single dword.
*/
static int qib_write_umem32(struct qib_devdata *dd, u32 regoffs,
const void __user *uaddr, size_t count)
{
u32 __iomem *reg_addr;
const u32 __iomem *reg_end;
u32 limit;
int ret;
reg_addr = qib_remap_ioaddr32(dd, regoffs, &limit);
if (reg_addr == NULL || limit == 0 || !(dd->flags & QIB_PRESENT)) {
ret = -EINVAL;
goto bail;
}
if (count >= limit)
count = limit;
reg_end = reg_addr + (count / sizeof(u32));
while (reg_addr < reg_end) {
u32 data;
if (copy_from_user(&data, uaddr, sizeof(data))) {
ret = -EFAULT;
goto bail;
}
writel(data, reg_addr);
reg_addr++;
uaddr += sizeof(u32);
}
ret = 0;
bail:
return ret;
}
static int qib_diag_open(struct inode *in, struct file *fp)
{
int unit = iminor(in) - QIB_DIAG_MINOR_BASE;
struct qib_devdata *dd;
struct qib_diag_client *dc;
int ret;
mutex_lock(&qib_mutex);
dd = qib_lookup(unit);
if (dd == NULL || !(dd->flags & QIB_PRESENT) ||
!dd->kregbase) {
ret = -ENODEV;
goto bail;
}
dc = get_client(dd);
if (!dc) {
ret = -ENOMEM;
goto bail;
}
dc->next = dd->diag_client;
dd->diag_client = dc;
fp->private_data = dc;
ret = 0;
bail:
mutex_unlock(&qib_mutex);
return ret;
}
/**
* qib_diagpkt_write - write an IB packet
* @fp: the diag data device file pointer
* @data: qib_diag_pkt structure saying where to get the packet
* @count: size of data to write
* @off: unused by this code
*/
static ssize_t qib_diagpkt_write(struct file *fp,
const char __user *data,
size_t count, loff_t *off)
{
u32 __iomem *piobuf;
u32 plen, pbufn, maxlen_reserve;
struct qib_diag_xpkt dp;
u32 *tmpbuf = NULL;
struct qib_devdata *dd;
struct qib_pportdata *ppd;
ssize_t ret = 0;
if (count != sizeof(dp)) {
ret = -EINVAL;
goto bail;
}
if (copy_from_user(&dp, data, sizeof(dp))) {
ret = -EFAULT;
goto bail;
}
dd = qib_lookup(dp.unit);
if (!dd || !(dd->flags & QIB_PRESENT) || !dd->kregbase) {
ret = -ENODEV;
goto bail;
}
if (!(dd->flags & QIB_INITTED)) {
/* no hardware, freeze, etc. */
ret = -ENODEV;
goto bail;
}
if (dp.version != _DIAG_XPKT_VERS) {
qib_dev_err(dd, "Invalid version %u for diagpkt_write\n",
dp.version);
ret = -EINVAL;
goto bail;
}
/* send count must be an exact number of dwords */
if (dp.len & 3) {
ret = -EINVAL;
goto bail;
}
if (!dp.port || dp.port > dd->num_pports) {
ret = -EINVAL;
goto bail;
}
ppd = &dd->pport[dp.port - 1];
/*
* need total length before first word written, plus 2 Dwords. One Dword
* is for padding so we get the full user data when not aligned on
* a word boundary. The other Dword is to make sure we have room for the
* ICRC which gets tacked on later.
*/
maxlen_reserve = 2 * sizeof(u32);
if (dp.len > ppd->ibmaxlen - maxlen_reserve) {
ret = -EINVAL;
goto bail;
}
plen = sizeof(u32) + dp.len;
tmpbuf = vmalloc(plen);
if (!tmpbuf) {
ret = -ENOMEM;
goto bail;
}
if (copy_from_user(tmpbuf,
u64_to_user_ptr(dp.data),
dp.len)) {
ret = -EFAULT;
goto bail;
}
plen >>= 2; /* in dwords */
if (dp.pbc_wd == 0)
dp.pbc_wd = plen;
piobuf = dd->f_getsendbuf(ppd, dp.pbc_wd, &pbufn);
if (!piobuf) {
ret = -EBUSY;
goto bail;
}
/* disarm it just to be extra sure */
dd->f_sendctrl(dd->pport, QIB_SENDCTRL_DISARM_BUF(pbufn));
/* disable header check on pbufn for this packet */
dd->f_txchk_change(dd, pbufn, 1, TXCHK_CHG_TYPE_DIS1, NULL);
writeq(dp.pbc_wd, piobuf);
/*
* Copy all but the trigger word, then flush, so it's written
* to chip before trigger word, then write trigger word, then
* flush again, so packet is sent.
*/
if (dd->flags & QIB_PIO_FLUSH_WC) {
qib_flush_wc();
qib_pio_copy(piobuf + 2, tmpbuf, plen - 1);
qib_flush_wc();
__raw_writel(tmpbuf[plen - 1], piobuf + plen + 1);
} else
qib_pio_copy(piobuf + 2, tmpbuf, plen);
if (dd->flags & QIB_USE_SPCL_TRIG) {
u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023;
qib_flush_wc();
__raw_writel(0xaebecede, piobuf + spcl_off);
}
/*
* Ensure buffer is written to the chip, then re-enable
* header checks (if supported by chip). The txchk
* code will ensure seen by chip before returning.
*/
qib_flush_wc();
qib_sendbuf_done(dd, pbufn);
dd->f_txchk_change(dd, pbufn, 1, TXCHK_CHG_TYPE_ENAB1, NULL);
ret = sizeof(dp);
bail:
vfree(tmpbuf);
return ret;
}
static int qib_diag_release(struct inode *in, struct file *fp)
{
mutex_lock(&qib_mutex);
return_client(fp->private_data);
fp->private_data = NULL;
mutex_unlock(&qib_mutex);
return 0;
}
/*
* Chip-specific code calls to register its interest in
* a specific range.
*/
struct diag_observer_list_elt {
struct diag_observer_list_elt *next;
const struct diag_observer *op;
};
int qib_register_observer(struct qib_devdata *dd,
const struct diag_observer *op)
{
struct diag_observer_list_elt *olp;
unsigned long flags;
if (!dd || !op)
return -EINVAL;
olp = vmalloc(sizeof(*olp));
if (!olp)
return -ENOMEM;
spin_lock_irqsave(&dd->qib_diag_trans_lock, flags);
olp->op = op;
olp->next = dd->diag_observer_list;
dd->diag_observer_list = olp;
spin_unlock_irqrestore(&dd->qib_diag_trans_lock, flags);
return 0;
}
/* Remove all registered observers when device is closed */
static void qib_unregister_observers(struct qib_devdata *dd)
{
struct diag_observer_list_elt *olp;
unsigned long flags;
spin_lock_irqsave(&dd->qib_diag_trans_lock, flags);
olp = dd->diag_observer_list;
while (olp) {
/* Pop one observer, let go of lock */
dd->diag_observer_list = olp->next;
spin_unlock_irqrestore(&dd->qib_diag_trans_lock, flags);
vfree(olp);
/* try again. */
spin_lock_irqsave(&dd->qib_diag_trans_lock, flags);
olp = dd->diag_observer_list;
}
spin_unlock_irqrestore(&dd->qib_diag_trans_lock, flags);
}
/*
* Find the observer, if any, for the specified address. Initial implementation
* is simple stack of observers. This must be called with diag transaction
* lock held.
*/
static const struct diag_observer *diag_get_observer(struct qib_devdata *dd,
u32 addr)
{
struct diag_observer_list_elt *olp;
const struct diag_observer *op = NULL;
olp = dd->diag_observer_list;
while (olp) {
op = olp->op;
if (addr >= op->bottom && addr <= op->top)
break;
olp = olp->next;
}
if (!olp)
op = NULL;
return op;
}
static ssize_t qib_diag_read(struct file *fp, char __user *data,
size_t count, loff_t *off)
{
struct qib_diag_client *dc = fp->private_data;
struct qib_devdata *dd = dc->dd;
ssize_t ret;
if (dc->pid != current->pid) {
ret = -EPERM;
goto bail;
}
if (count == 0)
ret = 0;
else if ((count % 4) || (*off % 4))
/* address or length is not 32-bit aligned, hence invalid */
ret = -EINVAL;
else if (dc->state < READY && (*off || count != 8))
ret = -EINVAL; /* prevent cat /dev/qib_diag* */
else {
unsigned long flags;
u64 data64 = 0;
int use_32;
const struct diag_observer *op;
use_32 = (count % 8) || (*off % 8);
ret = -1;
spin_lock_irqsave(&dd->qib_diag_trans_lock, flags);
/*
* Check for observer on this address range.
* we only support a single 32 or 64-bit read
* via observer, currently.
*/
op = diag_get_observer(dd, *off);
if (op) {
u32 offset = *off;
ret = op->hook(dd, op, offset, &data64, 0, use_32);
}
/*
* We need to release lock before any copy_to_user(),
* whether implicit in qib_read_umem* or explicit below.
*/
spin_unlock_irqrestore(&dd->qib_diag_trans_lock, flags);
if (!op) {
if (use_32)
/*
* Address or length is not 64-bit aligned;
* do 32-bit rd
*/
ret = qib_read_umem32(dd, data, (u32) *off,
count);
else
ret = qib_read_umem64(dd, data, (u32) *off,
count);
} else if (ret == count) {
/* Below finishes case where observer existed */
ret = copy_to_user(data, &data64, use_32 ?
sizeof(u32) : sizeof(u64));
if (ret)
ret = -EFAULT;
}
}
if (ret >= 0) {
*off += count;
ret = count;
if (dc->state == OPENED)
dc->state = INIT;
}
bail:
return ret;
}
static ssize_t qib_diag_write(struct file *fp, const char __user *data,
size_t count, loff_t *off)
{
struct qib_diag_client *dc = fp->private_data;
struct qib_devdata *dd = dc->dd;
ssize_t ret;
if (dc->pid != current->pid) {
ret = -EPERM;
goto bail;
}
if (count == 0)
ret = 0;
else if ((count % 4) || (*off % 4))
/* address or length is not 32-bit aligned, hence invalid */
ret = -EINVAL;
else if (dc->state < READY &&
((*off || count != 8) || dc->state != INIT))
/* No writes except second-step of init seq */
ret = -EINVAL; /* before any other write allowed */
else {
unsigned long flags;
const struct diag_observer *op = NULL;
int use_32 = (count % 8) || (*off % 8);
/*
* Check for observer on this address range.
* We only support a single 32 or 64-bit write
* via observer, currently. This helps, because
* we would otherwise have to jump through hoops
* to make "diag transaction" meaningful when we
* cannot do a copy_from_user while holding the lock.
*/
if (count == 4 || count == 8) {
u64 data64;
u32 offset = *off;
ret = copy_from_user(&data64, data, count);
if (ret) {
ret = -EFAULT;
goto bail;
}
spin_lock_irqsave(&dd->qib_diag_trans_lock, flags);
op = diag_get_observer(dd, *off);
if (op)
ret = op->hook(dd, op, offset, &data64, ~0Ull,
use_32);
spin_unlock_irqrestore(&dd->qib_diag_trans_lock, flags);
}
if (!op) {
if (use_32)
/*
* Address or length is not 64-bit aligned;
* do 32-bit write
*/
ret = qib_write_umem32(dd, (u32) *off, data,
count);
else
ret = qib_write_umem64(dd, (u32) *off, data,
count);
}
}
if (ret >= 0) {
*off += count;
ret = count;
if (dc->state == INIT)
dc->state = READY; /* all read/write OK now */
}
bail:
return ret;
}

798
drivers/infiniband/hw/qib/qib_driver.c
View file

@ -1,798 +0,0 @@
/*
* Copyright (c) 2021 Cornelis Networks. All rights reserved.
* Copyright (c) 2013 Intel Corporation. All rights reserved.
* Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/spinlock.h>
#include <linux/pci.h>
#include <linux/io.h>
#include <linux/delay.h>
#include <linux/netdevice.h>
#include <linux/vmalloc.h>
#include <linux/module.h>
#include <linux/prefetch.h>
#include "qib.h"
DEFINE_MUTEX(qib_mutex); /* general driver use */
unsigned qib_ibmtu;
module_param_named(ibmtu, qib_ibmtu, uint, S_IRUGO);
MODULE_PARM_DESC(ibmtu, "Set max IB MTU (0=2KB, 1=256, 2=512, ... 5=4096");
unsigned qib_compat_ddr_negotiate = 1;
module_param_named(compat_ddr_negotiate, qib_compat_ddr_negotiate, uint,
S_IWUSR | S_IRUGO);
MODULE_PARM_DESC(compat_ddr_negotiate,
"Attempt pre-IBTA 1.2 DDR speed negotiation");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Cornelis <support@cornelisnetworks.com>");
MODULE_DESCRIPTION("Cornelis IB driver");
/*
* QIB_PIO_MAXIBHDR is the max IB header size allowed for in our
* PIO send buffers. This is well beyond anything currently
* defined in the InfiniBand spec.
*/
#define QIB_PIO_MAXIBHDR 128
/*
* QIB_MAX_PKT_RCV is the max # if packets processed per receive interrupt.
*/
#define QIB_MAX_PKT_RECV 64
struct qlogic_ib_stats qib_stats;
struct pci_dev *qib_get_pci_dev(struct rvt_dev_info *rdi)
{
struct qib_ibdev *ibdev = container_of(rdi, struct qib_ibdev, rdi);
struct qib_devdata *dd = container_of(ibdev,
struct qib_devdata, verbs_dev);
return dd->pcidev;
}
/*
* Return count of units with at least one port ACTIVE.
*/
int qib_count_active_units(void)
{
struct qib_devdata *dd;
struct qib_pportdata *ppd;
unsigned long index, flags;
int pidx, nunits_active = 0;
xa_lock_irqsave(&qib_dev_table, flags);
xa_for_each(&qib_dev_table, index, dd) {
if (!(dd->flags & QIB_PRESENT) || !dd->kregbase)
continue;
for (pidx = 0; pidx < dd->num_pports; ++pidx) {
ppd = dd->pport + pidx;
if (ppd->lid && (ppd->lflags & (QIBL_LINKINIT |
QIBL_LINKARMED | QIBL_LINKACTIVE))) {
nunits_active++;
break;
}
}
}
xa_unlock_irqrestore(&qib_dev_table, flags);
return nunits_active;
}
/*
* Return count of all units, optionally return in arguments
* the number of usable (present) units, and the number of
* ports that are up.
*/
int qib_count_units(int *npresentp, int *nupp)
{
int nunits = 0, npresent = 0, nup = 0;
struct qib_devdata *dd;
unsigned long index, flags;
int pidx;
struct qib_pportdata *ppd;
xa_lock_irqsave(&qib_dev_table, flags);
xa_for_each(&qib_dev_table, index, dd) {
nunits++;
if ((dd->flags & QIB_PRESENT) && dd->kregbase)
npresent++;
for (pidx = 0; pidx < dd->num_pports; ++pidx) {
ppd = dd->pport + pidx;
if (ppd->lid && (ppd->lflags & (QIBL_LINKINIT |
QIBL_LINKARMED | QIBL_LINKACTIVE)))
nup++;
}
}
xa_unlock_irqrestore(&qib_dev_table, flags);
if (npresentp)
*npresentp = npresent;
if (nupp)
*nupp = nup;
return nunits;
}
/**
* qib_wait_linkstate - wait for an IB link state change to occur
* @ppd: the qlogic_ib device
* @state: the state to wait for
* @msecs: the number of milliseconds to wait
*
* wait up to msecs milliseconds for IB link state change to occur for
* now, take the easy polling route. Currently used only by
* qib_set_linkstate. Returns 0 if state reached, otherwise
* -ETIMEDOUT state can have multiple states set, for any of several
* transitions.
*/
int qib_wait_linkstate(struct qib_pportdata *ppd, u32 state, int msecs)
{
int ret;
unsigned long flags;
spin_lock_irqsave(&ppd->lflags_lock, flags);
if (ppd->state_wanted) {
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
ret = -EBUSY;
goto bail;
}
ppd->state_wanted = state;
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
wait_event_interruptible_timeout(ppd->state_wait,
(ppd->lflags & state),
msecs_to_jiffies(msecs));
spin_lock_irqsave(&ppd->lflags_lock, flags);
ppd->state_wanted = 0;
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
if (!(ppd->lflags & state))
ret = -ETIMEDOUT;
else
ret = 0;
bail:
return ret;
}
int qib_set_linkstate(struct qib_pportdata *ppd, u8 newstate)
{
u32 lstate;
int ret;
struct qib_devdata *dd = ppd->dd;
unsigned long flags;
switch (newstate) {
case QIB_IB_LINKDOWN_ONLY:
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LSTATE,
IB_LINKCMD_DOWN | IB_LINKINITCMD_NOP);
/* don't wait */
ret = 0;
goto bail;
case QIB_IB_LINKDOWN:
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LSTATE,
IB_LINKCMD_DOWN | IB_LINKINITCMD_POLL);
/* don't wait */
ret = 0;
goto bail;
case QIB_IB_LINKDOWN_SLEEP:
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LSTATE,
IB_LINKCMD_DOWN | IB_LINKINITCMD_SLEEP);
/* don't wait */
ret = 0;
goto bail;
case QIB_IB_LINKDOWN_DISABLE:
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LSTATE,
IB_LINKCMD_DOWN | IB_LINKINITCMD_DISABLE);
/* don't wait */
ret = 0;
goto bail;
case QIB_IB_LINKARM:
if (ppd->lflags & QIBL_LINKARMED) {
ret = 0;
goto bail;
}
if (!(ppd->lflags & (QIBL_LINKINIT | QIBL_LINKACTIVE))) {
ret = -EINVAL;
goto bail;
}
/*
* Since the port can be ACTIVE when we ask for ARMED,
* clear QIBL_LINKV so we can wait for a transition.
* If the link isn't ARMED, then something else happened
* and there is no point waiting for ARMED.
*/
spin_lock_irqsave(&ppd->lflags_lock, flags);
ppd->lflags &= ~QIBL_LINKV;
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LSTATE,
IB_LINKCMD_ARMED | IB_LINKINITCMD_NOP);
lstate = QIBL_LINKV;
break;
case QIB_IB_LINKACTIVE:
if (ppd->lflags & QIBL_LINKACTIVE) {
ret = 0;
goto bail;
}
if (!(ppd->lflags & QIBL_LINKARMED)) {
ret = -EINVAL;
goto bail;
}
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LSTATE,
IB_LINKCMD_ACTIVE | IB_LINKINITCMD_NOP);
lstate = QIBL_LINKACTIVE;
break;
default:
ret = -EINVAL;
goto bail;
}
ret = qib_wait_linkstate(ppd, lstate, 10);
bail:
return ret;
}
/*
* Get address of eager buffer from it's index (allocated in chunks, not
* contiguous).
*/
static inline void *qib_get_egrbuf(const struct qib_ctxtdata *rcd, u32 etail)
{
const u32 chunk = etail >> rcd->rcvegrbufs_perchunk_shift;
const u32 idx = etail & ((u32)rcd->rcvegrbufs_perchunk - 1);
return rcd->rcvegrbuf[chunk] + (idx << rcd->dd->rcvegrbufsize_shift);
}
/*
* Returns 1 if error was a CRC, else 0.
* Needed for some chip's synthesized error counters.
*/
static u32 qib_rcv_hdrerr(struct qib_ctxtdata *rcd, struct qib_pportdata *ppd,
u32 ctxt, u32 eflags, u32 l, u32 etail,
__le32 *rhf_addr, struct qib_message_header *rhdr)
{
u32 ret = 0;
if (eflags & (QLOGIC_IB_RHF_H_ICRCERR | QLOGIC_IB_RHF_H_VCRCERR))
ret = 1;
else if (eflags == QLOGIC_IB_RHF_H_TIDERR) {
/* For TIDERR and RC QPs premptively schedule a NAK */
struct ib_header *hdr = (struct ib_header *)rhdr;
struct ib_other_headers *ohdr = NULL;
struct qib_ibport *ibp = &ppd->ibport_data;
struct qib_devdata *dd = ppd->dd;
struct rvt_dev_info *rdi = &dd->verbs_dev.rdi;
struct rvt_qp *qp = NULL;
u32 tlen = qib_hdrget_length_in_bytes(rhf_addr);
u16 lid = be16_to_cpu(hdr->lrh[1]);
int lnh = be16_to_cpu(hdr->lrh[0]) & 3;
u32 qp_num;
u32 opcode;
u32 psn;
int diff;
/* Sanity check packet */
if (tlen < 24)
goto drop;
if (lid < be16_to_cpu(IB_MULTICAST_LID_BASE)) {
lid &= ~((1 << ppd->lmc) - 1);
if (unlikely(lid != ppd->lid))
goto drop;
}
/* Check for GRH */
if (lnh == QIB_LRH_BTH)
ohdr = &hdr->u.oth;
else if (lnh == QIB_LRH_GRH) {
u32 vtf;
ohdr = &hdr->u.l.oth;
if (hdr->u.l.grh.next_hdr != IB_GRH_NEXT_HDR)
goto drop;
vtf = be32_to_cpu(hdr->u.l.grh.version_tclass_flow);
if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
goto drop;
} else
goto drop;
/* Get opcode and PSN from packet */
opcode = be32_to_cpu(ohdr->bth[0]);
opcode >>= 24;
psn = be32_to_cpu(ohdr->bth[2]);
/* Get the destination QP number. */
qp_num = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
if (qp_num != QIB_MULTICAST_QPN) {
int ruc_res;
rcu_read_lock();
qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
if (!qp) {
rcu_read_unlock();
goto drop;
}
/*
* Handle only RC QPs - for other QP types drop error
* packet.
*/
spin_lock(&qp->r_lock);
/* Check for valid receive state. */
if (!(ib_rvt_state_ops[qp->state] &
RVT_PROCESS_RECV_OK)) {
ibp->rvp.n_pkt_drops++;
goto unlock;
}
switch (qp->ibqp.qp_type) {
case IB_QPT_RC:
ruc_res =
qib_ruc_check_hdr(
ibp, hdr,
lnh == QIB_LRH_GRH,
qp,
be32_to_cpu(ohdr->bth[0]));
if (ruc_res)
goto unlock;
/* Only deal with RDMA Writes for now */
if (opcode <
IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) {
diff = qib_cmp24(psn, qp->r_psn);
if (!qp->r_nak_state && diff >= 0) {
ibp->rvp.n_rc_seqnak++;
qp->r_nak_state =
IB_NAK_PSN_ERROR;
/* Use the expected PSN. */
qp->r_ack_psn = qp->r_psn;
/*
* Wait to send the sequence
* NAK until all packets
* in the receive queue have
* been processed.
* Otherwise, we end up
* propagating congestion.
*/
if (list_empty(&qp->rspwait)) {
qp->r_flags |=
RVT_R_RSP_NAK;
rvt_get_qp(qp);
list_add_tail(
&qp->rspwait,
&rcd->qp_wait_list);
}
} /* Out of sequence NAK */
} /* QP Request NAKs */
break;
case IB_QPT_SMI:
case IB_QPT_GSI:
case IB_QPT_UD:
case IB_QPT_UC:
default:
/* For now don't handle any other QP types */
break;
}
unlock:
spin_unlock(&qp->r_lock);
rcu_read_unlock();
} /* Unicast QP */
} /* Valid packet with TIDErr */
drop:
return ret;
}
/*
* qib_kreceive - receive a packet
* @rcd: the qlogic_ib context
* @llic: gets count of good packets needed to clear lli,
* (used with chips that need need to track crcs for lli)
*
* called from interrupt handler for errors or receive interrupt
* Returns number of CRC error packets, needed by some chips for
* local link integrity tracking. crcs are adjusted down by following
* good packets, if any, and count of good packets is also tracked.
*/
u32 qib_kreceive(struct qib_ctxtdata *rcd, u32 *llic, u32 *npkts)
{
struct qib_devdata *dd = rcd->dd;
struct qib_pportdata *ppd = rcd->ppd;
__le32 *rhf_addr;
void *ebuf;
const u32 rsize = dd->rcvhdrentsize; /* words */
const u32 maxcnt = dd->rcvhdrcnt * rsize; /* words */
u32 etail = -1, l, hdrqtail;
struct qib_message_header *hdr;
u32 eflags, etype, tlen, i = 0, updegr = 0, crcs = 0;
int last;
u64 lval;
struct rvt_qp *qp, *nqp;
l = rcd->head;
rhf_addr = (__le32 *) rcd->rcvhdrq + l + dd->rhf_offset;
if (dd->flags & QIB_NODMA_RTAIL) {
u32 seq = qib_hdrget_seq(rhf_addr);
if (seq != rcd->seq_cnt)
goto bail;
hdrqtail = 0;
} else {
hdrqtail = qib_get_rcvhdrtail(rcd);
if (l == hdrqtail)
goto bail;
smp_rmb(); /* prevent speculative reads of dma'ed hdrq */
}
for (last = 0, i = 1; !last; i += !last) {
hdr = dd->f_get_msgheader(dd, rhf_addr);
eflags = qib_hdrget_err_flags(rhf_addr);
etype = qib_hdrget_rcv_type(rhf_addr);
/* total length */
tlen = qib_hdrget_length_in_bytes(rhf_addr);
ebuf = NULL;
if ((dd->flags & QIB_NODMA_RTAIL) ?
qib_hdrget_use_egr_buf(rhf_addr) :
(etype != RCVHQ_RCV_TYPE_EXPECTED)) {
etail = qib_hdrget_index(rhf_addr);
updegr = 1;
if (tlen > sizeof(*hdr) ||
etype >= RCVHQ_RCV_TYPE_NON_KD) {
ebuf = qib_get_egrbuf(rcd, etail);
prefetch_range(ebuf, tlen - sizeof(*hdr));
}
}
if (!eflags) {
u16 lrh_len = be16_to_cpu(hdr->lrh[2]) << 2;
if (lrh_len != tlen) {
qib_stats.sps_lenerrs++;
goto move_along;
}
}
if (etype == RCVHQ_RCV_TYPE_NON_KD && !eflags &&
ebuf == NULL &&
tlen > (dd->rcvhdrentsize - 2 + 1 -
qib_hdrget_offset(rhf_addr)) << 2) {
goto move_along;
}
/*
* Both tiderr and qibhdrerr are set for all plain IB
* packets; only qibhdrerr should be set.
*/
if (unlikely(eflags))
crcs += qib_rcv_hdrerr(rcd, ppd, rcd->ctxt, eflags, l,
etail, rhf_addr, hdr);
else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
qib_ib_rcv(rcd, hdr, ebuf, tlen);
if (crcs)
crcs--;
else if (llic && *llic)
--*llic;
}
move_along:
l += rsize;
if (l >= maxcnt)
l = 0;
if (i == QIB_MAX_PKT_RECV)
last = 1;
rhf_addr = (__le32 *) rcd->rcvhdrq + l + dd->rhf_offset;
if (dd->flags & QIB_NODMA_RTAIL) {
u32 seq = qib_hdrget_seq(rhf_addr);
if (++rcd->seq_cnt > 13)
rcd->seq_cnt = 1;
if (seq != rcd->seq_cnt)
last = 1;
} else if (l == hdrqtail)
last = 1;
/*
* Update head regs etc., every 16 packets, if not last pkt,
* to help prevent rcvhdrq overflows, when many packets
* are processed and queue is nearly full.
* Don't request an interrupt for intermediate updates.
*/
lval = l;
if (!last && !(i & 0xf)) {
dd->f_update_usrhead(rcd, lval, updegr, etail, i);
updegr = 0;
}
}
rcd->head = l;
/*
* Iterate over all QPs waiting to respond.
* The list won't change since the IRQ is only run on one CPU.
*/
list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
list_del_init(&qp->rspwait);
if (qp->r_flags & RVT_R_RSP_NAK) {
qp->r_flags &= ~RVT_R_RSP_NAK;
qib_send_rc_ack(qp);
}
if (qp->r_flags & RVT_R_RSP_SEND) {
unsigned long flags;
qp->r_flags &= ~RVT_R_RSP_SEND;
spin_lock_irqsave(&qp->s_lock, flags);
if (ib_rvt_state_ops[qp->state] &
RVT_PROCESS_OR_FLUSH_SEND)
qib_schedule_send(qp);
spin_unlock_irqrestore(&qp->s_lock, flags);
}
rvt_put_qp(qp);
}
bail:
/* Report number of packets consumed */
if (npkts)
*npkts = i;
/*
* Always write head at end, and setup rcv interrupt, even
* if no packets were processed.
*/
lval = (u64)rcd->head | dd->rhdrhead_intr_off;
dd->f_update_usrhead(rcd, lval, updegr, etail, i);
return crcs;
}
/**
* qib_set_mtu - set the MTU
* @ppd: the perport data
* @arg: the new MTU
*
* We can handle "any" incoming size, the issue here is whether we
* need to restrict our outgoing size. For now, we don't do any
* sanity checking on this, and we don't deal with what happens to
* programs that are already running when the size changes.
* NOTE: changing the MTU will usually cause the IBC to go back to
* link INIT state...
*/
int qib_set_mtu(struct qib_pportdata *ppd, u16 arg)
{
u32 piosize;
int ret, chk;
if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
arg != 4096) {
ret = -EINVAL;
goto bail;
}
chk = ib_mtu_enum_to_int(qib_ibmtu);
if (chk > 0 && arg > chk) {
ret = -EINVAL;
goto bail;
}
piosize = ppd->ibmaxlen;
ppd->ibmtu = arg;
if (arg >= (piosize - QIB_PIO_MAXIBHDR)) {
/* Only if it's not the initial value (or reset to it) */
if (piosize != ppd->init_ibmaxlen) {
if (arg > piosize && arg <= ppd->init_ibmaxlen)
piosize = ppd->init_ibmaxlen - 2 * sizeof(u32);
ppd->ibmaxlen = piosize;
}
} else if ((arg + QIB_PIO_MAXIBHDR) != ppd->ibmaxlen) {
piosize = arg + QIB_PIO_MAXIBHDR - 2 * sizeof(u32);
ppd->ibmaxlen = piosize;
}
ppd->dd->f_set_ib_cfg(ppd, QIB_IB_CFG_MTU, 0);
ret = 0;
bail:
return ret;
}
int qib_set_lid(struct qib_pportdata *ppd, u32 lid, u8 lmc)
{
struct qib_devdata *dd = ppd->dd;
ppd->lid = lid;
ppd->lmc = lmc;
dd->f_set_ib_cfg(ppd, QIB_IB_CFG_LIDLMC,
lid | (~((1U << lmc) - 1)) << 16);
qib_devinfo(dd->pcidev, "IB%u:%u got a lid: 0x%x\n",
dd->unit, ppd->port, lid);
return 0;
}
/*
* Following deal with the "obviously simple" task of overriding the state
* of the LEDS, which normally indicate link physical and logical status.
* The complications arise in dealing with different hardware mappings
* and the board-dependent routine being called from interrupts.
* and then there's the requirement to _flash_ them.
*/
#define LED_OVER_FREQ_SHIFT 8
#define LED_OVER_FREQ_MASK (0xFF<<LED_OVER_FREQ_SHIFT)
/* Below is "non-zero" to force override, but both actual LEDs are off */
#define LED_OVER_BOTH_OFF (8)
static void qib_run_led_override(struct timer_list *t)
{
struct qib_pportdata *ppd = timer_container_of(ppd, t,
led_override_timer);
struct qib_devdata *dd = ppd->dd;
int timeoff;
int ph_idx;
if (!(dd->flags & QIB_INITTED))
return;
ph_idx = ppd->led_override_phase++ & 1;
ppd->led_override = ppd->led_override_vals[ph_idx];
timeoff = ppd->led_override_timeoff;
dd->f_setextled(ppd, 1);
/*
* don't re-fire the timer if user asked for it to be off; we let
* it fire one more time after they turn it off to simplify
*/
if (ppd->led_override_vals[0] || ppd->led_override_vals[1])
mod_timer(&ppd->led_override_timer, jiffies + timeoff);
}
void qib_set_led_override(struct qib_pportdata *ppd, unsigned int val)
{
struct qib_devdata *dd = ppd->dd;
int timeoff, freq;
if (!(dd->flags & QIB_INITTED))
return;
/* First check if we are blinking. If not, use 1HZ polling */
timeoff = HZ;
freq = (val & LED_OVER_FREQ_MASK) >> LED_OVER_FREQ_SHIFT;
if (freq) {
/* For blink, set each phase from one nybble of val */
ppd->led_override_vals[0] = val & 0xF;
ppd->led_override_vals[1] = (val >> 4) & 0xF;
timeoff = (HZ << 4)/freq;
} else {
/* Non-blink set both phases the same. */
ppd->led_override_vals[0] = val & 0xF;
ppd->led_override_vals[1] = val & 0xF;
}
ppd->led_override_timeoff = timeoff;
/*
* If the timer has not already been started, do so. Use a "quick"
* timeout so the function will be called soon, to look at our request.
*/
if (atomic_inc_return(&ppd->led_override_timer_active) == 1) {
/* Need to start timer */
timer_setup(&ppd->led_override_timer, qib_run_led_override, 0);
ppd->led_override_timer.expires = jiffies + 1;
add_timer(&ppd->led_override_timer);
} else {
if (ppd->led_override_vals[0] || ppd->led_override_vals[1])
mod_timer(&ppd->led_override_timer, jiffies + 1);
atomic_dec(&ppd->led_override_timer_active);
}
}
/**
* qib_reset_device - reset the chip if possible
* @unit: the device to reset
*
* Whether or not reset is successful, we attempt to re-initialize the chip
* (that is, much like a driver unload/reload). We clear the INITTED flag
* so that the various entry points will fail until we reinitialize. For
* now, we only allow this if no user contexts are open that use chip resources
*/
int qib_reset_device(int unit)
{
int ret, i;
struct qib_devdata *dd = qib_lookup(unit);
struct qib_pportdata *ppd;
unsigned long flags;
int pidx;
if (!dd) {
ret = -ENODEV;
goto bail;
}
qib_devinfo(dd->pcidev, "Reset on unit %u requested\n", unit);
if (!dd->kregbase || !(dd->flags & QIB_PRESENT)) {
qib_devinfo(dd->pcidev,
"Invalid unit number %u or not initialized or not present\n",
unit);
ret = -ENXIO;
goto bail;
}
spin_lock_irqsave(&dd->uctxt_lock, flags);
if (dd->rcd)
for (i = dd->first_user_ctxt; i < dd->cfgctxts; i++) {
if (!dd->rcd[i] || !dd->rcd[i]->cnt)
continue;
spin_unlock_irqrestore(&dd->uctxt_lock, flags);
ret = -EBUSY;
goto bail;
}
spin_unlock_irqrestore(&dd->uctxt_lock, flags);
for (pidx = 0; pidx < dd->num_pports; ++pidx) {
ppd = dd->pport + pidx;
if (atomic_read(&ppd->led_override_timer_active)) {
/* Need to stop LED timer, _then_ shut off LEDs */
timer_delete_sync(&ppd->led_override_timer);
atomic_set(&ppd->led_override_timer_active, 0);
}
/* Shut off LEDs after we are sure timer is not running */
ppd->led_override = LED_OVER_BOTH_OFF;
dd->f_setextled(ppd, 0);
if (dd->flags & QIB_HAS_SEND_DMA)
qib_teardown_sdma(ppd);
}
ret = dd->f_reset(dd);
if (ret == 1)
ret = qib_init(dd, 1);
else
ret = -EAGAIN;
if (ret)
qib_dev_err(dd,
"Reinitialize unit %u after reset failed with %d\n",
unit, ret);
else
qib_devinfo(dd->pcidev,
"Reinitialized unit %u after resetting\n",
unit);
bail:
return ret;
}

271
drivers/infiniband/hw/qib/qib_eeprom.c
View file

@ -1,271 +0,0 @@
/*
* Copyright (c) 2012 Intel Corporation. All rights reserved.
* Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/delay.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include "qib.h"
/*
* Functions specific to the serial EEPROM on cards handled by ib_qib.
* The actual serail interface code is in qib_twsi.c. This file is a client
*/
/**
* qib_eeprom_read - receives bytes from the eeprom via I2C
* @dd: the qlogic_ib device
* @eeprom_offset: address to read from
* @buff: where to store result
* @len: number of bytes to receive
*/
int qib_eeprom_read(struct qib_devdata *dd, u8 eeprom_offset,
void *buff, int len)
{
int ret;
ret = mutex_lock_interruptible(&dd->eep_lock);
if (!ret) {
ret = qib_twsi_reset(dd);
if (ret)
qib_dev_err(dd, "EEPROM Reset for read failed\n");
else
ret = qib_twsi_blk_rd(dd, dd->twsi_eeprom_dev,
eeprom_offset, buff, len);
mutex_unlock(&dd->eep_lock);
}
return ret;
}
/*
* Actually update the eeprom, first doing write enable if
* needed, then restoring write enable state.
* Must be called with eep_lock held
*/
static int eeprom_write_with_enable(struct qib_devdata *dd, u8 offset,
const void *buf, int len)
{
int ret, pwen;
pwen = dd->f_eeprom_wen(dd, 1);
ret = qib_twsi_reset(dd);
if (ret)
qib_dev_err(dd, "EEPROM Reset for write failed\n");
else
ret = qib_twsi_blk_wr(dd, dd->twsi_eeprom_dev,
offset, buf, len);
dd->f_eeprom_wen(dd, pwen);
return ret;
}
/**
* qib_eeprom_write - writes data to the eeprom via I2C
* @dd: the qlogic_ib device
* @eeprom_offset: where to place data
* @buff: data to write
* @len: number of bytes to write
*/
int qib_eeprom_write(struct qib_devdata *dd, u8 eeprom_offset,
const void *buff, int len)
{
int ret;
ret = mutex_lock_interruptible(&dd->eep_lock);
if (!ret) {
ret = eeprom_write_with_enable(dd, eeprom_offset, buff, len);
mutex_unlock(&dd->eep_lock);
}
return ret;
}
static u8 flash_csum(struct qib_flash *ifp, int adjust)
{
u8 *ip = (u8 *) ifp;
u8 csum = 0, len;
/*
* Limit length checksummed to max length of actual data.
* Checksum of erased eeprom will still be bad, but we avoid
* reading past the end of the buffer we were passed.
*/
len = ifp->if_length;
if (len > sizeof(struct qib_flash))
len = sizeof(struct qib_flash);
while (len--)
csum += *ip++;
csum -= ifp->if_csum;
csum = ~csum;
if (adjust)
ifp->if_csum = csum;
return csum;
}
/**
* qib_get_eeprom_info- get the GUID et al. from the TSWI EEPROM device
* @dd: the qlogic_ib device
*
* We have the capability to use the nguid field, and get
* the guid from the first chip's flash, to use for all of them.
*/
void qib_get_eeprom_info(struct qib_devdata *dd)
{
void *buf;
struct qib_flash *ifp;
__be64 guid;
int len, eep_stat;
u8 csum, *bguid;
int t = dd->unit;
struct qib_devdata *dd0 = qib_lookup(0);
if (t && dd0->nguid > 1 && t <= dd0->nguid) {
u8 oguid;
dd->base_guid = dd0->base_guid;
bguid = (u8 *) &dd->base_guid;
oguid = bguid[7];
bguid[7] += t;
if (oguid > bguid[7]) {
if (bguid[6] == 0xff) {
if (bguid[5] == 0xff) {
qib_dev_err(dd,
"Can't set GUID from base, wraps to OUI!\n");
dd->base_guid = 0;
goto bail;
}
bguid[5]++;
}
bguid[6]++;
}
dd->nguid = 1;
goto bail;
}
/*
* Read full flash, not just currently used part, since it may have
* been written with a newer definition.
* */
len = sizeof(struct qib_flash);
buf = vmalloc(len);
if (!buf)
goto bail;
/*
* Use "public" eeprom read function, which does locking and
* figures out device. This will migrate to chip-specific.
*/
eep_stat = qib_eeprom_read(dd, 0, buf, len);
if (eep_stat) {
qib_dev_err(dd, "Failed reading GUID from eeprom\n");
goto done;
}
ifp = (struct qib_flash *)buf;
csum = flash_csum(ifp, 0);
if (csum != ifp->if_csum) {
qib_devinfo(dd->pcidev,
"Bad I2C flash checksum: 0x%x, not 0x%x\n",
csum, ifp->if_csum);
goto done;
}
if (*(__be64 *) ifp->if_guid == cpu_to_be64(0) ||
*(__be64 *) ifp->if_guid == ~cpu_to_be64(0)) {
qib_dev_err(dd,
"Invalid GUID %llx from flash; ignoring\n",
*(unsigned long long *) ifp->if_guid);
/* don't allow GUID if all 0 or all 1's */
goto done;
}
/* complain, but allow it */
if (*(u64 *) ifp->if_guid == 0x100007511000000ULL)
qib_devinfo(dd->pcidev,
"Warning, GUID %llx is default, probably not correct!\n",
*(unsigned long long *) ifp->if_guid);
bguid = ifp->if_guid;
if (!bguid[0] && !bguid[1] && !bguid[2]) {
/*
* Original incorrect GUID format in flash; fix in
* core copy, by shifting up 2 octets; don't need to
* change top octet, since both it and shifted are 0.
*/
bguid[1] = bguid[3];
bguid[2] = bguid[4];
bguid[3] = 0;
bguid[4] = 0;
guid = *(__be64 *) ifp->if_guid;
} else
guid = *(__be64 *) ifp->if_guid;
dd->base_guid = guid;
dd->nguid = ifp->if_numguid;
/*
* Things are slightly complicated by the desire to transparently
* support both the Pathscale 10-digit serial number and the QLogic
* 13-character version.
*/
if ((ifp->if_fversion > 1) && ifp->if_sprefix[0] &&
((u8 *) ifp->if_sprefix)[0] != 0xFF) {
char *snp = dd->serial;
/*
* This board has a Serial-prefix, which is stored
* elsewhere for backward-compatibility.
*/
memcpy(snp, ifp->if_sprefix, sizeof(ifp->if_sprefix));
snp[sizeof(ifp->if_sprefix)] = '\0';
len = strlen(snp);
snp += len;
len = sizeof(dd->serial) - len;
if (len > sizeof(ifp->if_serial))
len = sizeof(ifp->if_serial);
memcpy(snp, ifp->if_serial, len);
} else {
memcpy(dd->serial, ifp->if_serial, sizeof(ifp->if_serial));
}
if (!strstr(ifp->if_comment, "Tested successfully"))
qib_dev_err(dd,
"Board SN %s did not pass functional test: %s\n",
dd->serial, ifp->if_comment);
done:
vfree(buf);
bail:;
}

2401
drivers/infiniband/hw/qib/qib_file_ops.c
View file

File diff suppressed because it is too large Load diff

549
drivers/infiniband/hw/qib/qib_fs.c
View file

@ -1,549 +0,0 @@
/*
* Copyright (c) 2012 Intel Corporation. All rights reserved.
* Copyright (c) 2006 - 2012 QLogic Corporation. All rights reserved.
* Copyright (c) 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/fs.h>
#include <linux/fs_context.h>
#include <linux/mount.h>
#include <linux/pagemap.h>
#include <linux/init.h>
#include <linux/namei.h>
#include "qib.h"
#define QIBFS_MAGIC 0x726a77
static struct super_block *qib_super;
#define private2dd(file) (file_inode(file)->i_private)
static int qibfs_mknod(struct inode *dir, struct dentry *dentry,
umode_t mode, const struct file_operations *fops,
void *data)
{
int error;
struct inode *inode = new_inode(dir->i_sb);
if (!inode) {
dput(dentry);
error = -EPERM;
goto bail;
}
inode->i_ino = get_next_ino();
inode->i_mode = mode;
inode->i_uid = GLOBAL_ROOT_UID;
inode->i_gid = GLOBAL_ROOT_GID;
inode->i_blocks = 0;
simple_inode_init_ts(inode);
inode->i_private = data;
if (S_ISDIR(mode)) {
inode->i_op = &simple_dir_inode_operations;
inc_nlink(inode);
inc_nlink(dir);
}
inode->i_fop = fops;
d_instantiate(dentry, inode);
error = 0;
bail:
return error;
}
static int create_file(const char *name, umode_t mode,
struct dentry *parent, struct dentry **dentry,
const struct file_operations *fops, void *data)
{
int error;
inode_lock(d_inode(parent));
*dentry = lookup_noperm(&QSTR(name), parent);
if (!IS_ERR(*dentry))
error = qibfs_mknod(d_inode(parent), *dentry,
mode, fops, data);
else
error = PTR_ERR(*dentry);
inode_unlock(d_inode(parent));
return error;
}
static ssize_t driver_stats_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
qib_stats.sps_ints = qib_sps_ints();
return simple_read_from_buffer(buf, count, ppos, &qib_stats,
sizeof(qib_stats));
}
/*
* driver stats field names, one line per stat, single string. Used by
* programs like ipathstats to print the stats in a way which works for
* different versions of drivers, without changing program source.
* if qlogic_ib_stats changes, this needs to change. Names need to be
* 12 chars or less (w/o newline), for proper display by ipathstats utility.
*/
static const char qib_statnames[] =
"KernIntr\n"
"ErrorIntr\n"
"Tx_Errs\n"
"Rcv_Errs\n"
"H/W_Errs\n"
"NoPIOBufs\n"
"CtxtsOpen\n"
"RcvLen_Errs\n"
"EgrBufFull\n"
"EgrHdrFull\n"
;
static ssize_t driver_names_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
return simple_read_from_buffer(buf, count, ppos, qib_statnames,
sizeof(qib_statnames) - 1); /* no null */
}
static const struct file_operations driver_ops[] = {
{ .read = driver_stats_read, .llseek = generic_file_llseek, },
{ .read = driver_names_read, .llseek = generic_file_llseek, },
};
/* read the per-device counters */
static ssize_t dev_counters_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
u64 *counters;
size_t avail;
struct qib_devdata *dd = private2dd(file);
avail = dd->f_read_cntrs(dd, *ppos, NULL, &counters);
return simple_read_from_buffer(buf, count, ppos, counters, avail);
}
/* read the per-device counters */
static ssize_t dev_names_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
char *names;
size_t avail;
struct qib_devdata *dd = private2dd(file);
avail = dd->f_read_cntrs(dd, *ppos, &names, NULL);
return simple_read_from_buffer(buf, count, ppos, names, avail);
}
static const struct file_operations cntr_ops[] = {
{ .read = dev_counters_read, .llseek = generic_file_llseek, },
{ .read = dev_names_read, .llseek = generic_file_llseek, },
};
/*
* Could use file_inode(file)->i_ino to figure out which file,
* instead of separate routine for each, but for now, this works...
*/
/* read the per-port names (same for each port) */
static ssize_t portnames_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
char *names;
size_t avail;
struct qib_devdata *dd = private2dd(file);
avail = dd->f_read_portcntrs(dd, *ppos, 0, &names, NULL);
return simple_read_from_buffer(buf, count, ppos, names, avail);
}
/* read the per-port counters for port 1 (pidx 0) */
static ssize_t portcntrs_1_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
u64 *counters;
size_t avail;
struct qib_devdata *dd = private2dd(file);
avail = dd->f_read_portcntrs(dd, *ppos, 0, NULL, &counters);
return simple_read_from_buffer(buf, count, ppos, counters, avail);
}
/* read the per-port counters for port 2 (pidx 1) */
static ssize_t portcntrs_2_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
u64 *counters;
size_t avail;
struct qib_devdata *dd = private2dd(file);
avail = dd->f_read_portcntrs(dd, *ppos, 1, NULL, &counters);
return simple_read_from_buffer(buf, count, ppos, counters, avail);
}
static const struct file_operations portcntr_ops[] = {
{ .read = portnames_read, .llseek = generic_file_llseek, },
{ .read = portcntrs_1_read, .llseek = generic_file_llseek, },
{ .read = portcntrs_2_read, .llseek = generic_file_llseek, },
};
/*
* read the per-port QSFP data for port 1 (pidx 0)
*/
static ssize_t qsfp_1_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
struct qib_devdata *dd = private2dd(file);
char *tmp;
int ret;
tmp = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!tmp)
return -ENOMEM;
ret = qib_qsfp_dump(dd->pport, tmp, PAGE_SIZE);
if (ret > 0)
ret = simple_read_from_buffer(buf, count, ppos, tmp, ret);
kfree(tmp);
return ret;
}
/*
* read the per-port QSFP data for port 2 (pidx 1)
*/
static ssize_t qsfp_2_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
struct qib_devdata *dd = private2dd(file);
char *tmp;
int ret;
if (dd->num_pports < 2)
return -ENODEV;
tmp = kmalloc(PAGE_SIZE, GFP_KERNEL);
if (!tmp)
return -ENOMEM;
ret = qib_qsfp_dump(dd->pport + 1, tmp, PAGE_SIZE);
if (ret > 0)
ret = simple_read_from_buffer(buf, count, ppos, tmp, ret);
kfree(tmp);
return ret;
}
static const struct file_operations qsfp_ops[] = {
{ .read = qsfp_1_read, .llseek = generic_file_llseek, },
{ .read = qsfp_2_read, .llseek = generic_file_llseek, },
};
static ssize_t flash_read(struct file *file, char __user *buf,
size_t count, loff_t *ppos)
{
struct qib_devdata *dd;
ssize_t ret;
loff_t pos;
char *tmp;
pos = *ppos;
if (pos < 0) {
ret = -EINVAL;
goto bail;
}
if (pos >= sizeof(struct qib_flash)) {
ret = 0;
goto bail;
}
if (count > sizeof(struct qib_flash) - pos)
count = sizeof(struct qib_flash) - pos;
tmp = kmalloc(count, GFP_KERNEL);
if (!tmp) {
ret = -ENOMEM;
goto bail;
}
dd = private2dd(file);
if (qib_eeprom_read(dd, pos, tmp, count)) {
qib_dev_err(dd, "failed to read from flash\n");
ret = -ENXIO;
goto bail_tmp;
}
if (copy_to_user(buf, tmp, count)) {
ret = -EFAULT;
goto bail_tmp;
}
*ppos = pos + count;
ret = count;
bail_tmp:
kfree(tmp);
bail:
return ret;
}
static ssize_t flash_write(struct file *file, const char __user *buf,
size_t count, loff_t *ppos)
{
struct qib_devdata *dd;
ssize_t ret;
loff_t pos;
char *tmp;
pos = *ppos;
if (pos != 0 || count != sizeof(struct qib_flash))
return -EINVAL;
tmp = memdup_user(buf, count);
if (IS_ERR(tmp))
return PTR_ERR(tmp);
dd = private2dd(file);
if (qib_eeprom_write(dd, pos, tmp, count)) {
ret = -ENXIO;
qib_dev_err(dd, "failed to write to flash\n");
goto bail_tmp;
}
*ppos = pos + count;
ret = count;
bail_tmp:
kfree(tmp);
return ret;
}
static const struct file_operations flash_ops = {
.read = flash_read,
.write = flash_write,
.llseek = default_llseek,
};
static int add_cntr_files(struct super_block *sb, struct qib_devdata *dd)
{
struct dentry *dir, *tmp;
char unit[10];
int ret, i;
/* create the per-unit directory */
snprintf(unit, sizeof(unit), "%u", dd->unit);
ret = create_file(unit, S_IFDIR|S_IRUGO|S_IXUGO, sb->s_root, &dir,
&simple_dir_operations, dd);
if (ret) {
pr_err("create_file(%s) failed: %d\n", unit, ret);
goto bail;
}
/* create the files in the new directory */
ret = create_file("counters", S_IFREG|S_IRUGO, dir, &tmp,
&cntr_ops[0], dd);
if (ret) {
pr_err("create_file(%s/counters) failed: %d\n",
unit, ret);
goto bail;
}
ret = create_file("counter_names", S_IFREG|S_IRUGO, dir, &tmp,
&cntr_ops[1], dd);
if (ret) {
pr_err("create_file(%s/counter_names) failed: %d\n",
unit, ret);
goto bail;
}
ret = create_file("portcounter_names", S_IFREG|S_IRUGO, dir, &tmp,
&portcntr_ops[0], dd);
if (ret) {
pr_err("create_file(%s/%s) failed: %d\n",
unit, "portcounter_names", ret);
goto bail;
}
for (i = 1; i <= dd->num_pports; i++) {
char fname[24];
sprintf(fname, "port%dcounters", i);
/* create the files in the new directory */
ret = create_file(fname, S_IFREG|S_IRUGO, dir, &tmp,
&portcntr_ops[i], dd);
if (ret) {
pr_err("create_file(%s/%s) failed: %d\n",
unit, fname, ret);
goto bail;
}
if (!(dd->flags & QIB_HAS_QSFP))
continue;
sprintf(fname, "qsfp%d", i);
ret = create_file(fname, S_IFREG|S_IRUGO, dir, &tmp,
&qsfp_ops[i - 1], dd);
if (ret) {
pr_err("create_file(%s/%s) failed: %d\n",
unit, fname, ret);
goto bail;
}
}
ret = create_file("flash", S_IFREG|S_IWUSR|S_IRUGO, dir, &tmp,
&flash_ops, dd);
if (ret)
pr_err("create_file(%s/flash) failed: %d\n",
unit, ret);
bail:
return ret;
}
static int remove_device_files(struct super_block *sb,
struct qib_devdata *dd)
{
struct dentry *dir;
char unit[10];
snprintf(unit, sizeof(unit), "%u", dd->unit);
dir = lookup_noperm_unlocked(&QSTR(unit), sb->s_root);
if (IS_ERR(dir)) {
pr_err("Lookup of %s failed\n", unit);
return PTR_ERR(dir);
}
simple_recursive_removal(dir, NULL);
dput(dir);
return 0;
}
/*
* This fills everything in when the fs is mounted, to handle umount/mount
* after device init. The direct add_cntr_files() call handles adding
* them from the init code, when the fs is already mounted.
*/
static int qibfs_fill_super(struct super_block *sb, struct fs_context *fc)
{
struct qib_devdata *dd;
unsigned long index;
int ret;
static const struct tree_descr files[] = {
[2] = {"driver_stats", &driver_ops[0], S_IRUGO},
[3] = {"driver_stats_names", &driver_ops[1], S_IRUGO},
{""},
};
ret = simple_fill_super(sb, QIBFS_MAGIC, files);
if (ret) {
pr_err("simple_fill_super failed: %d\n", ret);
goto bail;
}
xa_for_each(&qib_dev_table, index, dd) {
ret = add_cntr_files(sb, dd);
if (ret)
goto bail;
}
bail:
return ret;
}
static int qibfs_get_tree(struct fs_context *fc)
{
int ret = get_tree_single(fc, qibfs_fill_super);
if (ret == 0)
qib_super = fc->root->d_sb;
return ret;
}
static const struct fs_context_operations qibfs_context_ops = {
.get_tree = qibfs_get_tree,
};
static int qibfs_init_fs_context(struct fs_context *fc)
{
fc->ops = &qibfs_context_ops;
return 0;
}
static void qibfs_kill_super(struct super_block *s)
{
kill_litter_super(s);
qib_super = NULL;
}
int qibfs_add(struct qib_devdata *dd)
{
int ret;
/*
* On first unit initialized, qib_super will not yet exist
* because nobody has yet tried to mount the filesystem, so
* we can't consider that to be an error; if an error occurs
* during the mount, that will get a complaint, so this is OK.
* add_cntr_files() for all units is done at mount from
* qibfs_fill_super(), so one way or another, everything works.
*/
if (qib_super == NULL)
ret = 0;
else
ret = add_cntr_files(qib_super, dd);
return ret;
}
int qibfs_remove(struct qib_devdata *dd)
{
int ret = 0;
if (qib_super)
ret = remove_device_files(qib_super, dd);
return ret;
}
static struct file_system_type qibfs_fs_type = {
.owner = THIS_MODULE,
.name = "ipathfs",
.init_fs_context = qibfs_init_fs_context,
.kill_sb = qibfs_kill_super,
};
MODULE_ALIAS_FS("ipathfs");
int __init qib_init_qibfs(void)
{
return register_filesystem(&qibfs_fs_type);
}
int __exit qib_exit_qibfs(void)
{
return unregister_filesystem(&qibfs_fs_type);
}

3533
drivers/infiniband/hw/qib/qib_iba6120.c
View file

File diff suppressed because it is too large Load diff

4596
drivers/infiniband/hw/qib/qib_iba7220.c
View file

File diff suppressed because it is too large Load diff

8475
drivers/infiniband/hw/qib/qib_iba7322.c
View file

File diff suppressed because it is too large Load diff

1782
drivers/infiniband/hw/qib/qib_init.c
View file

File diff suppressed because it is too large Load diff

241
drivers/infiniband/hw/qib/qib_intr.c
View file

@ -1,241 +0,0 @@
/*
* Copyright (c) 2006, 2007, 2008, 2009, 2010 QLogic Corporation.
* All rights reserved.
* Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/pci.h>
#include <linux/delay.h>
#include "qib.h"
#include "qib_common.h"
/**
* qib_format_hwmsg - format a single hwerror message
* @msg: message buffer
* @msgl: length of message buffer
* @hwmsg: message to add to message buffer
*/
static void qib_format_hwmsg(char *msg, size_t msgl, const char *hwmsg)
{
strlcat(msg, "[", msgl);
strlcat(msg, hwmsg, msgl);
strlcat(msg, "]", msgl);
}
/**
* qib_format_hwerrors - format hardware error messages for display
* @hwerrs: hardware errors bit vector
* @hwerrmsgs: hardware error descriptions
* @nhwerrmsgs: number of hwerrmsgs
* @msg: message buffer
* @msgl: message buffer length
*/
void qib_format_hwerrors(u64 hwerrs, const struct qib_hwerror_msgs *hwerrmsgs,
size_t nhwerrmsgs, char *msg, size_t msgl)
{
int i;
for (i = 0; i < nhwerrmsgs; i++)
if (hwerrs & hwerrmsgs[i].mask)
qib_format_hwmsg(msg, msgl, hwerrmsgs[i].msg);
}
static void signal_ib_event(struct qib_pportdata *ppd, enum ib_event_type ev)
{
struct ib_event event;
struct qib_devdata *dd = ppd->dd;
event.device = &dd->verbs_dev.rdi.ibdev;
event.element.port_num = ppd->port;
event.event = ev;
ib_dispatch_event(&event);
}
void qib_handle_e_ibstatuschanged(struct qib_pportdata *ppd, u64 ibcs)
{
struct qib_devdata *dd = ppd->dd;
unsigned long flags;
u32 lstate;
u8 ltstate;
enum ib_event_type ev = 0;
lstate = dd->f_iblink_state(ibcs); /* linkstate */
ltstate = dd->f_ibphys_portstate(ibcs);
/*
* If linkstate transitions into INIT from any of the various down
* states, or if it transitions from any of the up (INIT or better)
* states into any of the down states (except link recovery), then
* call the chip-specific code to take appropriate actions.
*
* ppd->lflags could be 0 if this is the first time the interrupt
* handlers has been called but the link is already up.
*/
if (lstate >= IB_PORT_INIT &&
(!ppd->lflags || (ppd->lflags & QIBL_LINKDOWN)) &&
ltstate == IB_PHYSPORTSTATE_LINKUP) {
/* transitioned to UP */
if (dd->f_ib_updown(ppd, 1, ibcs))
goto skip_ibchange; /* chip-code handled */
} else if (ppd->lflags & (QIBL_LINKINIT | QIBL_LINKARMED |
QIBL_LINKACTIVE | QIBL_IB_FORCE_NOTIFY)) {
if (ltstate != IB_PHYSPORTSTATE_LINKUP &&
ltstate <= IB_PHYSPORTSTATE_CFG_TRAIN &&
dd->f_ib_updown(ppd, 0, ibcs))
goto skip_ibchange; /* chip-code handled */
qib_set_uevent_bits(ppd, _QIB_EVENT_LINKDOWN_BIT);
}
if (lstate != IB_PORT_DOWN) {
/* lstate is INIT, ARMED, or ACTIVE */
if (lstate != IB_PORT_ACTIVE) {
*ppd->statusp &= ~QIB_STATUS_IB_READY;
if (ppd->lflags & QIBL_LINKACTIVE)
ev = IB_EVENT_PORT_ERR;
spin_lock_irqsave(&ppd->lflags_lock, flags);
if (lstate == IB_PORT_ARMED) {
ppd->lflags |= QIBL_LINKARMED | QIBL_LINKV;
ppd->lflags &= ~(QIBL_LINKINIT |
QIBL_LINKDOWN | QIBL_LINKACTIVE);
} else {
ppd->lflags |= QIBL_LINKINIT | QIBL_LINKV;
ppd->lflags &= ~(QIBL_LINKARMED |
QIBL_LINKDOWN | QIBL_LINKACTIVE);
}
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
/* start a 75msec timer to clear symbol errors */
mod_timer(&ppd->symerr_clear_timer,
msecs_to_jiffies(75));
} else if (ltstate == IB_PHYSPORTSTATE_LINKUP &&
!(ppd->lflags & QIBL_LINKACTIVE)) {
/* active, but not active defered */
qib_hol_up(ppd); /* useful only for 6120 now */
*ppd->statusp |=
QIB_STATUS_IB_READY | QIB_STATUS_IB_CONF;
qib_clear_symerror_on_linkup(&ppd->symerr_clear_timer);
spin_lock_irqsave(&ppd->lflags_lock, flags);
ppd->lflags |= QIBL_LINKACTIVE | QIBL_LINKV;
ppd->lflags &= ~(QIBL_LINKINIT |
QIBL_LINKDOWN | QIBL_LINKARMED);
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
if (dd->flags & QIB_HAS_SEND_DMA)
qib_sdma_process_event(ppd,
qib_sdma_event_e30_go_running);
ev = IB_EVENT_PORT_ACTIVE;
dd->f_setextled(ppd, 1);
}
} else { /* down */
if (ppd->lflags & QIBL_LINKACTIVE)
ev = IB_EVENT_PORT_ERR;
spin_lock_irqsave(&ppd->lflags_lock, flags);
ppd->lflags |= QIBL_LINKDOWN | QIBL_LINKV;
ppd->lflags &= ~(QIBL_LINKINIT |
QIBL_LINKACTIVE | QIBL_LINKARMED);
spin_unlock_irqrestore(&ppd->lflags_lock, flags);
*ppd->statusp &= ~QIB_STATUS_IB_READY;
}
skip_ibchange:
ppd->lastibcstat = ibcs;
if (ev)
signal_ib_event(ppd, ev);
}
void qib_clear_symerror_on_linkup(struct timer_list *t)
{
struct qib_pportdata *ppd = timer_container_of(ppd, t,
symerr_clear_timer);
if (ppd->lflags & QIBL_LINKACTIVE)
return;
ppd->ibport_data.z_symbol_error_counter =
ppd->dd->f_portcntr(ppd, QIBPORTCNTR_IBSYMBOLERR);
}
/*
* Handle receive interrupts for user ctxts; this means a user
* process was waiting for a packet to arrive, and didn't want
* to poll.
*/
void qib_handle_urcv(struct qib_devdata *dd, u64 ctxtr)
{
struct qib_ctxtdata *rcd;
unsigned long flags;
int i;
spin_lock_irqsave(&dd->uctxt_lock, flags);
for (i = dd->first_user_ctxt; dd->rcd && i < dd->cfgctxts; i++) {
if (!(ctxtr & (1ULL << i)))
continue;
rcd = dd->rcd[i];
if (!rcd || !rcd->cnt)
continue;
if (test_and_clear_bit(QIB_CTXT_WAITING_RCV, &rcd->flag)) {
wake_up_interruptible(&rcd->wait);
dd->f_rcvctrl(rcd->ppd, QIB_RCVCTRL_INTRAVAIL_DIS,
rcd->ctxt);
} else if (test_and_clear_bit(QIB_CTXT_WAITING_URG,
&rcd->flag)) {
rcd->urgent++;
wake_up_interruptible(&rcd->wait);
}
}
spin_unlock_irqrestore(&dd->uctxt_lock, flags);
}
void qib_bad_intrstatus(struct qib_devdata *dd)
{
static int allbits;
/* separate routine, for better optimization of qib_intr() */
/*
* We print the message and disable interrupts, in hope of
* having a better chance of debugging the problem.
*/
qib_dev_err(dd,
"Read of chip interrupt status failed disabling interrupts\n");
if (allbits++) {
/* disable interrupt delivery, something is very wrong */
if (allbits == 2)
dd->f_set_intr_state(dd, 0);
if (allbits == 3) {
qib_dev_err(dd,
"2nd bad interrupt status, unregistering interrupts\n");
dd->flags |= QIB_BADINTR;
dd->flags &= ~QIB_INITTED;
dd->f_free_irq(dd);
}
}
}

2450
drivers/infiniband/hw/qib/qib_mad.c
View file

File diff suppressed because it is too large Load diff

Some files were not shown because too many files have changed in this diff Show more