From def46cf344dc9981a0901a1293d4513efafe14d2 Mon Sep 17 00:00:00 2001 From: Michael Brown Date: Sun, 31 Jan 2021 23:29:45 +0000 Subject: [hermon] Limit link poll frequency in DOWN state Some older versions of the hardware (and/or firmware) do not report an event when an Infiniband link reaches the INIT state. The driver works around this missing event by calling ib_smc_update() on each event queue poll while the link is in the DOWN state. Commit 6cb12ee ("[hermon] Increase polling rate for command completions") addressed this by speeding up the time taken to issue each command invoked by ib_smc_update(). Experimentation shows that the impact is still significant: for example, in a situation where an unplugged port is opened, the throughput on the other port can be reduced by over 99%. Fix by throttling the rate at which link polling is attempted. Debugged-by: Christian Iversen Signed-off-by: Michael Brown --- src/drivers/infiniband/hermon.c | 9 ++++++++- src/drivers/infiniband/hermon.h | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/drivers/infiniband/hermon.c b/src/drivers/infiniband/hermon.c index 2a9649d..50ba4f4 100644 --- a/src/drivers/infiniband/hermon.c +++ b/src/drivers/infiniband/hermon.c @@ -2130,6 +2130,8 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) { struct hermon_event_queue *hermon_eq = &hermon->eq; union hermonprm_event_entry *eqe; union hermonprm_doorbell_register db_reg; + unsigned long now; + unsigned long elapsed; unsigned int eqe_idx_mask; unsigned int event_type; @@ -2138,7 +2140,12 @@ static void hermon_poll_eq ( struct ib_device *ibdev ) { */ if ( ib_is_open ( ibdev ) && ( ibdev->port_state == IB_PORT_STATE_DOWN ) ) { - ib_smc_update ( ibdev, hermon_mad ); + now = currticks(); + elapsed = ( now - hermon->last_poll ); + if ( elapsed >= HERMON_LINK_POLL_INTERVAL ) { + hermon->last_poll = now; + ib_smc_update ( ibdev, hermon_mad ); + } } /* Poll event queue */ diff --git a/src/drivers/infiniband/hermon.h b/src/drivers/infiniband/hermon.h index 61f3b04..6d7471d 100644 --- a/src/drivers/infiniband/hermon.h +++ b/src/drivers/infiniband/hermon.h @@ -894,6 +894,8 @@ struct hermon { /** Event queue */ struct hermon_event_queue eq; + /** Last unsolicited link state poll */ + unsigned long last_poll; /** Unrestricted LKey * * Used to get unrestricted memory access. @@ -930,6 +932,13 @@ struct hermon { /** Memory key prefix */ #define HERMON_MKEY_PREFIX 0x77000000UL +/** Link poll interval + * + * Used when we need to poll for link state (rather than relying upon + * receiving an event). + */ +#define HERMON_LINK_POLL_INTERVAL ( TICKS_PER_SEC / 2 ) + /* * HCA commands * -- cgit v1.1