Commit 18d13584 authored by Tomer Tayar's avatar Tomer Tayar Committed by Oded Gabbay
Browse files

accel/habanalabs: enable graceful reset mechanism for compute-reset 



The graceful reset mechanism is currently enabled only for reset
requests that will end up with hard-reset.
In future, reset requests due to errors in some device engines, are
going to be modified to request compute-reset, as the much longer
hard-reset is not really needed there.
To allow it, enable graceful reset also for compute-reset, and reset
after user releases the device won't be escalated to hard-reset in those
cases.
If watchdog expires and user didn't release the device, hard-reset will
be initiated in any case.

Signed-off-by: default avatarTomer Tayar <ttayar@habana.ai>
Reviewed-by: default avatarOded Gabbay <ogabbay@kernel.org>
Signed-off-by: default avatarOded Gabbay <ogabbay@kernel.org>
Reviewed-by: default avatarStanislaw Gruszka <stanislaw.gruszka@linux.intel.com>
parent 57479adb

drivers/accel/habanalabs/common/device.c

+12 −14
Original line number Diff line number Diff line
@@ -778,14 +778,14 @@ static void device_hard_reset_pending(struct work_struct *work) 


static void device_release_watchdog_func(struct work_struct *work)

{

	struct hl_device_reset_work *device_release_watchdog_work =

	struct hl_device_reset_work *watchdog_work =

			container_of(work, struct hl_device_reset_work, reset_work.work);

	struct hl_device *hdev = device_release_watchdog_work->hdev;

	struct hl_device *hdev = watchdog_work->hdev;

	u32 flags;



	dev_dbg(hdev->dev, "Device wasn't released in time. Initiate device reset.\n");

	dev_dbg(hdev->dev, "Device wasn't released in time. Initiate hard-reset.\n");



	flags = device_release_watchdog_work->flags | HL_DRV_RESET_FROM_WD_THR;

	flags = watchdog_work->flags | HL_DRV_RESET_HARD | HL_DRV_RESET_FROM_WD_THR;



	hl_device_reset(hdev, flags);

}
@@ -1555,15 +1555,17 @@ int hl_device_reset(struct hl_device *hdev, u32 flags) 


		/* Cancel the device release watchdog work if required.

		 * In case of reset-upon-device-release while the release watchdog work is

		 * scheduled, do hard-reset instead of compute-reset.

		 * scheduled due to a hard-reset, do hard-reset instead of compute-reset.

		 */

		if ((hard_reset || from_dev_release) && hdev->reset_info.watchdog_active) {

			struct hl_device_reset_work *watchdog_work =

					&hdev->device_release_watchdog_work;



			hdev->reset_info.watchdog_active = 0;

			if (!from_watchdog_thread)

				cancel_delayed_work_sync(

						&hdev->device_release_watchdog_work.reset_work);

				cancel_delayed_work_sync(&watchdog_work->reset_work);



			if (from_dev_release) {

			if (from_dev_release && (watchdog_work->flags & HL_DRV_RESET_HARD)) {

				hdev->reset_info.in_compute_reset = 0;

				flags |= HL_DRV_RESET_HARD;

				flags &= ~HL_DRV_RESET_DEV_RELEASE;
@@ -1890,10 +1892,6 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask) 
{

	struct hl_ctx *ctx = NULL;



	/* Device release watchdog is only for hard reset */

	if (!(flags & HL_DRV_RESET_HARD) && hdev->asic_prop.allow_inference_soft_reset)

		goto device_reset;



	/* F/W reset cannot be postponed */

	if (flags & HL_DRV_RESET_BYPASS_REQ_TO_FW)

		goto device_reset;
@@ -1921,7 +1919,7 @@ int hl_device_cond_reset(struct hl_device *hdev, u32 flags, u64 event_mask) 
		goto out;



	hdev->device_release_watchdog_work.flags = flags;

	dev_dbg(hdev->dev, "Device is going to be reset in %u sec unless being released\n",

	dev_dbg(hdev->dev, "Device is going to be hard-reset in %u sec unless being released\n",

		hdev->device_release_watchdog_timeout_sec);

	schedule_delayed_work(&hdev->device_release_watchdog_work.reset_work,

				msecs_to_jiffies(hdev->device_release_watchdog_timeout_sec * 1000));