Commit 4d530e7d authored Mar 20, 2022 by Yuri Nudelman Committed by Greg Kroah-Hartman May 22, 2022

habanalabs: convert ts to use unified memory manager



With the introduction of the unified memory manager infrastructure, the
timestamp buffers can be converted to use it.

Signed-off-by: Yuri Nudelman <ynudelman@habana.ai>
Reviewed-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Oded Gabbay <ogabbay@kernel.org>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

parent babe8e7c

drivers/misc/habanalabs/common/command_submission.c

+27 −17

Original line number	Diff line number	Diff line
		@@ -957,9 +957,9 @@ wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt)

		spin_lock_irqsave(&interrupt->wait_list_lock, flags);
		list_for_each_entry_safe(pend, temp, &interrupt->wait_list_head, wait_list_node) {
		if (pend->ts_reg_info.ts_buff) {
		if (pend->ts_reg_info.buf) {
		list_del(&pend->wait_list_node);
		hl_ts_put(pend->ts_reg_info.ts_buff);
		hl_mmap_mem_buf_put(pend->ts_reg_info.buf);
		hl_cb_put(pend->ts_reg_info.cq_cb);
		} else {
		pend->fence.error = -EIO;
		@@ -2867,12 +2867,13 @@ static int hl_cs_wait_ioctl(struct hl_fpriv hpriv, void data)
		return 0;
		}

		static int ts_buff_get_kernel_ts_record(struct hl_ts_buff *ts_buff,
		static int ts_buff_get_kernel_ts_record(struct hl_mmap_mem_buf *buf,
		struct hl_cb *cq_cb,
		u64 ts_offset, u64 cq_offset, u64 target_value,
		spinlock_t *wait_list_lock,
		struct hl_user_pending_interrupt **pend)
		{
		struct hl_ts_buff *ts_buff = buf->private;
		struct hl_user_pending_interrupt *requested_offset_record =
		(struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address +
		ts_offset;
		@@ -2884,7 +2885,7 @@ static int ts_buff_get_kernel_ts_record(struct hl_ts_buff *ts_buff,

		/* Validate ts_offset not exceeding last max */
		if (requested_offset_record > cb_last) {
		dev_err(ts_buff->hdev->dev, "Ts offset exceeds max CB offset(0x%llx)\n",
		dev_err(buf->mmg->dev, "Ts offset exceeds max CB offset(0x%llx)\n",
		(u64)(uintptr_t)cb_last);
		return -EINVAL;
		}
		@@ -2903,18 +2904,21 @@ static int ts_buff_get_kernel_ts_record(struct hl_ts_buff *ts_buff,
		list_del(&requested_offset_record->wait_list_node);
		spin_unlock_irqrestore(wait_list_lock, flags);

		hl_ts_put(requested_offset_record->ts_reg_info.ts_buff);
		hl_mmap_mem_buf_put(requested_offset_record->ts_reg_info.buf);
		hl_cb_put(requested_offset_record->ts_reg_info.cq_cb);

		dev_dbg(ts_buff->hdev->dev, "ts node removed from interrupt list now can re-use\n");
		dev_dbg(buf->mmg->dev,
		"ts node removed from interrupt list now can re-use\n");
		} else {
		dev_dbg(ts_buff->hdev->dev, "ts node in middle of irq handling\n");
		dev_dbg(buf->mmg->dev,
		"ts node in middle of irq handling\n");

		/* irq handling in the middle give it time to finish */
		spin_unlock_irqrestore(wait_list_lock, flags);
		usleep_range(1, 10);
		if (++iter_counter == MAX_TS_ITER_NUM) {
		dev_err(ts_buff->hdev->dev, "handling registration interrupt took too long!!\n");
		dev_err(buf->mmg->dev,
		"handling registration interrupt took too long!!\n");
		return -EINVAL;
		}

		@@ -2926,7 +2930,7 @@ static int ts_buff_get_kernel_ts_record(struct hl_ts_buff *ts_buff,

		/* Fill up the new registration node info */
		requested_offset_record->ts_reg_info.in_use = 1;
		requested_offset_record->ts_reg_info.ts_buff = ts_buff;
		requested_offset_record->ts_reg_info.buf = buf;
		requested_offset_record->ts_reg_info.cq_cb = cq_cb;
		requested_offset_record->ts_reg_info.timestamp_kernel_addr =
		(u64 *) ts_buff->user_buff_address + ts_offset;
		@@ -2936,13 +2940,13 @@ static int ts_buff_get_kernel_ts_record(struct hl_ts_buff *ts_buff,

		*pend = requested_offset_record;

		dev_dbg(ts_buff->hdev->dev, "Found available node in TS kernel CB(0x%llx)\n",
		dev_dbg(buf->mmg->dev, "Found available node in TS kernel CB(0x%llx)\n",
		(u64)(uintptr_t)requested_offset_record);
		return 0;
		}

		static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx,
		struct hl_cb_mgr cb_mgr, struct hl_ts_mgr ts_mgr,
		struct hl_cb_mgr cb_mgr, struct hl_mem_mgr mmg,
		u64 timeout_us, u64 cq_counters_handle, u64 cq_counters_offset,
		u64 target_value, struct hl_user_interrupt *interrupt,
		bool register_ts_record, u64 ts_handle, u64 ts_offset,
		@@ -2950,7 +2954,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx,
		{
		u32 cq_patched_handle, ts_patched_handle;
		struct hl_user_pending_interrupt *pend;
		struct hl_ts_buff *ts_buff;
		struct hl_mmap_mem_buf *buf;
		struct hl_cb *cq_cb;
		unsigned long timeout, flags;
		long completion_rc;
		@@ -2971,15 +2975,21 @@ static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx,
		dev_dbg(hdev->dev, "Timestamp registration: interrupt id: %u, ts offset: %llu, cq_offset: %llu\n",
		interrupt->interrupt_id, ts_offset, cq_counters_offset);

		/* TODO:
		* See if this can be removed.
		* Embedding type in handle will no longer be needed as soon as we
		* switch to using a single memory manager for all memory types.
		* We may still need the page shift, though.
		*/
		ts_patched_handle = lower_32_bits(ts_handle >> PAGE_SHIFT);
		ts_buff = hl_ts_get(hdev, ts_mgr, ts_patched_handle);
		if (!ts_buff) {
		buf = hl_mmap_mem_buf_get(mmg, ts_patched_handle);
		if (!buf) {
		rc = -EINVAL;
		goto put_cq_cb;
		}

		/* Find first available record */
		rc = ts_buff_get_kernel_ts_record(ts_buff, cq_cb, ts_offset,
		rc = ts_buff_get_kernel_ts_record(buf, cq_cb, ts_offset,
		cq_counters_offset, target_value,
		&interrupt->wait_list_lock, &pend);
		if (rc)
		@@ -3086,7 +3096,7 @@ static int _hl_interrupt_wait_ioctl(struct hl_device hdev, struct hl_ctx ctx,
		return rc;

		put_ts_buff:
		hl_ts_put(ts_buff);
		hl_mmap_mem_buf_put(buf);
		put_cq_cb:
		hl_cb_put(cq_cb);
		put_ctx:
		@@ -3248,7 +3258,7 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv hpriv, void data)
		interrupt = &hdev->user_interrupt[interrupt_id - first_interrupt];

		if (args->in.flags & HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ)
		rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &hpriv->cb_mgr, &hpriv->ts_mem_mgr,
		rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &hpriv->cb_mgr, &hpriv->mem_mgr,
		args->in.interrupt_timeout_us, args->in.cq_counters_handle,
		args->in.cq_counters_offset,
		args->in.target, interrupt,

drivers/misc/habanalabs/common/device.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -149,7 +149,7 @@ static int hl_device_release(struct inode inode, struct file filp)
		hl_release_pending_user_interrupts(hpriv->hdev);

		hl_cb_mgr_fini(hdev, &hpriv->cb_mgr);
		hl_ts_mgr_fini(hpriv->hdev, &hpriv->ts_mem_mgr);
		hl_mem_mgr_fini(&hpriv->mem_mgr);
		hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);

		hdev->compute_ctx_in_release = 1;
		@@ -218,7 +218,7 @@ static int hl_mmap(struct file filp, struct vm_area_struct vma)
		return hl_hw_block_mmap(hpriv, vma);

		case HL_MMAP_TYPE_TS_BUFF:
		return hl_ts_mmap(hpriv, vma);
		return hl_mem_mgr_mmap(&hpriv->mem_mgr, vma, NULL);
		}

		return -EINVAL;

drivers/misc/habanalabs/common/habanalabs.h

+19 −24

Original line number	Diff line number	Diff line
		@@ -719,26 +719,14 @@ struct hl_ts_mgr {

		/**
		* struct hl_ts_buff - describes a timestamp buffer.
		* @refcount: reference counter for usage of the buffer.
		* @hdev: pointer to device this buffer belongs to.
		* @mmap: true if the buff is currently mapped to user.
		* @kernel_buff_address: Holds the internal buffer's kernel virtual address.
		* @user_buff_address: Holds the user buffer's kernel virtual address.
		* @id: the buffer ID.
		* @mmap_size: Holds the buffer size that was mmaped.
		* @kernel_buff_size: Holds the internal kernel buffer size.
		* @user_buff_size: Holds the user buffer size.
		*/
		struct hl_ts_buff {
		struct kref refcount;
		struct hl_device *hdev;
		atomic_t mmap;
		void *kernel_buff_address;
		void *user_buff_address;
		u32 id;
		u32 mmap_size;
		u32 kernel_buff_size;
		u32 user_buff_size;
		};

		struct hl_mmap_mem_buf;
		@@ -973,12 +961,12 @@ struct hl_user_interrupt {
		* struct timestamp_reg_free_node - holds the timestamp registration free objects node
		* @free_objects_node: node in the list free_obj_jobs
		* @cq_cb: pointer to cq command buffer to be freed
		* @ts_buff: pointer to timestamp buffer to be freed
		* @buf: pointer to timestamp buffer to be freed
		*/
		struct timestamp_reg_free_node {
		struct list_head free_objects_node;
		struct hl_cb *cq_cb;
		struct hl_ts_buff *ts_buff;
		struct hl_mmap_mem_buf *buf;
		};

		/* struct timestamp_reg_work_obj - holds the timestamp registration free objects job
		@@ -995,7 +983,7 @@ struct timestamp_reg_work_obj {
		};

		/* struct timestamp_reg_info - holds the timestamp registration related data.
		* @ts_buff: pointer to the timestamp buffer which include both user/kernel buffers.
		* @buf: pointer to the timestamp buffer which include both user/kernel buffers.
		* relevant only when doing timestamps records registration.
		* @cq_cb: pointer to CQ counter CB.
		* @timestamp_kernel_addr: timestamp handle address, where to set timestamp
		@@ -1007,7 +995,7 @@ struct timestamp_reg_work_obj {
		* allocating records dynamically.
		*/
		struct timestamp_reg_info {
		struct hl_ts_buff *ts_buff;
		struct hl_mmap_mem_buf *buf;
		struct hl_cb *cq_cb;
		u64 *timestamp_kernel_addr;
		u8 in_use;
		@@ -1966,7 +1954,7 @@ struct hl_debug_params {
		* @ctx: current executing context. TODO: remove for multiple ctx per process
		* @ctx_mgr: context manager to handle multiple context for this FD.
		* @cb_mgr: command buffer manager to handle multiple buffers for this FD.
		* @ts_mem_mgr: timestamp registration manager for alloc/free/map timestamp buffers.
		* @mem_mgr: manager descriptor for memory exportable via mmap
		* @debugfs_list: list of relevant ASIC debugfs.
		* @dev_node: node in the device list of file private data
		* @refcount: number of related contexts.
		@@ -1979,7 +1967,7 @@ struct hl_fpriv {
		struct hl_ctx *ctx;
		struct hl_ctx_mgr ctx_mgr;
		struct hl_cb_mgr cb_mgr;
		struct hl_ts_mgr ts_mem_mgr;
		struct hl_mem_mgr mem_mgr;
		struct list_head debugfs_list;
		struct list_head dev_node;
		struct kref refcount;
		@@ -3267,11 +3255,18 @@ __printf(4, 5) int hl_snprintf_resize(char *buf, size_t size, size_t *offset,
		const char *format, ...);
		char hl_format_as_binary(char buf, size_t buf_len, u32 n);
		const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type);
		void hl_ts_mgr_init(struct hl_ts_mgr *mgr);
		void hl_ts_mgr_fini(struct hl_device hdev, struct hl_ts_mgr mgr);
		int hl_ts_mmap(struct hl_fpriv hpriv, struct vm_area_struct vma);
		struct hl_ts_buff hl_ts_get(struct hl_device hdev, struct hl_ts_mgr *mgr, u32 handle);
		void hl_ts_put(struct hl_ts_buff *buff);

		void hl_mem_mgr_init(struct device dev, struct hl_mem_mgr mmg);
		void hl_mem_mgr_fini(struct hl_mem_mgr *mmg);
		int hl_mem_mgr_mmap(struct hl_mem_mgr mmg, struct vm_area_struct vma,
		void *args);
		struct hl_mmap_mem_buf hl_mmap_mem_buf_get(struct hl_mem_mgr mmg,
		u32 handle);
		int hl_mmap_mem_buf_put(struct hl_mmap_mem_buf *buf);
		struct hl_mmap_mem_buf *
		hl_mmap_mem_buf_alloc(struct hl_mem_mgr *mmg,
		struct hl_mmap_mem_buf_ops *behavior, gfp_t gfp,
		void *args);

		#ifdef CONFIG_DEBUG_FS

drivers/misc/habanalabs/common/habanalabs_drv.c

+2 −2

Original line number	Diff line number	Diff line
		@@ -140,7 +140,7 @@ int hl_device_open(struct inode inode, struct file filp)

		hl_cb_mgr_init(&hpriv->cb_mgr);
		hl_ctx_mgr_init(&hpriv->ctx_mgr);
		hl_ts_mgr_init(&hpriv->ts_mem_mgr);
		hl_mem_mgr_init(hpriv->hdev->dev, &hpriv->mem_mgr);

		hpriv->taskpid = get_task_pid(current, PIDTYPE_PID);

		@@ -198,7 +198,7 @@ int hl_device_open(struct inode inode, struct file filp)
		out_err:
		mutex_unlock(&hdev->fpriv_list_lock);
		hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
		hl_ts_mgr_fini(hpriv->hdev, &hpriv->ts_mem_mgr);
		hl_mem_mgr_fini(&hpriv->mem_mgr);
		hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
		filp->private_data = NULL;
		mutex_destroy(&hpriv->restore_phase_mutex);

drivers/misc/habanalabs/common/irq.c

+5 −5

Original line number	Diff line number	Diff line
		@@ -152,11 +152,11 @@ static void hl_ts_free_objects(struct work_struct *work)
		struct hl_device *hdev = job->hdev;

		list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) {
		dev_dbg(hdev->dev, "About to put refcount to ts_buff (%p) cq_cb(%p)\n",
		free_obj->ts_buff,
		dev_dbg(hdev->dev, "About to put refcount to buf (%p) cq_cb(%p)\n",
		free_obj->buf,
		free_obj->cq_cb);

		hl_ts_put(free_obj->ts_buff);
		hl_mmap_mem_buf_put(free_obj->buf);
		hl_cb_put(free_obj->cq_cb);
		kfree(free_obj);
		}
		@@ -210,7 +210,7 @@ static int handle_registration_node(struct hl_device *hdev, struct hl_user_pendi
		/* Putting the refcount for ts_buff and cq_cb objects will be handled
		* in workqueue context, just add job to free_list.
		*/
		free_node->ts_buff = pend->ts_reg_info.ts_buff;
		free_node->buf = pend->ts_reg_info.buf;
		free_node->cq_cb = pend->ts_reg_info.cq_cb;
		list_add(&free_node->free_objects_node, *free_list);

		@@ -244,7 +244,7 @@ static void handle_user_cq(struct hl_device *hdev,
		list_for_each_entry_safe(pend, temp_pend, &user_cq->wait_list_head, wait_list_node) {
		if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) \|\|
		!pend->cq_kernel_addr) {
		if (pend->ts_reg_info.ts_buff) {
		if (pend->ts_reg_info.buf) {
		if (!reg_node_handle_fail) {
		rc = handle_registration_node(hdev, pend,
		&ts_reg_free_list_head);