Commit d16945f6 authored by Tomer Tayar's avatar Tomer Tayar Committed by Oded Gabbay
Browse files

accel/habanalabs: add missing offset handling for dma-buf 



On devices with virtual device memory (Gaudi2 onwards), user can provide
an offset within an allocated device memory from which he wants to
export a dma-buf object.
The offset value is verified by driver, but it is not taken into
consideration when the importer driver maps the dma-buf and the SG table
it prepared.
Add the missing offset handling.

Signed-off-by: default avatarTomer Tayar <ttayar@habana.ai>
Reviewed-by: default avatarOded Gabbay <ogabbay@kernel.org>
Signed-off-by: default avatarOded Gabbay <ogabbay@kernel.org>
parent 878ebc14

drivers/accel/habanalabs/common/habanalabs.h

+3 −0
Original line number Diff line number Diff line
@@ -1811,6 +1811,8 @@ struct hl_cs_counters_atomic { 
 * @phys_pg_pack: pointer to physical page pack if the dma-buf was exported

 *                where virtual memory is supported.

 * @memhash_hnode: pointer to the memhash node. this object holds the export count.

 * @offset: the offset into the buffer from which the memory is exported.

 *          Relevant only if virtual memory is supported and phys_pg_pack is being used.

 * device_phys_addr: physical address of the device's memory. Relevant only

 *                   if phys_pg_pack is NULL (dma-buf was exported from address).

 *                   The total size can be taken from the dmabuf object.
@@ -1820,6 +1822,7 @@ struct hl_dmabuf_priv { 
	struct hl_ctx			*ctx;

	struct hl_vm_phys_pg_pack	*phys_pg_pack;

	struct hl_vm_hash_node		*memhash_hnode;

	u64				offset;

	u64				device_phys_addr;

};

drivers/accel/habanalabs/common/memory.c

+40 −35
Original line number Diff line number Diff line
@@ -1532,11 +1532,11 @@ static int set_dma_sg(struct scatterlist *sg, u64 bar_address, u64 chunk_size, 
}



static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64 *pages, u64 npages,

						u64 page_size, u64 exported_size,

						u64 page_size, u64 exported_size, u64 offset,

						struct device *dev, enum dma_data_direction dir)

{

	u64 dma_max_seg_size, curr_page, size, chunk_size, left_size_to_export, left_size_in_page,

		left_size_in_dma_seg, device_address, bar_address;

		left_size_in_dma_seg, device_address, bar_address, start_page;

	struct asic_fixed_properties *prop = &hdev->asic_prop;

	struct scatterlist *sg;

	unsigned int nents, i;
@@ -1557,11 +1557,20 @@ static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64 
	if (!sgt)

		return ERR_PTR(-ENOMEM);



	/* Use the offset to move to the actual first page that is exported */

	for (start_page = 0 ; start_page < npages ; ++start_page) {

		if (offset < page_size)

			break;



		/* The offset value was validated so there can't be an underflow */

		offset -= page_size;

	}



	/* Calculate the required number of entries for the SG table */

	curr_page = 0;

	curr_page = start_page;

	nents = 1;

	left_size_to_export = exported_size;

	left_size_in_page = page_size;

	left_size_in_page = page_size - offset;

	left_size_in_dma_seg = dma_max_seg_size;

	next_sg_entry = false;
@@ -1599,10 +1608,10 @@ static struct sg_table *alloc_sgt_from_device_pages(struct hl_device *hdev, u64 
		goto err_free_sgt;



	/* Prepare the SG table entries */

	curr_page = 0;

	device_address = pages[curr_page];

	curr_page = start_page;

	device_address = pages[curr_page] + offset;

	left_size_to_export = exported_size;

	left_size_in_page = page_size;

	left_size_in_page = page_size - offset;

	left_size_in_dma_seg = dma_max_seg_size;

	next_sg_entry = false;
@@ -1708,7 +1717,7 @@ static int hl_dmabuf_attach(struct dma_buf *dmabuf, 
static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment,

					enum dma_data_direction dir)

{

	u64 *pages, npages, page_size, exported_size;

	u64 *pages, npages, page_size, exported_size, offset;

	struct dma_buf *dma_buf = attachment->dmabuf;

	struct hl_vm_phys_pg_pack *phys_pg_pack;

	struct hl_dmabuf_priv *hl_dmabuf;
@@ -1724,6 +1733,7 @@ static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment, 
	}



	exported_size = hl_dmabuf->dmabuf->size;

	offset = hl_dmabuf->offset;

	phys_pg_pack = hl_dmabuf->phys_pg_pack;



	if (phys_pg_pack) {
@@ -1736,7 +1746,7 @@ static struct sg_table *hl_map_dmabuf(struct dma_buf_attachment *attachment, 
		page_size = hl_dmabuf->dmabuf->size;

	}



	sgt = alloc_sgt_from_device_pages(hdev, pages, npages, page_size, exported_size,

	sgt = alloc_sgt_from_device_pages(hdev, pages, npages, page_size, exported_size, offset,

						attachment->dev, dir);

	if (IS_ERR(sgt))

		dev_err(hdev->dev, "failed (%ld) to initialize sgt for dmabuf\n", PTR_ERR(sgt));
@@ -1882,12 +1892,12 @@ static int export_dmabuf(struct hl_ctx *ctx, 
	return rc;

}



static int validate_export_params_common(struct hl_device *hdev, u64 device_addr, u64 size)

static int validate_export_params_common(struct hl_device *hdev, u64 addr, u64 size, u64 offset)

{

	if (!PAGE_ALIGNED(device_addr)) {

	if (!PAGE_ALIGNED(addr)) {

		dev_dbg(hdev->dev,

			"exported device memory address 0x%llx should be aligned to PAGE_SIZE 0x%lx\n",

			device_addr, PAGE_SIZE);

			addr, PAGE_SIZE);

		return -EINVAL;

	}
@@ -1898,6 +1908,13 @@ static int validate_export_params_common(struct hl_device *hdev, u64 device_addr 
		return -EINVAL;

	}



	if (!PAGE_ALIGNED(offset)) {

		dev_dbg(hdev->dev,

			"exported device memory offset %llu should be a multiple of PAGE_SIZE %lu\n",

			offset, PAGE_SIZE);

		return -EINVAL;

	}



	return 0;

}
@@ -1907,7 +1924,7 @@ static int validate_export_params_no_mmu(struct hl_device *hdev, u64 device_addr 
	u64 bar_address;

	int rc;



	rc = validate_export_params_common(hdev, device_addr, size);

	rc = validate_export_params_common(hdev, device_addr, size, 0);

	if (rc)

		return rc;
@@ -1940,17 +1957,10 @@ static int validate_export_params(struct hl_device *hdev, u64 device_addr, u64 s 
	u64 bar_address;

	int i, rc;



	rc = validate_export_params_common(hdev, device_addr, size);

	rc = validate_export_params_common(hdev, device_addr, size, offset);

	if (rc)

		return rc;



	if (!PAGE_ALIGNED(offset)) {

		dev_dbg(hdev->dev,

			"exported device memory offset %llu should be a multiple of PAGE_SIZE %lu\n",

			offset, PAGE_SIZE);

		return -EINVAL;

	}



	if ((offset + size) > phys_pg_pack->total_size) {

		dev_dbg(hdev->dev, "offset %#llx and size %#llx exceed total map size %#llx\n",

			offset, size, phys_pg_pack->total_size);
@@ -1958,7 +1968,6 @@ static int validate_export_params(struct hl_device *hdev, u64 device_addr, u64 s 
	}



	for (i = 0 ; i < phys_pg_pack->npages ; i++) {



		bar_address = hdev->dram_pci_bar_start +

				(phys_pg_pack->pages[i] - prop->dram_base_address);
@@ -1967,9 +1976,7 @@ static int validate_export_params(struct hl_device *hdev, u64 device_addr, u64 s 
				(bar_address + phys_pg_pack->page_size) < bar_address) {

			dev_dbg(hdev->dev,

				"DRAM memory range 0x%llx (+0x%x) is outside of PCI BAR boundaries\n",

					phys_pg_pack->pages[i],

					phys_pg_pack->page_size);



				phys_pg_pack->pages[i], phys_pg_pack->page_size);

			return -EINVAL;

		}

	}
@@ -2025,7 +2032,6 @@ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 o 
	struct asic_fixed_properties *prop;

	struct hl_dmabuf_priv *hl_dmabuf;

	struct hl_device *hdev;

	u64 export_addr;

	int rc;



	hdev = ctx->hdev;
@@ -2037,8 +2043,6 @@ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 o 
		return -EINVAL;

	}



	export_addr = addr + offset;



	hl_dmabuf = kzalloc(sizeof(*hl_dmabuf), GFP_KERNEL);

	if (!hl_dmabuf)

		return -ENOMEM;
@@ -2054,18 +2058,19 @@ static int export_dmabuf_from_addr(struct hl_ctx *ctx, u64 addr, u64 size, u64 o 
			rc = PTR_ERR(phys_pg_pack);

			goto dec_memhash_export_cnt;

		}

		rc = validate_export_params(hdev, export_addr, size, offset, phys_pg_pack);

		rc = validate_export_params(hdev, addr, size, offset, phys_pg_pack);

		if (rc)

			goto dec_memhash_export_cnt;



		hl_dmabuf->phys_pg_pack = phys_pg_pack;

		hl_dmabuf->memhash_hnode = hnode;

		hl_dmabuf->offset = offset;

	} else {

		rc = validate_export_params_no_mmu(hdev, export_addr, size);

		rc = validate_export_params_no_mmu(hdev, addr, size);

		if (rc)

			goto err_free_dmabuf_wrapper;



		hl_dmabuf->device_phys_addr = export_addr;

		hl_dmabuf->device_phys_addr = addr;

	}



	rc = export_dmabuf(ctx, hl_dmabuf, size, flags, dmabuf_fd);