in habanalabs/common/memory.c [1063:1251]
static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args,
u64 *device_addr)
{
struct hl_device *hdev = ctx->hdev;
struct hl_vm *vm = &hdev->vm;
struct hl_vm_phys_pg_pack *phys_pg_pack;
struct hl_userptr *userptr = NULL;
struct hl_vm_hash_node *hnode;
struct hl_va_range *va_range;
enum vm_type *vm_type;
u64 ret_vaddr, hint_addr;
u32 handle = 0, va_block_align;
int rc;
bool is_userptr = args->flags & HL_MEM_USERPTR;
enum hl_va_range_type va_range_type = 0;
/* Assume failure */
*device_addr = 0;
if (is_userptr) {
u64 addr = args->map_host.host_virt_addr,
size = args->map_host.mem_size;
u32 page_size = hdev->asic_prop.pmmu.page_size,
huge_page_size = hdev->asic_prop.pmmu_huge.page_size;
rc = dma_map_host_va(hdev, addr, size, &userptr);
if (rc) {
dev_err(hdev->dev, "failed to get userptr from va\n");
return rc;
}
rc = init_phys_pg_pack_from_userptr(ctx, userptr,
&phys_pg_pack, false);
if (rc) {
dev_err(hdev->dev,
"unable to init page pack for vaddr 0x%llx\n",
addr);
goto init_page_pack_err;
}
vm_type = (enum vm_type *) userptr;
hint_addr = args->map_host.hint_addr;
handle = phys_pg_pack->handle;
/* get required alignment */
if (phys_pg_pack->page_size == page_size) {
va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST];
va_range_type = HL_VA_RANGE_TYPE_HOST;
/*
* huge page alignment may be needed in case of regular
* page mapping, depending on the host VA alignment
*/
if (addr & (huge_page_size - 1))
va_block_align = page_size;
else
va_block_align = huge_page_size;
} else {
/*
* huge page alignment is needed in case of huge page
* mapping
*/
va_range = ctx->va_range[HL_VA_RANGE_TYPE_HOST_HUGE];
va_range_type = HL_VA_RANGE_TYPE_HOST_HUGE;
va_block_align = huge_page_size;
}
} else {
handle = lower_32_bits(args->map_device.handle);
spin_lock(&vm->idr_lock);
phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle);
if (!phys_pg_pack) {
spin_unlock(&vm->idr_lock);
dev_err(hdev->dev,
"no match for handle %u\n", handle);
return -EINVAL;
}
/* increment now to avoid freeing device memory while mapping */
atomic_inc(&phys_pg_pack->mapping_cnt);
spin_unlock(&vm->idr_lock);
vm_type = (enum vm_type *) phys_pg_pack;
hint_addr = args->map_device.hint_addr;
/* DRAM VA alignment is the same as the MMU page size */
va_range = ctx->va_range[HL_VA_RANGE_TYPE_DRAM];
va_range_type = HL_VA_RANGE_TYPE_DRAM;
va_block_align = hdev->asic_prop.dmmu.page_size;
}
/*
* relevant for mapping device physical memory only, as host memory is
* implicitly shared
*/
if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) &&
phys_pg_pack->asid != ctx->asid) {
dev_err(hdev->dev,
"Failed to map memory, handle %u is not shared\n",
handle);
rc = -EPERM;
goto shared_err;
}
hnode = kzalloc(sizeof(*hnode), GFP_KERNEL);
if (!hnode) {
rc = -ENOMEM;
goto hnode_err;
}
if (hint_addr && phys_pg_pack->offset) {
if (args->flags & HL_MEM_FORCE_HINT) {
/* Fail if hint must be respected but it can't be */
dev_err(hdev->dev,
"Hint address 0x%llx cannot be respected because source memory is not aligned 0x%x\n",
hint_addr, phys_pg_pack->offset);
rc = -EINVAL;
goto va_block_err;
}
dev_dbg(hdev->dev,
"Hint address 0x%llx will be ignored because source memory is not aligned 0x%x\n",
hint_addr, phys_pg_pack->offset);
}
ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size,
hint_addr, va_block_align,
va_range_type, args->flags);
if (!ret_vaddr) {
dev_err(hdev->dev, "no available va block for handle %u\n",
handle);
rc = -ENOMEM;
goto va_block_err;
}
mutex_lock(&ctx->mmu_lock);
rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack);
if (rc) {
mutex_unlock(&ctx->mmu_lock);
dev_err(hdev->dev, "mapping page pack failed for handle %u\n",
handle);
goto map_err;
}
rc = hl_mmu_invalidate_cache_range(hdev, false, *vm_type | MMU_OP_SKIP_LOW_CACHE_INV,
ctx->asid, ret_vaddr, phys_pg_pack->total_size);
mutex_unlock(&ctx->mmu_lock);
if (rc)
goto map_err;
ret_vaddr += phys_pg_pack->offset;
hnode->ptr = vm_type;
hnode->vaddr = ret_vaddr;
mutex_lock(&ctx->mem_hash_lock);
hash_add(ctx->mem_hash, &hnode->node, ret_vaddr);
mutex_unlock(&ctx->mem_hash_lock);
*device_addr = ret_vaddr;
if (is_userptr)
rc = free_phys_pg_pack(hdev, phys_pg_pack);
return rc;
map_err:
if (add_va_block(hdev, va_range, ret_vaddr,
ret_vaddr + phys_pg_pack->total_size - 1))
dev_warn(hdev->dev,
"release va block failed for handle 0x%x, vaddr: 0x%llx\n",
handle, ret_vaddr);
va_block_err:
kfree(hnode);
hnode_err:
shared_err:
atomic_dec(&phys_pg_pack->mapping_cnt);
if (is_userptr)
free_phys_pg_pack(hdev, phys_pg_pack);
init_page_pack_err:
if (is_userptr)
dma_unmap_host_va(hdev, userptr);
return rc;
}