From aa3766def7506e5d9bd6c8387dcfe3629eb2a1f2 Mon Sep 17 00:00:00 2001 From: Oded Gabbay Date: Tue, 18 Jan 2022 14:58:30 +0200 Subject: habanalabs: expose number of user interrupts Currently we only expose to the user the ID of the first available user interrupt. To make user interrupts allocation truly dynamic, we need to also expose the number of user interrupts. Signed-off-by: Oded Gabbay --- include/uapi/misc/habanalabs.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/uapi/misc') diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 371dfc4243b3..12976f7a8d84 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -404,6 +404,8 @@ enum hl_server_type { * @cpucp_version: The CPUCP f/w version. * @card_name: The card name as passed by the f/w. * @dram_page_size: The DRAM physical page size. + * @number_of_user_interrupts: The number of interrupts that are available to the userspace + * application to use. Relevant for Gaudi2 and later. */ struct hl_info_hw_ip_info { __u64 sram_base_address; @@ -428,6 +430,9 @@ struct hl_info_hw_ip_info { __u8 card_name[HL_INFO_CARD_NAME_MAX_LEN]; __u64 reserved2; __u64 dram_page_size; + __u32 reserved3; + __u16 number_of_user_interrupts; + __u16 pad2; }; struct hl_info_dram_usage { -- cgit v1.2.3 From 9158bf69e74f98fea6847cca93bbf33a589bebcd Mon Sep 17 00:00:00 2001 From: farah kassabri Date: Thu, 23 Dec 2021 13:24:34 +0200 Subject: habanalabs: Timestamps buffers registration Timestamp registration API allows the user to register a timestamp record event which will make the driver set timestamp when CQ counter reaches the target value and write it to a specific location specified by the user. This is a non blocking API, unlike the wait_for_interrupt which is a blocking one. Signed-off-by: farah kassabri Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 197 +++++++++++++--- drivers/misc/habanalabs/common/device.c | 18 +- drivers/misc/habanalabs/common/habanalabs.h | 106 ++++++++- drivers/misc/habanalabs/common/habanalabs_drv.c | 2 + drivers/misc/habanalabs/common/irq.c | 127 +++++++++- drivers/misc/habanalabs/common/memory.c | 261 +++++++++++++++++++++ include/uapi/misc/habanalabs.h | 37 ++- 7 files changed, 690 insertions(+), 58 deletions(-) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index ba5215b77852..c7757c78d0b1 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -14,6 +14,8 @@ #define HL_CS_FLAGS_TYPE_MASK (HL_CS_FLAGS_SIGNAL | HL_CS_FLAGS_WAIT | \ HL_CS_FLAGS_COLLECTIVE_WAIT) +#define MAX_TS_ITER_NUM 10 + /** * enum hl_cs_wait_status - cs wait status * @CS_WAIT_STATUS_BUSY: cs was not completed yet @@ -924,7 +926,7 @@ void hl_cs_rollback_all(struct hl_device *hdev) int i; struct hl_cs *cs, *tmp; - flush_workqueue(hdev->sob_reset_wq); + flush_workqueue(hdev->ts_free_obj_wq); /* flush all completions before iterating over the CS mirror list in * order to avoid a race with the release functions @@ -948,13 +950,19 @@ void hl_cs_rollback_all(struct hl_device *hdev) static void wake_pending_user_interrupt_threads(struct hl_user_interrupt *interrupt) { - struct hl_user_pending_interrupt *pend; + struct hl_user_pending_interrupt *pend, *temp; unsigned long flags; spin_lock_irqsave(&interrupt->wait_list_lock, flags); - list_for_each_entry(pend, &interrupt->wait_list_head, wait_list_node) { - pend->fence.error = -EIO; - complete_all(&pend->fence.completion); + list_for_each_entry_safe(pend, temp, &interrupt->wait_list_head, wait_list_node) { + if (pend->ts_reg_info.ts_buff) { + list_del(&pend->wait_list_node); + hl_ts_put(pend->ts_reg_info.ts_buff); + hl_cb_put(pend->ts_reg_info.cq_cb); + } else { + pend->fence.error = -EIO; + complete_all(&pend->fence.completion); + } } spin_unlock_irqrestore(&interrupt->wait_list_lock, flags); } @@ -2857,43 +2865,133 @@ static int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data) return 0; } +static int ts_buff_get_kernel_ts_record(struct hl_ts_buff *ts_buff, + struct hl_cb *cq_cb, + u64 ts_offset, u64 cq_offset, u64 target_value, + spinlock_t *wait_list_lock, + struct hl_user_pending_interrupt **pend) +{ + struct hl_user_pending_interrupt *requested_offset_record = + (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + + ts_offset; + struct hl_user_pending_interrupt *cb_last = + (struct hl_user_pending_interrupt *)ts_buff->kernel_buff_address + + (ts_buff->kernel_buff_size / sizeof(struct hl_user_pending_interrupt)); + unsigned long flags, iter_counter = 0; + u64 current_cq_counter; + + /* Validate ts_offset not exceeding last max */ + if (requested_offset_record > cb_last) { + dev_err(ts_buff->hdev->dev, "Ts offset exceeds max CB offset(0x%llx)\n", + (u64)(uintptr_t)cb_last); + return -EINVAL; + } + +start_over: + spin_lock_irqsave(wait_list_lock, flags); + + /* Unregister only if we didn't reach the target value + * since in this case there will be no handling in irq context + * and then it's safe to delete the node out of the interrupt list + * then re-use it on other interrupt + */ + if (requested_offset_record->ts_reg_info.in_use) { + current_cq_counter = *requested_offset_record->cq_kernel_addr; + if (current_cq_counter < requested_offset_record->cq_target_value) { + list_del(&requested_offset_record->wait_list_node); + spin_unlock_irqrestore(wait_list_lock, flags); + + hl_ts_put(requested_offset_record->ts_reg_info.ts_buff); + hl_cb_put(requested_offset_record->ts_reg_info.cq_cb); + + dev_dbg(ts_buff->hdev->dev, "ts node removed from interrupt list now can re-use\n"); + } else { + dev_dbg(ts_buff->hdev->dev, "ts node in middle of irq handling\n"); + + /* irq handling in the middle give it time to finish */ + spin_unlock_irqrestore(wait_list_lock, flags); + usleep_range(1, 10); + if (++iter_counter == MAX_TS_ITER_NUM) { + dev_err(ts_buff->hdev->dev, "handling registration interrupt took too long!!\n"); + return -EINVAL; + } + + goto start_over; + } + } else { + spin_unlock_irqrestore(wait_list_lock, flags); + } + + /* Fill up the new registration node info */ + requested_offset_record->ts_reg_info.in_use = 1; + requested_offset_record->ts_reg_info.ts_buff = ts_buff; + requested_offset_record->ts_reg_info.cq_cb = cq_cb; + requested_offset_record->ts_reg_info.timestamp_kernel_addr = + (u64 *) ts_buff->user_buff_address + ts_offset; + requested_offset_record->cq_kernel_addr = + (u64 *) cq_cb->kernel_address + cq_offset; + requested_offset_record->cq_target_value = target_value; + + *pend = requested_offset_record; + + dev_dbg(ts_buff->hdev->dev, "Found available node in TS kernel CB(0x%llx)\n", + (u64)(uintptr_t)requested_offset_record); + return 0; +} + static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, - struct hl_cb_mgr *cb_mgr, u64 timeout_us, - u64 cq_counters_handle, u64 cq_counters_offset, + struct hl_cb_mgr *cb_mgr, struct hl_ts_mgr *ts_mgr, + u64 timeout_us, u64 cq_counters_handle, u64 cq_counters_offset, u64 target_value, struct hl_user_interrupt *interrupt, + bool register_ts_record, u64 ts_handle, u64 ts_offset, u32 *status, u64 *timestamp) { + u32 cq_patched_handle, ts_patched_handle; struct hl_user_pending_interrupt *pend; + struct hl_ts_buff *ts_buff; + struct hl_cb *cq_cb; unsigned long timeout, flags; long completion_rc; - struct hl_cb *cb; int rc = 0; - u32 handle; timeout = hl_usecs64_to_jiffies(timeout_us); hl_ctx_get(hdev, ctx); - cq_counters_handle >>= PAGE_SHIFT; - handle = (u32) cq_counters_handle; - - cb = hl_cb_get(hdev, cb_mgr, handle); - if (!cb) { - hl_ctx_put(ctx); - return -EINVAL; + cq_patched_handle = lower_32_bits(cq_counters_handle >> PAGE_SHIFT); + cq_cb = hl_cb_get(hdev, cb_mgr, cq_patched_handle); + if (!cq_cb) { + rc = -EINVAL; + goto put_ctx; } - pend = kzalloc(sizeof(*pend), GFP_KERNEL); - if (!pend) { - hl_cb_put(cb); - hl_ctx_put(ctx); - return -ENOMEM; - } + if (register_ts_record) { + dev_dbg(hdev->dev, "Timestamp registration: interrupt id: %u, ts offset: %llu, cq_offset: %llu\n", + interrupt->interrupt_id, ts_offset, cq_counters_offset); - hl_fence_init(&pend->fence, ULONG_MAX); + ts_patched_handle = lower_32_bits(ts_handle >> PAGE_SHIFT); + ts_buff = hl_ts_get(hdev, ts_mgr, ts_patched_handle); + if (!ts_buff) { + rc = -EINVAL; + goto put_cq_cb; + } - pend->cq_kernel_addr = (u64 *) cb->kernel_address + cq_counters_offset; - pend->cq_target_value = target_value; + /* Find first available record */ + rc = ts_buff_get_kernel_ts_record(ts_buff, cq_cb, ts_offset, + cq_counters_offset, target_value, + &interrupt->wait_list_lock, &pend); + if (rc) + goto put_ts_buff; + } else { + pend = kzalloc(sizeof(*pend), GFP_KERNEL); + if (!pend) { + rc = -ENOMEM; + goto put_cq_cb; + } + hl_fence_init(&pend->fence, ULONG_MAX); + pend->cq_kernel_addr = (u64 *) cq_cb->kernel_address + cq_counters_offset; + pend->cq_target_value = target_value; + } spin_lock_irqsave(&interrupt->wait_list_lock, flags); @@ -2901,13 +2999,19 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, * before we added the node to the wait list */ if (*pend->cq_kernel_addr >= target_value) { + if (register_ts_record) + pend->ts_reg_info.in_use = 0; spin_unlock_irqrestore(&interrupt->wait_list_lock, flags); *status = HL_WAIT_CS_STATUS_COMPLETED; - /* There was no interrupt, we assume the completion is now. */ - pend->fence.timestamp = ktime_get(); - goto set_timestamp; + if (register_ts_record) { + *pend->ts_reg_info.timestamp_kernel_addr = ktime_get_ns(); + goto put_ts_buff; + } else { + pend->fence.timestamp = ktime_get(); + goto set_timestamp; + } } else if (!timeout_us) { spin_unlock_irqrestore(&interrupt->wait_list_lock, flags); *status = HL_WAIT_CS_STATUS_BUSY; @@ -2916,11 +3020,19 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, } /* Add pending user interrupt to relevant list for the interrupt - * handler to monitor + * handler to monitor. + * Note that we cannot have sorted list by target value, + * in order to shorten the list pass loop, since + * same list could have nodes for different cq counter handle. */ list_add_tail(&pend->wait_list_node, &interrupt->wait_list_head); spin_unlock_irqrestore(&interrupt->wait_list_lock, flags); + if (register_ts_record) { + rc = *status = HL_WAIT_CS_STATUS_COMPLETED; + goto ts_registration_exit; + } + /* Wait for interrupt handler to signal completion */ completion_rc = wait_for_completion_interruptible_timeout(&pend->fence.completion, timeout); @@ -2952,15 +3064,30 @@ static int _hl_interrupt_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, } } + /* + * We keep removing the node from list here, and not at the irq handler + * for completion timeout case. and if it's a registration + * for ts record, the node will be deleted in the irq handler after + * we reach the target value. + */ spin_lock_irqsave(&interrupt->wait_list_lock, flags); list_del(&pend->wait_list_node); spin_unlock_irqrestore(&interrupt->wait_list_lock, flags); set_timestamp: *timestamp = ktime_to_ns(pend->fence.timestamp); - kfree(pend); - hl_cb_put(cb); + hl_cb_put(cq_cb); +ts_registration_exit: + hl_ctx_put(ctx); + + return rc; + +put_ts_buff: + hl_ts_put(ts_buff); +put_cq_cb: + hl_cb_put(cq_cb); +put_ctx: hl_ctx_put(ctx); return rc; @@ -3119,11 +3246,13 @@ static int hl_interrupt_wait_ioctl(struct hl_fpriv *hpriv, void *data) interrupt = &hdev->user_interrupt[interrupt_id - first_interrupt]; if (args->in.flags & HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ) - rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &hpriv->cb_mgr, + rc = _hl_interrupt_wait_ioctl(hdev, hpriv->ctx, &hpriv->cb_mgr, &hpriv->ts_mem_mgr, args->in.interrupt_timeout_us, args->in.cq_counters_handle, args->in.cq_counters_offset, - args->in.target, interrupt, &status, - ×tamp); + args->in.target, interrupt, + !!(args->in.flags & HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT), + args->in.timestamp_handle, args->in.timestamp_offset, + &status, ×tamp); else rc = _hl_interrupt_wait_ioctl_user_addr(hdev, hpriv->ctx, args->in.interrupt_timeout_us, args->in.addr, diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 727315b36b8b..60bfd737404b 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -145,6 +145,7 @@ static int hl_device_release(struct inode *inode, struct file *filp) hl_release_pending_user_interrupts(hpriv->hdev); hl_cb_mgr_fini(hdev, &hpriv->cb_mgr); + hl_ts_mgr_fini(hpriv->hdev, &hpriv->ts_mem_mgr); hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr); if (!hl_hpriv_put(hpriv)) @@ -209,6 +210,9 @@ static int hl_mmap(struct file *filp, struct vm_area_struct *vma) case HL_MMAP_TYPE_BLOCK: return hl_hw_block_mmap(hpriv, vma); + + case HL_MMAP_TYPE_TS_BUFF: + return hl_ts_mmap(hpriv, vma); } return -EINVAL; @@ -410,10 +414,10 @@ static int device_early_init(struct hl_device *hdev) goto free_cq_wq; } - hdev->sob_reset_wq = alloc_workqueue("hl-sob-reset", WQ_UNBOUND, 0); - if (!hdev->sob_reset_wq) { + hdev->ts_free_obj_wq = alloc_workqueue("hl-ts-free-obj", WQ_UNBOUND, 0); + if (!hdev->ts_free_obj_wq) { dev_err(hdev->dev, - "Failed to allocate SOB reset workqueue\n"); + "Failed to allocate Timestamp registration free workqueue\n"); rc = -ENOMEM; goto free_eq_wq; } @@ -422,7 +426,7 @@ static int device_early_init(struct hl_device *hdev) GFP_KERNEL); if (!hdev->hl_chip_info) { rc = -ENOMEM; - goto free_sob_reset_wq; + goto free_ts_free_wq; } rc = hl_mmu_if_set_funcs(hdev); @@ -461,8 +465,8 @@ free_cb_mgr: hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr); free_chip_info: kfree(hdev->hl_chip_info); -free_sob_reset_wq: - destroy_workqueue(hdev->sob_reset_wq); +free_ts_free_wq: + destroy_workqueue(hdev->ts_free_obj_wq); free_eq_wq: destroy_workqueue(hdev->eq_wq); free_cq_wq: @@ -501,7 +505,7 @@ static void device_early_fini(struct hl_device *hdev) kfree(hdev->hl_chip_info); - destroy_workqueue(hdev->sob_reset_wq); + destroy_workqueue(hdev->ts_free_obj_wq); destroy_workqueue(hdev->eq_wq); destroy_workqueue(hdev->device_reset_work.wq); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 995167020c27..b06e2b0812b6 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -31,14 +31,15 @@ #define HL_NAME "habanalabs" /* Use upper bits of mmap offset to store habana driver specific information. - * bits[63:61] - Encode mmap type + * bits[63:59] - Encode mmap type * bits[45:0] - mmap offset value * * NOTE: struct vm_area_struct.vm_pgoff uses offset in pages. Hence, these * defines are w.r.t to PAGE_SIZE */ -#define HL_MMAP_TYPE_SHIFT (61 - PAGE_SHIFT) -#define HL_MMAP_TYPE_MASK (0x7ull << HL_MMAP_TYPE_SHIFT) +#define HL_MMAP_TYPE_SHIFT (59 - PAGE_SHIFT) +#define HL_MMAP_TYPE_MASK (0x1full << HL_MMAP_TYPE_SHIFT) +#define HL_MMAP_TYPE_TS_BUFF (0x10ull << HL_MMAP_TYPE_SHIFT) #define HL_MMAP_TYPE_BLOCK (0x4ull << HL_MMAP_TYPE_SHIFT) #define HL_MMAP_TYPE_CB (0x2ull << HL_MMAP_TYPE_SHIFT) @@ -709,6 +710,40 @@ struct hl_cb_mgr { struct idr cb_handles; /* protected by cb_lock */ }; +/** + * struct hl_ts_mgr - describes the timestamp registration memory manager. + * @ts_lock: protects ts_handles. + * @ts_handles: an idr to hold all ts bufferes handles. + */ +struct hl_ts_mgr { + spinlock_t ts_lock; + struct idr ts_handles; +}; + +/** + * struct hl_ts_buff - describes a timestamp buffer. + * @refcount: reference counter for usage of the buffer. + * @hdev: pointer to device this buffer belongs to. + * @mmap: true if the buff is currently mapped to user. + * @kernel_buff_address: Holds the internal buffer's kernel virtual address. + * @user_buff_address: Holds the user buffer's kernel virtual address. + * @id: the buffer ID. + * @mmap_size: Holds the buffer size that was mmaped. + * @kernel_buff_size: Holds the internal kernel buffer size. + * @user_buff_size: Holds the user buffer size. + */ +struct hl_ts_buff { + struct kref refcount; + struct hl_device *hdev; + atomic_t mmap; + void *kernel_buff_address; + void *user_buff_address; + u32 id; + u32 mmap_size; + u32 kernel_buff_size; + u32 user_buff_size; +}; + /** * struct hl_cb - describes a Command Buffer. * @refcount: reference counter for usage of the CB. @@ -886,9 +921,54 @@ struct hl_user_interrupt { u32 interrupt_id; }; +/** + * struct timestamp_reg_free_node - holds the timestamp registration free objects node + * @free_objects_node: node in the list free_obj_jobs + * @cq_cb: pointer to cq command buffer to be freed + * @ts_buff: pointer to timestamp buffer to be freed + */ +struct timestamp_reg_free_node { + struct list_head free_objects_node; + struct hl_cb *cq_cb; + struct hl_ts_buff *ts_buff; +}; + +/* struct timestamp_reg_work_obj - holds the timestamp registration free objects job + * the job will be to pass over the free_obj_jobs list and put refcount to objects + * in each node of the list + * @free_obj: workqueue object to free timestamp registration node objects + * @hdev: pointer to the device structure + * @free_obj_head: list of free jobs nodes (node type timestamp_reg_free_node) + */ +struct timestamp_reg_work_obj { + struct work_struct free_obj; + struct hl_device *hdev; + struct list_head *free_obj_head; +}; + +/* struct timestamp_reg_info - holds the timestamp registration related data. + * @ts_buff: pointer to the timestamp buffer which include both user/kernel buffers. + * relevant only when doing timestamps records registration. + * @cq_cb: pointer to CQ counter CB. + * @timestamp_kernel_addr: timestamp handle address, where to set timestamp + * relevant only when doing timestamps records + * registration. + * @in_use: indicates if the node already in use. relevant only when doing + * timestamps records registration, since in this case the driver + * will have it's own buffer which serve as a records pool instead of + * allocating records dynamically. + */ +struct timestamp_reg_info { + struct hl_ts_buff *ts_buff; + struct hl_cb *cq_cb; + u64 *timestamp_kernel_addr; + u8 in_use; +}; + /** * struct hl_user_pending_interrupt - holds a context to a user thread * pending on an interrupt + * @ts_reg_info: holds the timestamps registration nodes info * @wait_list_node: node in the list of user threads pending on an interrupt * @fence: hl fence object for interrupt completion * @cq_target_value: CQ target value @@ -896,10 +976,11 @@ struct hl_user_interrupt { * handler for taget value comparison */ struct hl_user_pending_interrupt { - struct list_head wait_list_node; - struct hl_fence fence; - u64 cq_target_value; - u64 *cq_kernel_addr; + struct timestamp_reg_info ts_reg_info; + struct list_head wait_list_node; + struct hl_fence fence; + u64 cq_target_value; + u64 *cq_kernel_addr; }; /** @@ -1833,6 +1914,7 @@ struct hl_debug_params { * @ctx: current executing context. TODO: remove for multiple ctx per process * @ctx_mgr: context manager to handle multiple context for this FD. * @cb_mgr: command buffer manager to handle multiple buffers for this FD. + * @ts_mem_mgr: timestamp registration manager for alloc/free/map timestamp buffers. * @debugfs_list: list of relevant ASIC debugfs. * @dev_node: node in the device list of file private data * @refcount: number of related contexts. @@ -1845,6 +1927,7 @@ struct hl_fpriv { struct hl_ctx *ctx; struct hl_ctx_mgr ctx_mgr; struct hl_cb_mgr cb_mgr; + struct hl_ts_mgr ts_mem_mgr; struct list_head debugfs_list; struct list_head dev_node; struct kref refcount; @@ -2517,7 +2600,7 @@ struct hl_reset_info { * @cq_wq: work queues of completion queues for executing work in process * context. * @eq_wq: work queue of event queue for executing work in process context. - * @sob_reset_wq: work queue for sob reset executions. + * @ts_free_obj_wq: work queue for timestamp registration objects release. * @kernel_ctx: Kernel driver context structure. * @kernel_queues: array of hl_hw_queue. * @cs_mirror_list: CS mirror list for TDR. @@ -2645,7 +2728,7 @@ struct hl_device { struct hl_user_interrupt common_user_interrupt; struct workqueue_struct **cq_wq; struct workqueue_struct *eq_wq; - struct workqueue_struct *sob_reset_wq; + struct workqueue_struct *ts_free_obj_wq; struct hl_ctx *kernel_ctx; struct hl_hw_queue *kernel_queues; struct list_head cs_mirror_list; @@ -3128,6 +3211,11 @@ __printf(4, 5) int hl_snprintf_resize(char **buf, size_t *size, size_t *offset, const char *format, ...); char *hl_format_as_binary(char *buf, size_t buf_len, u32 n); const char *hl_sync_engine_to_string(enum hl_sync_engine_type engine_type); +void hl_ts_mgr_init(struct hl_ts_mgr *mgr); +void hl_ts_mgr_fini(struct hl_device *hdev, struct hl_ts_mgr *mgr); +int hl_ts_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma); +struct hl_ts_buff *hl_ts_get(struct hl_device *hdev, struct hl_ts_mgr *mgr, u32 handle); +void hl_ts_put(struct hl_ts_buff *buff); #ifdef CONFIG_DEBUG_FS diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index 017bfc4551da..ca404ed9d9a7 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -140,6 +140,7 @@ int hl_device_open(struct inode *inode, struct file *filp) hl_cb_mgr_init(&hpriv->cb_mgr); hl_ctx_mgr_init(&hpriv->ctx_mgr); + hl_ts_mgr_init(&hpriv->ts_mem_mgr); hpriv->taskpid = get_task_pid(current, PIDTYPE_PID); @@ -184,6 +185,7 @@ int hl_device_open(struct inode *inode, struct file *filp) out_err: mutex_unlock(&hdev->fpriv_list_lock); hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr); + hl_ts_mgr_fini(hpriv->hdev, &hpriv->ts_mem_mgr); hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr); filp->private_data = NULL; mutex_destroy(&hpriv->restore_phase_mutex); diff --git a/drivers/misc/habanalabs/common/irq.c b/drivers/misc/habanalabs/common/irq.c index 1b6bdc900c26..c28f3a1c7a03 100644 --- a/drivers/misc/habanalabs/common/irq.c +++ b/drivers/misc/habanalabs/common/irq.c @@ -137,22 +137,137 @@ irqreturn_t hl_irq_handler_cq(int irq, void *arg) return IRQ_HANDLED; } +/* + * hl_ts_free_objects - handler of the free objects workqueue. + * This function should put refcount to objects that the registration node + * took refcount to them. + * @work: workqueue object pointer + */ +static void hl_ts_free_objects(struct work_struct *work) +{ + struct timestamp_reg_work_obj *job = + container_of(work, struct timestamp_reg_work_obj, free_obj); + struct timestamp_reg_free_node *free_obj, *temp_free_obj; + struct list_head *free_list_head = job->free_obj_head; + struct hl_device *hdev = job->hdev; + + list_for_each_entry_safe(free_obj, temp_free_obj, free_list_head, free_objects_node) { + dev_dbg(hdev->dev, "About to put refcount to ts_buff (%p) cq_cb(%p)\n", + free_obj->ts_buff, + free_obj->cq_cb); + + hl_ts_put(free_obj->ts_buff); + hl_cb_put(free_obj->cq_cb); + kfree(free_obj); + } + + kfree(free_list_head); + kfree(job); +} + +/* + * This function called with spin_lock of wait_list_lock taken + * This function will set timestamp and delete the registration node from the + * wait_list_lock. + * and since we're protected with spin_lock here, so we cannot just put the refcount + * for the objects here, since the release function may be called and it's also a long + * logic (which might sleep also) that cannot be handled in irq context. + * so here we'll be filling a list with nodes of "put" jobs and then will send this + * list to a dedicated workqueue to do the actual put. + */ +int handle_registration_node(struct hl_device *hdev, struct hl_user_pending_interrupt *pend, + struct list_head **free_list) +{ + struct timestamp_reg_free_node *free_node; + u64 timestamp; + + if (!(*free_list)) { + /* Alloc/Init the timestamp registration free objects list */ + *free_list = kmalloc(sizeof(struct list_head), GFP_ATOMIC); + if (!(*free_list)) + return -ENOMEM; + + INIT_LIST_HEAD(*free_list); + } + + free_node = kmalloc(sizeof(*free_node), GFP_ATOMIC); + if (!free_node) + return -ENOMEM; + + timestamp = ktime_get_ns(); + + *pend->ts_reg_info.timestamp_kernel_addr = timestamp; + + dev_dbg(hdev->dev, "Timestamp is set to ts cb address (%p), ts: 0x%llx\n", + pend->ts_reg_info.timestamp_kernel_addr, + *(u64 *)pend->ts_reg_info.timestamp_kernel_addr); + + list_del(&pend->wait_list_node); + + /* Mark kernel CB node as free */ + pend->ts_reg_info.in_use = 0; + + /* Putting the refcount for ts_buff and cq_cb objects will be handled + * in workqueue context, just add job to free_list. + */ + free_node->ts_buff = pend->ts_reg_info.ts_buff; + free_node->cq_cb = pend->ts_reg_info.cq_cb; + list_add(&free_node->free_objects_node, *free_list); + + return 0; +} + static void handle_user_cq(struct hl_device *hdev, struct hl_user_interrupt *user_cq) { - struct hl_user_pending_interrupt *pend; + struct hl_user_pending_interrupt *pend, *temp_pend; + struct list_head *ts_reg_free_list_head = NULL; + struct timestamp_reg_work_obj *job; + bool reg_node_handle_fail = false; ktime_t now = ktime_get(); + int rc; + + /* For registration nodes: + * As part of handling the registration nodes, we should put refcount to + * some objects. the problem is that we cannot do that under spinlock + * or in irq handler context at all (since release functions are long and + * might sleep), so we will need to handle that part in workqueue context. + * To avoid handling kmalloc failure which compels us rolling back actions + * and move nodes hanged on the free list back to the interrupt wait list + * we always alloc the job of the WQ at the beginning. + */ + job = kmalloc(sizeof(*job), GFP_ATOMIC); + if (!job) + return; spin_lock(&user_cq->wait_list_lock); - list_for_each_entry(pend, &user_cq->wait_list_head, wait_list_node) { - if ((pend->cq_kernel_addr && - *(pend->cq_kernel_addr) >= pend->cq_target_value) || + list_for_each_entry_safe(pend, temp_pend, &user_cq->wait_list_head, wait_list_node) { + if ((pend->cq_kernel_addr && *(pend->cq_kernel_addr) >= pend->cq_target_value) || !pend->cq_kernel_addr) { - pend->fence.timestamp = now; - complete_all(&pend->fence.completion); + if (pend->ts_reg_info.ts_buff) { + if (!reg_node_handle_fail) { + rc = handle_registration_node(hdev, pend, + &ts_reg_free_list_head); + if (rc) + reg_node_handle_fail = true; + } + } else { + /* Handle wait target value node */ + pend->fence.timestamp = now; + complete_all(&pend->fence.completion); + } } } spin_unlock(&user_cq->wait_list_lock); + + if (ts_reg_free_list_head) { + INIT_WORK(&job->free_obj, hl_ts_free_objects); + job->free_obj_head = ts_reg_free_list_head; + job->hdev = hdev; + queue_work(hdev->ts_free_obj_wq, &job->free_obj); + } else { + kfree(job); + } } /** diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index f60a9638d9ea..c0904081f37e 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -20,6 +20,9 @@ MODULE_IMPORT_NS(DMA_BUF); /* use small pages for supporting non-pow2 (32M/40M/48M) DRAM phys page sizes */ #define DRAM_POOL_PAGE_SIZE SZ_8M +static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, + struct hl_mem_in *args, u64 *handle); + /* * The va ranges in context object contain a list with the available chunks of * device virtual memory. @@ -2021,6 +2024,9 @@ static int mem_ioctl_no_mmu(struct hl_fpriv *hpriv, union hl_mem_args *args) rc = -EPERM; break; + case HL_MEM_OP_TS_ALLOC: + rc = allocate_timestamps_buffers(hpriv, &args->in, &args->out.handle); + break; default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -EINVAL; @@ -2031,6 +2037,258 @@ out: return rc; } +static void ts_buff_release(struct kref *ref) +{ + struct hl_ts_buff *buff; + + buff = container_of(ref, struct hl_ts_buff, refcount); + + vfree(buff->kernel_buff_address); + vfree(buff->user_buff_address); + kfree(buff); +} + +struct hl_ts_buff *hl_ts_get(struct hl_device *hdev, struct hl_ts_mgr *mgr, + u32 handle) +{ + struct hl_ts_buff *buff; + + spin_lock(&mgr->ts_lock); + buff = idr_find(&mgr->ts_handles, handle); + if (!buff) { + spin_unlock(&mgr->ts_lock); + dev_warn(hdev->dev, + "TS buff get failed, no match to handle 0x%x\n", handle); + return NULL; + } + kref_get(&buff->refcount); + spin_unlock(&mgr->ts_lock); + + return buff; +} + +void hl_ts_put(struct hl_ts_buff *buff) +{ + kref_put(&buff->refcount, ts_buff_release); +} + +static void buff_vm_close(struct vm_area_struct *vma) +{ + struct hl_ts_buff *buff = (struct hl_ts_buff *) vma->vm_private_data; + long new_mmap_size; + + new_mmap_size = buff->mmap_size - (vma->vm_end - vma->vm_start); + + if (new_mmap_size > 0) { + buff->mmap_size = new_mmap_size; + return; + } + + atomic_set(&buff->mmap, 0); + hl_ts_put(buff); + vma->vm_private_data = NULL; +} + +static const struct vm_operations_struct ts_buff_vm_ops = { + .close = buff_vm_close +}; + +int hl_ts_mmap(struct hl_fpriv *hpriv, struct vm_area_struct *vma) +{ + struct hl_device *hdev = hpriv->hdev; + struct hl_ts_buff *buff; + u32 handle, user_buff_size; + int rc; + + /* We use the page offset to hold the idr and thus we need to clear + * it before doing the mmap itself + */ + handle = vma->vm_pgoff; + vma->vm_pgoff = 0; + + buff = hl_ts_get(hdev, &hpriv->ts_mem_mgr, handle); + if (!buff) { + dev_err(hdev->dev, + "TS buff mmap failed, no match to handle 0x%x\n", handle); + return -EINVAL; + } + + /* Validation check */ + user_buff_size = vma->vm_end - vma->vm_start; + if (user_buff_size != ALIGN(buff->user_buff_size, PAGE_SIZE)) { + dev_err(hdev->dev, + "TS buff mmap failed, mmap size 0x%x != 0x%x buff size\n", + user_buff_size, ALIGN(buff->user_buff_size, PAGE_SIZE)); + rc = -EINVAL; + goto put_buff; + } + +#ifdef _HAS_TYPE_ARG_IN_ACCESS_OK + if (!access_ok(VERIFY_WRITE, + (void __user *) (uintptr_t) vma->vm_start, user_buff_size)) { +#else + if (!access_ok((void __user *) (uintptr_t) vma->vm_start, + user_buff_size)) { +#endif + dev_err(hdev->dev, + "user pointer is invalid - 0x%lx\n", + vma->vm_start); + + rc = -EINVAL; + goto put_buff; + } + + if (atomic_cmpxchg(&buff->mmap, 0, 1)) { + dev_err(hdev->dev, "TS buff memory mmap failed, already mmaped to user\n"); + rc = -EINVAL; + goto put_buff; + } + + vma->vm_ops = &ts_buff_vm_ops; + vma->vm_private_data = buff; + vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY | VM_NORESERVE; + rc = remap_vmalloc_range(vma, buff->user_buff_address, 0); + if (rc) { + atomic_set(&buff->mmap, 0); + goto put_buff; + } + + buff->mmap_size = buff->user_buff_size; + vma->vm_pgoff = handle; + + return 0; + +put_buff: + hl_ts_put(buff); + return rc; +} + +void hl_ts_mgr_init(struct hl_ts_mgr *mgr) +{ + spin_lock_init(&mgr->ts_lock); + idr_init(&mgr->ts_handles); +} + +void hl_ts_mgr_fini(struct hl_device *hdev, struct hl_ts_mgr *mgr) +{ + struct hl_ts_buff *buff; + struct idr *idp; + u32 id; + + idp = &mgr->ts_handles; + + idr_for_each_entry(idp, buff, id) { + if (kref_put(&buff->refcount, ts_buff_release) != 1) + dev_err(hdev->dev, "TS buff handle %d for CTX is still alive\n", + id); + } + + idr_destroy(&mgr->ts_handles); +} + +static struct hl_ts_buff *hl_ts_alloc_buff(struct hl_device *hdev, u32 num_elements) +{ + struct hl_ts_buff *ts_buff = NULL; + u32 size; + void *p; + + ts_buff = kzalloc(sizeof(*ts_buff), GFP_KERNEL); + if (!ts_buff) + return NULL; + + /* Allocate the user buffer */ + size = num_elements * sizeof(u64); + p = vmalloc_user(size); + if (!p) + goto free_mem; + + ts_buff->user_buff_address = p; + ts_buff->user_buff_size = size; + + /* Allocate the internal kernel buffer */ + size = num_elements * sizeof(struct hl_user_pending_interrupt); + p = vmalloc(size); + if (!p) + goto free_user_buff; + + ts_buff->kernel_buff_address = p; + ts_buff->kernel_buff_size = size; + + return ts_buff; + +free_user_buff: + vfree(ts_buff->user_buff_address); +free_mem: + kfree(ts_buff); + return NULL; +} + +/** + * allocate_timestamps_buffers() - allocate timestamps buffers + * This function will allocate ts buffer that will later on be mapped to the user + * in order to be able to read the timestamp. + * in additon it'll allocate an extra buffer for registration management. + * since we cannot fail during registration for out-of-memory situation, so + * we'll prepare a pool which will be used as user interrupt nodes and instead + * of dynamically allocating nodes while registration we'll pick the node from + * this pool. in addtion it'll add node to the mapping hash which will be used + * to map user ts buffer to the internal kernel ts buffer. + * @hpriv: pointer to the private data of the fd + * @args: ioctl input + * @handle: user timestamp buffer handle as an output + */ +static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, struct hl_mem_in *args, u64 *handle) +{ + struct hl_ts_mgr *ts_mgr = &hpriv->ts_mem_mgr; + struct hl_device *hdev = hpriv->hdev; + struct hl_ts_buff *ts_buff; + int rc = 0; + + if (args->num_of_elements > TS_MAX_ELEMENTS_NUM) { + dev_err(hdev->dev, "Num of elements exceeds Max allowed number (0x%x > 0x%x)\n", + args->num_of_elements, TS_MAX_ELEMENTS_NUM); + return -EINVAL; + } + + /* Allocate ts buffer object + * This object will contain two buffers one that will be mapped to the user + * and another internal buffer for the driver use only, which won't be mapped + * to the user. + */ + ts_buff = hl_ts_alloc_buff(hdev, args->num_of_elements); + if (!ts_buff) { + rc = -ENOMEM; + goto out_err; + } + + spin_lock(&ts_mgr->ts_lock); + rc = idr_alloc(&ts_mgr->ts_handles, ts_buff, 1, 0, GFP_ATOMIC); + spin_unlock(&ts_mgr->ts_lock); + if (rc < 0) { + dev_err(hdev->dev, "Failed to allocate IDR for a new ts buffer\n"); + goto release_ts_buff; + } + + ts_buff->id = rc; + ts_buff->hdev = hdev; + + kref_init(&ts_buff->refcount); + + /* idr is 32-bit so we can safely OR it with a mask that is above 32 bit */ + *handle = (u64) ts_buff->id | HL_MMAP_TYPE_TS_BUFF; + *handle <<= PAGE_SHIFT; + + dev_dbg(hdev->dev, "Created ts buff object handle(%u)\n", ts_buff->id); + + return 0; + +release_ts_buff: + kref_put(&ts_buff->refcount, ts_buff_release); +out_err: + *handle = 0; + return rc; +} + int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) { enum hl_device_status status; @@ -2146,6 +2404,9 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) args->out.fd = dmabuf_fd; break; + case HL_MEM_OP_TS_ALLOC: + rc = allocate_timestamps_buffers(hpriv, &args->in, &args->out.handle); + break; default: dev_err(hdev->dev, "Unknown opcode for memory IOCTL\n"); rc = -EINVAL; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 12976f7a8d84..e21db03196ae 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note * - * Copyright 2016-2020 HabanaLabs, Ltd. + * Copyright 2016-2021 HabanaLabs, Ltd. * All Rights Reserved. * */ @@ -30,6 +30,9 @@ */ #define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 72 +/* Max number of elements in timestamps registration buffers */ +#define TS_MAX_ELEMENTS_NUM (1 << 20) /* 1MB */ + /* * Goya queue Numbering * @@ -695,10 +698,12 @@ struct hl_cb_in { __u64 cb_handle; /* HL_CB_OP_* */ __u32 op; + /* Size of CB. Maximum size is HL_MAX_CB_SIZE. The minimum size that * will be allocated, regardless of this parameter's value, is PAGE_SIZE */ __u32 cb_size; + /* Context ID - Currently not in use */ __u32 ctx_id; /* HL_CB_FLAGS_* */ @@ -964,6 +969,7 @@ union hl_cs_args { #define HL_WAIT_CS_FLAGS_INTERRUPT_MASK 0xFFF00000 #define HL_WAIT_CS_FLAGS_MULTI_CS 0x4 #define HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ 0x10 +#define HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT 0x20 #define HL_WAIT_MULTI_CS_LIST_MAX_LEN 32 @@ -1036,6 +1042,20 @@ struct hl_wait_cs_in { * relevant only when HL_WAIT_CS_FLAGS_INTERRUPT_KERNEL_CQ is set */ __u64 cq_counters_offset; + + /* + * Timestamp_handle timestamps buffer handle. + * relevant only when HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT is set + */ + __u64 timestamp_handle; + + /* + * Timestamp_offset is offset inside the timestamp buffer pointed by timestamp_handle above. + * upon interrupt, if the cq reached the target value then driver will write + * timestamp to this offset. + * relevant only when HL_WAIT_CS_FLAGS_REGISTER_INTERRUPT is set + */ + __u64 timestamp_offset; }; #define HL_WAIT_CS_STATUS_COMPLETED 0 @@ -1082,6 +1102,14 @@ union hl_wait_cs_args { */ #define HL_MEM_OP_EXPORT_DMABUF_FD 5 +/* Opcode to create timestamps pool for user interrupts registration support + * The memory will be allocated by the kernel driver, A timestamp buffer which the user + * will get handle to it for mmap, and another internal buffer used by the + * driver for registration management + * The memory will be freed when the user closes the file descriptor(ctx close) + */ +#define HL_MEM_OP_TS_ALLOC 6 + /* Memory flags */ #define HL_MEM_CONTIGUOUS 0x1 #define HL_MEM_SHARED 0x2 @@ -1173,9 +1201,14 @@ struct hl_mem_in { * DMA-BUF file/FD flags. */ __u32 flags; + /* Context ID - Currently not in use */ __u32 ctx_id; - __u32 pad; + + /* number of timestamp elements + * used only when HL_MEM_OP_TS_ALLOC opcode + */ + __u32 num_of_elements; }; struct hl_mem_out { -- cgit v1.2.3 From 9349a321d327d08a4e91f9b6aca9c1f3a3a25306 Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Mon, 14 Feb 2022 15:20:08 +0200 Subject: habanalabs: use kernel-doc for memory ioctl documentation Re-format the comments for the memory ioctl structure to be in kernel-doc style. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- include/uapi/misc/habanalabs.h | 111 +++++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 54 deletions(-) (limited to 'include/uapi/misc') diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index e21db03196ae..a2a953a91b5e 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -1116,98 +1116,101 @@ union hl_wait_cs_args { #define HL_MEM_USERPTR 0x4 #define HL_MEM_FORCE_HINT 0x8 +/** + * structure hl_mem_in - structure that handle input args for memory IOCTL + * @union arg: union of structures to be used based on the input operation + * @op: specify the requested memory operation (one of the HL_MEM_OP_* definitions). + * @flags: flags for the memory operation (one of the HL_MEM_* definitions). + * For the HL_MEM_OP_EXPORT_DMABUF_FD opcode, this field holds the DMA-BUF file/FD flags. + * @ctx_id: context ID - currently not in use. + * @num_of_elements: number of timestamp elements used only with HL_MEM_OP_TS_ALLOC opcode. + */ struct hl_mem_in { union { - /* HL_MEM_OP_ALLOC- allocate device memory */ + /** + * structure for device memory allocation (used with the HL_MEM_OP_ALLOC op) + * @mem_size: memory size to allocate + */ struct { - /* Size to alloc */ __u64 mem_size; } alloc; - /* HL_MEM_OP_FREE - free device memory */ + /** + * structure for free-ing device memory (used with the HL_MEM_OP_FREE op) + * @handle: handle returned from HL_MEM_OP_ALLOC + */ struct { - /* Handle returned from HL_MEM_OP_ALLOC */ __u64 handle; } free; - /* HL_MEM_OP_MAP - map device memory */ + /** + * structure for mapping device memory (used with the HL_MEM_OP_MAP op) + * @hint_addr: requested virtual address of mapped memory. + * the driver will try to map the requested region to this hint + * address, as long as the address is valid and not already mapped. + * the user should check the returned address of the IOCTL to make + * sure he got the hint address. + * passing 0 here means that the driver will choose the address itself. + * @handle: handle returned from HL_MEM_OP_ALLOC. + */ struct { - /* - * Requested virtual address of mapped memory. - * The driver will try to map the requested region to - * this hint address, as long as the address is valid - * and not already mapped. The user should check the - * returned address of the IOCTL to make sure he got - * the hint address. Passing 0 here means that the - * driver will choose the address itself. - */ __u64 hint_addr; - /* Handle returned from HL_MEM_OP_ALLOC */ __u64 handle; } map_device; - /* HL_MEM_OP_MAP - map host memory */ + /** + * structure for mapping host memory (used with the HL_MEM_OP_MAP op) + * @host_virt_addr: address of allocated host memory. + * @hint_addr: requested virtual address of mapped memory. + * the driver will try to map the requested region to this hint + * address, as long as the address is valid and not already mapped. + * the user should check the returned address of the IOCTL to make + * sure he got the hint address. + * passing 0 here means that the driver will choose the address itself. + * @size: size of allocated host memory. + */ struct { - /* Address of allocated host memory */ __u64 host_virt_addr; - /* - * Requested virtual address of mapped memory. - * The driver will try to map the requested region to - * this hint address, as long as the address is valid - * and not already mapped. The user should check the - * returned address of the IOCTL to make sure he got - * the hint address. Passing 0 here means that the - * driver will choose the address itself. - */ __u64 hint_addr; - /* Size of allocated host memory */ __u64 mem_size; } map_host; - /* HL_MEM_OP_MAP_BLOCK - map a hw block */ + /** + * structure for mapping hw block (used with the HL_MEM_OP_MAP_BLOCK op) + * @block_addr:HW block address to map, a handle and size will be returned + * to the user and will be used to mmap the relevant block. + * only addresses from configuration space are allowed. + */ struct { - /* - * HW block address to map, a handle and size will be - * returned to the user and will be used to mmap the - * relevant block. Only addresses from configuration - * space are allowed. - */ __u64 block_addr; } map_block; - /* HL_MEM_OP_UNMAP - unmap host memory */ + /** + * structure for unmapping host memory (used with the HL_MEM_OP_UNMAP op) + * @device_virt_addr: virtual address returned from HL_MEM_OP_MAP + */ struct { - /* Virtual address returned from HL_MEM_OP_MAP */ __u64 device_virt_addr; } unmap; - /* HL_MEM_OP_EXPORT_DMABUF_FD */ + /** + * structure for exporting DMABUF object (used with + * the HL_MEM_OP_EXPORT_DMABUF_FD op) + * @handle: handle returned from HL_MEM_OP_ALLOC. + * in Gaudi, where we don't have MMU for the device memory, the + * driver expects a physical address (instead of a handle) in the + * device memory space. + * @mem_size: size of memory allocation. Relevant only for GAUDI + */ struct { - /* Handle returned from HL_MEM_OP_ALLOC. In Gaudi, - * where we don't have MMU for the device memory, the - * driver expects a physical address (instead of - * a handle) in the device memory space. - */ __u64 handle; - /* Size of memory allocation. Relevant only for GAUDI */ __u64 mem_size; } export_dmabuf_fd; }; - /* HL_MEM_OP_* */ __u32 op; - /* HL_MEM_* flags. - * For the HL_MEM_OP_EXPORT_DMABUF_FD opcode, this field holds the - * DMA-BUF file/FD flags. - */ __u32 flags; - - /* Context ID - Currently not in use */ __u32 ctx_id; - - /* number of timestamp elements - * used only when HL_MEM_OP_TS_ALLOC opcode - */ __u32 num_of_elements; }; -- cgit v1.2.3 From f23f280277d5a701de99c6652623b6bf8801c534 Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Mon, 14 Feb 2022 15:22:00 +0200 Subject: habanalabs: allow user to set allocation page size In future ASICs the MMU will be able to work with multiple page sizes, thus a new flag is added to allow the user to set the requested page size. This flag is added since the whole DRAM is allocated for the user and the user also should be familiar with the memory usage use case. As such, the user may choose to "over allocate" memory in favor of performance (for instance- large page allocations covers more memory in less TLB entries). For example: say available page sizes are of 1MB and 32MB. If user wants to allocate 40MB the user can either set page size to 1MB and allocate the exact amount of memory (but will result in 40 TLB entries) or the user can use 32MB pages, "waste" 8MB of physical memory but occupy only 2 TLB entries. Note that this feature will be available only to ASIC that supports multiple DRAM page sizes. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/habanalabs.h | 5 +++++ drivers/misc/habanalabs/common/memory.c | 31 ++++++++++++++++++++++++++++- drivers/misc/habanalabs/gaudi/gaudi.c | 3 ++- drivers/misc/habanalabs/goya/goya.c | 1 + include/uapi/misc/habanalabs.h | 5 ++++- 5 files changed, 42 insertions(+), 3 deletions(-) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index b5055ab528b8..677ae4ff922c 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -563,6 +563,7 @@ struct hl_hints_range { * devices) * @configurable_stop_on_err: is stop-on-error option configurable via debugfs. * @set_max_power_on_device_init: true if need to set max power in F/W on device init. + * @supports_user_set_page_size: true if user can set the allocation page size. */ struct asic_fixed_properties { struct hw_queue_properties *hw_queues_props; @@ -648,6 +649,7 @@ struct asic_fixed_properties { u8 allow_inference_soft_reset; u8 configurable_stop_on_err; u8 set_max_power_on_device_init; + u8 supports_user_set_page_size; }; /** @@ -1311,6 +1313,8 @@ struct fw_load_mgr { * @get_sob_addr: get SOB base address offset. * @set_pci_memory_regions: setting properties of PCI memory regions * @get_stream_master_qid_arr: get pointer to stream masters QID array + * @is_valid_dram_page_size: return true if page size is supported in device + * memory allocation, otherwise false. */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -1436,6 +1440,7 @@ struct hl_asic_funcs { u32 (*get_sob_addr)(struct hl_device *hdev, u32 sob_id); void (*set_pci_memory_regions)(struct hl_device *hdev); u32* (*get_stream_master_qid_arr)(void); + bool (*is_valid_dram_page_size)(u32 page_size); }; diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 7bd4cae87a54..385bf3448c73 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -24,6 +24,31 @@ MODULE_IMPORT_NS(DMA_BUF); static int allocate_timestamps_buffers(struct hl_fpriv *hpriv, struct hl_mem_in *args, u64 *handle); +static int set_alloc_page_size(struct hl_device *hdev, struct hl_mem_in *args, u32 *page_size) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + u32 psize; + + /* + * for ASIC that supports setting the allocation page size by user we will address + * user's choice only if it is not 0 (as 0 means taking the default page size) + */ + if (prop->supports_user_set_page_size && args->alloc.page_size) { + psize = args->alloc.page_size; + + if (!hdev->asic_funcs->is_valid_dram_page_size(psize)) { + dev_err(hdev->dev, "user page size (%#x) is not valid\n", psize); + return -EINVAL; + } + } else { + psize = hdev->asic_prop.dram_page_size; + } + + *page_size = psize; + + return 0; +} + /* * The va ranges in context object contain a list with the available chunks of * device virtual memory. @@ -69,7 +94,11 @@ static int alloc_device_memory(struct hl_ctx *ctx, struct hl_mem_in *args, bool contiguous; num_curr_pgs = 0; - page_size = hdev->asic_prop.dram_page_size; + + rc = set_alloc_page_size(hdev, args, &page_size); + if (rc) + return rc; + num_pgs = DIV_ROUND_UP_ULL(args->alloc.mem_size, page_size); total_size = num_pgs * page_size; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index ad640a268a3c..a60c0e23d23c 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -9432,7 +9432,8 @@ static const struct hl_asic_funcs gaudi_funcs = { .state_dump_init = gaudi_state_dump_init, .get_sob_addr = gaudi_get_sob_addr, .set_pci_memory_regions = gaudi_set_pci_memory_regions, - .get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr + .get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr, + .is_valid_dram_page_size = NULL }; /** diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index c08d96e43c9f..ec9358bcbf0b 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5764,6 +5764,7 @@ static const struct hl_asic_funcs goya_funcs = { .get_sob_addr = &goya_get_sob_addr, .set_pci_memory_regions = goya_set_pci_memory_regions, .get_stream_master_qid_arr = goya_get_stream_master_qid_arr, + .is_valid_dram_page_size = NULL }; /* diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index a2a953a91b5e..1d6b4f0c4159 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note * - * Copyright 2016-2021 HabanaLabs, Ltd. + * Copyright 2016-2022 HabanaLabs, Ltd. * All Rights Reserved. * */ @@ -1130,9 +1130,12 @@ struct hl_mem_in { /** * structure for device memory allocation (used with the HL_MEM_OP_ALLOC op) * @mem_size: memory size to allocate + * @page_size: page size to use on allocation. when the value is 0 the default page + * size will be taken. */ struct { __u64 mem_size; + __u64 page_size; } alloc; /** -- cgit v1.2.3 From 5c1b97c7d7b736e6439af4f43a65837bc72f56c1 Mon Sep 17 00:00:00 2001 From: Jeya R Date: Mon, 14 Feb 2022 16:09:52 +0000 Subject: misc: fastrpc: add support for FASTRPC_IOCTL_MEM_MAP/UNMAP Add support for IOCTL requests to map and unmap on DSP based on map flags. Signed-off-by: Jeya R Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20220214161002.6831-3-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/fastrpc.c | 154 ++++++++++++++++++++++++++++++++++++++++++++ include/uapi/misc/fastrpc.h | 51 +++++++++++++++ 2 files changed, 205 insertions(+) (limited to 'include/uapi/misc') diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index ee5eb87e9f87..7d8150179b1b 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -72,6 +72,8 @@ #define FASTRPC_RMID_INIT_CREATE 6 #define FASTRPC_RMID_INIT_CREATE_ATTR 7 #define FASTRPC_RMID_INIT_CREATE_STATIC 8 +#define FASTRPC_RMID_INIT_MEM_MAP 10 +#define FASTRPC_RMID_INIT_MEM_UNMAP 11 /* Protection Domain(PD) ids */ #define AUDIO_PD (0) /* also GUEST_OS PD? */ @@ -108,12 +110,29 @@ struct fastrpc_mmap_req_msg { s32 num; }; +struct fastrpc_mem_map_req_msg { + s32 pgid; + s32 fd; + s32 offset; + u32 flags; + u64 vaddrin; + s32 num; + s32 data_len; +}; + struct fastrpc_munmap_req_msg { s32 pgid; u64 vaddr; u64 size; }; +struct fastrpc_mem_unmap_req_msg { + s32 pgid; + s32 fd; + u64 vaddrin; + u64 len; +}; + struct fastrpc_msg { int pid; /* process group id */ int tid; /* thread id */ @@ -170,6 +189,7 @@ struct fastrpc_map { u64 size; void *va; u64 len; + u64 raddr; struct kref refcount; }; @@ -1500,6 +1520,134 @@ err_invoke: return err; } +static int fastrpc_req_mem_unmap_impl(struct fastrpc_user *fl, struct fastrpc_mem_unmap *req) +{ + struct fastrpc_invoke_args args[1] = { [0] = { 0 } }; + struct fastrpc_map *map = NULL, *m; + struct fastrpc_mem_unmap_req_msg req_msg = { 0 }; + int err = 0; + u32 sc; + struct device *dev = fl->sctx->dev; + + spin_lock(&fl->lock); + list_for_each_entry_safe(map, m, &fl->maps, node) { + if ((req->fd < 0 || map->fd == req->fd) && (map->raddr == req->vaddr)) + break; + map = NULL; + } + + spin_unlock(&fl->lock); + + if (!map) { + dev_err(dev, "map not in list\n"); + return -EINVAL; + } + + req_msg.pgid = fl->tgid; + req_msg.len = map->len; + req_msg.vaddrin = map->raddr; + req_msg.fd = map->fd; + + args[0].ptr = (u64) (uintptr_t) &req_msg; + args[0].length = sizeof(req_msg); + + sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_UNMAP, 1, 0); + err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, + &args[0]); + fastrpc_map_put(map); + if (err) + dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr); + + return err; +} + +static int fastrpc_req_mem_unmap(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_mem_unmap req; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + return fastrpc_req_mem_unmap_impl(fl, &req); +} + +static int fastrpc_req_mem_map(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_invoke_args args[4] = { [0 ... 3] = { 0 } }; + struct fastrpc_mem_map_req_msg req_msg = { 0 }; + struct fastrpc_mmap_rsp_msg rsp_msg = { 0 }; + struct fastrpc_mem_unmap req_unmap = { 0 }; + struct fastrpc_phy_page pages = { 0 }; + struct fastrpc_mem_map req; + struct device *dev = fl->sctx->dev; + struct fastrpc_map *map = NULL; + int err; + u32 sc; + + if (copy_from_user(&req, argp, sizeof(req))) + return -EFAULT; + + /* create SMMU mapping */ + err = fastrpc_map_create(fl, req.fd, req.length, &map); + if (err) { + dev_err(dev, "failed to map buffer, fd = %d\n", req.fd); + return err; + } + + req_msg.pgid = fl->tgid; + req_msg.fd = req.fd; + req_msg.offset = req.offset; + req_msg.vaddrin = req.vaddrin; + map->va = (void *) (uintptr_t) req.vaddrin; + req_msg.flags = req.flags; + req_msg.num = sizeof(pages); + req_msg.data_len = 0; + + args[0].ptr = (u64) (uintptr_t) &req_msg; + args[0].length = sizeof(req_msg); + + pages.addr = map->phys; + pages.size = map->size; + + args[1].ptr = (u64) (uintptr_t) &pages; + args[1].length = sizeof(pages); + + args[2].ptr = (u64) (uintptr_t) &pages; + args[2].length = 0; + + args[3].ptr = (u64) (uintptr_t) &rsp_msg; + args[3].length = sizeof(rsp_msg); + + sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_MEM_MAP, 3, 1); + err = fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, sc, &args[0]); + if (err) { + dev_err(dev, "mem mmap error, fd %d, vaddr %llx, size %lld\n", + req.fd, req.vaddrin, map->size); + goto err_invoke; + } + + /* update the buffer to be able to deallocate the memory on the DSP */ + map->raddr = rsp_msg.vaddr; + + /* let the client know the address to use */ + req.vaddrout = rsp_msg.vaddr; + + if (copy_to_user((void __user *)argp, &req, sizeof(req))) { + /* unmap the memory and release the buffer */ + req_unmap.vaddr = (uintptr_t) rsp_msg.vaddr; + req_unmap.length = map->size; + fastrpc_req_mem_unmap_impl(fl, &req_unmap); + return -EFAULT; + } + + return 0; + +err_invoke: + fastrpc_map_put(map); + + return err; +} + static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -1529,6 +1677,12 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, case FASTRPC_IOCTL_MUNMAP: err = fastrpc_req_munmap(fl, argp); break; + case FASTRPC_IOCTL_MEM_MAP: + err = fastrpc_req_mem_map(fl, argp); + break; + case FASTRPC_IOCTL_MEM_UNMAP: + err = fastrpc_req_mem_unmap(fl, argp); + break; default: err = -ENOTTY; break; diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index 0a89f95463f6..d248eeb20e67 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -13,6 +13,37 @@ #define FASTRPC_IOCTL_MMAP _IOWR('R', 6, struct fastrpc_req_mmap) #define FASTRPC_IOCTL_MUNMAP _IOWR('R', 7, struct fastrpc_req_munmap) #define FASTRPC_IOCTL_INIT_ATTACH_SNS _IO('R', 8) +#define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map) +#define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap) + +/** + * enum fastrpc_map_flags - control flags for mapping memory on DSP user process + * @FASTRPC_MAP_STATIC: Map memory pages with RW- permission and CACHE WRITEBACK. + * The driver is responsible for cache maintenance when passed + * the buffer to FastRPC calls. Same virtual address will be + * assigned for subsequent FastRPC calls. + * @FASTRPC_MAP_RESERVED: Reserved + * @FASTRPC_MAP_FD: Map memory pages with RW- permission and CACHE WRITEBACK. + * Mapping tagged with a file descriptor. User is responsible for + * CPU and DSP cache maintenance for the buffer. Get virtual address + * of buffer on DSP using HAP_mmap_get() and HAP_mmap_put() APIs. + * @FASTRPC_MAP_FD_DELAYED: Mapping delayed until user call HAP_mmap() and HAP_munmap() + * functions on DSP. It is useful to map a buffer with cache modes + * other than default modes. User is responsible for CPU and DSP + * cache maintenance for the buffer. + * @FASTRPC_MAP_FD_NOMAP: This flag is used to skip CPU mapping, + * otherwise behaves similar to FASTRPC_MAP_FD_DELAYED flag. + * @FASTRPC_MAP_MAX: max count for flags + * + */ +enum fastrpc_map_flags { + FASTRPC_MAP_STATIC = 0, + FASTRPC_MAP_RESERVED, + FASTRPC_MAP_FD = 2, + FASTRPC_MAP_FD_DELAYED, + FASTRPC_MAP_FD_NOMAP = 16, + FASTRPC_MAP_MAX, +}; struct fastrpc_invoke_args { __u64 ptr; @@ -49,9 +80,29 @@ struct fastrpc_req_mmap { __u64 vaddrout; /* dsp virtual address */ }; +struct fastrpc_mem_map { + __s32 version; + __s32 fd; /* fd */ + __s32 offset; /* buffer offset */ + __u32 flags; /* flags defined in enum fastrpc_map_flags */ + __u64 vaddrin; /* buffer virtual address */ + __u64 length; /* buffer length */ + __u64 vaddrout; /* [out] remote virtual address */ + __s32 attrs; /* buffer attributes used for SMMU mapping */ + __s32 reserved[4]; +}; + struct fastrpc_req_munmap { __u64 vaddrout; /* address to unmap */ __u64 size; /* size */ }; +struct fastrpc_mem_unmap { + __s32 vesion; + __s32 fd; /* fd */ + __u64 vaddr; /* remote process (dsp) virtual address */ + __u64 length; /* buffer size */ + __s32 reserved[5]; +}; + #endif /* __QCOM_FASTRPC_H__ */ -- cgit v1.2.3 From 6c16fd8bdd4058d4a6aaca9d5a7b40e4cb281d5a Mon Sep 17 00:00:00 2001 From: Jeya R Date: Mon, 14 Feb 2022 16:09:53 +0000 Subject: misc: fastrpc: Add support to get DSP capabilities Add support to get DSP capabilities. The capability information is cached on driver. Signed-off-by: Jeya R Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20220214161002.6831-4-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/fastrpc.c | 112 ++++++++++++++++++++++++++++++++++++++++++++ include/uapi/misc/fastrpc.h | 8 ++++ 2 files changed, 120 insertions(+) (limited to 'include/uapi/misc') diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 7d8150179b1b..4b1a4c1c5535 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -31,10 +31,15 @@ #define FASTRPC_PHYS(p) ((p) & 0xffffffff) #define FASTRPC_CTX_MAX (256) #define FASTRPC_INIT_HANDLE 1 +#define FASTRPC_DSP_UTILITIES_HANDLE 2 #define FASTRPC_CTXID_MASK (0xFF0) #define INIT_FILELEN_MAX (2 * 1024 * 1024) #define FASTRPC_DEVICE_NAME "fastrpc" #define ADSP_MMAP_ADD_PAGES 0x1000 +#define DSP_UNSUPPORTED_API (0x80000414) +/* MAX NUMBER of DSP ATTRIBUTES SUPPORTED */ +#define FASTRPC_MAX_DSP_ATTRIBUTES (256) +#define FASTRPC_MAX_DSP_ATTRIBUTES_LEN (sizeof(u32) * FASTRPC_MAX_DSP_ATTRIBUTES) /* Retrives number of input buffers from the scalars parameter */ #define REMOTE_SCALARS_INBUFS(sc) (((sc) >> 16) & 0x0ff) @@ -233,6 +238,9 @@ struct fastrpc_channel_ctx { struct idr ctx_idr; struct list_head users; struct kref refcount; + /* Flag if dsp attributes are cached */ + bool valid_attributes; + u32 dsp_attributes[FASTRPC_MAX_DSP_ATTRIBUTES]; struct fastrpc_device *fdevice; }; @@ -1378,6 +1386,107 @@ static int fastrpc_invoke(struct fastrpc_user *fl, char __user *argp) return err; } +static int fastrpc_get_info_from_dsp(struct fastrpc_user *fl, uint32_t *dsp_attr_buf, + uint32_t dsp_attr_buf_len) +{ + struct fastrpc_invoke_args args[2] = { 0 }; + + /* Capability filled in userspace */ + dsp_attr_buf[0] = 0; + + args[0].ptr = (u64)(uintptr_t)&dsp_attr_buf_len; + args[0].length = sizeof(dsp_attr_buf_len); + args[0].fd = -1; + args[1].ptr = (u64)(uintptr_t)&dsp_attr_buf[1]; + args[1].length = dsp_attr_buf_len; + args[1].fd = -1; + fl->pd = 1; + + return fastrpc_internal_invoke(fl, true, FASTRPC_DSP_UTILITIES_HANDLE, + FASTRPC_SCALARS(0, 1, 1), args); +} + +static int fastrpc_get_info_from_kernel(struct fastrpc_ioctl_capability *cap, + struct fastrpc_user *fl) +{ + struct fastrpc_channel_ctx *cctx = fl->cctx; + uint32_t attribute_id = cap->attribute_id; + uint32_t *dsp_attributes; + unsigned long flags; + uint32_t domain = cap->domain; + int err; + + spin_lock_irqsave(&cctx->lock, flags); + /* check if we already have queried dsp for attributes */ + if (cctx->valid_attributes) { + spin_unlock_irqrestore(&cctx->lock, flags); + goto done; + } + spin_unlock_irqrestore(&cctx->lock, flags); + + dsp_attributes = kzalloc(FASTRPC_MAX_DSP_ATTRIBUTES_LEN, GFP_KERNEL); + if (!dsp_attributes) + return -ENOMEM; + + err = fastrpc_get_info_from_dsp(fl, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN); + if (err == DSP_UNSUPPORTED_API) { + dev_info(&cctx->rpdev->dev, + "Warning: DSP capabilities not supported on domain: %d\n", domain); + kfree(dsp_attributes); + return -EOPNOTSUPP; + } else if (err) { + dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err); + kfree(dsp_attributes); + return err; + } + + spin_lock_irqsave(&cctx->lock, flags); + memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN); + cctx->valid_attributes = true; + spin_unlock_irqrestore(&cctx->lock, flags); + kfree(dsp_attributes); +done: + cap->capability = cctx->dsp_attributes[attribute_id]; + return 0; +} + +static int fastrpc_get_dsp_info(struct fastrpc_user *fl, char __user *argp) +{ + struct fastrpc_ioctl_capability cap = {0}; + int err = 0; + + if (copy_from_user(&cap, argp, sizeof(cap))) + return -EFAULT; + + cap.capability = 0; + if (cap.domain >= FASTRPC_DEV_MAX) { + dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n", + cap.domain, err); + return -ECHRNG; + } + + /* Fastrpc Capablities does not support modem domain */ + if (cap.domain == MDSP_DOMAIN_ID) { + dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err); + return -ECHRNG; + } + + if (cap.attribute_id >= FASTRPC_MAX_DSP_ATTRIBUTES) { + dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n", + cap.attribute_id, err); + return -EOVERFLOW; + } + + err = fastrpc_get_info_from_kernel(&cap, fl); + if (err) + return err; + + if (copy_to_user(argp, &cap.capability, sizeof(cap.capability))) + return -EFAULT; + + return 0; +} + static int fastrpc_req_munmap_impl(struct fastrpc_user *fl, struct fastrpc_req_munmap *req) { @@ -1683,6 +1792,9 @@ static long fastrpc_device_ioctl(struct file *file, unsigned int cmd, case FASTRPC_IOCTL_MEM_UNMAP: err = fastrpc_req_mem_unmap(fl, argp); break; + case FASTRPC_IOCTL_GET_DSP_INFO: + err = fastrpc_get_dsp_info(fl, argp); + break; default: err = -ENOTTY; break; diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index d248eeb20e67..7cc9d342078a 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -15,6 +15,7 @@ #define FASTRPC_IOCTL_INIT_ATTACH_SNS _IO('R', 8) #define FASTRPC_IOCTL_MEM_MAP _IOWR('R', 10, struct fastrpc_mem_map) #define FASTRPC_IOCTL_MEM_UNMAP _IOWR('R', 11, struct fastrpc_mem_unmap) +#define FASTRPC_IOCTL_GET_DSP_INFO _IOWR('R', 13, struct fastrpc_ioctl_capability) /** * enum fastrpc_map_flags - control flags for mapping memory on DSP user process @@ -105,4 +106,11 @@ struct fastrpc_mem_unmap { __s32 reserved[5]; }; +struct fastrpc_ioctl_capability { + __u32 domain; + __u32 attribute_id; + __u32 capability; /* dsp capability */ + __u32 reserved[4]; +}; + #endif /* __QCOM_FASTRPC_H__ */ -- cgit v1.2.3 From 7f1f481263c3ce5387d4fd5ad63ddaa8a295aab2 Mon Sep 17 00:00:00 2001 From: Jeya R Date: Mon, 14 Feb 2022 16:09:56 +0000 Subject: misc: fastrpc: check before loading process to the DSP Reject session if DSP domain is secure, device node is non-secure and signed PD is requested. Secure device node can access DSP without any restriction. Unsigned PD offload is only allowed for the DSP domain that can support unsigned offloading. Signed-off-by: Jeya R Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20220214161002.6831-7-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/fastrpc.c | 33 +++++++++++++++++++++++++++++++++ include/uapi/misc/fastrpc.h | 17 +++++++++++++++++ 2 files changed, 50 insertions(+) (limited to 'include/uapi/misc') diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 3b7731f158eb..8fbb24d763bd 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -244,6 +244,7 @@ struct fastrpc_channel_ctx { struct fastrpc_device *secure_fdevice; struct fastrpc_device *fdevice; bool secure; + bool unsigned_support; }; struct fastrpc_device { @@ -264,6 +265,7 @@ struct fastrpc_user { int tgid; int pd; + bool is_secure_dev; /* Lock for lists */ spinlock_t lock; /* lock for allocations */ @@ -1052,6 +1054,24 @@ bail: return err; } +static bool is_session_rejected(struct fastrpc_user *fl, bool unsigned_pd_request) +{ + /* Check if the device node is non-secure and channel is secure*/ + if (!fl->is_secure_dev && fl->cctx->secure) { + /* + * Allow untrusted applications to offload only to Unsigned PD when + * channel is configured as secure and block untrusted apps on channel + * that does not support unsigned PD offload + */ + if (!fl->cctx->unsigned_support || !unsigned_pd_request) { + dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD"); + return true; + } + } + + return false; +} + static int fastrpc_init_create_process(struct fastrpc_user *fl, char __user *argp) { @@ -1071,6 +1091,7 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl, u32 siglen; } inbuf; u32 sc; + bool unsigned_module = false; args = kcalloc(FASTRPC_CREATE_PROCESS_NARGS, sizeof(*args), GFP_KERNEL); if (!args) @@ -1081,6 +1102,14 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl, goto err; } + if (init.attrs & FASTRPC_MODE_UNSIGNED_MODULE) + unsigned_module = true; + + if (is_session_rejected(fl, unsigned_module)) { + err = -ECONNREFUSED; + goto err; + } + if (init.filelen > INIT_FILELEN_MAX) { err = -EINVAL; goto err; @@ -1280,6 +1309,7 @@ static int fastrpc_device_open(struct inode *inode, struct file *filp) INIT_LIST_HEAD(&fl->user); fl->tgid = current->tgid; fl->cctx = cctx; + fl->is_secure_dev = fdevice->secure; fl->sctx = fastrpc_session_alloc(cctx); if (!fl->sctx) { @@ -1958,11 +1988,14 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) case ADSP_DOMAIN_ID: case MDSP_DOMAIN_ID: case SDSP_DOMAIN_ID: + /* Unsigned PD offloading is only supported on CDSP*/ + data->unsigned_support = false; err = fastrpc_device_register(rdev, data, secure_dsp, domains[domain_id]); if (err) goto fdev_error; break; case CDSP_DOMAIN_ID: + data->unsigned_support = true; /* Create both device nodes so that we can allow both Signed and Unsigned PD */ err = fastrpc_device_register(rdev, data, true, domains[domain_id]); if (err) diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index 7cc9d342078a..f39edac20305 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -46,6 +46,23 @@ enum fastrpc_map_flags { FASTRPC_MAP_MAX, }; +enum fastrpc_proc_attr { + /* Macro for Debug attr */ + FASTRPC_MODE_DEBUG = (1 << 0), + /* Macro for Ptrace */ + FASTRPC_MODE_PTRACE = (1 << 1), + /* Macro for CRC Check */ + FASTRPC_MODE_CRC = (1 << 2), + /* Macro for Unsigned PD */ + FASTRPC_MODE_UNSIGNED_MODULE = (1 << 3), + /* Macro for Adaptive QoS */ + FASTRPC_MODE_ADAPTIVE_QOS = (1 << 4), + /* Macro for System Process */ + FASTRPC_MODE_SYSTEM_PROCESS = (1 << 5), + /* Macro for Prvileged Process */ + FASTRPC_MODE_PRIVILEGED = (1 << 6), +}; + struct fastrpc_invoke_args { __u64 ptr; __u64 length; -- cgit v1.2.3 From e90d911906196bf987492c94e38f10ca611dfd7b Mon Sep 17 00:00:00 2001 From: Vamsi Krishna Gattupalli Date: Mon, 14 Feb 2022 16:09:58 +0000 Subject: misc: fastrpc: Add support to secure memory map This patch adds support to secure memory allocations for DSP. It repurposes the reserved field in struct fastrpc_invoke_args to add attributes to invoke request, for example to setup a secure memory map for dsp. Secure memory is assigned to DSP Virtual Machine IDs using Qualcomm SCM calls. Signed-off-by: Vamsi Krishna Gattupalli Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20220214161002.6831-9-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/misc/Kconfig | 1 + drivers/misc/fastrpc.c | 65 ++++++++++++++++++++++++++++++++++++++------- include/uapi/misc/fastrpc.h | 5 +++- 3 files changed, 61 insertions(+), 10 deletions(-) (limited to 'include/uapi/misc') diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index a2b26426efba..41d2bb0ae23a 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -259,6 +259,7 @@ config QCOM_FASTRPC depends on ARCH_QCOM || COMPILE_TEST depends on RPMSG select DMA_SHARED_BUFFER + select QCOM_SCM help Provides a communication mechanism that allows for clients to make remote method invocations across processor boundary to diff --git a/drivers/misc/fastrpc.c b/drivers/misc/fastrpc.c index 8fbb24d763bd..ae7586d7956f 100644 --- a/drivers/misc/fastrpc.c +++ b/drivers/misc/fastrpc.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #define ADSP_DOMAIN_ID (0) @@ -25,6 +26,7 @@ #define CDSP_DOMAIN_ID (3) #define FASTRPC_DEV_MAX 4 /* adsp, mdsp, slpi, cdsp*/ #define FASTRPC_MAX_SESSIONS 13 /*12 compute, 1 cpz*/ +#define FASTRPC_MAX_VMIDS 16 #define FASTRPC_ALIGN 128 #define FASTRPC_MAX_FDLIST 16 #define FASTRPC_MAX_CRCLIST 64 @@ -195,6 +197,7 @@ struct fastrpc_map { void *va; u64 len; u64 raddr; + u32 attr; struct kref refcount; }; @@ -232,6 +235,9 @@ struct fastrpc_session_ctx { struct fastrpc_channel_ctx { int domain_id; int sesscount; + int vmcount; + u32 perms; + struct qcom_scm_vmperm vmperms[FASTRPC_MAX_VMIDS]; struct rpmsg_device *rpdev; struct fastrpc_session_ctx session[FASTRPC_MAX_SESSIONS]; spinlock_t lock; @@ -279,6 +285,20 @@ static void fastrpc_free_map(struct kref *ref) map = container_of(ref, struct fastrpc_map, refcount); if (map->table) { + if (map->attr & FASTRPC_ATTR_SECUREMAP) { + struct qcom_scm_vmperm perm; + int err = 0; + + perm.vmid = QCOM_SCM_VMID_HLOS; + perm.perm = QCOM_SCM_PERM_RWX; + err = qcom_scm_assign_mem(map->phys, map->size, + &(map->fl->cctx->vmperms[0].vmid), &perm, 1); + if (err) { + dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", + map->phys, map->size, err); + return; + } + } dma_buf_unmap_attachment(map->attach, map->table, DMA_BIDIRECTIONAL); dma_buf_detach(map->buf, map->attach); @@ -655,7 +675,7 @@ static const struct dma_buf_ops fastrpc_dma_buf_ops = { }; static int fastrpc_map_create(struct fastrpc_user *fl, int fd, - u64 len, struct fastrpc_map **ppmap) + u64 len, u32 attr, struct fastrpc_map **ppmap) { struct fastrpc_session_ctx *sess = fl->sctx; struct fastrpc_map *map = NULL; @@ -697,6 +717,22 @@ static int fastrpc_map_create(struct fastrpc_user *fl, int fd, map->len = len; kref_init(&map->refcount); + if (attr & FASTRPC_ATTR_SECUREMAP) { + /* + * If subsystem VMIDs are defined in DTSI, then do + * hyp_assign from HLOS to those VM(s) + */ + unsigned int perms = BIT(QCOM_SCM_VMID_HLOS); + + map->attr = attr; + err = qcom_scm_assign_mem(map->phys, (u64)map->size, &perms, + fl->cctx->vmperms, fl->cctx->vmcount); + if (err) { + dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d", + map->phys, map->size, err); + goto map_err; + } + } spin_lock(&fl->lock); list_add_tail(&map->node, &fl->maps); spin_unlock(&fl->lock); @@ -781,16 +817,13 @@ static int fastrpc_create_maps(struct fastrpc_invoke_ctx *ctx) int i, err; for (i = 0; i < ctx->nscalars; ++i) { - /* Make sure reserved field is set to 0 */ - if (ctx->args[i].reserved) - return -EINVAL; if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 || ctx->args[i].length == 0) continue; err = fastrpc_map_create(ctx->fl, ctx->args[i].fd, - ctx->args[i].length, &ctx->maps[i]); + ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); if (err) { dev_err(dev, "Error Creating map %d\n", err); return -EINVAL; @@ -1124,7 +1157,7 @@ static int fastrpc_init_create_process(struct fastrpc_user *fl, fl->pd = USER_PD; if (init.filelen && init.filefd) { - err = fastrpc_map_create(fl, init.filefd, init.filelen, &map); + err = fastrpc_map_create(fl, init.filefd, init.filelen, 0, &map); if (err) goto err; } @@ -1233,7 +1266,6 @@ static int fastrpc_release_current_dsp_process(struct fastrpc_user *fl) args[0].ptr = (u64)(uintptr_t) &tgid; args[0].length = sizeof(tgid); args[0].fd = -1; - args[0].reserved = 0; sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_RELEASE, 1, 0); return fastrpc_internal_invoke(fl, true, FASTRPC_INIT_HANDLE, @@ -1381,7 +1413,6 @@ static int fastrpc_init_attach(struct fastrpc_user *fl, int pd) args[0].ptr = (u64)(uintptr_t) &tgid; args[0].length = sizeof(tgid); args[0].fd = -1; - args[0].reserved = 0; sc = FASTRPC_SCALARS(FASTRPC_RMID_INIT_ATTACH, 1, 0); fl->pd = pd; @@ -1954,9 +1985,10 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) { struct device *rdev = &rpdev->dev; struct fastrpc_channel_ctx *data; - int i, err, domain_id = -1; + int i, err, domain_id = -1, vmcount; const char *domain; bool secure_dsp; + unsigned int vmids[FASTRPC_MAX_VMIDS]; err = of_property_read_string(rdev->of_node, "label", &domain); if (err) { @@ -1976,10 +2008,25 @@ static int fastrpc_rpmsg_probe(struct rpmsg_device *rpdev) return -EINVAL; } + vmcount = of_property_read_variable_u32_array(rdev->of_node, + "qcom,vmids", &vmids[0], 0, FASTRPC_MAX_VMIDS); + if (vmcount < 0) + vmcount = 0; + else if (!qcom_scm_is_available()) + return -EPROBE_DEFER; + data = kzalloc(sizeof(*data), GFP_KERNEL); if (!data) return -ENOMEM; + if (vmcount) { + data->vmcount = vmcount; + data->perms = BIT(QCOM_SCM_VMID_HLOS); + for (i = 0; i < data->vmcount; i++) { + data->vmperms[i].vmid = vmids[i]; + data->vmperms[i].perm = QCOM_SCM_PERM_RWX; + } + } secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); data->secure = secure_dsp; diff --git a/include/uapi/misc/fastrpc.h b/include/uapi/misc/fastrpc.h index f39edac20305..5e29f2cfa42d 100644 --- a/include/uapi/misc/fastrpc.h +++ b/include/uapi/misc/fastrpc.h @@ -63,11 +63,14 @@ enum fastrpc_proc_attr { FASTRPC_MODE_PRIVILEGED = (1 << 6), }; +/* Fastrpc attribute for memory protection of buffers */ +#define FASTRPC_ATTR_SECUREMAP (1) + struct fastrpc_invoke_args { __u64 ptr; __u64 length; __s32 fd; - __u32 reserved; + __u32 attr; }; struct fastrpc_invoke { -- cgit v1.2.3 From 1359fcbe0f4aa2cd6ea684727a5a111eebeeed3a Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Wed, 23 Feb 2022 11:47:05 +0200 Subject: habanalabs: add DRAM default page size to HW info When using the device memory allocation API the user ought to know what is the default allocation page size. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay Signed-off-by: Greg Kroah-Hartman --- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 1 + include/uapi/misc/habanalabs.h | 3 +++ 2 files changed, 4 insertions(+) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index c13a3c2a7013..14c58579b9cd 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -76,6 +76,7 @@ static int hw_ip_info(struct hl_device *hdev, struct hl_info_args *args) if (hw_ip.dram_size > PAGE_SIZE) hw_ip.dram_enabled = 1; hw_ip.dram_page_size = prop->dram_page_size; + hw_ip.device_mem_alloc_default_page_size = prop->device_mem_alloc_default_page_size; hw_ip.num_of_events = prop->num_of_events; memcpy(hw_ip.cpucp_version, prop->cpucp_info.cpucp_version, diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 1d6b4f0c4159..ae2441521467 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -409,6 +409,7 @@ enum hl_server_type { * @dram_page_size: The DRAM physical page size. * @number_of_user_interrupts: The number of interrupts that are available to the userspace * application to use. Relevant for Gaudi2 and later. + * @device_mem_alloc_default_page_size: default page size used in device memory allocation. */ struct hl_info_hw_ip_info { __u64 sram_base_address; @@ -436,6 +437,8 @@ struct hl_info_hw_ip_info { __u32 reserved3; __u16 number_of_user_interrupts; __u16 pad2; + __u64 reserved4; + __u64 device_mem_alloc_default_page_size; }; struct hl_info_dram_usage { -- cgit v1.2.3 From 050a6f349a09d3cefb14f4114bfa047b2c5b2a65 Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Wed, 23 Feb 2022 13:37:08 +0200 Subject: habanalabs: add user API to get valid DRAM page sizes Future devices will support multiple device memory page sizes. In addition, an API for the user was added for it to be able to control the device memory allocation page size. This patch is a complementary patch to inform the user of the available page size supported by the device. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay Signed-off-by: Greg Kroah-Hartman --- drivers/misc/habanalabs/common/habanalabs.h | 2 + drivers/misc/habanalabs/common/habanalabs_ioctl.c | 24 +++++++++ drivers/misc/habanalabs/gaudi/gaudi.c | 7 +++ drivers/misc/habanalabs/goya/goya.c | 7 +++ include/uapi/misc/habanalabs.h | 63 +++++++++++++---------- 5 files changed, 77 insertions(+), 26 deletions(-) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 6eb35e4124c2..564797766f42 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1304,6 +1304,7 @@ struct fw_load_mgr { * @get_stream_master_qid_arr: get pointer to stream masters QID array * @is_valid_dram_page_size: return true if page size is supported in device * memory allocation, otherwise false. + * @get_valid_dram_page_orders: get valid device memory allocation page orders */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -1432,6 +1433,7 @@ struct hl_asic_funcs { bool (*is_valid_dram_page_size)(u32 page_size); int (*mmu_get_real_page_size)(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop, u32 page_size, u32 *real_page_size, bool is_dram_addr); + void (*get_valid_dram_page_orders)(struct hl_info_dev_memalloc_page_sizes *info); }; diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 14c58579b9cd..c6fe35ae1238 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -591,6 +591,27 @@ static int razwi_info(struct hl_fpriv *hpriv, struct hl_info_args *args) return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; } +static int dev_mem_alloc_page_sizes_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + struct hl_info_dev_memalloc_page_sizes info = {0}; + struct hl_device *hdev = hpriv->hdev; + u32 max_size = args->return_size; + + if ((!max_size) || (!out)) + return -EINVAL; + + /* + * Future ASICs that will support multiple DRAM page sizes will support only "powers of 2" + * pages (unlike some of the ASICs before supporting multiple page sizes). + * For this reason for all ASICs that not support multiple page size the function will + * return an empty bitmask indicating that multiple page sizes is not supported. + */ + hdev->asic_funcs->get_valid_dram_page_orders(&info); + + return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -641,6 +662,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_RAZWI_EVENT: return razwi_info(hpriv, args); + case HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES: + return dev_mem_alloc_page_sizes_info(hpriv, args); + default: break; } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 5979434d1905..64cb195bc26e 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -9378,6 +9378,12 @@ static u32 *gaudi_get_stream_master_qid_arr(void) return gaudi_stream_master; } +static void gaudi_get_valid_dram_page_orders(struct hl_info_dev_memalloc_page_sizes *info) +{ + /* set 0 since multiple pages are not supported */ + info->page_order_bitmask = 0; +} + static ssize_t infineon_ver_show(struct device *dev, struct device_attribute *attr, char *buf) { struct hl_device *hdev = dev_get_drvdata(dev); @@ -9489,6 +9495,7 @@ static const struct hl_asic_funcs gaudi_funcs = { .get_stream_master_qid_arr = gaudi_get_stream_master_qid_arr, .is_valid_dram_page_size = NULL, .mmu_get_real_page_size = hl_mmu_get_real_page_size, + .get_valid_dram_page_orders = gaudi_get_valid_dram_page_orders, }; /** diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index ec347bd3bb69..b18288070754 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5679,6 +5679,12 @@ static u32 *goya_get_stream_master_qid_arr(void) return NULL; } +static void goya_get_valid_dram_page_orders(struct hl_info_dev_memalloc_page_sizes *info) +{ + /* set 0 since multiple pages are not supported */ + info->page_order_bitmask = 0; +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5767,6 +5773,7 @@ static const struct hl_asic_funcs goya_funcs = { .get_stream_master_qid_arr = goya_get_stream_master_qid_arr, .is_valid_dram_page_size = NULL, .mmu_get_real_page_size = hl_mmu_get_real_page_size, + .get_valid_dram_page_orders = goya_get_valid_dram_page_orders, }; /* diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index ae2441521467..f474e7fb018d 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -348,33 +348,35 @@ enum hl_server_type { * The address which accessing it caused the razwi. * Razwi initiator. * Razwi cause, was it a page fault or MMU access error. + * HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES - Retrieve valid page sizes for device memory allocation */ -#define HL_INFO_HW_IP_INFO 0 -#define HL_INFO_HW_EVENTS 1 -#define HL_INFO_DRAM_USAGE 2 -#define HL_INFO_HW_IDLE 3 -#define HL_INFO_DEVICE_STATUS 4 -#define HL_INFO_DEVICE_UTILIZATION 6 -#define HL_INFO_HW_EVENTS_AGGREGATE 7 -#define HL_INFO_CLK_RATE 8 -#define HL_INFO_RESET_COUNT 9 -#define HL_INFO_TIME_SYNC 10 -#define HL_INFO_CS_COUNTERS 11 -#define HL_INFO_PCI_COUNTERS 12 -#define HL_INFO_CLK_THROTTLE_REASON 13 -#define HL_INFO_SYNC_MANAGER 14 -#define HL_INFO_TOTAL_ENERGY 15 -#define HL_INFO_PLL_FREQUENCY 16 -#define HL_INFO_POWER 17 -#define HL_INFO_OPEN_STATS 18 -#define HL_INFO_DRAM_REPLACED_ROWS 21 -#define HL_INFO_DRAM_PENDING_ROWS 22 -#define HL_INFO_LAST_ERR_OPEN_DEV_TIME 23 -#define HL_INFO_CS_TIMEOUT_EVENT 24 -#define HL_INFO_RAZWI_EVENT 25 - -#define HL_INFO_VERSION_MAX_LEN 128 -#define HL_INFO_CARD_NAME_MAX_LEN 16 +#define HL_INFO_HW_IP_INFO 0 +#define HL_INFO_HW_EVENTS 1 +#define HL_INFO_DRAM_USAGE 2 +#define HL_INFO_HW_IDLE 3 +#define HL_INFO_DEVICE_STATUS 4 +#define HL_INFO_DEVICE_UTILIZATION 6 +#define HL_INFO_HW_EVENTS_AGGREGATE 7 +#define HL_INFO_CLK_RATE 8 +#define HL_INFO_RESET_COUNT 9 +#define HL_INFO_TIME_SYNC 10 +#define HL_INFO_CS_COUNTERS 11 +#define HL_INFO_PCI_COUNTERS 12 +#define HL_INFO_CLK_THROTTLE_REASON 13 +#define HL_INFO_SYNC_MANAGER 14 +#define HL_INFO_TOTAL_ENERGY 15 +#define HL_INFO_PLL_FREQUENCY 16 +#define HL_INFO_POWER 17 +#define HL_INFO_OPEN_STATS 18 +#define HL_INFO_DRAM_REPLACED_ROWS 21 +#define HL_INFO_DRAM_PENDING_ROWS 22 +#define HL_INFO_LAST_ERR_OPEN_DEV_TIME 23 +#define HL_INFO_CS_TIMEOUT_EVENT 24 +#define HL_INFO_RAZWI_EVENT 25 +#define HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES 26 + +#define HL_INFO_VERSION_MAX_LEN 128 +#define HL_INFO_CARD_NAME_MAX_LEN 16 /** * struct hl_info_hw_ip_info - hardware information on various IPs in the ASIC @@ -643,6 +645,15 @@ struct hl_info_razwi_event { __u8 pad[2]; }; +/** + * struct hl_info_dev_memalloc_page_sizes - valid page sizes in device mem alloc information. + * @page_order_bitmask: bitmap in which a set bit represents the order of the supported page size + * (e.g. 0x2100000 means that 1MB and 32MB pages are supported). + */ +struct hl_info_dev_memalloc_page_sizes { + __u64 page_order_bitmask; +}; + enum gaudi_dcores { HL_GAUDI_WS_DCORE, HL_GAUDI_WN_DCORE, -- cgit v1.2.3 From fdec56c1a416c6947b1db22617da15cb89f46c6c Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 3 Mar 2022 09:43:10 +0200 Subject: habanalabs: expose compute ctx status through info ioctl In order for the user to know if he can try and open device, we expose the compute ctx state. The user can now know if the context is used by another process or whether the device is still ongoing through cleanup or reset and will be available soon. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay Signed-off-by: Greg Kroah-Hartman --- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 2 ++ include/uapi/misc/habanalabs.h | 5 +++++ 2 files changed, 7 insertions(+) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index c6fe35ae1238..bfb5cfe68110 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -498,6 +498,8 @@ static int open_stats_info(struct hl_fpriv *hpriv, struct hl_info_args *args) open_stats_info.last_open_period_ms = jiffies64_to_msecs( hdev->last_open_session_duration_jif); open_stats_info.open_counter = hdev->open_counter; + open_stats_info.is_compute_ctx_active = hdev->is_compute_ctx_active; + open_stats_info.compute_ctx_in_release = hdev->compute_ctx_in_release; return copy_to_user(out, &open_stats_info, min((size_t) max_size, sizeof(open_stats_info))) ? -EFAULT : 0; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index f474e7fb018d..ca2af5f98056 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -543,10 +543,15 @@ struct hl_pll_frequency_info { * struct hl_open_stats_info - device open statistics information * @open_counter: ever growing counter, increased on each successful dev open * @last_open_period_ms: duration (ms) device was open last time + * @is_compute_ctx_active: Whether there is an active compute context executing + * @compute_ctx_in_release: true if the current compute context is being released */ struct hl_open_stats_info { __u64 open_counter; __u64 last_open_period_ms; + __u8 is_compute_ctx_active; + __u8 compute_ctx_in_release; + __u8 pad[6]; }; /** -- cgit v1.2.3 From 5d1a0de2c778f369970dd50f6713e95068926a8b Mon Sep 17 00:00:00 2001 From: Ohad Sharabi Date: Sun, 10 Apr 2022 11:19:42 +0300 Subject: habanalabs: add prefetch flag to the MAP operation This patch let the user decide whether the translations done in the page tables will be fetched directly to the STLB right after the map. We want to let the user control whether to perform prefetch upon map operation. To do so a memory flag was added, to be used in the MAP ioctl, called HL_MEM_PREFETCH and if set- the mappings will be fetched directly to the STLB after map operation. Signed-off-by: Ohad Sharabi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay Signed-off-by: Greg Kroah-Hartman --- drivers/misc/habanalabs/common/memory.c | 11 ++++++----- include/uapi/misc/habanalabs.h | 1 + 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 326c2179628f..6face45c57e3 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -1250,11 +1250,12 @@ static int map_device_va(struct hl_ctx *ctx, struct hl_mem_in *args, if (rc) goto map_err; - /* already prefetch the relevant translations to the cache */ - rc = hl_mmu_prefetch_cache_range(hdev, *vm_type, ctx->asid, ret_vaddr, - phys_pg_pack->total_size); - if (rc) - goto map_err; + if (args->flags & HL_MEM_PREFETCH) { + rc = hl_mmu_prefetch_cache_range(hdev, *vm_type, ctx->asid, ret_vaddr, + phys_pg_pack->total_size); + if (rc) + goto map_err; + } mutex_unlock(&ctx->mmu_lock); diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index ca2af5f98056..3576bf2b4841 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -1134,6 +1134,7 @@ union hl_wait_cs_args { #define HL_MEM_SHARED 0x2 #define HL_MEM_USERPTR 0x4 #define HL_MEM_FORCE_HINT 0x8 +#define HL_MEM_PREFETCH 0x40 /** * structure hl_mem_in - structure that handle input args for memory IOCTL -- cgit v1.2.3 From 422ef171038d4855ffe938137039a8f3b3e84293 Mon Sep 17 00:00:00 2001 From: Tal Cohen Date: Thu, 28 Apr 2022 13:45:18 +0300 Subject: habanalabs: add support for notification via eventfd The driver will be able to send notification events towards a user process, using user's registered event file descriptor. The driver uses the notification mechanism to inform the user about an occurred event. A user thread can wait until a notification is received from the driver. The driver stores the occurred event until the user reads it, using HL_INFO_GET_EVENTS - new ioctl opcode in the INFO ioctl. Gaudi specific implementation includes sending a notification on a TPC assertion event that is received from f/w. Signed-off-by: Tal Cohen Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay Signed-off-by: Greg Kroah-Hartman --- drivers/misc/habanalabs/common/device.c | 52 ++++++++++++++++++ drivers/misc/habanalabs/common/habanalabs.h | 40 +++++++++----- drivers/misc/habanalabs/common/habanalabs_drv.c | 9 ++++ drivers/misc/habanalabs/common/habanalabs_ioctl.c | 65 +++++++++++++++++++++++ drivers/misc/habanalabs/gaudi/gaudi.c | 14 ++++- include/uapi/misc/habanalabs.h | 15 ++++++ 6 files changed, 182 insertions(+), 13 deletions(-) (limited to 'include/uapi/misc') diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 15df89b31e1b..315510aaca35 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -285,6 +285,14 @@ static void hpriv_release(struct kref *ref) hdev->compute_ctx_in_release = 0; + /* release the eventfd */ + if (hpriv->notifier_event.eventfd) { + eventfd_ctx_put(hpriv->notifier_event.eventfd); + hpriv->notifier_event.eventfd = 0; + } + + mutex_destroy(&hpriv->notifier_event.lock); + kfree(hpriv); } @@ -355,6 +363,13 @@ static int hl_device_release_ctrl(struct inode *inode, struct file *filp) list_del(&hpriv->dev_node); mutex_unlock(&hdev->fpriv_ctrl_list_lock); out: + /* release the eventfd */ + if (hpriv->notifier_event.eventfd) { + eventfd_ctx_put(hpriv->notifier_event.eventfd); + hpriv->notifier_event.eventfd = 0; + } + + mutex_destroy(&hpriv->notifier_event.lock); put_pid(hpriv->taskpid); kfree(hpriv); @@ -1506,6 +1521,43 @@ out_err: return rc; } +static void hl_notifier_event_send(struct hl_notifier_event *notifier_event, u64 event) +{ + mutex_lock(¬ifier_event->lock); + notifier_event->events_mask |= event; + if (notifier_event->eventfd) + eventfd_signal(notifier_event->eventfd, 1); + + mutex_unlock(¬ifier_event->lock); +} + +/* + * hl_notifier_event_send_all - notify all user processes via eventfd + * + * @hdev: pointer to habanalabs device structure + * @event: the occurred event + * Returns 0 for success or an error on failure. + */ +void hl_notifier_event_send_all(struct hl_device *hdev, u64 event) +{ + struct hl_fpriv *hpriv; + + mutex_lock(&hdev->fpriv_list_lock); + + list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) + hl_notifier_event_send(&hpriv->notifier_event, event); + + mutex_unlock(&hdev->fpriv_list_lock); + + /* control device */ + mutex_lock(&hdev->fpriv_ctrl_list_lock); + + list_for_each_entry(hpriv, &hdev->fpriv_ctrl_list, dev_node) + hl_notifier_event_send(&hpriv->notifier_event, event); + + mutex_unlock(&hdev->fpriv_ctrl_list_lock); +} + /* * hl_device_init - main initialization function for habanalabs device * diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 918e8a04acab..8977ec67dba7 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1932,6 +1933,18 @@ struct hl_debug_params { bool enable; }; +/** + * struct hl_notifier_event - holds the notifier data structure + * @eventfd: the event file descriptor to raise the notifications + * @lock: mutex lock to protect the notifier data flows + * @events_mask: indicates the bitmap events + */ +struct hl_notifier_event { + struct eventfd_ctx *eventfd; + struct mutex lock; + u64 events_mask; +}; + /* * FILE PRIVATE STRUCTURE */ @@ -1943,24 +1956,25 @@ struct hl_debug_params { * @taskpid: current process ID. * @ctx: current executing context. TODO: remove for multiple ctx per process * @ctx_mgr: context manager to handle multiple context for this FD. - * @cb_mgr: command buffer manager to handle multiple buffers for this FD. * @mem_mgr: manager descriptor for memory exportable via mmap + * @notifier_event: notifier eventfd towards user process * @debugfs_list: list of relevant ASIC debugfs. * @dev_node: node in the device list of file private data * @refcount: number of related contexts. * @restore_phase_mutex: lock for context switch and restore phase. */ struct hl_fpriv { - struct hl_device *hdev; - struct file *filp; - struct pid *taskpid; - struct hl_ctx *ctx; - struct hl_ctx_mgr ctx_mgr; - struct hl_mem_mgr mem_mgr; - struct list_head debugfs_list; - struct list_head dev_node; - struct kref refcount; - struct mutex restore_phase_mutex; + struct hl_device *hdev; + struct file *filp; + struct pid *taskpid; + struct hl_ctx *ctx; + struct hl_ctx_mgr ctx_mgr; + struct hl_mem_mgr mem_mgr; + struct hl_notifier_event notifier_event; + struct list_head debugfs_list; + struct list_head dev_node; + struct kref refcount; + struct mutex restore_phase_mutex; }; @@ -2676,8 +2690,8 @@ struct hl_reset_info { * @state_dump_specs: constants and dictionaries needed to dump system state. * @multi_cs_completion: array of multi-CS completion. * @clk_throttling: holds information about current/previous clock throttling events - * @reset_info: holds current device reset information. * @last_error: holds information about last session in which CS timeout or razwi error occurred. + * @reset_info: holds current device reset information. * @stream_master_qid_arr: pointer to array with QIDs of master streams. * @fw_major_version: major version of current loaded preboot * @dram_used_mem: current DRAM memory consumption. @@ -3071,6 +3085,8 @@ int hl_device_utilization(struct hl_device *hdev, u32 *utilization); int hl_build_hwmon_channel_info(struct hl_device *hdev, struct cpucp_sensor *sensors_arr); +void hl_notifier_event_send_all(struct hl_device *hdev, u64 event); + int hl_sysfs_init(struct hl_device *hdev); void hl_sysfs_fini(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index 1210de39d661..c97173e9507d 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -134,6 +134,10 @@ int hl_device_open(struct inode *inode, struct file *filp) hpriv->hdev = hdev; filp->private_data = hpriv; hpriv->filp = filp; + hpriv->notifier_event.events_mask = 0; + hpriv->notifier_event.eventfd = 0; + + mutex_init(&hpriv->notifier_event.lock); mutex_init(&hpriv->restore_phase_mutex); kref_init(&hpriv->refcount); nonseekable_open(inode, filp); @@ -208,6 +212,7 @@ out_err: hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr); filp->private_data = NULL; mutex_destroy(&hpriv->restore_phase_mutex); + mutex_destroy(&hpriv->notifier_event.lock); put_pid(hpriv->taskpid); kfree(hpriv); @@ -241,6 +246,10 @@ int hl_device_open_ctrl(struct inode *inode, struct file *filp) hpriv->hdev = hdev; filp->private_data = hpriv; hpriv->filp = filp; + hpriv->notifier_event.events_mask = 0; + hpriv->notifier_event.eventfd = 0; + + mutex_init(&hpriv->notifier_event.lock); nonseekable_open(inode, filp); hpriv->taskpid = get_task_pid(current, PIDTYPE_PID); diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index bfb5cfe68110..d1ef56a8d3ac 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -116,6 +116,25 @@ static int hw_events_info(struct hl_device *hdev, bool aggregate, return copy_to_user(out, arr, min(max_size, size)) ? -EFAULT : 0; } +static int events_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + int rc; + u32 max_size = args->return_size; + u64 events_mask; + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + + if ((max_size < sizeof(u64)) || (!out)) + return -EINVAL; + + mutex_lock(&hpriv->notifier_event.lock); + events_mask = hpriv->notifier_event.events_mask; + hpriv->notifier_event.events_mask = 0; + mutex_unlock(&hpriv->notifier_event.lock); + + rc = copy_to_user(out, &events_mask, sizeof(u64)); + return rc; +} + static int dram_usage_info(struct hl_fpriv *hpriv, struct hl_info_args *args) { struct hl_device *hdev = hpriv->hdev; @@ -614,6 +633,43 @@ static int dev_mem_alloc_page_sizes_info(struct hl_fpriv *hpriv, struct hl_info_ return copy_to_user(out, &info, min_t(size_t, max_size, sizeof(info))) ? -EFAULT : 0; } +static int eventfd_register(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + int rc; + + /* check if there is already a registered on that process */ + mutex_lock(&hpriv->notifier_event.lock); + if (hpriv->notifier_event.eventfd) { + mutex_unlock(&hpriv->notifier_event.lock); + return -EINVAL; + } + + hpriv->notifier_event.eventfd = eventfd_ctx_fdget(args->eventfd); + if (IS_ERR(hpriv->notifier_event.eventfd)) { + rc = PTR_ERR(hpriv->notifier_event.eventfd); + hpriv->notifier_event.eventfd = 0; + mutex_unlock(&hpriv->notifier_event.lock); + return rc; + } + + mutex_unlock(&hpriv->notifier_event.lock); + return 0; +} + +static int eventfd_unregister(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + mutex_lock(&hpriv->notifier_event.lock); + if (!hpriv->notifier_event.eventfd) { + mutex_unlock(&hpriv->notifier_event.lock); + return -EINVAL; + } + + eventfd_ctx_put(hpriv->notifier_event.eventfd); + hpriv->notifier_event.eventfd = 0; + mutex_unlock(&hpriv->notifier_event.lock); + return 0; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -667,6 +723,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES: return dev_mem_alloc_page_sizes_info(hpriv, args); + case HL_INFO_GET_EVENTS: + return events_info(hpriv, args); + default: break; } @@ -717,6 +776,12 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_DRAM_PENDING_ROWS: return dram_pending_rows_info(hpriv, args); + case HL_INFO_REGISTER_EVENTFD: + return eventfd_register(hpriv, args); + + case HL_INFO_UNREGISTER_EVENTFD: + return eventfd_unregister(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -EINVAL; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 08cd60300b4f..1c388537de33 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -7879,7 +7879,6 @@ static void gaudi_handle_eqe(struct hl_device *hdev, case GAUDI_EVENT_MMU_PAGE_FAULT: case GAUDI_EVENT_MMU_WR_PERM: case GAUDI_EVENT_RAZWI_OR_ADC: - case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: case GAUDI_EVENT_MME0_QM ... GAUDI_EVENT_MME2_QM: case GAUDI_EVENT_DMA0_QM ... GAUDI_EVENT_DMA7_QM: fallthrough; @@ -7899,6 +7898,19 @@ static void gaudi_handle_eqe(struct hl_device *hdev, hl_fw_unmask_irq(hdev, event_type); break; + case GAUDI_EVENT_TPC0_QM ... GAUDI_EVENT_TPC7_QM: + gaudi_print_irq_info(hdev, event_type, true); + gaudi_handle_qman_err(hdev, event_type); + hl_fw_unmask_irq(hdev, event_type); + + /* In TPC QM event, notify on TPC assertion. While there isn't + * a specific event for assertion yet, the FW generates QM event. + * The SW upper layer will inspect an internal mapped area to indicate + * if the event is a tpc assertion or tpc QM. + */ + hl_notifier_event_send_all(hdev, HL_NOTIFIER_EVENT_TPC_ASSERT); + break; + case GAUDI_EVENT_RAZWI_OR_ADC_SW: gaudi_print_irq_info(hdev, event_type, true); goto reset_device; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 3576bf2b4841..52540d5b4fc9 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -349,6 +349,9 @@ enum hl_server_type { * Razwi initiator. * Razwi cause, was it a page fault or MMU access error. * HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES - Retrieve valid page sizes for device memory allocation + * HL_INFO_REGISTER_EVENTFD - Register eventfd for event notifications. + * HL_INFO_UNREGISTER_EVENTFD - Unregister eventfd + * HL_INFO_GET_EVENTS - Retrieve the last occurred events */ #define HL_INFO_HW_IP_INFO 0 #define HL_INFO_HW_EVENTS 1 @@ -374,6 +377,9 @@ enum hl_server_type { #define HL_INFO_CS_TIMEOUT_EVENT 24 #define HL_INFO_RAZWI_EVENT 25 #define HL_INFO_DEV_MEM_ALLOC_PAGE_SIZES 26 +#define HL_INFO_REGISTER_EVENTFD 28 +#define HL_INFO_UNREGISTER_EVENTFD 29 +#define HL_INFO_GET_EVENTS 30 #define HL_INFO_VERSION_MAX_LEN 128 #define HL_INFO_CARD_NAME_MAX_LEN 16 @@ -679,6 +685,7 @@ enum gaudi_dcores { * @period_ms: Period value, in milliseconds, for utilization rate in range 100ms - 1000ms in 100 ms * resolution. Currently not in use. * @pll_index: Index as defined in hl__pll_index enumeration. + * @eventfd: event file descriptor for event notifications. * @pad: Padding to 64 bit. */ struct hl_info_args { @@ -691,6 +698,7 @@ struct hl_info_args { __u32 ctx_id; __u32 period_ms; __u32 pll_index; + __u32 eventfd; }; __u32 pad; @@ -1390,6 +1398,13 @@ struct hl_debug_args { __u32 ctx_id; }; +/* + * Notifier event values - for the notification mechanism and the HL_INFO_GET_EVENTS command + * + * HL_NOTIFIER_EVENT_TPC_ASSERT - Indicates TPC assert event + */ +#define HL_NOTIFIER_EVENT_TPC_ASSERT (1 << 0) + /* * Various information operations such as: * - H/W IP information -- cgit v1.2.3